| author | |
| committer | |
| log | 3a5777b6b10927eca53ae1e607011a9779f2e7b1 |
| tree | b5b3d2cf0db306d439f29bbf07fe68d41ce04d01 |
| parent | 722aa2a8ab4aeac4f47d79569d5d0d73406fa0f2 |
build summary now tells why a cache miss occurred, if it was an uncommon
case:
* invalid cache manifest
* a specific file was invalidated
* directory contents changed
* metadata changed
Cache miss due to it being the first time a particular build step is
being executed remains silent.
Not all kinds of build steps have this extra data about which files
caused cache invalidation, most notably Compile steps.10 files changed, 102 insertions(+), 41 deletions(-)
lib/compiler/Maker.zig+40-19| ... | ... | @@ -2618,7 +2618,7 @@ fn markFailedStepsDirty(maker: *Maker) void { |
| 2618 | 2618 | for (all_steps) |step_index| { |
| 2619 | 2619 | const step = maker.stepByIndex(step_index); |
| 2620 | 2620 | switch (step.state) { |
| 2621 | .success => step.result_cached = true, | |
| 2621 | .success => step.result_cache_status = .hit, | |
| 2622 | 2622 | else => continue, |
| 2623 | 2623 | } |
| 2624 | 2624 | } |
| ... | ... | @@ -3004,7 +3004,7 @@ fn makeSteps( |
| 3004 | 3004 | const found = switch (maker.summary) { |
| 3005 | 3005 | .all, .line, .none => unreachable, |
| 3006 | 3006 | .failures => step.state != .success, |
| 3007 | .new => !step.result_cached, | |
| 3007 | .new => step.result_cache_status != .hit, | |
| 3008 | 3008 | }; |
| 3009 | 3009 | if (found) break :blk i; |
| 3010 | 3010 | } |
| ... | ... | @@ -3242,7 +3242,7 @@ fn printTreeStep( |
| 3242 | 3242 | const skip = switch (summary) { |
| 3243 | 3243 | .none, .line => unreachable, |
| 3244 | 3244 | .all => false, |
| 3245 | .new => make_step.result_cached, | |
| 3245 | .new => make_step.result_cache_status == .hit, | |
| 3246 | 3246 | .failures => make_step.state == .success, |
| 3247 | 3247 | }; |
| 3248 | 3248 | if (skip) return; |
| ... | ... | @@ -3268,7 +3268,7 @@ fn printTreeStep( |
| 3268 | 3268 | const found = switch (summary) { |
| 3269 | 3269 | .all, .line, .none => unreachable, |
| 3270 | 3270 | .failures => dep.state != .success, |
| 3271 | .new => !dep.result_cached, | |
| 3271 | .new => dep.result_cache_status != .hit, | |
| 3272 | 3272 | }; |
| 3273 | 3273 | if (found) break :blk i; |
| 3274 | 3274 | } |
| ... | ... | @@ -3312,24 +3312,45 @@ fn printStepStatus(maker: *Maker, step_index: Configuration.Step.Index, stderr: |
| 3312 | 3312 | }, |
| 3313 | 3313 | |
| 3314 | 3314 | .success => { |
| 3315 | try stderr.setColor(.green); | |
| 3316 | if (s.result_cached) { | |
| 3317 | try writer.writeAll(" cached"); | |
| 3318 | } else if (s.test_results.test_count > 0) { | |
| 3319 | const pass_count = s.test_results.passCount(); | |
| 3320 | assert(s.test_results.test_count == pass_count + s.test_results.skip_count); | |
| 3321 | try writer.print(" {d} pass", .{pass_count}); | |
| 3322 | if (s.test_results.skip_count > 0) { | |
| 3315 | cs: { | |
| 3316 | switch (s.result_cache_status) { | |
| 3317 | .hit => { | |
| 3318 | try stderr.setColor(.green); | |
| 3319 | try writer.writeAll(" cached"); | |
| 3320 | break :cs; | |
| 3321 | }, | |
| 3322 | .incomplete_manifest => { | |
| 3323 | // Common case: first-time cache miss, print nothing. | |
| 3324 | try stderr.setColor(.green); | |
| 3325 | }, | |
| 3326 | .invalid_manifest => { | |
| 3327 | try stderr.setColor(.blue); | |
| 3328 | try writer.writeAll(" invalid_manifest"); | |
| 3329 | try stderr.setColor(.green); | |
| 3330 | }, | |
| 3331 | .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |path, tag| { | |
| 3332 | try stderr.setColor(.blue); | |
| 3333 | try writer.print(" {t}: {s}", .{ tag, path }); | |
| 3334 | try stderr.setColor(.green); | |
| 3335 | }, | |
| 3336 | } | |
| 3337 | if (s.test_results.test_count > 0) { | |
| 3338 | const pass_count = s.test_results.passCount(); | |
| 3339 | assert(s.test_results.test_count == pass_count + s.test_results.skip_count); | |
| 3340 | try writer.print(" {d} pass", .{pass_count}); | |
| 3341 | if (s.test_results.skip_count > 0) { | |
| 3342 | try stderr.setColor(.reset); | |
| 3343 | try writer.writeAll(", "); | |
| 3344 | try stderr.setColor(.yellow); | |
| 3345 | try writer.print("{d} skip", .{s.test_results.skip_count}); | |
| 3346 | } | |
| 3323 | 3347 | try stderr.setColor(.reset); |
| 3324 | try writer.writeAll(", "); | |
| 3325 | try stderr.setColor(.yellow); | |
| 3326 | try writer.print("{d} skip", .{s.test_results.skip_count}); | |
| 3348 | try writer.print(" ({d} total)", .{s.test_results.test_count}); | |
| 3349 | } else { | |
| 3350 | try writer.writeAll(" success"); | |
| 3327 | 3351 | } |
| 3328 | try stderr.setColor(.reset); | |
| 3329 | try writer.print(" ({d} total)", .{s.test_results.test_count}); | |
| 3330 | } else { | |
| 3331 | try writer.writeAll(" success"); | |
| 3332 | 3352 | } |
| 3353 | ||
| 3333 | 3354 | try stderr.setColor(.reset); |
| 3334 | 3355 | if (s.result_duration_ns) |ns| { |
| 3335 | 3356 | try stderr.setColor(.dim); |
lib/compiler/Maker/Step.zig+46-12| ... | ... | @@ -59,7 +59,8 @@ result_error_msgs: std.ArrayList([]const u8) = .empty, |
| 59 | 59 | result_error_bundle: std.zig.ErrorBundle = .empty, |
| 60 | 60 | /// Owned by `Maker.gpa`. |
| 61 | 61 | result_stderr: []const u8 = "", |
| 62 | result_cached: bool = false, | |
| 62 | /// Owned by `Maker.gpa`. | |
| 63 | result_cache_status: ResultCacheStatus = .incomplete_manifest, | |
| 63 | 64 | /// Indicates error information is missing due to allocation failure. |
| 64 | 65 | result_oom: bool = false, |
| 65 | 66 | result_duration_ns: ?u64 = null, |
| ... | ... | @@ -78,6 +79,16 @@ comptime { |
| 78 | 79 | if (std.atomic.cache_line <= 128) assert(@sizeOf(@This()) <= 128 * 3); |
| 79 | 80 | } |
| 80 | 81 | |
| 82 | const ResultCacheStatus = union(enum) { | |
| 83 | hit, | |
| 84 | incomplete_manifest, | |
| 85 | invalid_manifest, | |
| 86 | path_deleted: [*:0]const u8, | |
| 87 | directory_status_changed: [*:0]const u8, | |
| 88 | metadata_changed: [*:0]const u8, | |
| 89 | contents_changed: [*:0]const u8, | |
| 90 | }; | |
| 91 | ||
| 81 | 92 | pub const Extended = union(enum) { |
| 82 | 93 | check_file: CheckFile, |
| 83 | 94 | compile: Compile, |
| ... | ... | @@ -314,6 +325,7 @@ pub fn make( |
| 314 | 325 | |
| 315 | 326 | pub fn deinit(step: *Step, gpa: Allocator, io: Io) void { |
| 316 | 327 | step.clearResultStderr(gpa); |
| 328 | step.clearResultCacheStatus(gpa); | |
| 317 | 329 | step.clearFailedCommand(gpa); |
| 318 | 330 | step.clearErrorBundle(gpa); |
| 319 | 331 | step.inputs.deinit(gpa); |
| ... | ... | @@ -344,8 +356,8 @@ pub fn reset(step: *Step, maker: *Maker) void { |
| 344 | 356 | |
| 345 | 357 | clearFailedCommand(step, gpa); |
| 346 | 358 | clearResultStderr(step, gpa); |
| 359 | clearResultCacheStatus(step, gpa); | |
| 347 | 360 | step.result_error_msgs.clearRetainingCapacity(); |
| 348 | step.result_cached = false; | |
| 349 | 361 | step.result_duration_ns = null; |
| 350 | 362 | step.result_peak_rss = 0; |
| 351 | 363 | step.test_results = .{}; |
| ... | ... | @@ -650,7 +662,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi |
| 650 | 662 | .emit_digest => { |
| 651 | 663 | const emit_digest = body_r.takeStruct(std.zig.Server.Message.EmitDigest, .little) catch unreachable; |
| 652 | 664 | const digest = body_r.takeArray(Cache.bin_digest_len) catch unreachable; |
| 653 | s.result_cached = emit_digest.flags.cache_hit; | |
| 665 | s.result_cache_status = if (emit_digest.flags.cache_hit) .hit else .incomplete_manifest; | |
| 654 | 666 | result = .{ .bin = digest.* }; |
| 655 | 667 | }, |
| 656 | 668 | .file_system_inputs => { |
| ... | ... | @@ -779,9 +791,7 @@ pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_n |
| 779 | 791 | var diag: Cache.Manifest.CheckDiagnostic = undefined; |
| 780 | 792 | const status = man.check(&diag, parent_progress_node) catch |err| |
| 781 | 793 | return failWithCacheError(s, maker, man, &diag, err); |
| 782 | const hit = .hit == status; // TODO cache miss reason in the build summary | |
| 783 | s.result_cached = hit; | |
| 784 | return hit; | |
| 794 | return setResultCacheStatus(s, man, status); | |
| 785 | 795 | } |
| 786 | 796 | |
| 787 | 797 | /// Clears previous watch inputs, if any, and then populates watch inputs from |
| ... | ... | @@ -792,12 +802,36 @@ pub fn cacheHitWatched(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_pro |
| 792 | 802 | var diag: Cache.Manifest.CheckDiagnostic = undefined; |
| 793 | 803 | const status = man.check(&diag, parent_progress_node) catch |err| |
| 794 | 804 | return failWithCacheError(s, maker, man, &diag, err); |
| 795 | const hit = .hit == status; // TODO cache miss reason in the build summary | |
| 796 | s.result_cached = hit; | |
| 797 | // The above call to hit() populates the manifest with files, so in case of | |
| 798 | // a hit, we need to populate watch inputs. | |
| 799 | if (hit) try setWatchInputsFromManifest(s, maker, man); | |
| 800 | return hit; | |
| 805 | // The above call to check() populates the manifest with files, so in case of a hit, we need to | |
| 806 | // populate watch inputs. | |
| 807 | if (status == .hit) try setWatchInputsFromManifest(s, maker, man); | |
| 808 | return setResultCacheStatus(s, man, status); | |
| 809 | } | |
| 810 | ||
| 811 | fn setResultCacheStatus(s: *Step, man: *const Cache.Manifest, status: Cache.Manifest.CheckResult) Allocator.Error!bool { | |
| 812 | const gpa = man.cache.gpa; | |
| 813 | clearResultCacheStatus(s, gpa); | |
| 814 | s.result_cache_status = switch (status) { | |
| 815 | .hit => .hit, | |
| 816 | .incomplete_manifest => .incomplete_manifest, | |
| 817 | .invalid_manifest => .invalid_manifest, | |
| 818 | inline .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |off, tag| @unionInit( | |
| 819 | ResultCacheStatus, | |
| 820 | @tagName(tag), | |
| 821 | try gpa.dupeSentinel(u8, off.path(man.contents.items), 0), | |
| 822 | ), | |
| 823 | }; | |
| 824 | return status == .hit; | |
| 825 | } | |
| 826 | ||
| 827 | fn clearResultCacheStatus(s: *Step, gpa: Allocator) void { | |
| 828 | switch (s.result_cache_status) { | |
| 829 | .hit, .incomplete_manifest, .invalid_manifest => {}, | |
| 830 | .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |path| { | |
| 831 | gpa.free(std.mem.sliceTo(path, 0)); | |
| 832 | }, | |
| 833 | } | |
| 834 | s.result_cache_status = .incomplete_manifest; | |
| 801 | 835 | } |
| 802 | 836 | |
| 803 | 837 | fn failWithCacheError( |
lib/compiler/Maker/Step/InstallArtifact.zig+3-1| ... | ... | @@ -137,5 +137,7 @@ pub fn make( |
| 137 | 137 | }; |
| 138 | 138 | } |
| 139 | 139 | |
| 140 | step.result_cached = all_cached; | |
| 140 | // TODO instead of a bool, make this a proper Step.ResultCacheStatus that tracks the file path that caused | |
| 141 | // a miss for better reporting. | |
| 142 | step.result_cache_status = if (all_cached) .hit else .incomplete_manifest; | |
| 141 | 143 | } |
lib/compiler/Maker/Step/InstallDir.zig+3-1| ... | ... | @@ -95,5 +95,7 @@ pub fn make( |
| 95 | 95 | } |
| 96 | 96 | } |
| 97 | 97 | |
| 98 | step.result_cached = all_cached; | |
| 98 | // TODO instead of a bool, make this a proper Step.ResultCacheStatus that tracks the file path that caused | |
| 99 | // a miss for better reporting. | |
| 100 | step.result_cache_status = if (all_cached) .hit else .incomplete_manifest; | |
| 99 | 101 | } |
lib/compiler/Maker/Step/InstallFile.zig+1-1| ... | ... | @@ -22,5 +22,5 @@ pub fn make( |
| 22 | 22 | |
| 23 | 23 | try step.singleUnchangingWatchInput(maker, arena, conf_if.source.get(conf)); |
| 24 | 24 | const p = try maker.installLazyPathSub(arena, conf_if.source, conf_if.dest_dir, conf_if.dest_sub_path.slice(conf), step_index); |
| 25 | step.result_cached = p == .fresh; | |
| 25 | step.result_cache_status = if (p == .fresh) .hit else .incomplete_manifest; | |
| 26 | 26 | } |
lib/compiler/Maker/Step/Options.zig+1-1| ... | ... | @@ -80,7 +80,7 @@ pub fn make( |
| 80 | 80 | if (try step.cacheHitWatched(maker, &man, progress_node)) { |
| 81 | 81 | const digest = man.hitDigestHex(); |
| 82 | 82 | _ = try maker.setGeneratedPath(conf_options.generated_file, .local_cache, &.{ "o", &digest, basename }); |
| 83 | step.result_cached = true; | |
| 83 | step.result_cache_status = .hit; | |
| 84 | 84 | return; |
| 85 | 85 | } |
| 86 | 86 |
lib/compiler/Maker/Step/Run.zig+1-1| ... | ... | @@ -573,7 +573,7 @@ pub fn make( |
| 573 | 573 | const digest = man.hitDigestHex(); |
| 574 | 574 | try populateGeneratedStdIo(maker, &conf_run, &digest); |
| 575 | 575 | try populateGeneratedPaths(maker, output_placeholders.items, &digest); |
| 576 | step.result_cached = true; | |
| 576 | step.result_cache_status = .hit; | |
| 577 | 577 | return; |
| 578 | 578 | } |
| 579 | 579 |
lib/compiler/Maker/Step/UpdateSourceFiles.zig+3-1| ... | ... | @@ -84,5 +84,7 @@ pub fn make( |
| 84 | 84 | progress_node.completeOne(); |
| 85 | 85 | } |
| 86 | 86 | |
| 87 | step.result_cached = !any_miss; | |
| 87 | // TODO instead of a bool, make this a proper Step.ResultCacheStatus that tracks the file path that caused | |
| 88 | // a miss for better reporting. | |
| 89 | step.result_cache_status = if (!any_miss) .hit else .incomplete_manifest; | |
| 88 | 90 | } |
lib/compiler/Maker/Step/WriteFile.zig+3-3| ... | ... | @@ -110,7 +110,7 @@ pub fn make( |
| 110 | 110 | if (try step.cacheHit(maker, &man, progress_node)) { |
| 111 | 111 | const digest = man.hitDigestHex(); |
| 112 | 112 | _ = try maker.setGeneratedPath(conf_wf.generated_directory, .local_cache, &.{ "o", &digest }); |
| 113 | assert(step.result_cached); | |
| 113 | assert(step.result_cache_status == .hit); | |
| 114 | 114 | return; |
| 115 | 115 | } |
| 116 | 116 | |
| ... | ... | @@ -122,7 +122,7 @@ pub fn make( |
| 122 | 122 | try step.finalizeManifest(maker, &man); |
| 123 | 123 | }, |
| 124 | 124 | .tmp => { |
| 125 | step.result_cached = false; | |
| 125 | step.result_cache_status = .incomplete_manifest; | |
| 126 | 126 | |
| 127 | 127 | var rand_int: u64 = undefined; |
| 128 | 128 | io.random(@ptrCast(&rand_int)); |
| ... | ... | @@ -135,7 +135,7 @@ pub fn make( |
| 135 | 135 | try operate(maker, step_index, open_dir_cache, out_path, progress_node); |
| 136 | 136 | }, |
| 137 | 137 | .mutate => { |
| 138 | step.result_cached = false; | |
| 138 | step.result_cache_status = .incomplete_manifest; | |
| 139 | 139 | const root_path = try maker.resolveLazyPathIndex(arena, conf_wf.mutate_path.value.?, step_index); |
| 140 | 140 | try operate(maker, step_index, open_dir_cache, root_path, progress_node); |
| 141 | 141 | _ = try maker.setGeneratedPathPath(conf_wf.generated_directory, root_path); |
lib/std/Build/Cache.zig+1-1| ... | ... | @@ -978,7 +978,7 @@ pub const Manifest = struct { |
| 978 | 978 | // this is a miss, we'll learn those from future calls to `addDiscoveredPath` etc. As such, the state of |
| 979 | 979 | // `man.hash.hasher` after this function depends on whether this is a hit or a miss. |
| 980 | 980 | // |
| 981 | // If we return `CacheStatus.hit`, then `man.hash.hasher` must already include the digests of the discovered | |
| 981 | // If we return `CheckResult.hit`, then `man.hash.hasher` must already include the digests of the discovered | |
| 982 | 982 | // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests |
| 983 | 983 | // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered |
| 984 | 984 | // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user |