From 15f00d1756f1503da29151cef5213858f6667f4f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 30 Jun 2026 17:44:50 -0700 Subject: [PATCH] std.Build.Cache: make Progress.Node for hits Occasionally, two different Zig processes are waiting on the same compilation artifacts. For example, if the user runs `zig build` for the first time in two different terminals with the same version of Zig, one of them will win the race on compiler_rt and start building it, while the other process waits. Then they both use the same cache artifacts. Now, the fact that this is happening is communicated to the user by displaying it in a progress node during the checking for cache hit. --- lib/compiler/Maker.zig | 2 +- lib/compiler/Maker/Step.zig | 12 +++++------ lib/compiler/Maker/Step/ConfigHeader.zig | 3 +-- lib/compiler/Maker/Step/ObjCopy.zig | 2 +- lib/compiler/Maker/Step/Options.zig | 5 +---- lib/compiler/Maker/Step/Run.zig | 2 +- lib/compiler/Maker/Step/WriteFile.zig | 2 +- lib/std/Build/Cache.zig | 26 +++++++++++++++--------- src/Compilation.zig | 10 ++++----- src/libs/freebsd.zig | 2 +- src/libs/glibc.zig | 2 +- src/libs/mingw.zig | 2 +- src/libs/netbsd.zig | 2 +- src/libs/openbsd.zig | 2 +- src/main.zig | 2 +- 15 files changed, 39 insertions(+), 37 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 46653431cb447f1d773c6838681a06e7c1c26cf0..ef7c995d801e48b7fa6020e6481c7a47d7c0e857 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1216,7 +1216,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { defer compile_prog_node.end(); switch (options.cache_poison) { - .pure, .disallowed, .ignored => if (try config_man.hit()) { + .pure, .disallowed, .ignored => if (try config_man.hit(compile_prog_node)) { const digest = config_man.final(); break :cp .{ .{ diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 8ccb1415d8f17a1df24dfa9505d0ac699304e97f..11f9095da07883568b5cb956f8b2b524ccb683b0 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -711,10 +711,10 @@ pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.T if (!term.success()) return s.fail(maker, "process {f}", .{term}); } -/// Prefer `cacheHitAndWatch` unless you already added watch inputs +/// Prefer `cacheHitWatched` unless you already added watch inputs /// separately from using the cache system. -pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest) !bool { - s.result_cached = man.hit() catch |err| return failWithCacheError(s, maker, man, err); +pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { + s.result_cached = man.hit(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err); return s.result_cached; } @@ -722,8 +722,8 @@ pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest) !bool { /// the full set of files picked up by the cache manifest. /// /// Must be accompanied with `writeManifestAndWatch`. -pub fn cacheHitAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !bool { - const is_hit = man.hit() catch |err| return failWithCacheError(s, maker, man, err); +pub fn cacheHitWatched(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { + const is_hit = man.hit(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err); s.result_cached = is_hit; // The above call to hit() populates the manifest with files, so in case of // a hit, we need to populate watch inputs. @@ -770,7 +770,7 @@ pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { /// Clears previous watch inputs, if any, and then populates watch inputs from /// the full set of files picked up by the cache manifest. /// -/// Must be accompanied with `cacheHitAndWatch`. +/// Must be accompanied with `cacheHitWatched`. pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { try writeManifest(s, maker, man); try setWatchInputsFromManifest(s, maker, man); diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index e0fde54d54b88297e7033627da8a39d3f895ae6a..2863ced63f5f96def9a22d8870f35e999a021a7f 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -27,7 +27,6 @@ pub fn make( progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { _ = config_header; - _ = progress_node; const graph = maker.graph; const step = maker.stepByIndex(step_index); const io = graph.io; @@ -111,7 +110,7 @@ pub fn make( const output = aw.written(); man.hash.addBytes(output); - if (try step.cacheHit(maker, &man)) { + if (try step.cacheHit(maker, &man, progress_node)) { const digest = man.final(); maker.generatedPath(conf_ch.generated_dir).* = .{ .root_dir = cache_root, diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index a6c94e687fd8dfa0c6252aafb95752b9f53489e1..49424d4b35daea41bb6a485e1fb2a37acfe5e468 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -46,7 +46,7 @@ pub fn make( const basename = opt_basename orelse Io.Dir.path.basename(input_path.sub_path); - if (try step.cacheHit(maker, &man)) { + if (try step.cacheHit(maker, &man, progress_node)) { // Cache hit, skip subprocess execution. const digest = man.final(); maker.generatedPath(conf_oc.output_file).* = .{ diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index b44c9af4057e8d3a7b50a0d58ae705da397df79c..7c31f144d59a20f70f1e6550320b8f8c05237276 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -16,9 +16,6 @@ pub fn make( ) Step.ExtendedMakeError!void { _ = options; - // This step completes so quickly that no progress reporting is necessary. - _ = progress_node; - const graph = maker.graph; const step = maker.stepByIndex(step_index); const io = graph.io; @@ -56,7 +53,7 @@ pub fn make( const basename = "options.zig"; - if (try step.cacheHitAndWatch(maker, &man)) { + if (try step.cacheHitWatched(maker, &man, progress_node)) { const digest = man.final(); maker.generatedPath(conf_options.generated_file).* = .{ .root_dir = cache_root, diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 14da2a4bc22342d9edc5c50666b200f45e274521..b03f0336c0257dcdc9dcc0c0d323a07ca75a8eb0 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -254,7 +254,7 @@ pub fn make( .check, .zig_test => false, }; - if (!has_side_effects and try step.cacheHitAndWatch(maker, &man)) { + if (!has_side_effects and try step.cacheHitWatched(maker, &man, progress_node)) { // Cache hit; skip running command. const digest = man.final(); try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index 53bd71eda99ce058926660731b581f682e13c8ef..65b19bdbae8137e94b407fded33c31af71805bc2 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -104,7 +104,7 @@ pub fn make( } } - if (try step.cacheHit(maker, &man)) { + if (try step.cacheHit(maker, &man, progress_node)) { const digest = man.final(); maker.generatedPath(conf_wf.generated_directory).* = .{ .root_dir = cache_root, diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index f731f5667c30b6406f0b3721d1249b1b3437fd77..9b9e808aaeb72a3423844688c3fb14eb2c100579 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -501,7 +501,13 @@ pub const Manifest = struct { /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn hit(self: *Manifest) HitError!bool { + pub fn hit(man: *Manifest, parent_progress_node: std.Progress.Node) HitError!bool { + const node = parent_progress_node.start("Reusing Cache Artifacts", 0); + defer node.end(); + return hitInner(man); + } + + pub fn hitInner(self: *Manifest) HitError!bool { assert(self.manifest_file == null); self.diagnostic = .none; @@ -1380,7 +1386,7 @@ test "cache file and then recall it" { _ = try ch.addFile(temp_file, null); // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit()); + try testing.expectEqual(false, try ch.hit(.none)); digest1 = ch.final(); try ch.writeManifest(); @@ -1395,7 +1401,7 @@ test "cache file and then recall it" { _ = try ch.addFile(temp_file, null); // Cache hit! We just "built" the same file - try testing.expect(try ch.hit()); + try testing.expect(try ch.hit(.none)); digest2 = ch.final(); try testing.expectEqual(false, ch.have_exclusive_lock); @@ -1448,7 +1454,7 @@ test "check that changing a file makes cache fail" { const temp_file_idx = try ch.addFile(temp_file, 100); // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit()); + try testing.expectEqual(false, try ch.hit(.none)); try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.keys()[temp_file_idx].contents.?)); @@ -1467,7 +1473,7 @@ test "check that changing a file makes cache fail" { const temp_file_idx = try ch.addFile(temp_file, 100); // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit()); + try testing.expectEqual(false, try ch.hit(.none)); // The cache system does not keep the contents of re-hashed input files. try testing.expect(ch.files.keys()[temp_file_idx].contents == null); @@ -1511,7 +1517,7 @@ test "no file inputs" { man.hash.addBytes("1234"); // There should be nothing in the cache - try testing.expectEqual(false, try man.hit()); + try testing.expectEqual(false, try man.hit(.none)); digest1 = man.final(); @@ -1523,7 +1529,7 @@ test "no file inputs" { man.hash.addBytes("1234"); - try testing.expect(try man.hit()); + try testing.expect(try man.hit(.none)); digest2 = man.final(); try testing.expectEqual(false, man.have_exclusive_lock); } @@ -1575,7 +1581,7 @@ test "Manifest with files added after initial hash work" { _ = try ch.addFile(temp_file1, null); // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit()); + try testing.expectEqual(false, try ch.hit(.none)); _ = try ch.addFilePost(temp_file2); @@ -1589,7 +1595,7 @@ test "Manifest with files added after initial hash work" { ch.hash.addBytes("1234"); _ = try ch.addFile(temp_file1, null); - try testing.expect(try ch.hit()); + try testing.expect(try ch.hit(.none)); digest2 = ch.final(); try testing.expectEqual(false, ch.have_exclusive_lock); @@ -1613,7 +1619,7 @@ test "Manifest with files added after initial hash work" { _ = try ch.addFile(temp_file1, null); // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit()); + try testing.expectEqual(false, try ch.hit(.none)); _ = try ch.addFilePost(temp_file2); diff --git a/src/Compilation.zig b/src/Compilation.zig index 1e9b81d493d62ceea21fb74b3786677b119eb9b4..35eee9c648f7f3bd5c6b8409f38d23c0ad33f94c 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2755,7 +2755,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE man.want_shared_lock = false; } - const is_hit = man.hit() catch |err| switch (err) { + const is_hit = man.hit(main_progress_node) catch |err| switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, .manifest_create, .manifest_read, .manifest_lock => |e| return comp.setMiscFailure( @@ -5435,7 +5435,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr const target = comp.getTarget(); assert(target.ofmt != .c); const o_ext = target.ofmt.fileExt(target.cpu.arch); - const digest = if (!comp.disable_c_depfile and try man.hit()) man.final() else blk: { + const digest = if (!comp.disable_c_depfile and try man.hit(child_progress_node)) man.final() else blk: { var argv: std.array_list.Managed([]const u8) = .init(gpa); defer argv.deinit(); @@ -5696,7 +5696,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr } // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. - if (comp.disable_c_depfile) _ = try man.hit(); + if (comp.disable_c_depfile) _ = try man.hit(child_progress_node); // Rename into place. const digest = man.final(); @@ -5784,7 +5784,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); - const digest = if (try man.hit()) man.final() else blk: { + const digest = if (try man.hit(child_progress_node)) man.final() else blk: { // The digest only depends on the .manifest file, so we can // get the digest now and write the .res directly to the cache const digest = man.final(); @@ -5877,7 +5877,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename_noext = src_basename[0 .. src_basename.len - fs.path.extension(src_basename).len]; - const digest = if (try man.hit()) man.final() else blk: { + const digest = if (try man.hit(child_progress_node)) man.final() else blk: { var zig_cache_tmp_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, "tmp", .{}); defer zig_cache_tmp_dir.close(io); diff --git a/src/libs/freebsd.zig b/src/libs/freebsd.zig index ecc15e0dab899e21093c047e58a7b02467a29ac6..8fcfb043b3fd7d6d2fcd060909cc270f27c8b520 100644 --- a/src/libs/freebsd.zig +++ b/src/libs/freebsd.zig @@ -462,7 +462,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye const full_abilists_path = try comp.dirs.zig_lib.join(arena, &.{abilists_path}); const abilists_index = try man.addFile(full_abilists_path, abilists_max_size); - if (try man.hit()) { + if (try man.hit(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/libs/glibc.zig b/src/libs/glibc.zig index af80133d3332e33b69334b6c2b3dcc1b3f075979..a23757c2f6107cf8b2f48b732d5505563127a9d5 100644 --- a/src/libs/glibc.zig +++ b/src/libs/glibc.zig @@ -703,7 +703,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye const full_abilists_path = try comp.dirs.zig_lib.join(arena, &.{abilists_path}); const abilists_index = try man.addFile(full_abilists_path, abilists_max_size); - if (try man.hit()) { + if (try man.hit(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index a19a83e044ce28800257497b1e76bb9fc6cbf9cb..41a372fdc4013083a25669056058f2799743e4e0 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -255,7 +255,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P const final_lib_basename = try std.fmt.allocPrint(gpa, "{s}.lib", .{lib_name}); errdefer gpa.free(final_lib_basename); - if (try man.hit()) { + if (try man.hit(prog_node)) { const digest = man.final(); const sub_path = try std.fs.path.join(gpa, &.{ "o", &digest, final_lib_basename }); errdefer gpa.free(sub_path); diff --git a/src/libs/netbsd.zig b/src/libs/netbsd.zig index c3e0a38ffb6cd6cb98cfd51bb0cc74151c627bd4..3b7162363f148c5332dd721923ef8ff046faa46d 100644 --- a/src/libs/netbsd.zig +++ b/src/libs/netbsd.zig @@ -409,7 +409,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye const full_abilists_path = try comp.dirs.zig_lib.join(arena, &.{abilists_path}); const abilists_index = try man.addFile(full_abilists_path, abilists_max_size); - if (try man.hit()) { + if (try man.hit(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/libs/openbsd.zig b/src/libs/openbsd.zig index ee50196c8169bcb52f0a1bbd455f4b14c20f93e8..9cbd679a3efae3bd3aadf574edadaf5479eee836 100644 --- a/src/libs/openbsd.zig +++ b/src/libs/openbsd.zig @@ -332,7 +332,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye const full_abilists_path = try comp.dirs.zig_lib.join(arena, &.{abilists_path}); const abilists_index = try man.addFile(full_abilists_path, abilists_max_size); - if (try man.hit()) { + if (try man.hit(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/main.zig b/src/main.zig index dbe4c34265ea7cba91200748d35c2cd99df427f8..56266b1724c15a1370dacbf4499a7bff47239f87 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4792,7 +4792,7 @@ fn cmdTranslateC( Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| fatal("unable to process {q}: {t}", .{ c_source_file.src_path, err }); - const result: Compilation.TranslateCResult = if (try man.hit()) .{ + const result: Compilation.TranslateCResult = if (try man.hit(prog_node)) .{ .digest = man.finalBin(), .cache_hit = true, .errors = std.zig.ErrorBundle.empty, -- 2.54.0