authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-04 21:03:03-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-04 21:03:03-07:00
logc16c0c997f4e4d623eed112f16f1eba1fcaf4146
tree3f7a76f0a1167985b8241a7bcb9ba15b92a1f716
parent96adf07fdeb0c10fa829ab4e645ee91d4022f900

std.Build.Cache: eliminate unbounded concurrency

would be nice if Io.Select.async had a variant where it returned the eager result rather than trying to put it in the queue so that the queue buffer doesn't need to be dynamically allocated

1 files changed, 31 insertions(+), 40 deletions(-)

lib/std/Build/Cache.zig+31-40
...@@ -866,15 +866,7 @@ pub const Manifest = struct {...@@ -866,15 +866,7 @@ pub const Manifest = struct {
866 .contents = contents,866 .contents = contents,
867 });867 });
868868
869 // This group we would like to cancel as soon as a cache miss is discovered.869 // Validate and count discovered files.
870 const PostResult = union(enum) {
871 checkFile: CheckFileError!Check.Status,
872 };
873 var post_select_buffer: [16]PostResult = undefined;
874 var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer);
875 var post_select_remaining: usize = 0;
876 defer post_select.cancelDiscard();
877
878 while (off + 1 < contents.len) {870 while (off + 1 < contents.len) {
879 const file_off: File.Offset = @fromBackingInt(@intCast(off));871 const file_off: File.Offset = @fromBackingInt(@intCast(off));
880 const file = try file_off.getFallible(contents);872 const file = try file_off.getFallible(contents);
...@@ -884,34 +876,6 @@ pub const Manifest = struct {...@@ -884,34 +876,6 @@ pub const Manifest = struct {
884876
885 try m.files.putContext(gpa, file_off, {}, .{ .contents = contents });877 try m.files.putContext(gpa, file_off, {}, .{ .contents = contents });
886878
887 // In order to call async here we would need to ensure `post_select_buffer`
888 // has capacity for as many elements as files being checked. Since we don't want
889 // to dynamically allocate that buffer, we use concurrent + fallback here.
890 if (post_select.concurrent(.checkFile, checkFile, .{ m, &c, file_off, path })) |_| {
891 post_select_remaining += 1;
892 } else |err| switch (err) {
893 error.ConcurrencyUnavailable => {
894 // Detect if input group already had a miss. In this case we still wait
895 // for those digests to be updated, but cancel the non input group.
896 switch (@atomicLoad(Check.Status, &c.status, .unordered)) {
897 .miss => {
898 post_select.cancelDiscard();
899 try input_group.await(io);
900 return .miss;
901 },
902 .hit => {},
903 }
904 switch (try checkFile(m, &c, file_off, path)) {
905 .hit => continue,
906 .miss => {
907 post_select.cancelDiscard();
908 try input_group.await(io);
909 return .miss;
910 },
911 }
912 },
913 }
914
915 off += File.sizeOf(path.len);879 off += File.sizeOf(path.len);
916 }880 }
917881
...@@ -925,6 +889,36 @@ pub const Manifest = struct {...@@ -925,6 +889,36 @@ pub const Manifest = struct {
925889
926 // Don't track the trailing zero byte in contents.890 // Don't track the trailing zero byte in contents.
927 m.contents.items.len -= 1;891 m.contents.items.len -= 1;
892 // Needed due to the length mutation above.
893 const refreshed_contents = m.contents.items;
894
895 // This group we would like to cancel as soon as a cache miss is discovered.
896 const PostResult = union(enum) {
897 checkFile: CheckFileError!Check.Status,
898 };
899 // In order to call async in the loop we need to ensure this buffer has capacity for as many elements as files
900 // being checked, otherwise a deadlock could occur since writing to the queue is waiting on the same task as
901 // would read from it.
902 const post_select_buffer = try gpa.alloc(PostResult, m.files.count() - m.input_paths.items.len);
903 defer gpa.free(post_select_buffer);
904
905 var post_select: Io.Select(PostResult) = .init(io, post_select_buffer);
906 defer post_select.cancelDiscard();
907
908 var post_select_remaining: usize = 0;
909 for (m.files.keys()[m.input_paths.items.len..]) |file_off| {
910 post_select.async(.checkFile, checkFile, .{ m, &c, file_off, filePath(refreshed_contents, file_off) });
911 post_select_remaining += 1;
912 // In case the async checkFile runs eagerly.
913 switch (@atomicLoad(Check.Status, &c.status, .unordered)) {
914 .miss => {
915 post_select.cancelDiscard();
916 try input_group.await(io);
917 return .miss;
918 },
919 .hit => {},
920 }
921 }
928922
929 var post_await_buffer: [16]PostResult = undefined;923 var post_await_buffer: [16]PostResult = undefined;
930 while (post_select_remaining > 0) {924 while (post_select_remaining > 0) {
...@@ -958,9 +952,6 @@ pub const Manifest = struct {...@@ -958,9 +952,6 @@ pub const Manifest = struct {
958 if (c.status == .miss) return .miss;952 if (c.status == .miss) return .miss;
959 if (m.diagnostic != .none) return error.CacheCheckFailed;953 if (m.diagnostic != .none) return error.CacheCheckFailed;
960954
961 // Needed due to the length mutation above.
962 const refreshed_contents = m.contents.items;
963
964 for (m.files.keys()) |file_off| {955 for (m.files.keys()) |file_off| {
965 m.hash.hasher.update(&file_off.get(refreshed_contents).digest);956 m.hash.hasher.update(&file_off.get(refreshed_contents).digest);
966 }957 }