authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-03 20:12:52-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-16 13:12:47-07:00
log2a82193af49b514c2c784746abd22fdec7c31500
tree11a2c43467339023000ff788f2897157fe739b30
parent4dbe872dfed8ef46b986a7eebe4bc7475d2c5db6

std.Build.Cache: fix deadlock from using Io.Select

Io.Select.async has a fatal flaw: if the queue size isn't big enough to hold the result of an eager async call, then it deadlocks. The workaround in this thread is using concurrent() which should be reverted because it will spawn too many threads.

1 files changed, 34 insertions(+), 7 deletions(-)

lib/std/Build/Cache.zig+34-7
......@@ -795,10 +795,12 @@ pub const Manifest = struct {
795795
796796 fn shrinkFilesToInput(m: *Manifest) void {
797797 if (m.files.count() <= m.input_paths.items.len) return;
798 // Reads from files hash map whose data is destroyed on the next line.
798799 const off = m.files.keys()[m.input_paths.items.len];
800 // Reads from the unshrunken contents whose data is destroyed on the next line.
801 m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items });
799802 m.contents.shrinkRetainingCapacity(@backingInt(off));
800803 assert(mem.isAligned(m.contents.items.len, @alignOf(File)));
801 m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items });
802804 }
803805
804806 /// Assumes that `self.hash.hasher` has been updated only with the original digest and that
......@@ -852,7 +854,7 @@ pub const Manifest = struct {
852854 const PostResult = union(enum) {
853855 checkFile: CheckFileError!Check.Status,
854856 };
855 var post_select_buffer: [10]PostResult = undefined;
857 var post_select_buffer: [16]PostResult = undefined;
856858 var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer);
857859 var post_select_remaining: usize = 0;
858860 defer post_select.cancelDiscard();
......@@ -866,8 +868,33 @@ pub const Manifest = struct {
866868
867869 try m.files.putContext(gpa, file_off, {}, .{ .contents = contents });
868870
869 post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path });
870 post_select_remaining += 1;
871 // In order to call async here we would need to ensure `post_select_buffer`
872 // has capacity for as many elements as files being checked. Since we don't want
873 // to dynamically allocate that buffer, we use concurrent + fallback here.
874 if (post_select.concurrent(.checkFile, checkFile, .{ m, &c, file_off, path })) |_| {
875 post_select_remaining += 1;
876 } else |err| switch (err) {
877 error.ConcurrencyUnavailable => {
878 // Detect if input group already had a miss. In this case we still wait
879 // for those digests to be updated, but cancel the non input group.
880 switch (@atomicLoad(Check.Status, &c.status, .unordered)) {
881 .miss => {
882 post_select.cancelDiscard();
883 try input_group.await(io);
884 return .miss;
885 },
886 .hit => {},
887 }
888 switch (try checkFile(m, &c, file_off, path)) {
889 .hit => continue,
890 .miss => {
891 post_select.cancelDiscard();
892 try input_group.await(io);
893 return .miss;
894 },
895 }
896 },
897 }
871898
872899 off += File.sizeOf(path.len);
873900 }
......@@ -883,7 +910,7 @@ pub const Manifest = struct {
883910 // Don't track the trailing zero byte in contents.
884911 m.contents.items.len -= 1;
885912
886 var post_await_buffer: [10]PostResult = undefined;
913 var post_await_buffer: [16]PostResult = undefined;
887914 while (post_select_remaining > 0) {
888915 const n = try post_select.awaitMany(&post_await_buffer, 1);
889916 post_select_remaining -= n;
......@@ -896,11 +923,11 @@ pub const Manifest = struct {
896923 try input_group.await(io);
897924 return .miss;
898925 },
899 .hit => continue,
926 .hit => {},
900927 }
901928
902929 for (post_await_buffer[0..n]) |u| switch (u) {
903 .checkFile => |result| switch (result) {
930 .checkFile => |result| switch (try result) {
904931 .hit => continue,
905932 .miss => {
906933 post_select.cancelDiscard();