authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-03 20:12:52-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-04 17:42:22-07:00
log44007a5cfd05fc8415b4b0e1a1a4629f2fd191c1
treea96db476f0e189af25f4a817a1f09dc8ec4af505
parent91e1010a5afe3a28512aeb4ee1fd20c8afb81bc3

std.Build.Cache: fix deadlock from using Io.Select

Io.Select.async has a fatal flaw: if the queue size isn't big enough to hold the result of an eager async call, then it deadlocks. The workaround in this thread is using concurrent() which should be reverted because it will spawn too many threads.

1 files changed, 34 insertions(+), 7 deletions(-)

lib/std/Build/Cache.zig+34-7
......@@ -795,10 +795,12 @@ pub const Manifest = struct {
795795
796796 fn shrinkFilesToInput(m: *Manifest) void {
797797 if (m.files.count() <= m.input_paths.items.len) return;
798 // Reads from files hash map whose data is destroyed on the next line.
798799 const off = m.files.keys()[m.input_paths.items.len];
800 // Reads from the unshrunken contents whose data is destroyed on the next line.
801 m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items });
799802 m.contents.shrinkRetainingCapacity(@backingInt(off));
800803 assert(mem.isAligned(m.contents.items.len, @alignOf(File)));
801 m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items });
802804 }
803805
804806 /// Assumes that `self.hash.hasher` has been updated only with the original digest and that
......@@ -852,7 +854,7 @@ pub const Manifest = struct {
852854 const PostResult = union(enum) {
853855 checkFile: CheckFileError!Check.Status,
854856 };
855 var post_select_buffer: [10]PostResult = undefined;
857 var post_select_buffer: [16]PostResult = undefined;
856858 var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer);
857859 var post_select_remaining: usize = 0;
858860 defer post_select.cancelDiscard();
......@@ -866,8 +868,33 @@ pub const Manifest = struct {
866868
867869 try m.files.putContext(gpa, file_off, {}, .{ .contents = contents });
868870
869 post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path });
870 post_select_remaining += 1;
871 // In order to call async here we would need to ensure `post_select_buffer`
872 // has capacity for as many elements as files being checked. Since we don't want
873 // to dynamically allocate that buffer, we use concurrent + fallback here.
874 if (post_select.concurrent(.checkFile, checkFile, .{ m, &c, file_off, path })) |_| {
875 post_select_remaining += 1;
876 } else |err| switch (err) {
877 error.ConcurrencyUnavailable => {
878 // Detect if input group already had a miss. In this case we still wait
879 // for those digests to be updated, but cancel the non input group.
880 switch (@atomicLoad(Check.Status, &c.status, .unordered)) {
881 .miss => {
882 post_select.cancelDiscard();
883 try input_group.await(io);
884 return .miss;
885 },
886 .hit => {},
887 }
888 switch (try checkFile(m, &c, file_off, path)) {
889 .hit => continue,
890 .miss => {
891 post_select.cancelDiscard();
892 try input_group.await(io);
893 return .miss;
894 },
895 }
896 },
897 }
871898
872899 off += File.sizeOf(path.len);
873900 }
......@@ -883,7 +910,7 @@ pub const Manifest = struct {
883910 // Don't track the trailing zero byte in contents.
884911 m.contents.items.len -= 1;
885912
886 var post_await_buffer: [10]PostResult = undefined;
913 var post_await_buffer: [16]PostResult = undefined;
887914 while (post_select_remaining > 0) {
888915 const n = try post_select.awaitMany(&post_await_buffer, 1);
889916 post_select_remaining -= n;
......@@ -896,11 +923,11 @@ pub const Manifest = struct {
896923 try input_group.await(io);
897924 return .miss;
898925 },
899 .hit => continue,
926 .hit => {},
900927 }
901928
902929 for (post_await_buffer[0..n]) |u| switch (u) {
903 .checkFile => |result| switch (result) {
930 .checkFile => |result| switch (try result) {
904931 .hit => continue,
905932 .miss => {
906933 post_select.cancelDiscard();