| ... | @@ -839,13 +839,10 @@ pub const Manifest = struct { | ... | @@ -839,13 +839,10 @@ pub const Manifest = struct { |
| 839 | var off: usize = 0; | 839 | var off: usize = 0; |
| 840 | var c: Check = .{}; | 840 | var c: Check = .{}; |
| 841 | | 841 | |
| 842 | // This group we always want to compute the hash digests, even on a | 842 | // We always want to compute the input file hash digests, even on a cache miss, because they will be |
| 843 | // cache miss, because they will be used in the manifest digest. | 843 | // used in the manifest digest. |
| 844 | var input_group: Io.Group = .init; | | |
| 845 | defer input_group.cancel(io); | | |
| 846 | | 844 | |
| 847 | // First the input files section, which must match our input files, | 845 | // First the input files section, which must match our input files, otherwise it's invalid format. |
| 848 | // otherwise it's invalid format. | | |
| 849 | for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| { | 846 | for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| { |
| 850 | if (off + 1 >= contents.len) return error.InvalidFormat; | 847 | if (off + 1 >= contents.len) return error.InvalidFormat; |
| 851 | const file_off: File.Offset = @fromBackingInt(@intCast(off)); | 848 | const file_off: File.Offset = @fromBackingInt(@intCast(off)); |
| ... | @@ -855,11 +852,14 @@ pub const Manifest = struct { | ... | @@ -855,11 +852,14 @@ pub const Manifest = struct { |
| 855 | if (path.len == 0) return error.InvalidFormat; | 852 | if (path.len == 0) return error.InvalidFormat; |
| 856 | if (input_file_off != file_off) return error.InvalidFormat; | 853 | if (input_file_off != file_off) return error.InvalidFormat; |
| 857 | | 854 | |
| 858 | input_group.async(io, checkInputFile, .{ m, &c, file_off, path, input_path }); | 855 | try checkInputFile(m, &c, file_off, path, input_path); |
| 859 | | 856 | |
| 860 | off += File.sizeOf(path.len); | 857 | off += File.sizeOf(path.len); |
| 861 | } | 858 | } |
| 862 | | 859 | |
| | 860 | if (c.status == .miss) return .miss; |
| | 861 | if (m.diagnostic != .none) return error.CacheCheckFailed; |
| | 862 | |
| 863 | // Guess number of files based on manifest contents len to reduce allocations. | 863 | // Guess number of files based on manifest contents len to reduce allocations. |
| 864 | // This is not an upper bound; subsequent insertions may potentially allocate. | 864 | // This is not an upper bound; subsequent insertions may potentially allocate. |
| 865 | try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{ | 865 | try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{ |
| ... | @@ -875,6 +875,10 @@ pub const Manifest = struct { | ... | @@ -875,6 +875,10 @@ pub const Manifest = struct { |
| 875 | if (path.len == 0) return error.InvalidFormat; | 875 | if (path.len == 0) return error.InvalidFormat; |
| 876 | | 876 | |
| 877 | try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); | 877 | try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); |
| | 878 | switch (try checkFile(m, &c, file_off, filePath(contents, file_off))) { |
| | 879 | .hit => {}, |
| | 880 | .miss => return .miss, |
| | 881 | } |
| 878 | | 882 | |
| 879 | off += File.sizeOf(path.len); | 883 | off += File.sizeOf(path.len); |
| 880 | } | 884 | } |
| ... | @@ -882,79 +886,12 @@ pub const Manifest = struct { | ... | @@ -882,79 +886,12 @@ pub const Manifest = struct { |
| 882 | // Final terminating zero byte to distinguish empty manifest file from | 886 | // Final terminating zero byte to distinguish empty manifest file from |
| 883 | // manifest with zero files. | 887 | // manifest with zero files. |
| 884 | const file_valid = off + 1 == contents.len and contents[off] == 0; | 888 | const file_valid = off + 1 == contents.len and contents[off] == 0; |
| 885 | if (!file_valid) { | 889 | if (!file_valid) return .miss; |
| 886 | try input_group.await(io); | 890 | |
| 887 | return .miss; | 891 | for (m.files.keys()) |file_off| m.hash.hasher.update(&file_off.get(contents).digest); |
| 888 | } | | |
| 889 | | 892 | |
| 890 | // Don't track the trailing zero byte in contents. | 893 | // Don't track the trailing zero byte in contents. |
| 891 | m.contents.items.len -= 1; | 894 | m.contents.items.len -= 1; |
| 892 | // Needed due to the length mutation above. | | |
| 893 | const refreshed_contents = m.contents.items; | | |
| 894 | | | |
| 895 | // This group we would like to cancel as soon as a cache miss is discovered. | | |
| 896 | const PostResult = union(enum) { | | |
| 897 | checkFile: CheckFileError!Check.Status, | | |
| 898 | }; | | |
| 899 | // In order to call async in the loop we need to ensure this buffer has capacity for as many elements as files | | |
| 900 | // being checked, otherwise a deadlock could occur since writing to the queue is waiting on the same task as | | |
| 901 | // would read from it. | | |
| 902 | const post_select_buffer = try gpa.alloc(PostResult, m.files.count() - m.input_paths.items.len); | | |
| 903 | defer gpa.free(post_select_buffer); | | |
| 904 | | | |
| 905 | var post_select: Io.Select(PostResult) = .init(io, post_select_buffer); | | |
| 906 | defer post_select.cancelDiscard(); | | |
| 907 | | | |
| 908 | var post_select_remaining: usize = 0; | | |
| 909 | for (m.files.keys()[m.input_paths.items.len..]) |file_off| { | | |
| 910 | post_select.async(.checkFile, checkFile, .{ m, &c, file_off, filePath(refreshed_contents, file_off) }); | | |
| 911 | post_select_remaining += 1; | | |
| 912 | // In case the async checkFile runs eagerly. | | |
| 913 | switch (@atomicLoad(Check.Status, &c.status, .unordered)) { | | |
| 914 | .miss => { | | |
| 915 | post_select.cancelDiscard(); | | |
| 916 | try input_group.await(io); | | |
| 917 | return .miss; | | |
| 918 | }, | | |
| 919 | .hit => {}, | | |
| 920 | } | | |
| 921 | } | | |
| 922 | | | |
| 923 | var post_await_buffer: [16]PostResult = undefined; | | |
| 924 | while (post_select_remaining > 0) { | | |
| 925 | const n = try post_select.awaitMany(&post_await_buffer, 1); | | |
| 926 | post_select_remaining -= n; | | |
| 927 | | | |
| 928 | // Detect if input group already had a miss. In this case we still wait | | |
| 929 | // for those digests to be updated, but cancel the non input group. | | |
| 930 | switch (@atomicLoad(Check.Status, &c.status, .unordered)) { | | |
| 931 | .miss => { | | |
| 932 | post_select.cancelDiscard(); | | |
| 933 | try input_group.await(io); | | |
| 934 | return .miss; | | |
| 935 | }, | | |
| 936 | .hit => {}, | | |
| 937 | } | | |
| 938 | | | |
| 939 | for (post_await_buffer[0..n]) |u| switch (u) { | | |
| 940 | .checkFile => |result| switch (try result) { | | |
| 941 | .hit => continue, | | |
| 942 | .miss => { | | |
| 943 | post_select.cancelDiscard(); | | |
| 944 | try input_group.await(io); | | |
| 945 | return .miss; | | |
| 946 | }, | | |
| 947 | }, | | |
| 948 | }; | | |
| 949 | } | | |
| 950 | | | |
| 951 | try input_group.await(io); | | |
| 952 | if (c.status == .miss) return .miss; | | |
| 953 | if (m.diagnostic != .none) return error.CacheCheckFailed; | | |
| 954 | | | |
| 955 | for (m.files.keys()) |file_off| { | | |
| 956 | m.hash.hasher.update(&file_off.get(refreshed_contents).digest); | | |
| 957 | } | | |
| 958 | | 895 | |
| 959 | return .hit; | 896 | return .hit; |
| 960 | } | 897 | } |
| ... | @@ -965,17 +902,14 @@ pub const Manifest = struct { | ... | @@ -965,17 +902,14 @@ pub const Manifest = struct { |
| 965 | file_off: File.Offset, | 902 | file_off: File.Offset, |
| 966 | file_path: [:0]const u8, | 903 | file_path: [:0]const u8, |
| 967 | input_path: *InputPath, | 904 | input_path: *InputPath, |
| 968 | ) Io.Cancelable!void { | 905 | ) CheckFileError!void { |
| 969 | if (input_path.have_handle) @panic("TODO"); | 906 | if (input_path.have_handle) @panic("TODO"); |
| 970 | if (input_path.have_stat) @panic("TODO"); | 907 | if (input_path.have_stat) @panic("TODO"); |
| 971 | if (input_path.contents != .not_requested) @panic("TODO"); | 908 | if (input_path.contents != .not_requested) @panic("TODO"); |
| 972 | if (input_path.request_handle) @panic("TODO"); | 909 | if (input_path.request_handle) @panic("TODO"); |
| 973 | if (checkFile(m, c, file_off, file_path)) |status| switch (status) { | 910 | switch (try checkFile(m, c, file_off, file_path)) { |
| 974 | .hit => return, | 911 | .hit => {}, |
| 975 | .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered), | 912 | .miss => c.status = .miss, |
| 976 | } else |err| switch (err) { | | |
| 977 | error.CacheCheckFailed => assert(m.diagnostic != .none), | | |
| 978 | else => |e| return e, | | |
| 979 | } | 913 | } |
| 980 | } | 914 | } |
| 981 | | 915 | |
Andrew Kelley <andrew@ziglang.org> 2026-09-07 14:37:32-07:00