authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-09 20:25:37-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-16 13:12:47-07:00
loga24b4758f4cabe7c690a5707f2fe2fda913b549c
treeecddec90230411d8430a6300a15aae1e21471b21
parent3f092f8c286b2462cda2922120dd732cd558c91c

std.Build.Cache: rework the core logic again

- implement the logic for populating remaining input file digests when a miss occurs while checking one of the input files - make `Manifest.check` return not just "miss" but also explain why it was a miss and which file caused it - fix fundamental design flaw which clobbered input file digest, path, and stat information that was needed to compare against on disk manifest entries - implement the logic for when the Manifest API user provides an already available handle, stat, or contents, and when they request the handle or contents. - rename `Manifest.writeManifest` to `Manifest.finalize` - rename `Manifest.manifest_dirty` to `Manifest.dirty`

1 files changed, 546 insertions(+), 181 deletions(-)

lib/std/Build/Cache.zig+546-181
......@@ -48,7 +48,7 @@ pub fn obtain(cache: *Cache) Manifest {
4848 .cache = cache,
4949 .hash = cache.hash,
5050 .manifest_file = null,
51 .manifest_dirty = false,
51 .dirty = false,
5252 };
5353}
5454
......@@ -274,7 +274,8 @@ pub const Manifest = struct {
274274 hash: HashHelper,
275275 /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase.
276276 manifest_file: ?Io.File,
277 manifest_dirty: bool,
277 /// `finalize` has not been called yet.
278 dirty: bool,
278279 /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code
279280 /// using the cache will not modify the files within the cache directory. This allows multiple processes
280281 /// to utilize the same cache directory at the same time.
......@@ -306,30 +307,6 @@ pub const Manifest = struct {
306307 all_input_content: std.ArrayList(u8) = .empty,
307308 max_input_content_len: usize = std.math.maxInt(u32),
308309
309 /// State that exists only during check.
310 pub const Check = struct {
311 /// Protects `Manifest.diagnostic` from data races.
312 diagnostic_lock: bool = false,
313 status: Status = .hit,
314
315 pub const Status = enum { hit, miss };
316
317 pub const Error = error{
318 /// Unable to check the cache for a reason that has been recorded into
319 /// the `diagnostic` field.
320 CacheCheckFailed,
321 /// A cache manifest file exists however it could not be parsed.
322 InvalidFormat,
323 } || Allocator.Error || Io.Cancelable;
324
325 fn fail(c: *Check, m: *Manifest, diagnostic: Diagnostic) error{CacheCheckFailed} {
326 if (!@atomicRmw(bool, &c.diagnostic_lock, .Xchg, true, .monotonic)) {
327 m.diagnostic = diagnostic;
328 }
329 return error.CacheCheckFailed;
330 }
331 };
332
333310 pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false);
334311
335312 /// Source files and directories whose prefix and relative path are
......@@ -340,7 +317,9 @@ pub const Manifest = struct {
340317 /// but the other fields of `File` will be populated depending on the
341318 /// fields of `InputPath`.
342319 pub const InputPath = struct {
320 /// Determines whether `check` will keep the handle open or close it.
343321 request_handle: bool,
322 /// Determines whether `handle` is populated.
344323 have_handle: bool,
345324 /// Determines whether `File.size`, `File.inode`, and `File.mtime` are populated.
346325 have_stat: bool,
......@@ -362,6 +341,13 @@ pub const Manifest = struct {
362341 off: u32,
363342 len: u32,
364343 },
344
345 pub fn get(this: @This(), all_contents: []const u8) ?[]const u8 {
346 return if (this.status.populated())
347 all_contents[this.status.populated.off..][0..this.status.populated.len]
348 else
349 null;
350 }
365351 } align(@alignOf(u32)),
366352 /// `have_handle` determines whether this is populated.
367353 handle: union {
......@@ -369,6 +355,12 @@ pub const Manifest = struct {
369355 dir: Io.Dir,
370356 },
371357
358 pub const Handle = union(enum) {
359 none,
360 file: Io.File,
361 dir: Io.Dir,
362 };
363
372364 /// Index into `Manifest.input_paths`.
373365 pub const Index = enum(u32) {
374366 _,
......@@ -387,6 +379,13 @@ pub const Manifest = struct {
387379 return manifest.all_input_content.items[populated.off..][0..populated.len];
388380 }
389381 };
382
383 pub fn getHandle(this: @This(), file: *const File) Handle {
384 return if (!this.have_handle) .none else switch (file.flags.is_directory) {
385 false => .{ .file = this.handle.file },
386 true => .{ .dir = this.handle.dir },
387 };
388 }
390389 };
391390
392391 /// The data per tracked input file that is stored in the manifest file.
......@@ -449,6 +448,24 @@ pub const Manifest = struct {
449448 if (!mem.isAligned(@backingInt(offset), @alignOf(File))) return error.InvalidFormat;
450449 return getConst(offset, contents);
451450 }
451
452 pub fn flagsAndPath(off: File.Offset, contents: []const u8) [:0]const u8 {
453 const flags_off = @offsetOf(File, "flags");
454 comptime assert(@offsetOf(File, "path_start") - flags_off == 1);
455 const hash_start = @backingInt(off) + flags_off;
456 const hash_end = mem.findScalarPos(u8, contents, hash_start, 0).?;
457 return contents[hash_start..hash_end];
458 }
459
460 pub fn pathFallible(off: File.Offset, contents: []const u8) error{InvalidFormat}![:0]const u8 {
461 const path_start = @backingInt(off) + @offsetOf(File, "path_start");
462 const path_end = mem.findScalarPos(u8, contents, path_start, 0) orelse return error.InvalidFormat;
463 return contents[path_start..path_end :0];
464 }
465
466 pub fn path(off: File.Offset, contents: []const u8) [:0]const u8 {
467 return pathFallible(contents, off) catch unreachable;
468 }
452469 };
453470
454471 /// Intentionally matches if the files are different only by flags other than prefix.
......@@ -457,7 +474,7 @@ pub const Manifest = struct {
457474
458475 pub fn hash(this: @This(), off: Offset) u32 {
459476 const file_prefix = off.getConst(this.contents).flags.prefix;
460 const file_path = filePath(this.contents, off);
477 const file_path = off.path(this.contents);
461478 return @truncate(std.hash.Wyhash.hash(file_prefix, file_path));
462479 }
463480
......@@ -466,8 +483,8 @@ pub const Manifest = struct {
466483 const a_prefix = a_off.getConst(this.contents).flags.prefix;
467484 const b_prefix = b_off.getConst(this.contents).flags.prefix;
468485 if (a_prefix != b_prefix) return false;
469 const a_path = filePath(this.contents, a_off);
470 const b_path = filePath(this.contents, b_off);
486 const a_path = a_off.path(this.contents);
487 const b_path = b_off.path(this.contents);
471488 return mem.eql(u8, a_path, b_path);
472489 }
473490 };
......@@ -480,12 +497,24 @@ pub const Manifest = struct {
480497 hasher.final(&file.digest);
481498 }
482499
483 fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void {
484 file.size = stat.size;
485 file.inode = stat.inode;
486 file.mtime = @intCast(stat.mtime.toNanoseconds());
500 fn stat(file: *const File) Stat {
501 return .{
502 .size = file.size,
503 .inode = file.inode,
504 .mtime = file.mtime,
505 };
506 }
507
508 fn setStatUnchecked(file: *File, s: Stat) void {
509 file.size = s.size;
510 file.inode = s.inode;
511 file.mtime = @intCast(s.mtime.toNanoseconds());
512 }
513
514 fn setStat(file: *File, m: *Manifest, s: Stat) Io.Cancelable!void {
515 setStatUnchecked(file, s);
487516
488 if (try m.isProblematicTimestamp(stat.mtime)) {
517 if (try m.isProblematicTimestamp(s.mtime)) {
489518 // The actual file has an unreliable timestamp; force it to be hashed.
490519 file.mtime = 0;
491520 file.inode = 0;
......@@ -493,14 +522,14 @@ pub const Manifest = struct {
493522 }
494523
495524 /// Returns true if the stat was changed. Updates the `file` with the new stat value.
496 fn setStatChanged(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!bool {
497 if (stat.size == file.size and
498 stat.mtime.nanoseconds == file.mtime and
499 stat.inode == file.inode)
525 fn setStatChanged(file: *File, m: *Manifest, s: Stat) Io.Cancelable!bool {
526 if (s.size == file.size and
527 s.mtime.nanoseconds == file.mtime and
528 s.inode == file.inode)
500529 {
501530 return false;
502531 } else {
503 try setStat(file, m, stat);
532 try setStat(file, m, s);
504533 return true;
505534 }
506535 }
......@@ -535,7 +564,7 @@ pub const Manifest = struct {
535564 const prefix = fo.file_offset.get(contents).flags.prefix;
536565 return .{
537566 .root_dir = manifest.cache.prefixes()[prefix],
538 .sub_path = filePath(contents, fo.file_offset),
567 .sub_path = fo.file_offset.path(contents),
539568 };
540569 }
541570 };
......@@ -680,6 +709,21 @@ pub const Manifest = struct {
680709 _ = try addInputPath(m, opt_path orelse return, options);
681710 }
682711
712 pub const CheckResult = union(enum) {
713 hit,
714 incomplete_manifest,
715 path_deleted: File.Offset,
716 directory_status_changed: File.Offset,
717 metadata_changed: File.Offset,
718 contents_changed: File.Offset,
719 };
720
721 pub const CheckError = error{
722 /// Unable to check the cache for a reason that has been recorded into
723 /// the `diagnostic` field.
724 CacheCheckFailed,
725 } || Allocator.Error || Io.Cancelable;
726
683727 /// Check the cache to see if the input exists in it.
684728 /// A hex encoding of its hash is available by calling `final`.
685729 ///
......@@ -692,13 +736,13 @@ pub const Manifest = struct {
692736 /// The lock on the manifest file is released when `deinit` is called. As another
693737 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
694738 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
695 pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) Check.Error!Check.Status {
739 pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) CheckError!CheckResult {
696740 const node = parent_progress_node.start("Reusing Cache Artifacts", 0);
697741 defer node.end();
698742 return checkProgressless(man);
699743 }
700744
701 pub fn checkProgressless(man: *Manifest) Check.Error!Check.Status {
745 pub fn checkProgressless(man: *Manifest) CheckError!CheckResult {
702746 assert(man.manifest_file == null);
703747
704748 // This is *not* hashing the contents of the input files. It is the
......@@ -798,16 +842,18 @@ pub const Manifest = struct {
798842 // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests
799843 // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered
800844 // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user
801 // can use `final`, and will at some point `writeManifest` the file list to disk.
845 // can use `final`, and will at some point `finalize` the file list to disk.
802846
803847 man.hash.hasher = hasher_init;
804848 man.hash.hasher.update(&input_digest);
805849
806850 hit: {
807 digests: {
808 switch (try man.checkLocked()) {
809 .hit => break :hit,
810 .miss => if (!try man.upgradeToExclusiveLock()) break :digests,
851 const miss_result = miss: {
852 const result = try man.checkLocked();
853 if (result == .hit) {
854 break :hit;
855 } else if (!try man.upgradeToExclusiveLock()) {
856 break :miss result;
811857 }
812858 // We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone
813859 // else might have modified the digest, so we need to check again before deciding to miss.
......@@ -816,18 +862,16 @@ pub const Manifest = struct {
816862 man.hash.hasher = hasher_init;
817863 man.hash.hasher.update(&input_digest);
818864 man.shrinkFilesToInput();
819 switch (try man.checkLocked()) {
820 .hit => break :hit,
821 .miss => break :digests,
822 }
823 }
865 const refreshed_result = try man.checkLocked();
866 if (refreshed_result == .hit) break :hit;
867 break :miss refreshed_result;
868 };
824869
825 // Cache miss. `checkLocked` guarantees that all input files have their digests populated
826 // unless it returns an error.
827 man.manifest_dirty = true;
828 // All input file digests are already populated by `checkLocked`, so we can call `unhit` directly.
870 // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a
871 // cache miss, which is needed because they will be used in the manifest digest.
872 man.dirty = true;
829873 unhit(man, &input_digest);
830 return .miss;
874 return miss_result;
831875 }
832876
833877 if (man.want_shared_lock) {
......@@ -852,127 +896,454 @@ pub const Manifest = struct {
852896
853897 /// Assumes that `self.hash.hasher` has been updated only with the original digest and that
854898 /// `self.files` contains only the original input files.
855 fn checkLocked(m: *Manifest) Check.Error!Check.Status {
899 fn checkLocked(m: *Manifest) CheckError!CheckResult {
856900 const gpa = m.cache.gpa;
857901 const io = m.cache.io;
858902 const manifest_file = m.manifest_file.?;
859903
860904 const manifest_size = if (manifest_file.stat(io)) |stat| stat.size else |err| switch (err) {
861905 error.Canceled => |e| return e,
862 else => |e| {
863 m.diagnostic = .{ .manifest_stat = e };
864 return error.CacheCheckFailed;
865 },
906 else => |e| return fail(&m.diagnostic, .{ .manifest_stat = e }),
866907 };
867908
868 // We always want to compute the input file hash digests, even on a cache miss, because they will be
869 // used in the manifest digest.
870
871909 if (manifest_size == 0 or manifest_size < m.contents.items.len) {
872 // Don't clobber our input files by overwriting contents since it's a cache miss.
910 // Manifest file was never finalized.
873911 try m.contents.ensureUnusedCapacity(gpa, 1);
874 return .miss;
912 return .incomplete_manifest;
875913 }
876914
877 // Already supposedly includes the extra null byte that we have to maintain.
878 try m.contents.resize(gpa, manifest_size);
879
915 // We must not clobber existing `Manifest.contents` because it possibly contains prepopulated stat and digest
916 // information which we must compare with the file system. Since non input files are typically greater in number
917 // than input files, the strategy here is to move the contents to make room for entirety of the manifest
918 // contents on disk, and then later truncate the contents array.
919 //
920 // The file on disk already supposedly includes the extra null byte that we have to maintain.
921 const input_contents_len = m.contents.len;
922 // After this resize, every return statement needs to modify m.contents length.
923 try m.contents.resize(gpa, input_contents_len + manifest_size);
924 errdefer m.contents.shrinkRetainingCapacity(input_contents_len);
925 @memcpy(m.contents.items[manifest_size..][0..input_contents_len], m.contents.items[0..input_contents_len]);
880926 {
881927 const n = manifest_file.readPositionalAll(io, m.contents.items, 0) catch |err| switch (err) {
882928 error.Canceled => |e| return e,
883 else => |e| {
884 m.diagnostic = .{ .manifest_read = e };
885 return error.CacheCheckFailed;
886 },
929 else => |e| return fail(&m.diagnostic, .{ .manifest_read = e }),
887930 };
888 m.contents.shrinkRetainingCapacity(n);
931 if (n != manifest_size) return checkLockedMiss(m, input_contents_len, .incomplete_manifest);
889932 }
890 const contents = m.contents.items;
933 const disk_contents = m.contents.items[0..manifest_size];
934 const input_contents = m.contents.items[manifest_size..input_contents_len];
891935
936 // First the input files section, which must match our input files, otherwise it's invalid or incomplete.
892937 var off: usize = 0;
893 var c: Check = .{};
894
895 // First the input files section, which must match our input files, otherwise it's invalid format.
896 for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| {
897 if (off + 1 >= contents.len) return error.InvalidFormat;
898 const file_off: File.Offset = @fromBackingInt(@intCast(off));
899 const file = try file_off.getFallible(contents);
900 if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat;
901 const path = try filePathFallible(contents, file_off);
902 if (path.len == 0) return error.InvalidFormat;
903 if (input_file_off != file_off) return error.InvalidFormat;
904
905 try checkInputFile(m, &c, file_off, path, input_path);
906
907 off += File.sizeOf(path.len);
938 for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len], 0..) |*input_path, file_off, i| {
939 if (@backingInt(file_off) + 1 >= disk_contents.len)
940 return missInput(m, i, input_contents_len, manifest_size, .incomplete_manifest);
941 if (!mem.eql(u8, file_off.flagsAndPath(disk_contents), file_off.flagsAndPath(input_contents)))
942 return missInput(m, i, input_contents_len, manifest_size, .invalid_manifest);
943
944 const result = try checkInputPath(m, file_off, input_path, disk_contents, input_contents);
945 if (result != .hit) return missInput(m, i + 1, input_contents_len, manifest_size, result);
946 off = @backingInt(file_off);
908947 }
909948
910 if (c.status == .miss) return .miss;
911 if (m.diagnostic != .none) return error.CacheCheckFailed;
912
913949 // Guess number of files based on manifest contents len to reduce allocations.
914950 // This is not an upper bound; subsequent insertions may potentially allocate.
915 try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{
916 .contents = contents,
951 try m.files.ensureUnusedCapacityContext(gpa, disk_contents.len / (@sizeOf(File) + 32), .{
952 .contents = disk_contents,
917953 });
918954
919 // Validate and count discovered files.
920 while (off + 1 < contents.len) {
955 // Validate and check discovered files.
956 while (off + 1 < disk_contents.len) {
921957 const file_off: File.Offset = @fromBackingInt(@intCast(off));
922 const file = try file_off.getFallible(contents);
923 if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat;
924 const path = try filePathFallible(contents, file_off);
925 if (path.len == 0) return error.InvalidFormat;
926
927 try m.files.putContext(gpa, file_off, {}, .{ .contents = contents });
928 switch (try checkFile(m, &c, file_off, filePath(contents, file_off))) {
929 .hit => {},
930 .miss => return .miss,
931 }
958 const file = try file_off.getFallible(disk_contents);
959 if (file.flags.prefix >= m.cache.prefixes_len)
960 return checkLockedMiss(m, input_contents_len, .invalid_manifest);
961 const path = try file_off.pathFallible(disk_contents);
962 if (path.len == 0) return checkLockedMiss(m, input_contents_len, .invalid_manifest);
963
964 try m.files.putContext(gpa, file_off, {}, .{ .contents = disk_contents });
965 const result = try checkDiscoveredPath(file_off, file_off.path(disk_contents));
966 if (result != .hit) return checkLockedMiss(m, input_contents_len, result);
932967
933968 off += File.sizeOf(path.len);
934969 }
935970
936971 // Final terminating zero byte to distinguish empty manifest file from
937972 // manifest with zero files.
938 const file_valid = off + 1 == contents.len and contents[off] == 0;
939 if (!file_valid) return .miss;
973 const file_valid = off + 1 == disk_contents.len and disk_contents[off] == 0;
974 if (!file_valid) return checkLockedMiss(m, input_contents_len, .incomplete_manifest);
940975
941 for (m.files.keys()) |file_off| m.hash.hasher.update(&file_off.get(contents).digest);
942
943 // Don't track the trailing zero byte in contents.
944 m.contents.items.len -= 1;
976 for (m.files.keys()) |file_off| m.hash.hasher.update(&file_off.get(disk_contents).digest);
945977
978 // Since it's a cache hit, we accept the input file contents from disk and discard the other copy.
979 // Furthermore, don't track the trailing zero byte in contents.
980 m.contents.shrinkRetainingCapacity(manifest_size - 1);
946981 return .hit;
947982 }
948983
949 fn checkInputFile(
984 /// Restore input file contents to prepare for cache miss workflow. We want to keep everything prior to `off` to
985 /// avoid redundant computation of digest and stat.
986 ///
987 /// Furthermore, this function is responsible for populating the remaining input file digests and metadata after the
988 /// missed one.
989 fn missInput(
990 m: *Manifest,
991 next_file_index: usize,
992 input_contents_len: usize,
993 manifest_size: usize,
994 result: CheckResult,
995 ) CheckError!CheckResult {
996 const file_offs = m.files.keys();
997 const off = if (file_offs.len - next_file_index == 0) input_contents_len else file_offs[next_file_index];
998 const copy_len = input_contents_len - off;
999 @memcpy(
1000 m.contents.items[off..][0..copy_len],
1001 m.contents.items[manifest_size + off ..][0..copy_len],
1002 );
1003 m.contents.shrinkRetainingCapacity(input_contents_len);
1004 // And now iterate over remaining input files and populate the missing input file hashes.
1005 const contents = m.contents.items;
1006 for (file_offs[next_file_index..], m.input_paths.items) |input_file_off, *input_path| {
1007 assert(@backingInt(input_file_off) == off);
1008 try populateInputPath(m, input_file_off, input_path, contents);
1009 }
1010 return result;
1011 }
1012
1013 fn checkLockedMiss(m: *Manifest, input_contents_len: usize, result: CheckResult) CheckResult {
1014 m.contents.shrinkRetainingCapacity(input_contents_len);
1015 return result;
1016 }
1017
1018 /// Similar to `checkInputPath` and `checkDiscoveredPath` but does not compare metadata or digest against file
1019 /// system. Only ensures that metadata and digest are available.
1020 fn populateInputPath(
9501021 m: *Manifest,
951 c: *Check,
9521022 file_off: File.Offset,
953 file_path: [:0]const u8,
9541023 input_path: *InputPath,
955 ) CheckFileError!void {
956 if (input_path.have_handle) @panic("TODO");
957 if (input_path.have_stat) @panic("TODO");
958 if (input_path.contents.status != .unrequested) @panic("TODO");
959 if (input_path.request_handle) @panic("TODO");
960 switch (try checkFile(m, c, file_off, file_path)) {
961 .hit => {},
962 .miss => c.status = .miss,
1024 contents: []const u8,
1025 ) CheckError!void {
1026 const cache = m.cache;
1027 const io = cache.io;
1028 const input_file = file_off.get(contents);
1029 const parent_dir = cache.prefixes()[input_file.flags.prefix].handle;
1030 const file_path = file_off.path(contents);
1031
1032 if (input_path.have_digest) return;
1033
1034 const stat_path_ok = input_path.contents.status.populated() or input_file.flags.metadata_only;
1035 if (!input_path.have_stat and stat_path_ok) {
1036 // Since this is an input file, FileNotFound counts as a failure, not a cache miss.
1037 const actual_stat = switch (input_path.getHandle(input_file)) {
1038 .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) {
1039 error.Canceled => |e| return e,
1040 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1041 .file_offset = file_off,
1042 .err = e,
1043 } }),
1044 },
1045 .file => |opened_file| opened_file.stat(io) catch |err| switch (err) {
1046 error.Canceled => |e| return e,
1047 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1048 .file_offset = file_off,
1049 .err = e,
1050 } }),
1051 },
1052 .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) {
1053 error.Canceled => |e| return e,
1054 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1055 .file_offset = file_off,
1056 .err = e,
1057 } }),
1058 },
1059 };
1060 const actual_is_directory = actual_stat.kind == .directory;
1061 if (actual_is_directory != input_file.flags.is_directory) {
1062 // Since this is an input file, this is a failure, not a cache miss.
1063 return fail(&m.diagnostic, .{ .file_stat = .{
1064 .file_offset = file_off,
1065 .err = if (actual_is_directory) error.IsDir else error.NotDir,
1066 } });
1067 }
1068 try input_file.setStat(m, .init(actual_stat));
1069 }
1070
1071 if (input_file.flags.metadata_only) {
1072 assert(stat_path_ok);
1073 input_file.hashFromMetadata();
1074 return;
1075 }
1076
1077 if (input_path.contents.get()) |file_or_dir_contents| {
1078 assert(stat_path_ok);
1079 var hasher = hasher_init;
1080 hasher.update(file_or_dir_contents);
1081 hasher.final(&input_file.digest);
1082 return;
1083 }
1084
1085 if (input_file.flags.is_directory) {
1086 const opened_dir = if (input_path.have_handle)
1087 input_path.handle.dir
1088 else
1089 parent_dir.openDir(io, file_path, .{
1090 .iterate = true,
1091 .access_sub_paths = false,
1092 }) catch |err| switch (err) {
1093 error.Canceled => |e| return e,
1094 else => |e| return fail(&m.diagnostic, .{ .file_open = .{
1095 .file_offset = file_off,
1096 .err = e,
1097 } }),
1098 };
1099 defer if (!input_path.request_handle) opened_dir.close(io) else {
1100 input_path.handle = .{ .dir = opened_dir };
1101 input_path.have_handle = true;
1102 };
1103
1104 if (!input_path.have_stat) {
1105 assert(!stat_path_ok);
1106 const actual_stat = opened_dir.stat(io) catch |err| switch (err) {
1107 error.Canceled => |e| return e,
1108 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1109 .file_offset = file_off,
1110 .err = e,
1111 } }),
1112 };
1113 try input_file.setStat(m, .init(actual_stat));
1114 }
1115
1116 const dir_contents_start = m.all_input_content.items.len;
1117 const gpa = cache.gpa;
1118 hashDir(gpa, io, opened_dir, &input_file.digest, &m.all_input_content) catch |err| switch (err) {
1119 error.Canceled, error.OutOfMemory => |e| return e,
1120 else => |e| return fail(&m.diagnostic, .{ .file_read = .{
1121 .file_offset = file_off,
1122 .err = e,
1123 } }),
1124 };
1125 defer if (input_path.contents.status == .requested) {
1126 input_path.contents = .{ .populated = .{
1127 .off = @intCast(dir_contents_start),
1128 .len = @intCast(m.all_input_content.items.len - dir_contents_start),
1129 } };
1130 } else m.all_input_content.shrinkRetainingCapacity(dir_contents_start);
1131 } else {
1132 const opened_file = if (input_path.have_handle)
1133 input_path.handle.file
1134 else
1135 parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) {
1136 error.Canceled => |e| return e,
1137 else => |e| return fail(&m.diagnostic, .{ .file_open = .{
1138 .file_offset = file_off,
1139 .err = e,
1140 } }),
1141 };
1142 defer if (!input_path.request_handle) opened_file.close(io) else {
1143 input_path.handle = .{ .file = opened_file };
1144 input_path.have_handle = true;
1145 };
1146
1147 if (!input_path.have_stat) {
1148 assert(!stat_path_ok);
1149 const actual_stat = opened_file.stat(io) catch |err| switch (err) {
1150 error.Canceled => |e| return e,
1151 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1152 .file_offset = file_off,
1153 .err = e,
1154 } }),
1155 };
1156 try input_file.setStat(m, .init(actual_stat));
1157 }
1158
1159 hashFile(io, opened_file, &input_file.digest) catch |err| switch (err) {
1160 error.Canceled => |e| return e,
1161 else => |e| return fail(&m.diagnostic, .{ .file_read = .{
1162 .file_offset = file_off,
1163 .err = e,
1164 } }),
1165 };
1166
1167 if (input_path.contents.status == .requested) @panic("TODO");
9631168 }
9641169 }
9651170
966 const CheckFileError = error{ Canceled, CacheCheckFailed };
1171 /// Upon return, disk_contents digest and stat will be populated regardless of whether hit or miss occurs.
1172 fn checkInputPath(
1173 m: *Manifest,
1174 file_off: File.Offset,
1175 input_path: *InputPath,
1176 disk_contents: []const u8,
1177 input_contents: []const u8,
1178 ) CheckError!CheckResult {
1179 const cache = m.cache;
1180 const io = cache.io;
1181 const disk_file = file_off.get(disk_contents);
1182 const input_file = file_off.get(input_contents);
1183 const parent_dir = cache.prefixes()[disk_file.flags.prefix].handle;
1184 const file_path = file_off.path(disk_contents);
1185
1186 assert(disk_file.flags == input_file.flags);
1187
1188 if (input_path.have_digest) {
1189 if (mem.eql(u8, &disk_file.digest, &input_file.digest))
1190 return .hit;
1191 }
1192
1193 if (input_path.have_stat) {
1194 const changed = try disk_file.setStatChanged(m, input_file.stat);
1195 if (!changed) return .hit;
1196 }
1197
1198 if (disk_file.flags.metadata_only) {
1199 if (!input_path.have_stat) {
1200 // Since this is an input file, FileNotFound counts as a failure, not a cache miss.
1201 const actual_stat = switch (input_path.getHandle(input_file)) {
1202 .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) {
1203 error.Canceled => |e| return e,
1204 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1205 .file_offset = file_off,
1206 .err = e,
1207 } }),
1208 },
1209 .file => |opened_file| opened_file.stat(io) catch |err| switch (err) {
1210 error.Canceled => |e| return e,
1211 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1212 .file_offset = file_off,
1213 .err = e,
1214 } }),
1215 },
1216 .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) {
1217 error.Canceled => |e| return e,
1218 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1219 .file_offset = file_off,
1220 .err = e,
1221 } }),
1222 },
1223 };
1224 // In the other case (have_stat=true), this check is redundant since we already asserted the flags are
1225 // identical, which contains the prefix.
1226 const actual_is_directory = actual_stat.kind == .directory;
1227 if (actual_is_directory != disk_file.flags.is_directory) {
1228 // Since this is an input file, this is a failure, not a cache miss.
1229 return fail(&m.diagnostic, .{ .file_stat = .{
1230 .file_offset = file_off,
1231 .err = if (actual_is_directory) error.IsDir else error.NotDir,
1232 } });
1233 }
1234 const stat: Stat = .init(actual_stat);
1235 const changed = try disk_file.setStatChanged(m, stat);
1236 disk_file.hashFromMetadata();
1237 if (!changed) return .hit;
1238 }
1239 return .{ .metadata_changed = file_off };
1240 }
1241
1242 if (input_path.contents.get()) |file_or_dir_contents| {
1243 var hasher = hasher_init;
1244 hasher.update(file_or_dir_contents);
1245 hasher.final(&disk_file.digest);
1246
1247 if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit;
1248 return .{ .contents_changed = file_off };
1249 }
1250
1251 if (disk_file.flags.is_directory) {
1252 const opened_dir = if (input_path.have_handle)
1253 input_path.handle.dir
1254 else
1255 parent_dir.openDir(io, file_path, .{
1256 .iterate = true,
1257 .access_sub_paths = false,
1258 }) catch |err| switch (err) {
1259 error.Canceled => |e| return e,
1260 else => |e| return fail(&m.diagnostic, .{ .file_open = .{
1261 .file_offset = file_off,
1262 .err = e,
1263 } }),
1264 };
1265 defer if (!input_path.request_handle) opened_dir.close(io) else {
1266 input_path.handle = .{ .dir = opened_dir };
1267 input_path.have_handle = true;
1268 };
1269
1270 if (!input_path.have_stat) {
1271 const actual_stat = opened_dir.stat(io) catch |err| switch (err) {
1272 error.Canceled => |e| return e,
1273 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1274 .file_offset = file_off,
1275 .err = e,
1276 } }),
1277 };
1278 if (!try disk_file.setStatChanged(m, .init(actual_stat))) return .hit;
1279 }
1280
1281 const dir_contents_start = m.all_input_content.items.len;
1282 const gpa = cache.gpa;
1283 hashDir(gpa, io, opened_dir, &disk_file.digest, &m.all_input_content) catch |err| switch (err) {
1284 error.Canceled, error.OutOfMemory => |e| return e,
1285 else => |e| return fail(&m.diagnostic, .{ .file_read = .{
1286 .file_offset = file_off,
1287 .err = e,
1288 } }),
1289 };
1290 defer if (input_path.contents.status == .requested) {
1291 input_path.contents = .{ .populated = .{
1292 .off = @intCast(dir_contents_start),
1293 .len = @intCast(m.all_input_content.items.len - dir_contents_start),
1294 } };
1295 } else m.all_input_content.shrinkRetainingCapacity(dir_contents_start);
1296
1297 if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit;
1298 return .{ .contents_changed = file_off };
1299 } else {
1300 const opened_file = if (input_path.have_handle)
1301 input_path.handle.file
1302 else
1303 parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) {
1304 error.Canceled => |e| return e,
1305 else => |e| return fail(&m.diagnostic, .{ .file_open = .{
1306 .file_offset = file_off,
1307 .err = e,
1308 } }),
1309 };
1310 defer if (!input_path.request_handle) opened_file.close(io) else {
1311 input_path.handle = .{ .file = opened_file };
1312 input_path.have_handle = true;
1313 };
1314
1315 if (!input_path.have_stat) {
1316 const actual_stat = opened_file.stat(io) catch |err| switch (err) {
1317 error.Canceled => |e| return e,
1318 else => |e| return fail(&m.diagnostic, .{ .file_stat = .{
1319 .file_offset = file_off,
1320 .err = e,
1321 } }),
1322 };
1323 if (!try disk_file.setStatChanged(m, .init(actual_stat))) return .hit;
1324 }
9671325
968 /// Runs concurrently with other `checkFile`.
969 fn checkFile(
1326 hashFile(io, opened_file, &disk_file.digest) catch |err| switch (err) {
1327 error.Canceled => |e| return e,
1328 else => |e| return fail(&m.diagnostic, .{ .file_read = .{
1329 .file_offset = file_off,
1330 .err = e,
1331 } }),
1332 };
1333 if (input_path.contents.status == .requested) @panic("TODO");
1334
1335 if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit;
1336 return .{ .contents_changed = file_off };
1337 }
1338 }
1339
1340 fn checkDiscoveredPath(
9701341 m: *Manifest,
971 c: *Check,
9721342 file_off: File.Offset,
9731343 file_path: [:0]const u8,
974 ) CheckFileError!Check.Status {
975 const file = file_off.get(m.contents.items);
1344 contents: []const u8,
1345 ) CheckError!CheckResult {
1346 const file = file_off.get(contents);
9761347 const cache = m.cache;
9771348 const gpa = cache.gpa;
9781349 const io = cache.io;
......@@ -980,20 +1351,21 @@ pub const Manifest = struct {
9801351
9811352 if (file.flags.metadata_only) {
9821353 const actual_stat = parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) {
983 error.FileNotFound => return .miss,
1354 error.FileNotFound => return .{ .path_deleted = file_off },
9841355 error.Canceled => |e| return e,
985 else => |e| return c.fail(m, .{ .file_stat = .{
1356 else => |e| return fail(m, .{ .file_stat = .{
9861357 .file_offset = file_off,
9871358 .err = e,
9881359 } }),
9891360 };
9901361
9911362 const actual_is_directory = actual_stat.kind == .directory;
992 if (actual_is_directory != file.flags.is_directory) return .miss;
1363 if (actual_is_directory != file.flags.is_directory) return .{ .directory_status_changed = file_off };
9931364
9941365 const changed = try file.setStatChanged(m, .init(actual_stat));
9951366 file.hashFromMetadata();
996 return if (changed) .miss else .hit;
1367 if (changed) return .{ .metadata_changed = file_off };
1368 return .hit;
9971369 }
9981370
9991371 if (file.flags.is_directory) {
......@@ -1001,9 +1373,10 @@ pub const Manifest = struct {
10011373 .iterate = true,
10021374 .access_sub_paths = false,
10031375 }) catch |err| switch (err) {
1004 error.FileNotFound, error.NotDir => return .miss,
1376 error.FileNotFound => return .{ .path_deleted = file_off },
1377 error.NotDir => return .{ .directory_status_changed = file_off },
10051378 error.Canceled => |e| return e,
1006 else => |e| return c.fail(m, .{ .file_open = .{
1379 else => |e| return fail(m, .{ .file_open = .{
10071380 .file_offset = file_off,
10081381 .err = e,
10091382 } }),
......@@ -1012,32 +1385,33 @@ pub const Manifest = struct {
10121385
10131386 const actual_stat = opened_dir.stat(io) catch |err| switch (err) {
10141387 error.Canceled => |e| return e,
1015 else => |e| return c.fail(m, .{ .file_stat = .{
1388 else => |e| return fail(m, .{ .file_stat = .{
10161389 .file_offset = file_off,
10171390 .err = e,
10181391 } }),
10191392 };
10201393 if (try file.setStatChanged(m, .init(actual_stat))) {
10211394 const prev_digest: BinDigest = file.digest;
1022 var contents: std.ArrayList(u8) = .empty;
1023 defer contents.deinit(gpa);
1024 hashDir(gpa, io, opened_dir, &file.digest, &contents) catch |err| switch (err) {
1025 error.Canceled => |e| return e,
1026 else => |e| return c.fail(m, .{ .file_read = .{
1395 const dir_contents_start = m.all_input_content.items.len;
1396 defer m.all_input_content.shrinkRetainingCapacity(dir_contents_start);
1397 hashDir(gpa, io, opened_dir, &file.digest, &m.all_input_contents) catch |err| switch (err) {
1398 error.Canceled, error.OutOfMemory => |e| return e,
1399 else => |e| return fail(m, .{ .file_read = .{
10271400 .file_offset = file_off,
10281401 .err = e,
10291402 } }),
10301403 };
10311404
1032 if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss;
1405 if (!mem.eql(u8, &file.digest, &prev_digest)) return .{ .contents_changed = file_off };
10331406 }
10341407 return .hit;
10351408 }
10361409
10371410 const opened_file = parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) {
1038 error.FileNotFound, error.IsDir => return .miss,
1411 error.FileNotFound => return .{ .path_deleted = file_off },
1412 error.IsDir => return .{ .directory_status_changed = file_off },
10391413 error.Canceled => |e| return e,
1040 else => |e| return c.fail(m, .{ .file_open = .{
1414 else => |e| return fail(m, .{ .file_open = .{
10411415 .file_offset = file_off,
10421416 .err = e,
10431417 } }),
......@@ -1046,7 +1420,7 @@ pub const Manifest = struct {
10461420
10471421 const actual_stat = opened_file.stat(io) catch |err| switch (err) {
10481422 error.Canceled => |e| return e,
1049 else => |e| return c.fail(m, .{ .file_stat = .{
1423 else => |e| return fail(m, .{ .file_stat = .{
10501424 .file_offset = file_off,
10511425 .err = e,
10521426 } }),
......@@ -1056,19 +1430,25 @@ pub const Manifest = struct {
10561430 const prev_digest: BinDigest = file.digest;
10571431 hashFile(io, opened_file, &file.digest) catch |err| switch (err) {
10581432 error.Canceled => |e| return e,
1059 else => |e| return c.fail(m, .{ .file_read = .{
1433 else => |e| return fail(m, .{ .file_read = .{
10601434 .file_offset = file_off,
10611435 .err = e,
10621436 } }),
10631437 };
10641438
1065 if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss;
1439 if (!mem.eql(u8, &file.digest, &prev_digest)) return .{ .contents_changed = file_off };
10661440 }
10671441
10681442 return .hit;
10691443 }
10701444
1071 /// Reset `man.hash.hasher` to the state it should be in after `check` returns `Check.Status.miss`.
1445 fn fail(ptr: *Diagnostic, d: Diagnostic) error{CacheCheckFailed} {
1446 assert(ptr.* == .none);
1447 ptr.* = d;
1448 return error.CacheCheckFailed;
1449 }
1450
1451 /// Reset `man.hash.hasher` to the state it should be in after `check` returns `CheckResult.miss`.
10721452 /// The hasher contains the original input digest, and all original input file digests (i.e.
10731453 /// not including discovered files).
10741454 ///
......@@ -1231,7 +1611,7 @@ pub const Manifest = struct {
12311611 try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic);
12321612 } else {
12331613 const dir = cache.prefixes()[prefix].handle;
1234 const sub_path = filePath(m.contents.items, file_offset);
1614 const sub_path = file_offset.path(m.contents.items);
12351615 const handle = dir.openDir(io, sub_path, .{
12361616 .access_sub_paths = false,
12371617 .iterate = true,
......@@ -1250,7 +1630,7 @@ pub const Manifest = struct {
12501630 try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic);
12511631 } else {
12521632 const dir = cache.prefixes()[prefix].handle;
1253 const sub_path = filePath(m.contents.items, file_offset);
1633 const sub_path = file_offset.path(m.contents.items);
12541634 const handle = dir.openFile(io, sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
12551635 error.Canceled => |e| return e,
12561636 else => |e| {
......@@ -1420,18 +1800,18 @@ pub const Manifest = struct {
14201800
14211801 /// If `want_shared_lock` is true, this function automatically downgrades the
14221802 /// lock from exclusive to shared.
1423 pub fn writeManifest(m: *Manifest) !void {
1803 pub fn finalize(m: *Manifest) !void {
14241804 assert(m.have_exclusive_lock);
14251805 const io = m.cache.io;
14261806 const manifest_file = m.manifest_file.?;
1427 if (m.manifest_dirty) {
1807 if (m.dirty) {
14281808 m.contents.appendAssumeCapacity(0);
14291809 defer _ = m.contents.pop().?;
14301810
14311811 try manifest_file.setLength(io, m.contents.items.len);
14321812 try manifest_file.writePositionalAll(io, m.contents.items, 0);
14331813
1434 m.manifest_dirty = false;
1814 m.dirty = false;
14351815 }
14361816
14371817 if (m.want_shared_lock) {
......@@ -1473,7 +1853,7 @@ pub const Manifest = struct {
14731853 /// Obtain only the data needed to maintain a lock on the manifest file.
14741854 /// The `Manifest` remains safe to deinit.
14751855 ///
1476 /// Don't forget to call `writeManifest` before this!
1856 /// Don't forget to call `finalize` before this!
14771857 pub fn toOwnedLock(self: *Manifest) Lock {
14781858 defer self.manifest_file = null;
14791859 return .{ .manifest_file = self.manifest_file.? };
......@@ -1496,7 +1876,7 @@ pub const Manifest = struct {
14961876 }
14971877
14981878 pub fn path(scf: *const SelfContainedFiles, file_offset: File.Offset) [:0]const u8 {
1499 return filePath(scf.contents.items, file_offset);
1879 return file_offset.path(scf.contents.items);
15001880 }
15011881 };
15021882
......@@ -1516,7 +1896,7 @@ pub const Manifest = struct {
15161896 /// Releases the manifest file and frees any memory the Manifest was using.
15171897 /// `Manifest.hit` must be called first.
15181898 ///
1519 /// Don't forget to call `writeManifest` before this!
1899 /// Don't forget to call `finalize` before this!
15201900 pub fn deinit(m: *Manifest) void {
15211901 const io = m.cache.io;
15221902 const gpa = m.cache.gpa;
......@@ -1544,7 +1924,7 @@ pub const Manifest = struct {
15441924 if (files.len > 0) {
15451925 const contents = man.contents.items;
15461926 for (files) |file| {
1547 const file_path = filePath(contents, file);
1927 const file_path = file.path(contents);
15481928 try buf.ensureUnusedCapacity(gpa, file_path.len + 2);
15491929 buf.appendAssumeCapacity(@as(u8, file.get(contents).flags.prefix) + 1);
15501930 buf.appendSliceAssumeCapacity(file_path);
......@@ -1675,22 +2055,7 @@ pub const Manifest = struct {
16752055 }
16762056
16772057 fn hashFlagsAndPath(m: *const Manifest, off: File.Offset, hasher: *Hasher) void {
1678 const contents = m.contents.items;
1679 const flags_off = @offsetOf(File, "flags");
1680 comptime assert(@offsetOf(File, "path_start") - flags_off == 1);
1681 const hash_start = @backingInt(off) + flags_off;
1682 const hash_end = mem.findScalarPos(u8, contents, hash_start, 0).?;
1683 hasher.update(contents[hash_start..hash_end]);
1684 }
1685
1686 pub fn filePathFallible(contents: []const u8, off: File.Offset) error{InvalidFormat}![:0]const u8 {
1687 const path_start = @backingInt(off) + @offsetOf(File, "path_start");
1688 const path_end = mem.findScalarPos(u8, contents, path_start, 0) orelse return error.InvalidFormat;
1689 return contents[path_start..path_end :0];
1690 }
1691
1692 fn filePath(contents: []const u8, off: File.Offset) [:0]const u8 {
1693 return filePathFallible(contents, off) catch unreachable;
2058 hasher.update(off.flagsAndPath(m.contents.items));
16942059 }
16952060};
16962061
......@@ -1756,7 +2121,7 @@ test "cache file and then recall it" {
17562121 try testing.expectEqual(.miss, try ch.check(.none));
17572122
17582123 digest1 = ch.final();
1759 try ch.writeManifest();
2124 try ch.finalize();
17602125 }
17612126 {
17622127 var ch = cache.obtain();
......@@ -1827,7 +2192,7 @@ test "check that changing a file makes cache fail" {
18272192
18282193 digest1 = man.final();
18292194
1830 try man.writeManifest();
2195 try man.finalize();
18312196 }
18322197
18332198 try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = updated_temp_file_contents });
......@@ -1847,7 +2212,7 @@ test "check that changing a file makes cache fail" {
18472212
18482213 digest2 = man.final();
18492214
1850 try man.writeManifest();
2215 try man.finalize();
18512216 }
18522217
18532218 try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
......@@ -1888,7 +2253,7 @@ test "no file inputs" {
18882253
18892254 digest1 = man.final();
18902255
1891 try man.writeManifest();
2256 try man.finalize();
18922257 }
18932258 {
18942259 var man = cache.obtain();
......@@ -1953,7 +2318,7 @@ test "Manifest with files added after initial hash work" {
19532318 _ = try ch.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } });
19542319
19552320 digest1 = ch.final();
1956 try ch.writeManifest();
2321 try ch.finalize();
19572322 }
19582323 {
19592324 var ch = cache.obtain();
......@@ -1992,7 +2357,7 @@ test "Manifest with files added after initial hash work" {
19922357
19932358 digest3 = ch.final();
19942359
1995 try ch.writeManifest();
2360 try ch.finalize();
19962361 }
19972362
19982363 try testing.expect(!mem.eql(u8, &digest1, &digest3));