authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-08-24 20:08:32-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-04 17:41:27-07:00
log8ba8b0af6bfa93760ae325734a347670b815ab42
treeb6f54f5c4dd5f699e942db1cb47ee5997124644a
parent91c5eaeab9e2c5d71bda4614c8748d6ebf715835

std.Build.Cache: implement is_directory and metadata_only

1 files changed, 317 insertions(+), 97 deletions(-)

lib/std/Build/Cache.zig+317-97
...@@ -49,7 +49,6 @@ pub fn obtain(cache: *Cache) Manifest {...@@ -49,7 +49,6 @@ pub fn obtain(cache: *Cache) Manifest {
49 .hash = cache.hash,49 .hash = cache.hash,
50 .manifest_file = null,50 .manifest_file = null,
51 .manifest_dirty = false,51 .manifest_dirty = false,
52 .hex_digest = undefined,
53 };52 };
54}53}
5554
...@@ -251,7 +250,7 @@ pub const HashHelper = struct {...@@ -251,7 +250,7 @@ pub const HashHelper = struct {
251250
252pub fn binToHex(bin_digest: BinDigest) HexDigest {251pub fn binToHex(bin_digest: BinDigest) HexDigest {
253 var out_digest: HexDigest = undefined;252 var out_digest: HexDigest = undefined;
254 var w: std.Io.Writer = .fixed(&out_digest);253 var w: Io.Writer = .fixed(&out_digest);
255 w.printHex(&bin_digest, .lower) catch unreachable;254 w.printHex(&bin_digest, .lower) catch unreachable;
256 return out_digest;255 return out_digest;
257}256}
...@@ -278,7 +277,6 @@ pub const Manifest = struct {...@@ -278,7 +277,6 @@ pub const Manifest = struct {
278 cache: *Cache,277 cache: *Cache,
279 /// Current state for incremental hashing.278 /// Current state for incremental hashing.
280 hash: HashHelper,279 hash: HashHelper,
281 hex_digest: HexDigest,
282 /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase.280 /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase.
283 manifest_file: ?Io.File,281 manifest_file: ?Io.File,
284 manifest_dirty: bool,282 manifest_dirty: bool,
...@@ -309,9 +307,34 @@ pub const Manifest = struct {...@@ -309,9 +307,34 @@ pub const Manifest = struct {
309 /// All contents from all `input_files` whose contents were requested,307 /// All contents from all `input_files` whose contents were requested,
310 /// concatenated. Total byte size will be less than `max_input_content_len`308 /// concatenated. Total byte size will be less than `max_input_content_len`
311 /// otherwise an error is returned.309 /// otherwise an error is returned.
310 ///
311 /// Data is invalidated when `addFilePost` is called.
312 all_input_content: std.ArrayList(u8) = .empty,312 all_input_content: std.ArrayList(u8) = .empty,
313 max_input_content_len: usize = std.math.maxInt(u32),313 max_input_content_len: usize = std.math.maxInt(u32),
314314
315 /// State that exists only during check.
316 pub const Check = struct {
317 /// Protects `Manifest.diagnostic` from data races.
318 diagnostic_lock: bool = false,
319 status: Status = .hit,
320
321 pub const Status = enum { hit, miss };
322
323 pub const Error = error{
324 /// Unable to check the cache for a reason that has been recorded into
325 /// the `diagnostic` field.
326 CacheCheckFailed,
327 /// A cache manifest file exists however it could not be parsed.
328 InvalidFormat,
329 } || Allocator.Error || Io.Cancelable;
330
331 fn fail(c: *Check, m: *Manifest, diagnostic: Diagnostic) void {
332 if (!@atomicRmw(bool, &c.diagnostic_lock, .Xchg, true, .unordered)) {
333 m.diagnostic = diagnostic;
334 }
335 }
336 };
337
315 pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false);338 pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false);
316339
317 /// Source files whose prefix and relative path are included when computing340 /// Source files whose prefix and relative path are included when computing
...@@ -328,7 +351,7 @@ pub const Manifest = struct {...@@ -328,7 +351,7 @@ pub const Manifest = struct {
328 have_stat: bool,351 have_stat: bool,
329 /// Determines whether `File.digest` is populated.352 /// Determines whether `File.digest` is populated.
330 have_digest: bool,353 have_digest: bool,
331 contents: enum (usize) {354 contents: enum(usize) {
332 requested = std.math.maxInt(u32) - 1,355 requested = std.math.maxInt(u32) - 1,
333 not_requested = std.math.maxInt(u32),356 not_requested = std.math.maxInt(u32),
334 /// Byte offset index into `Manifest.all_input_content`.357 /// Byte offset index into `Manifest.all_input_content`.
...@@ -356,12 +379,29 @@ pub const Manifest = struct {...@@ -356,12 +379,29 @@ pub const Manifest = struct {
356 /// Terminated by zero byte, then followed by padding until 8-byte aligned.379 /// Terminated by zero byte, then followed by padding until 8-byte aligned.
357 path_start: [0]u8,380 path_start: [0]u8,
358381
359 pub const Flags = packed struct (u8) {382 pub const Flags = packed struct(u8) {
360 is_directory: bool,383 is_directory: bool,
361 metadata_only: bool,384 metadata_only: bool,
362 prefix: u6,385 prefix: u6,
363 };386 };
364387
388 /// Prefixes path names in encoded directory contents. Starts numbering
389 /// at `1` so that null byte can be used unambiguously as entry
390 /// separator.
391 pub const Kind = enum(u8) {
392 file = 1,
393 directory = 2,
394 other = 3,
395
396 pub fn fromStat(kind: Io.File.Kind) @This() {
397 return switch (kind) {
398 .file => .file,
399 .directory => .directory,
400 else => .other,
401 };
402 }
403 };
404
365 /// Byte index within `Manifest.contents` where the entry starts.405 /// Byte index within `Manifest.contents` where the entry starts.
366 pub const Offset = enum(u32) {406 pub const Offset = enum(u32) {
367 _,407 _,
...@@ -392,7 +432,6 @@ pub const Manifest = struct {...@@ -392,7 +432,6 @@ pub const Manifest = struct {
392 }432 }
393 };433 };
394434
395
396 pub fn path(file: *const File) [:0]const u8 {435 pub fn path(file: *const File) [:0]const u8 {
397 return pathFallible(file) catch unreachable;436 return pathFallible(file) catch unreachable;
398 }437 }
...@@ -408,7 +447,7 @@ pub const Manifest = struct {...@@ -408,7 +447,7 @@ pub const Manifest = struct {
408 const path_len = mem.findScalar(u8, path_ptr, 0).?;447 const path_len = mem.findScalar(u8, path_ptr, 0).?;
409 comptime assert(@offsetOf(File, "path_start") - @offsetOf(File, "flags") == 1);448 comptime assert(@offsetOf(File, "path_start") - @offsetOf(File, "flags") == 1);
410 // Includes flags and sentinel.449 // Includes flags and sentinel.
411 const hash_string = (path_ptr - 1)[0..path_len + 2];450 const hash_string = (path_ptr - 1)[0 .. path_len + 2];
412 hasher.update(hash_string);451 hasher.update(hash_string);
413 }452 }
414453
...@@ -424,9 +463,20 @@ pub const Manifest = struct {...@@ -424,9 +463,20 @@ pub const Manifest = struct {
424 }463 }
425 }464 }
426465
466 /// Returns true if the stat was changed. Updates the `file` with the new stat value.
467 fn setStatChanged(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!bool {
468 if (stat.size == file.size and
469 stat.mtime.nanoseconds == file.mtime and
470 stat.inode == file.inode)
471 {
472 return false;
473 } else {
474 setStat(file, m, stat);
475 return true;
476 }
477 }
427 };478 };
428479
429
430 pub const Diagnostic = union(enum) {480 pub const Diagnostic = union(enum) {
431 none,481 none,
432 manifest_create: Io.File.OpenError,482 manifest_create: Io.File.OpenError,
...@@ -438,7 +488,7 @@ pub const Manifest = struct {...@@ -438,7 +488,7 @@ pub const Manifest = struct {
438 file_hash: FileOp,488 file_hash: FileOp,
439489
440 pub const FileOp = struct {490 pub const FileOp = struct {
441 file_index: usize,491 file_offset: File.Offset,
442 err: anyerror,492 err: anyerror,
443 };493 };
444 };494 };
...@@ -450,16 +500,24 @@ pub const Manifest = struct {...@@ -450,16 +500,24 @@ pub const Manifest = struct {
450 };500 };
451501
452 pub const AddInputFileOptions = struct {502 pub const AddInputFileOptions = struct {
503 /// If `is_directory` is true, this handle must be opened with
504 /// iteration capability.
453 handle: ?Io.File = null,505 handle: ?Io.File = null,
454 stat: ?Stat = null,506 stat: ?Stat = null,
455 request_handle: bool = false,507 request_handle: bool = false,
508 /// Can request file or directory contents depending on `is_directory`.
456 request_contents: bool = false,509 request_contents: bool = false,
510 /// Contents of a directory are considered to be the sorted list of
511 /// file names of direct entries, separated by null byte. Each file name
512 /// is prefixed by `Io.File.Kind` byte, +1 so that the zero tag is
513 /// not aliased by the entry separator.
457 is_directory: bool = false,514 is_directory: bool = false,
515 /// Content hashing skipped; any difference in metadata implies cache
516 /// miss.
458 metadata_only: bool = false,517 metadata_only: bool = false,
459
460 };518 };
461519
462 pub const AddInputFileError = error {520 pub const AddInputFileError = error{
463 /// The same file path has been added to the cache manifest both as a521 /// The same file path has been added to the cache manifest both as a
464 /// directory and as a normal file, making the intended caching522 /// directory and as a normal file, making the intended caching
465 /// behavior ambiguous.523 /// behavior ambiguous.
...@@ -544,16 +602,6 @@ pub const Manifest = struct {...@@ -544,16 +602,6 @@ pub const Manifest = struct {
544 _ = try addInputFile(m, opt_path orelse return, options);602 _ = try addInputFile(m, opt_path orelse return, options);
545 }603 }
546604
547 pub const CheckError = error{
548 /// Unable to check the cache for a reason that has been recorded into
549 /// the `diagnostic` field.
550 CacheCheckFailed,
551 /// A cache manifest file exists however it could not be parsed.
552 InvalidFormat,
553 } || Allocator.Error || Io.Cancelable;
554
555 pub const CheckStatus = enum { hit, miss };
556
557 /// Check the cache to see if the input exists in it.605 /// Check the cache to see if the input exists in it.
558 /// A hex encoding of its hash is available by calling `final`.606 /// A hex encoding of its hash is available by calling `final`.
559 ///607 ///
...@@ -566,13 +614,13 @@ pub const Manifest = struct {...@@ -566,13 +614,13 @@ pub const Manifest = struct {
566 /// The lock on the manifest file is released when `deinit` is called. As another614 /// The lock on the manifest file is released when `deinit` is called. As another
567 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent615 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
568 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.616 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
569 pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) CheckError!CheckStatus {617 pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) Check.Error!Check.Status {
570 const node = parent_progress_node.start("Reusing Cache Artifacts", 0);618 const node = parent_progress_node.start("Reusing Cache Artifacts", 0);
571 defer node.end();619 defer node.end();
572 return checkProgressless(man);620 return checkProgressless(man);
573 }621 }
574622
575 pub fn checkProgressless(man: *Manifest) CheckError!CheckStatus {623 pub fn checkProgressless(man: *Manifest) Check.Error!Check.Status {
576 assert(man.manifest_file == null);624 assert(man.manifest_file == null);
577625
578 for (man.files.keys()[0..man.input_files.items.len]) |file_off| {626 for (man.files.keys()[0..man.input_files.items.len]) |file_off| {
...@@ -583,9 +631,8 @@ pub const Manifest = struct {...@@ -583,9 +631,8 @@ pub const Manifest = struct {
583631
584 var bin_digest: BinDigest = undefined;632 var bin_digest: BinDigest = undefined;
585 man.hash.hasher.final(&bin_digest);633 man.hash.hasher.final(&bin_digest);
586 man.hex_digest = binToHex(bin_digest);634 const hex_digest = binToHex(bin_digest);
587635 const manifest_file_path = &hex_digest;
588 const manifest_file_path = &man.hex_digest;
589 const io = man.cache.io;636 const io = man.cache.io;
590637
591 // We'll try to open the cache with an exclusive lock, but if that would block638 // We'll try to open the cache with an exclusive lock, but if that would block
...@@ -724,7 +771,7 @@ pub const Manifest = struct {...@@ -724,7 +771,7 @@ pub const Manifest = struct {
724771
725 /// Assumes that `self.hash.hasher` has been updated only with the original digest and that772 /// Assumes that `self.hash.hasher` has been updated only with the original digest and that
726 /// `self.files` contains only the original input files.773 /// `self.files` contains only the original input files.
727 fn checkLocked(m: *Manifest) CheckError!CheckStatus {774 fn checkLocked(m: *Manifest) Check.Error!Check.Status {
728 const gpa = m.cache.gpa;775 const gpa = m.cache.gpa;
729 const io = m.cache.io;776 const io = m.cache.io;
730777
...@@ -750,11 +797,12 @@ pub const Manifest = struct {...@@ -750,11 +797,12 @@ pub const Manifest = struct {
750797
751 // This group we would like to cancel as soon as a cache miss is discovered.798 // This group we would like to cancel as soon as a cache miss is discovered.
752 const PostResult = union(enum) {799 const PostResult = union(enum) {
753 checkFile: CheckFileResult,800 checkFile: Check.Status,
754 };801 };
755 var post_select_buffer: [10]PostResult = undefined;802 var post_select_buffer: [10]PostResult = undefined;
756 var post_select: Io.Select(PostResult) = .init(&post_select_buffer);803 var post_select: Io.Select(PostResult) = .init(&post_select_buffer);
757 var post_select_remaining: usize = 0;804 var post_select_remaining: usize = 0;
805 var c: Check = .{};
758 defer post_select.cancel(io);806 defer post_select.cancel(io);
759807
760 while (off + 1 < m.contents.len) {808 while (off + 1 < m.contents.len) {
...@@ -767,11 +815,11 @@ pub const Manifest = struct {...@@ -767,11 +815,11 @@ pub const Manifest = struct {
767 if (file_index < m.input_files.items.len) {815 if (file_index < m.input_files.items.len) {
768 if (m.files.keys()[file_index] != file_off) return error.InvalidFormat;816 if (m.files.keys()[file_index] != file_off) return error.InvalidFormat;
769817
770 input_group.async(io, checkFile, .{m.cache, file, path});818 input_group.async(io, checkInputFile, .{ m, &c, file_off, path });
771 } else {819 } else {
772 try m.files.put(gpa, file_off);820 try m.files.put(gpa, file_off);
773821
774 post_select.async(.checkFile, checkFile, .{m.cache, file, path});822 post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path });
775 post_select_remaining += 1;823 post_select_remaining += 1;
776 }824 }
777825
...@@ -794,6 +842,18 @@ pub const Manifest = struct {...@@ -794,6 +842,18 @@ pub const Manifest = struct {
794 while (post_select_remaining > 0) {842 while (post_select_remaining > 0) {
795 const n = try post_select.awaitMany(&post_await_buffer, 1);843 const n = try post_select.awaitMany(&post_await_buffer, 1);
796 post_select_remaining -= n;844 post_select_remaining -= n;
845
846 // Detect if input group already had a miss. In this case we still wait
847 // for those digests to be updated, but cancel the non input group.
848 switch (@atomicLoad(Check.Status, &c.status, .unordered)) {
849 .miss => {
850 post_select.cancelDiscard();
851 try input_group.await(io);
852 return .miss;
853 },
854 .hit => continue,
855 }
856
797 for (post_await_buffer[0..n]) |u| switch (u) {857 for (post_await_buffer[0..n]) |u| switch (u) {
798 .checkFile => |result| switch (result) {858 .checkFile => |result| switch (result) {
799 .hit => continue,859 .hit => continue,
...@@ -811,6 +871,7 @@ pub const Manifest = struct {...@@ -811,6 +871,7 @@ pub const Manifest = struct {
811 }871 }
812872
813 try input_group.await(io);873 try input_group.await(io);
874 if (c.status == .miss) return .miss;
814875
815 for (m.files.keys()) |file_off| {876 for (m.files.keys()) |file_off| {
816 m.hash.hasher.update(&file_off.get(m).digest);877 m.hash.hasher.update(&file_off.get(m).digest);
...@@ -819,54 +880,121 @@ pub const Manifest = struct {...@@ -819,54 +880,121 @@ pub const Manifest = struct {
819 return .hit;880 return .hit;
820 }881 }
821882
822 const CheckFileResult = union(enum) {883 fn checkInputFile(m: *Manifest, c: *Check, file_off: File.Offset, file_path: [:0]const u8) Io.Cancelable!void {
823 hit,884 // TODO use already open handle
824 miss,885 // TODO use already provided stat
825 fail: Diagnostic,886 // TODO implement request_handle
826 };887 // TODO implement request_contents
888 switch (try checkFile(m, c, file_off, file_path)) {
889 .hit => return,
890 .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered),
891 }
892 }
827893
828 /// Runs concurrently with other `checkFile`.894 /// Runs concurrently with other `checkFile`.
829 fn checkFile(cache: *const Cache, file: *File, file_path: [:0]const u8) Io.Cancelable!CheckFileResult {895 fn checkFile(
896 m: *Manifest,
897 c: *Check,
898 file_off: File.Offset,
899 file_path: [:0]const u8,
900 ) Io.Cancelable!Check.Status {
901 const file = file_off.get(m);
902 const cache = m.cache;
903 const gpa = cache.gpa;
830 const io = cache.io;904 const io = cache.io;
831 const dir = cache.prefixes()[file.flags.prefix].handle;905 const parent_dir = cache.prefixes()[file.flags.prefix].handle;
906
907 if (file.flags.metadata_only) {
908 const actual_stat = parent_dir.statFile() catch |err| switch (err) {
909 error.FileNotFound => return .miss,
910 error.Canceled => |e| return e,
911 else => |e| return c.fail(m, .{ .file_stat = .{
912 .file_offset = file_off,
913 .err = e,
914 } }),
915 };
916
917 const actual_is_directory = actual_stat.kind == .directory;
918 if (actual_is_directory != file.flags.is_directory) return .miss;
919
920 if (try file.setStatChanged(m, actual_stat)) return .miss;
921
922 return .hit;
923 }
832924
833 const this_file = dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) {925 if (file.flags.is_directory) {
834 error.FileNotFound => return .miss,926 const opened_dir = parent_dir.openDir(io, file_path, .{
927 .iterate = true,
928 .access_sub_paths = false,
929 }) catch |err| switch (err) {
930 error.FileNotFound, error.NotDir => return .miss,
931 error.Canceled => |e| return e,
932 else => |e| return c.fail(m, .{ .file_open = .{
933 .file_offset = file_off,
934 .err = e,
935 } }),
936 };
937 defer opened_dir.close(io);
938
939 const actual_stat = opened_dir.stat(io) catch |err| switch (err) {
940 error.Canceled => |e| return e,
941 else => |e| return c.fail(m, .{ .file_stat = .{
942 .file_offset = file_off,
943 .err = e,
944 } }),
945 };
946 if (try file.setStatChanged(m, actual_stat)) {
947 const prev_digest: BinDigest = file.digest;
948 var contents: std.ArrayList(u8) = .empty;
949 defer contents.deinit(gpa);
950 hashDir(gpa, io, opened_dir, &file.digest, &contents) catch |err| switch (err) {
951 error.Canceled => |e| return e,
952 else => |e| return c.fail(m, .{ .file_read = .{
953 .file_offset = file_off,
954 .err = e,
955 } }),
956 };
957
958 if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss;
959 }
960 return .hit;
961 }
962
963 const opened_file = parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) {
964 error.FileNotFound, error.IsDir => return .miss,
835 error.Canceled => |e| return e,965 error.Canceled => |e| return e,
836 else => |e| return .{ .fail = .{ .file_open = .{966 else => |e| return c.fail(m, .{ .file_open = .{
837 .file_index = file_index,967 .file_offset = file_off,
838 .err = e,968 .err = e,
839 } }},969 } }),
840 };970 };
841 defer this_file.close(io);971 defer opened_file.close(io);
842972
843 const actual_stat = this_file.stat(io) catch |err| return .{ .fail = .{ .file_stat = .{973 const actual_stat = opened_file.stat(io) catch |err| switch (err) {
844 .file_index = file_index,974 error.Canceled => |e| return e,
845 .err = err,975 else => |e| return c.fail(m, .{ .file_stat = .{
846 } }};976 .file_offset = file_off,
847 const size_match = actual_stat.size == file.size;977 .err = e,
848 const mtime_match = actual_stat.mtime.nanoseconds == file.mtime;978 } }),
849 const inode_match = actual_stat.inode == file.inode;979 };
850980
851 if (!size_match or !mtime_match or !inode_match) {981 if (try file.setStatChanged(m, actual_stat)) {
852 try file.setStat(actual_stat);982 const prev_digest: BinDigest = file.digest;
853983 hashFile(io, opened_file, &file.digest) catch |err| switch (err) {
854 var actual_digest: BinDigest = undefined;984 error.Canceled => |e| return e,
855 hashFile(io, this_file, &actual_digest) catch |err| return .{ .fail = .{ .file_read = .{985 else => |e| return c.fail(m, .{ .file_read = .{
856 .file_index = file_index,986 .file_offset = file_off,
857 .err = err,987 .err = e,
858 } }};988 } }),
859989 };
860 if (!mem.eql(u8, &file.digest, &actual_digest)) {990
861 file.digest = actual_digest;991 if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss;
862 return .miss;
863 }
864 }992 }
865993
866 return .hit;994 return .hit;
867 }995 }
868996
869 /// Reset `man.hash.hasher` to the state it should be in after `hit` returns `CheckStatus.miss`.997 /// Reset `man.hash.hasher` to the state it should be in after `hit` returns `Check.Status.miss`.
870 /// The hasher contains the original input digest, and all original input file digests (i.e.998 /// The hasher contains the original input digest, and all original input file digests (i.e.
871 /// not including post files).999 /// not including post files).
872 ///1000 ///
...@@ -931,16 +1059,18 @@ pub const Manifest = struct {...@@ -931,16 +1059,18 @@ pub const Manifest = struct {
931 dir: ?Io.Dir,1059 dir: ?Io.Dir,
932 } = .{ .file = null },1060 } = .{ .file = null },
933 stat: ?Stat = null,1061 stat: ?Stat = null,
1062 /// If it is a directory, there is a special encoding required for contents, which
1063 /// is null-separated sorted entries, each one prefixed with `File.Kind`.
934 contents: ?[]const u8 = null,1064 contents: ?[]const u8 = null,
935 metadata_only: bool = false,1065 metadata_only: bool = false,
936 };1066 };
9371067
938 pub const AddFilePostError = error {1068 pub const AddFilePostError = error{
939 /// The same file path has been added to the cache manifest both as a1069 /// The same file path has been added to the cache manifest both as a
940 /// directory and as a normal file, making the intended caching1070 /// directory and as a normal file, making the intended caching
941 /// behavior ambiguous.1071 /// behavior ambiguous.
942 IsDirectoryAmbiguous,1072 IsDirectoryAmbiguous,
943 } || Allocator.Error;1073 } || Io.Cancelable || Allocator.Error;
9441074
945 /// Add a file as a dependency of process being cached, after cache miss1075 /// Add a file as a dependency of process being cached, after cache miss
946 /// occurs.1076 /// occurs.
...@@ -1006,7 +1136,10 @@ pub const Manifest = struct {...@@ -1006,7 +1136,10 @@ pub const Manifest = struct {
1006 try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);1136 try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);
1007 } else {1137 } else {
1008 const dir = cache.prefixes()[header.flags.prefix].handle;1138 const dir = cache.prefixes()[header.flags.prefix].handle;
1009 const handle = try dir.openDir(io, header.path(), .{ .access_sub_paths = false, .iterate = true, });1139 const handle = try dir.openDir(io, header.path(), .{
1140 .access_sub_paths = false,
1141 .iterate = true,
1142 });
1010 defer handle.close(io);1143 defer handle.close(io);
1011 try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);1144 try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);
1012 },1145 },
...@@ -1022,7 +1155,14 @@ pub const Manifest = struct {...@@ -1022,7 +1155,14 @@ pub const Manifest = struct {
1022 }1155 }
1023 }1156 }
10241157
1025 fn populateFile(m: *Manifest, file: *File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void {1158 fn populateFile(
1159 m: *Manifest,
1160 file: *File,
1161 need_stat: bool,
1162 handle: Io.File,
1163 contents: ?[]const u8,
1164 metadata_only: bool,
1165 ) !void {
1026 const io = m.cache.io;1166 const io = m.cache.io;
10271167
1028 if (need_stat) {1168 if (need_stat) {
...@@ -1039,14 +1179,32 @@ pub const Manifest = struct {...@@ -1039,14 +1179,32 @@ pub const Manifest = struct {
1039 }1179 }
1040 }1180 }
10411181
1042 fn populateDirectory(m: *Manifest, file: *File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void {1182 fn populateDirectory(
1043 _ = m;1183 m: *Manifest,
1044 _ = file;1184 file: *File,
1045 _ = need_stat;1185 need_stat: bool,
1046 _ = handle;1186 handle: Io.Dir,
1047 _ = contents;1187 contents: ?[]const u8,
1048 _ = metadata_only;1188 metadata_only: bool,
1049 @panic("TODO");1189 ) !void {
1190 const cache = m.cache;
1191 const io = cache.io;
1192 const gpa = cache.gpa;
1193
1194 if (need_stat) {
1195 const stat = try handle.stat(io);
1196 try file.setStat(m, stat);
1197 }
1198 if (metadata_only) return;
1199 if (contents) |bytes| {
1200 var hasher = hasher_init;
1201 hasher.update(bytes);
1202 hasher.final(&file.digest);
1203 } else {
1204 const prev_contents_len = m.all_input_content.items.len;
1205 defer m.all_input_content.shrinkRetainingCapacity(prev_contents_len);
1206 try hashDir(gpa, io, handle, &file.digest, &m.all_input_content);
1207 }
1050 }1208 }
10511209
1052 pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {1210 pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
...@@ -1062,7 +1220,7 @@ pub const Manifest = struct {...@@ -1062,7 +1220,7 @@ pub const Manifest = struct {
1062 fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {1220 fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
1063 const gpa = self.cache.gpa;1221 const gpa = self.cache.gpa;
1064 const io = self.cache.io;1222 const io = self.cache.io;
1065 const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(file_size_max));1223 const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .unlimited);
1066 defer gpa.free(dep_file_contents);1224 defer gpa.free(dep_file_contents);
10671225
1068 var error_buf: std.ArrayList(u8) = .empty;1226 var error_buf: std.ArrayList(u8) = .empty;
...@@ -1124,7 +1282,6 @@ pub const Manifest = struct {...@@ -1124,7 +1282,6 @@ pub const Manifest = struct {
1124 const io = m.cache.io;1282 const io = m.cache.io;
1125 const manifest_file = m.manifest_file.?;1283 const manifest_file = m.manifest_file.?;
1126 if (m.manifest_dirty) {1284 if (m.manifest_dirty) {
1127
1128 m.contents.appendAssumeCapacity(0);1285 m.contents.appendAssumeCapacity(0);
1129 defer _ = m.contents.pop().?;1286 defer _ = m.contents.pop().?;
11301287
...@@ -1264,22 +1421,85 @@ pub const Manifest = struct {...@@ -1264,22 +1421,85 @@ pub const Manifest = struct {
1264 other_file.prefix = prefix_map[other_file.prefix];1421 other_file.prefix = prefix_map[other_file.prefix];
1265 }1422 }
1266 }1423 }
1267};
12681424
1269fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void {1425 fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void {
1270 var buffer: [2048]u8 = undefined;1426 var buffer: [2048]u8 = undefined;
1271 var hasher = hasher_init;1427 var hasher = hasher_init;
1272 var offset: u64 = 0;1428 var offset: u64 = 0;
1273 while (true) {1429 while (true) {
1274 const n = try file.readPositional(io, &.{&buffer}, offset);1430 const n = try file.readPositional(io, &.{&buffer}, offset);
1275 if (n == 0) break;1431 if (n == 0) break;
1276 hasher.update(buffer[0..n]);1432 hasher.update(buffer[0..n]);
1277 offset += n;1433 offset += n;
1434 }
1435 hasher.final(bin_digest);
1278 }1436 }
1279 hasher.final(bin_digest);
1280}
12811437
1282// Create/Write a file, close it, then grab its stat.mtime timestamp.1438 const HashDirError = Io.Dir.Reader.Error || Allocator.Error;
1439
1440 /// Appends the sorted, encoded directory entries to `contents`.
1441 fn hashDir(
1442 gpa: Allocator,
1443 io: Io,
1444 dir: Io.Dir,
1445 bin_digest: *[Hasher.mac_length]u8,
1446 contents: *std.ArrayList(u8),
1447 ) HashDirError!void {
1448 var buffer: [@max(2048, Io.Dir.Reader.min_buffer_len)]u8 align(@alignOf(usize)) = undefined;
1449 var reader: Io.Dir.Reader = .init(dir, &buffer);
1450 var entry_buffer: [16]Io.Dir.Entry = undefined;
1451
1452 const contents_start = contents.items.len;
1453 errdefer contents.shrinkRetainingCapacity(contents_start);
1454
1455 // Each index points into `contents`.
1456 var entries_list: std.ArrayList(u32) = .empty;
1457 defer entries_list.deinit(gpa);
1458
1459 while (true) {
1460 const entries = entry_buffer[0..try reader.read(io, &entry_buffer)];
1461 for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| {
1462 off.* = contents.items.len;
1463 // As an optimization, make the reservation also count the duplication
1464 // of the contents buffer that will be required after sorting.
1465 try contents.ensureUnusedCapacity(gpa, (contents.items.len + entry.name.len + 2 - contents_start) * 2);
1466 contents.appendAssumeCapacity(@backingInt(Manifest.File.Kind.fromStat(entry.kind)));
1467 contents.appendSliceAssumeCapacity(entry.name);
1468 contents.appendAssumeCapacity(0);
1469 }
1470 }
1471
1472 const Sort = struct {
1473 contents: []const u8,
1474 pub fn lessThan(this: @This(), lhs: u32, rhs: u32) bool {
1475 return mem.lessThanZ(u8, this.contents[lhs + 1 ..], this.contents[rhs + 1 ..]); // +1 for kind byte
1476 }
1477 };
1478 mem.sortUnstable(u32, entries_list.items, @as(Sort, .{ .contents = contents.items }), Sort.lessThan);
1479
1480 // Duplicate the contents such that we may refer to it while creating a
1481 // sorted copy in the original position (at contents_start). We will then
1482 // offset all the entries_list offsets by contents len when reading from the unsorted copy.
1483 const contents_len = contents.items.len - contents_start;
1484 @memcpy(contents.addManyAsSliceAssumeCapacity(contents_len), contents.items[contents_start..][0..contents_len]);
1485
1486 var new_offset: usize = contents_start;
1487 for (entries_list.items) |wrong_offset| {
1488 const offset = wrong_offset + contents_len;
1489 // Includes the kind prefix which we also want to copy.
1490 const entry: [*:0]const u8 = @ptrCast(contents.items[offset..]);
1491 new_offset += mem.copySentinel(u8, 0, contents.items[new_offset..], entry);
1492 }
1493 assert(new_offset == contents_start + contents_len);
1494 contents.shrinkRetainingCapacity(contents_start + contents_len);
1495
1496 var hasher = hasher_init;
1497 hasher.update(contents.items[contents_start..][0..contents_len]);
1498 hasher.final(bin_digest);
1499 }
1500};
1501
1502/// Create/Write a file, close it, then grab its stat.mtime timestamp.
1283fn testGetCurrentFileTimestamp(io: Io, dir: Io.Dir) !Io.Timestamp {1503fn testGetCurrentFileTimestamp(io: Io, dir: Io.Dir) !Io.Timestamp {
1284 const test_out_file = "test-filetimestamp.tmp";1504 const test_out_file = "test-filetimestamp.tmp";
12851505
...@@ -1312,7 +1532,7 @@ test "cache file and then recall it" {...@@ -1312,7 +1532,7 @@ test "cache file and then recall it" {
1312 // Wait for file timestamps to tick1532 // Wait for file timestamps to tick
1313 const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir);1533 const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir);
1314 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) {1534 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) {
1315 try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);1535 try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);
1316 }1536 }
13171537
1318 var digest1: HexDigest = undefined;1538 var digest1: HexDigest = undefined;
...@@ -1382,7 +1602,7 @@ test "check that changing a file makes cache fail" {...@@ -1382,7 +1602,7 @@ test "check that changing a file makes cache fail" {
1382 // Wait for file timestamps to tick1602 // Wait for file timestamps to tick
1383 const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir);1603 const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir);
1384 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) {1604 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) {
1385 try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);1605 try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);
1386 }1606 }
13871607
1388 var digest1: HexDigest = undefined;1608 var digest1: HexDigest = undefined;
...@@ -1508,7 +1728,7 @@ test "Manifest with files added after initial hash work" {...@@ -1508,7 +1728,7 @@ test "Manifest with files added after initial hash work" {
1508 // Wait for file timestamps to tick1728 // Wait for file timestamps to tick
1509 const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir);1729 const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir);
1510 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) {1730 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) {
1511 try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);1731 try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);
1512 }1732 }
15131733
1514 var digest1: HexDigest = undefined;1734 var digest1: HexDigest = undefined;
...@@ -1560,7 +1780,7 @@ test "Manifest with files added after initial hash work" {...@@ -1560,7 +1780,7 @@ test "Manifest with files added after initial hash work" {
1560 // Wait for file timestamps to tick1780 // Wait for file timestamps to tick
1561 const initial_time2 = try testGetCurrentFileTimestamp(io, tmp.dir);1781 const initial_time2 = try testGetCurrentFileTimestamp(io, tmp.dir);
1562 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time2.nanoseconds) {1782 while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time2.nanoseconds) {
1563 try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);1783 try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io);
1564 }1784 }
15651785
1566 {1786 {