authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-03 18:34:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-04 17:42:22-07:00
log61e2e70b1e8c3e3ee9b6f524187c4453308f98f5
tree98de65fc77c26c2bc1ea4f77a9700e33a8b337eb
parentcef0857634686c0b4968d9d5a90991f0feeef839

std.Build.Cache: get it compiling again


4 files changed, 217 insertions(+), 162 deletions(-)

lib/compiler/Maker.zig+2-1
...@@ -1498,7 +1498,8 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1498,7 +1498,8 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
14981498
1499 if (config_man) |man| for (configuration.path_deps) |path_dep| {1499 if (config_man) |man| for (configuration.path_deps) |path_dep| {
1500 const path = try confPathDepToCachePath(arena, graph, &configuration, path_dep);1500 const path = try confPathDepToCachePath(arena, graph, &configuration, path_dep);
1501 try man.addPathPost(path, .{1501 try man.addPathPost(.{
1502 .path = .{ .unresolved = path },
1502 .handle = if (path_dep.flags.is_directory) .{ .dir = null } else .{ .file = null },1503 .handle = if (path_dep.flags.is_directory) .{ .dir = null } else .{ .file = null },
1503 .metadata_only = path_dep.flags.metadata_only,1504 .metadata_only = path_dep.flags.metadata_only,
1504 });1505 });
lib/compiler/Maker/Step.zig+1-1
...@@ -787,7 +787,7 @@ pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !voi...@@ -787,7 +787,7 @@ pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !voi
787}787}
788788
789pub fn setWatchInputsFromManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void {789pub fn setWatchInputsFromManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void {
790 return setWatchInputsFromManifestFiles(s, maker, &man.files, man.cache.prefixes());790 return setWatchInputsFromManifestFiles(s, maker, &man.borrowFiles(), man.cache.prefixes());
791}791}
792792
793pub fn setWatchInputsFromManifestFiles(793pub fn setWatchInputsFromManifestFiles(
lib/std/Build/Cache.zig+208-154
...@@ -56,71 +56,60 @@ pub fn prefixes(cache: *const Cache) []const Directory {...@@ -56,71 +56,60 @@ pub fn prefixes(cache: *const Cache) []const Directory {
56 return cache.prefixes_buffer[0..cache.prefixes_len];56 return cache.prefixes_buffer[0..cache.prefixes_len];
57}57}
5858
59pub const PrefixedPath = struct {59const PrefixIndex = u6;
60 prefix: u8,
61 sub_path: []const u8,
62
63 fn eql(a: PrefixedPath, b: PrefixedPath) bool {
64 return a.prefix == b.prefix and mem.eql(u8, a.sub_path, b.sub_path);
65 }
6660
67 fn hash(pp: PrefixedPath) u32 {61const PrefixedPath = struct {
68 return @truncate(std.hash.Wyhash.hash(pp.prefix, pp.sub_path));62 prefix: PrefixIndex,
69 }63 sub_path: []const u8,
70};64};
7165
72fn findPrefixPath(cache: *const Cache, path: Path) !PrefixedPath {66fn appendPrefixedPath(cache: *const Cache, contents: *std.ArrayList(u8), prefixed_path: PrefixedPath) !PrefixIndex {
73 const gpa = cache.gpa;67 const end = contents.items.len + prefixed_path.sub_path.len;
74 const resolved_path = try std.fs.path.resolve(gpa, &.{68 const needed_alignment = @alignOf(Manifest.File) - (end % @alignOf(Manifest.File));
75 cache.cwd, path.root_dir.path orelse ".", path.subPathOrDot(),69 assert(needed_alignment >= 1); // Always need at least a null byte.
76 });70 try contents.ensureTotalCapacity(cache.gpa, end + needed_alignment);
77 errdefer gpa.free(resolved_path);71 contents.appendSliceAssumeCapacity(prefixed_path.sub_path);
78 return findPrefixResolved(cache, resolved_path);72 contents.appendNTimesAssumeCapacity(0, needed_alignment);
79}73 return prefixed_path.prefix;
80
81fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
82 const gpa = cache.gpa;
83 const resolved_path = try std.fs.path.resolve(gpa, &.{file_path});
84 errdefer gpa.free(resolved_path);
85 return findPrefixResolved(cache, resolved_path);
86}74}
8775
88/// Takes ownership of `resolved_path` on success.76fn resolveAppendPath(cache: *const Cache, contents: *std.ArrayList(u8), path: Path) !PrefixIndex {
89fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
90 const gpa = cache.gpa;77 const gpa = cache.gpa;
91 const cwd = cache.cwd;78 const cwd = cache.cwd;
79 const path_start = contents.items.len;
80
81 const resolved_path = try std.fs.path.resolveAlloc(gpa, &.{
82 path.root_dir.path orelse cwd,
83 path.subPathOrDot(),
84 });
85 defer gpa.free(resolved_path);
86
92 for (cache.prefixes(), 0..) |prefix, i| {87 for (cache.prefixes(), 0..) |prefix, i| {
93 const p = prefix.path orelse continue;88 const pp = prefix.path orelse continue;
94 const sub_path = getPrefixSubpath(gpa, cwd, p, resolved_path) catch |err| switch (err) {89 contents.shrinkRetainingCapacity(path_start);
95 error.NotASubPath => continue,90 try std.fs.path.relativeAppend(gpa, contents, cwd, null, pp, resolved_path);
96 else => |e| return e,91 const relative = contents.items[path_start..];
97 };
98 // Free the resolved path since we're not going to return it
99 gpa.free(resolved_path);
100 return .{
101 .prefix = @intCast(i),
102 .sub_path = sub_path,
103 };
104 }
10592
106 return .{93 var component_iterator: std.fs.path.NativeComponentIterator = .init(relative);
107 .prefix = 0,94 if (component_iterator.root() != null) continue;
108 .sub_path = resolved_path,95 const first_component = component_iterator.first();
109 };96 if (first_component != null and mem.eql(u8, first_component.?.name, "..")) continue;
110}
11197
112fn getPrefixSubpath(gpa: Allocator, cwd: []const u8, prefix: []const u8, path: []u8) ![]u8 {98 const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File));
113 const relative = try std.fs.path.relative(gpa, cwd, null, prefix, path);99 assert(needed_alignment >= 1); // Always need at least a null byte.
114 errdefer gpa.free(relative);100 try contents.appendNTimes(gpa, 0, needed_alignment);
115 var component_iterator: std.fs.path.NativeComponentIterator = .init(relative);101
116 if (component_iterator.root() != null) {102 return @intCast(i);
117 return error.NotASubPath;
118 }
119 const first_component = component_iterator.first();
120 if (first_component != null and mem.eql(u8, first_component.?.name, "..")) {
121 return error.NotASubPath;
122 }103 }
123 return relative;104
105 contents.shrinkRetainingCapacity(path_start);
106 try contents.appendSlice(gpa, resolved_path);
107
108 const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File));
109 assert(needed_alignment >= 1); // Always need at least a null byte.
110 try contents.appendNTimes(gpa, 0, needed_alignment);
111
112 return 0;
124}113}
125114
126/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6115/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
...@@ -359,7 +348,10 @@ pub const Manifest = struct {...@@ -359,7 +348,10 @@ pub const Manifest = struct {
359 _,348 _,
360 },349 },
361 /// `have_handle` determines whether this is populated.350 /// `have_handle` determines whether this is populated.
362 handle: Io.File,351 handle: union {
352 file: Io.File,
353 dir: Io.Dir,
354 },
363355
364 /// Index into `Manifest.input_paths`.356 /// Index into `Manifest.input_paths`.
365 pub const Index = enum(u32) {357 pub const Index = enum(u32) {
...@@ -383,7 +375,7 @@ pub const Manifest = struct {...@@ -383,7 +375,7 @@ pub const Manifest = struct {
383 pub const Flags = packed struct(u8) {375 pub const Flags = packed struct(u8) {
384 is_directory: bool,376 is_directory: bool,
385 metadata_only: bool,377 metadata_only: bool,
386 prefix: u6,378 prefix: PrefixIndex,
387 };379 };
388380
389 /// Prefixes path names in encoded directory contents. Starts numbering381 /// Prefixes path names in encoded directory contents. Starts numbering
...@@ -408,40 +400,49 @@ pub const Manifest = struct {...@@ -408,40 +400,49 @@ pub const Manifest = struct {
408 _,400 _,
409401
410 pub fn get(offset: Offset, contents: []u8) *File {402 pub fn get(offset: Offset, contents: []u8) *File {
411 return @ptrCast(@alignCast(contents.items[@backingInt(offset)..][0..@sizeOf(File)]));403 return @constCast(getConst(offset, contents));
404 }
405
406 pub fn getConst(offset: Offset, contents: []const u8) *const File {
407 return @ptrCast(@alignCast(contents[@backingInt(offset)..][0..@sizeOf(File)]));
412 }408 }
413409
414 pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File {410 pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File {
415 if (@backingInt(offset) + @sizeOf(File) >= contents.items.len) return error.InvalidFormat;411 if (@backingInt(offset) + @sizeOf(File) >= contents.len) return error.InvalidFormat;
416 return get(offset, contents);412 return get(offset, contents);
417 }413 }
418 };414 };
419415
416 /// Intentionally matches if the files are different only by flags other than prefix.
420 pub const HashContext = struct {417 pub const HashContext = struct {
421 manifest: *const Manifest,418 contents: []const u8,
422419
423 pub fn hash(this: @This(), off: Offset) u32 {420 pub fn hash(this: @This(), off: Offset) u32 {
424 const file = off.get(this.manifest);421 const file_prefix = off.getConst(this.contents).flags.prefix;
425 return @truncate(std.hash.Wyhash.hash(file.prefix, file.path()));422 const file_path = filePath(this.contents, off);
423 return @truncate(std.hash.Wyhash.hash(file_prefix, file_path));
426 }424 }
427425
428 pub fn eql(this: @This(), a_off: Offset, b_off: Offset, b_index: usize) bool {426 pub fn eql(this: @This(), a_off: Offset, b_off: Offset, b_index: usize) bool {
429 _ = b_index;427 _ = b_index;
430 const a = a_off.get(this.manifest);428 const a_prefix = a_off.getConst(this.contents).flags.prefix;
431 const b = b_off.get(this.manifest);429 const b_prefix = b_off.getConst(this.contents).flags.prefix;
432 return a.prefix == b.prefix and mem.eql(u8, a.path(), b.path());430 if (a_prefix != b_prefix) return false;
431 const a_path = filePath(this.contents, a_off);
432 const b_path = filePath(this.contents, b_off);
433 return mem.eql(u8, a_path, b_path);
433 }434 }
434 };435 };
435436
436 fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void {437 fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void {
437 file.size = stat.size;438 file.size = stat.size;
438 file.inode = stat.inode;439 file.inode = stat.inode;
439 file.mtime = stat.mtime;440 file.mtime = @intCast(stat.mtime.toNanoseconds());
440441
441 if (try m.isProblematicTimestamp(stat.mtime)) {442 if (try m.isProblematicTimestamp(stat.mtime)) {
442 // The actual file has an unreliable timestamp; force it to be hashed.443 // The actual file has an unreliable timestamp; force it to be hashed.
443 file.stat.mtime = 0;444 file.mtime = 0;
444 file.stat.inode = 0;445 file.inode = 0;
445 }446 }
446 }447 }
447448
...@@ -453,7 +454,7 @@ pub const Manifest = struct {...@@ -453,7 +454,7 @@ pub const Manifest = struct {
453 {454 {
454 return false;455 return false;
455 } else {456 } else {
456 setStat(file, m, stat);457 try setStat(file, m, stat);
457 return true;458 return true;
458 }459 }
459 }460 }
...@@ -489,12 +490,31 @@ pub const Manifest = struct {...@@ -489,12 +490,31 @@ pub const Manifest = struct {
489 size: u64,490 size: u64,
490 inode: Io.File.INode,491 inode: Io.File.INode,
491 mtime: Io.Timestamp,492 mtime: Io.Timestamp,
493
494 pub fn init(other: Io.File.Stat) Stat {
495 return .{
496 .size = other.size,
497 .inode = other.inode,
498 .mtime = other.mtime,
499 };
500 }
492 };501 };
493502
494 pub const PathHandle = union(enum) {503 pub const PathHandle = union(enum) {
495 file: ?Io.File,504 file: ?Io.File,
496 /// If provided, this handle must be opened with iteration capability.505 /// If provided, this handle must be opened with iteration capability.
497 dir: ?Io.Dir,506 dir: ?Io.Dir,
507
508 pub fn isDirectory(this: @This()) bool {
509 return this == .dir;
510 }
511
512 pub fn have(this: @This()) bool {
513 return switch (this) {
514 .file => |opt_file| opt_file != null,
515 .dir => |opt_dir| opt_dir != null,
516 };
517 }
498 };518 };
499519
500 pub const AddInputPathOptions = struct {520 pub const AddInputPathOptions = struct {
...@@ -524,59 +544,73 @@ pub const Manifest = struct {...@@ -524,59 +544,73 @@ pub const Manifest = struct {
524 /// See also:544 /// See also:
525 /// * `addPathPost`545 /// * `addPathPost`
526 pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index {546 pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index {
527 const gpa = m.cache.gpa;547 const cache = m.cache;
528 try m.files.ensureUnusedCapacity(gpa, 1);548 const gpa = cache.gpa;
549 try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items });
529 try m.input_paths.ensureUnusedCapacity(gpa, 1);550 try m.input_paths.ensureUnusedCapacity(gpa, 1);
530551
531 const prev_contents_len = m.contents.items.len;552 const prev_contents_len = m.contents.items.len;
532 const header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)));553 const header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))));
533 errdefer m.contents.shrinkRetainingCapacity(prev_contents_len);554 errdefer m.contents.shrinkRetainingCapacity(prev_contents_len);
534555
535 header.* = .{556 header.* = .{
536 .flags = .{557 .flags = .{
537 .prefix = try m.cache.findAppendPrefixedPath(&m.contents, path),558 .prefix = try cache.resolveAppendPath(&m.contents, path),
538 .is_directory = options.is_directory,559 .is_directory = options.handle.isDirectory(),
539 .metadata_only = options.metadata_only,560 .metadata_only = options.metadata_only,
540 },561 },
541 .size = undefined,562 .size = undefined,
542 .inode = undefined,563 .inode = undefined,
543 .mtime = undefined,564 .mtime = undefined,
544 .digest = undefined,565 .digest = undefined,
566 .path_start = .{},
545 };567 };
546 assert(m.contents.items.len % @alignOf(File) == 0);568 assert(mem.isAligned(m.contents.items.len, @alignOf(File)));
547569
548 const gop = try m.files.getOrPutAssumeCapacityContext(@fromBackingInt(prev_contents_len), .{570 const gop = m.files.getOrPutAssumeCapacityContext(@fromBackingInt(@intCast(prev_contents_len)), .{
549 .manifest = m,571 .contents = m.contents.items,
550 });572 });
573 m.files.lockPointers();
574 defer m.files.unlockPointers();
575
551 if (gop.found_existing) {576 if (gop.found_existing) {
552 m.contents.shrinkRetainingCapacity(prev_contents_len);577 m.contents.shrinkRetainingCapacity(prev_contents_len);
553 const existing_input_file = &m.input_paths.items[gop.index];578 const existing_input_file = &m.input_paths.items[gop.index];
554 if (options.handle) |handle| {579 switch (options.handle) {
555 existing_input_file.handle = handle;580 .file => |opt_file| if (opt_file) |file| {
556 existing_input_file.have_handle = true;581 existing_input_file.handle = .{ .file = file };
582 existing_input_file.have_handle = true;
583 },
584 .dir => |opt_dir| if (opt_dir) |dir| {
585 existing_input_file.handle = .{ .dir = dir };
586 existing_input_file.have_handle = true;
587 },
557 }588 }
558 if (options.request_contents) switch (existing_input_file.contents) {589 if (options.request_contents) switch (existing_input_file.contents) {
559 .requested, .not_requested => existing_input_file.contents = .requested,590 .requested, .not_requested => existing_input_file.contents = .requested,
560 _ => {},591 _ => {},
561 };592 };
562 const existing_header = &m.files.keys()[gop.index];593 const existing_header = m.files.keys()[gop.index].get(m.contents.items);
563 if (options.stat) |stat| {594 if (options.stat) |stat| {
564 existing_input_file.have_stat = true;595 existing_input_file.have_stat = true;
565 existing_header.size = stat.size;596 existing_header.size = stat.size;
566 existing_header.inode = stat.inode;597 existing_header.inode = stat.inode;
567 existing_header.mtime = stat.mtime;598 existing_header.mtime = @intCast(stat.mtime.toNanoseconds());
568 }599 }
569 // If it trips, the same file path has been added to the cache600 // If it trips, the same file path has been added to the cache
570 // manifest both as a directory and as a normal file, making the601 // manifest both as a directory and as a normal file, making the
571 // intended caching behavior ambiguous.602 // intended caching behavior ambiguous.
572 assert(existing_header.flags.is_directory == options.is_directory);603 assert(existing_header.flags.is_directory == options.handle.isDirectory());
573 if (!options.metadata_only)604 if (!options.metadata_only)
574 existing_header.flags.metadata_only = false;605 existing_header.flags.metadata_only = false;
575 } else {606 } else {
576 m.input_paths.appendAssumeCapacity(.{607 m.input_paths.appendAssumeCapacity(.{
577 .request_handle = options.request_handle,608 .request_handle = options.request_handle,
578 .have_handle = options.handle != null,609 .have_handle = options.handle.have(),
579 .handle = if (options.handle) |handle| handle else undefined,610 .handle = switch (options.handle) {
611 .file => |opt_file| if (opt_file) |file| .{ .file = file } else undefined,
612 .dir => |opt_dir| if (opt_dir) |dir| .{ .dir = dir } else undefined,
613 },
580 .contents = if (options.request_contents) .requested else .not_requested,614 .contents = if (options.request_contents) .requested else .not_requested,
581 .have_digest = false,615 .have_digest = false,
582 .have_stat = options.stat != null,616 .have_stat = options.stat != null,
...@@ -585,10 +619,10 @@ pub const Manifest = struct {...@@ -585,10 +619,10 @@ pub const Manifest = struct {
585 if (options.stat) |stat| {619 if (options.stat) |stat| {
586 header.size = stat.size;620 header.size = stat.size;
587 header.inode = stat.inode;621 header.inode = stat.inode;
588 header.mtime = stat.mtime;622 header.mtime = @intCast(stat.mtime.toNanoseconds());
589 }623 }
590 }624 }
591 return @fromBackingInt(gop.index);625 return @fromBackingInt(@intCast(gop.index));
592 }626 }
593627
594 pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void {628 pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void {
...@@ -759,8 +793,8 @@ pub const Manifest = struct {...@@ -759,8 +793,8 @@ pub const Manifest = struct {
759 if (m.files.count() <= m.input_paths.items.len) return;793 if (m.files.count() <= m.input_paths.items.len) return;
760 const off = m.files.keys()[m.input_paths.items.len];794 const off = m.files.keys()[m.input_paths.items.len];
761 m.contents.shrinkRetainingCapacity(@backingInt(off));795 m.contents.shrinkRetainingCapacity(@backingInt(off));
762 assert(m.contents.items.len % @alignOf(File) == 0);796 assert(mem.isAligned(m.contents.items.len, @alignOf(File)));
763 m.files.shrinkRetainingCapacity(m.input_paths.items.len);797 m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items });
764 }798 }
765799
766 /// Assumes that `self.hash.hasher` has been updated only with the original digest and that800 /// Assumes that `self.hash.hasher` has been updated only with the original digest and that
...@@ -806,11 +840,13 @@ pub const Manifest = struct {...@@ -806,11 +840,13 @@ pub const Manifest = struct {
806840
807 // Guess number of files based on manifest contents len to reduce allocations.841 // Guess number of files based on manifest contents len to reduce allocations.
808 // This is not an upper bound; subsequent insertions may potentially allocate.842 // This is not an upper bound; subsequent insertions may potentially allocate.
809 try m.files.ensureUnusedCapacity(gpa, contents.len / (@sizeOf(File) + 32));843 try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{
844 .contents = contents,
845 });
810846
811 // This group we would like to cancel as soon as a cache miss is discovered.847 // This group we would like to cancel as soon as a cache miss is discovered.
812 const PostResult = union(enum) {848 const PostResult = union(enum) {
813 checkFile: Check.Status,849 checkFile: CheckFileError!Check.Status,
814 };850 };
815 var post_select_buffer: [10]PostResult = undefined;851 var post_select_buffer: [10]PostResult = undefined;
816 var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer);852 var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer);
...@@ -819,12 +855,12 @@ pub const Manifest = struct {...@@ -819,12 +855,12 @@ pub const Manifest = struct {
819855
820 while (off + 1 < contents.len) {856 while (off + 1 < contents.len) {
821 const file_off: File.Offset = @fromBackingInt(off);857 const file_off: File.Offset = @fromBackingInt(off);
822 const file = try file_off.getFallible(m);858 const file = try file_off.getFallible(contents);
823 if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat;859 if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat;
824 const path = try filePathFallible(contents, file_off);860 const path = try filePathFallible(contents, file_off);
825 if (path.len == 0) return error.InvalidFormat;861 if (path.len == 0) return error.InvalidFormat;
826862
827 try m.files.put(gpa, file_off, {});863 try m.files.putContext(gpa, file_off, {}, .{ .contents = contents });
828864
829 post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path });865 post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path });
830 post_select_remaining += 1;866 post_select_remaining += 1;
...@@ -867,19 +903,19 @@ pub const Manifest = struct {...@@ -867,19 +903,19 @@ pub const Manifest = struct {
867 try input_group.await(io);903 try input_group.await(io);
868 return .miss;904 return .miss;
869 },905 },
870 .fail => |diagnostic| {
871 m.diagnostic = diagnostic;
872 return error.CacheCheckFailed;
873 },
874 },906 },
875 };907 };
876 }908 }
877909
878 try input_group.await(io);910 try input_group.await(io);
879 if (c.status == .miss) return .miss;911 if (c.status == .miss) return .miss;
912 if (m.diagnostic != .none) return error.CacheCheckFailed;
913
914 // Needed due to the length mutation above.
915 const refreshed_contents = m.contents.items;
880916
881 for (m.files.keys()) |file_off| {917 for (m.files.keys()) |file_off| {
882 m.hash.hasher.update(&file_off.get(m).digest);918 m.hash.hasher.update(&file_off.get(refreshed_contents).digest);
883 }919 }
884920
885 return .hit;921 return .hit;
...@@ -896,20 +932,25 @@ pub const Manifest = struct {...@@ -896,20 +932,25 @@ pub const Manifest = struct {
896 if (input_path.have_stat) @panic("TODO");932 if (input_path.have_stat) @panic("TODO");
897 if (input_path.contents != .not_requested) @panic("TODO");933 if (input_path.contents != .not_requested) @panic("TODO");
898 if (input_path.request_handle) @panic("TODO");934 if (input_path.request_handle) @panic("TODO");
899 switch (try checkFile(m, c, file_off, file_path)) {935 if (checkFile(m, c, file_off, file_path)) |status| switch (status) {
900 .hit => return,936 .hit => return,
901 .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered),937 .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered),
938 } else |err| switch (err) {
939 error.CacheCheckFailed => assert(m.diagnostic != .none),
940 else => |e| return e,
902 }941 }
903 }942 }
904943
944 const CheckFileError = error{ Canceled, CacheCheckFailed };
945
905 /// Runs concurrently with other `checkFile`.946 /// Runs concurrently with other `checkFile`.
906 fn checkFile(947 fn checkFile(
907 m: *Manifest,948 m: *Manifest,
908 c: *Check,949 c: *Check,
909 file_off: File.Offset,950 file_off: File.Offset,
910 file_path: [:0]const u8,951 file_path: [:0]const u8,
911 ) error{ Canceled, CacheCheckFailed }!Check.Status {952 ) CheckFileError!Check.Status {
912 const file = file_off.get(m);953 const file = file_off.get(m.contents.items);
913 const cache = m.cache;954 const cache = m.cache;
914 const gpa = cache.gpa;955 const gpa = cache.gpa;
915 const io = cache.io;956 const io = cache.io;
...@@ -928,7 +969,7 @@ pub const Manifest = struct {...@@ -928,7 +969,7 @@ pub const Manifest = struct {
928 const actual_is_directory = actual_stat.kind == .directory;969 const actual_is_directory = actual_stat.kind == .directory;
929 if (actual_is_directory != file.flags.is_directory) return .miss;970 if (actual_is_directory != file.flags.is_directory) return .miss;
930971
931 if (try file.setStatChanged(m, actual_stat)) return .miss;972 if (try file.setStatChanged(m, .init(actual_stat))) return .miss;
932973
933 return .hit;974 return .hit;
934 }975 }
...@@ -954,7 +995,7 @@ pub const Manifest = struct {...@@ -954,7 +995,7 @@ pub const Manifest = struct {
954 .err = e,995 .err = e,
955 } }),996 } }),
956 };997 };
957 if (try file.setStatChanged(m, actual_stat)) {998 if (try file.setStatChanged(m, .init(actual_stat))) {
958 const prev_digest: BinDigest = file.digest;999 const prev_digest: BinDigest = file.digest;
959 var contents: std.ArrayList(u8) = .empty;1000 var contents: std.ArrayList(u8) = .empty;
960 defer contents.deinit(gpa);1001 defer contents.deinit(gpa);
...@@ -989,7 +1030,7 @@ pub const Manifest = struct {...@@ -989,7 +1030,7 @@ pub const Manifest = struct {
989 } }),1030 } }),
990 };1031 };
9911032
992 if (try file.setStatChanged(m, actual_stat)) {1033 if (try file.setStatChanged(m, .init(actual_stat))) {
993 const prev_digest: BinDigest = file.digest;1034 const prev_digest: BinDigest = file.digest;
994 hashFile(io, opened_file, &file.digest) catch |err| switch (err) {1035 hashFile(io, opened_file, &file.digest) catch |err| switch (err) {
995 error.Canceled => |e| return e,1036 error.Canceled => |e| return e,
...@@ -1015,8 +1056,9 @@ pub const Manifest = struct {...@@ -1015,8 +1056,9 @@ pub const Manifest = struct {
1015 man.hash.hasher = hasher_init;1056 man.hash.hasher = hasher_init;
1016 man.hash.hasher.update(bin_digest);1057 man.hash.hasher.update(bin_digest);
1017 man.shrinkFilesToInput();1058 man.shrinkFilesToInput();
1059 const contents = man.contents.items;
1018 for (man.files.keys()) |off| {1060 for (man.files.keys()) |off| {
1019 const file = off.get(man);1061 const file = off.get(contents);
1020 man.hash.hasher.update(&file.digest);1062 man.hash.hasher.update(&file.digest);
1021 }1063 }
1022 }1064 }
...@@ -1065,6 +1107,10 @@ pub const Manifest = struct {...@@ -1065,6 +1107,10 @@ pub const Manifest = struct {
1065 }1107 }
10661108
1067 pub const AddPathPostOptions = struct {1109 pub const AddPathPostOptions = struct {
1110 path: union(enum) {
1111 unresolved: Path,
1112 prefixed: PrefixedPath,
1113 },
1068 handle: PathHandle = .{ .file = null },1114 handle: PathHandle = .{ .file = null },
1069 stat: ?Stat = null,1115 stat: ?Stat = null,
1070 /// If it is a directory, there is a special encoding required for contents, which1116 /// If it is a directory, there is a special encoding required for contents, which
...@@ -1073,29 +1119,30 @@ pub const Manifest = struct {...@@ -1073,29 +1119,30 @@ pub const Manifest = struct {
1073 metadata_only: bool = false,1119 metadata_only: bool = false,
1074 };1120 };
10751121
1076 pub const AddPathPostError = Io.Cancelable || Allocator.Error;
1077
1078 /// Add a file as a dependency of process being cached, after cache miss1122 /// Add a file as a dependency of process being cached, after cache miss
1079 /// occurs.1123 /// occurs.
1080 ///1124 ///
1081 /// See also:1125 /// See also:
1082 /// * `addInputPath`1126 /// * `addInputPath`
1083 pub fn addPathPost(m: *Manifest, path: Path, options: AddPathPostOptions) AddPathPostError!void {1127 pub fn addPathPost(m: *Manifest, options: AddPathPostOptions) !void {
1084 assert(m.manifest_file != null);1128 assert(m.manifest_file != null);
1085 const cache = m.cache;1129 const cache = m.cache;
1086 const gpa = cache.gpa;1130 const gpa = cache.gpa;
1087 const io = cache.io;1131 const io = cache.io;
1088 const is_directory = options.handle == .dir;1132 const is_directory = options.handle == .dir;
10891133
1090 try m.files.ensureUnusedCapacity(gpa, 1);1134 try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items });
10911135
1092 const prev_contents_len = m.contents.items.len;1136 const new_file_offset: File.Offset = @fromBackingInt(@intCast(m.contents.items.len));
1093 const new_header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)));1137 const new_header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))));
1094 errdefer m.contents.shrinkRetainingCapacity(prev_contents_len);1138 errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset));
10951139
1096 new_header.* = .{1140 new_header.* = .{
1097 .flags = .{1141 .flags = .{
1098 .prefix = try cache.findAppendPrefixedPath(&m.contents, path),1142 .prefix = switch (options.path) {
1143 .unresolved => |unresolved| try cache.resolveAppendPath(&m.contents, unresolved),
1144 .prefixed => |prefixed| try cache.appendPrefixedPath(&m.contents, prefixed),
1145 },
1099 .is_directory = is_directory,1146 .is_directory = is_directory,
1100 .metadata_only = options.metadata_only,1147 .metadata_only = options.metadata_only,
1101 },1148 },
...@@ -1103,31 +1150,32 @@ pub const Manifest = struct {...@@ -1103,31 +1150,32 @@ pub const Manifest = struct {
1103 .inode = undefined,1150 .inode = undefined,
1104 .mtime = undefined,1151 .mtime = undefined,
1105 .digest = @splat(0),1152 .digest = @splat(0),
1153 .path_start = .{},
1106 };1154 };
1107 assert(m.contents.items.len % @alignOf(File) == 0);1155 assert(mem.isAligned(m.contents.items.len, @alignOf(File)));
11081156
1109 const gop = m.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{1157 const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{
1110 .manifest = m,1158 .contents = m.contents.items,
1111 });1159 });
1112 m.files.lockPointers();1160 m.files.lockPointers();
1113 defer m.files.unlockPointers();1161 defer m.files.unlockPointers();
11141162
1115 const header = if (gop.found_existing) h: {1163 const header, const file_offset = if (gop.found_existing) h: {
1116 m.contents.shrinkRetainingCapacity(prev_contents_len);1164 m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset));
1117 const existing_off = gop.key_ptr.*;1165 const existing_off = gop.key_ptr.*;
1118 const header = existing_off.get(m);1166 const header = existing_off.get(m.contents.items);
1119 // If it trips, the same file path has been added to the cache1167 // If it trips, the same file path has been added to the cache
1120 // manifest both as a directory and as a normal file, making the1168 // manifest both as a directory and as a normal file, making the
1121 // intended caching behavior ambiguous.1169 // intended caching behavior ambiguous.
1122 assert(header.flags.is_directory == is_directory);1170 assert(header.flags.is_directory == is_directory);
1123 if (!options.metadata_only)1171 if (!options.metadata_only)
1124 header.flags.metadata_only = false;1172 header.flags.metadata_only = false;
1125 break :h header;1173 break :h .{ header, existing_off };
1126 } else new_header;1174 } else .{ new_header, new_file_offset };
11271175
1128 if (options.stat) |stat| {1176 if (options.stat) |stat| {
1129 try header.setStat(m, stat);1177 try header.setStat(m, stat);
1130 if (header.metadata_only) {1178 if (header.flags.metadata_only) {
1131 return;1179 return;
1132 } else if (options.contents) |contents| {1180 } else if (options.contents) |contents| {
1133 var hasher = hasher_init;1181 var hasher = hasher_init;
...@@ -1138,27 +1186,31 @@ pub const Manifest = struct {...@@ -1138,27 +1186,31 @@ pub const Manifest = struct {
1138 }1186 }
11391187
1140 const need_stat = options.stat == null;1188 const need_stat = options.stat == null;
1189 const metadata_only = header.flags.metadata_only;
1190 const prefix = header.flags.prefix;
11411191
1142 switch (options.handle) {1192 switch (options.handle) {
1143 .dir => |opt_handle| if (opt_handle) |handle| {1193 .dir => |opt_handle| if (opt_handle) |handle| {
1144 try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);1194 try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only);
1145 } else {1195 } else {
1146 const dir = cache.prefixes()[header.flags.prefix].handle;1196 const dir = cache.prefixes()[prefix].handle;
1147 const handle = try dir.openDir(io, header.path(), .{1197 const sub_path = filePath(m.contents.items, file_offset);
1198 const handle = try dir.openDir(io, sub_path, .{
1148 .access_sub_paths = false,1199 .access_sub_paths = false,
1149 .iterate = true,1200 .iterate = true,
1150 });1201 });
1151 defer handle.close(io);1202 defer handle.close(io);
1152 try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);1203 try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only);
1153 },1204 },
11541205
1155 .file => |opt_handle| if (opt_handle) |handle| {1206 .file => |opt_handle| if (opt_handle) |handle| {
1156 try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only);1207 try populateFile(m, header, need_stat, handle, options.contents, metadata_only);
1157 } else {1208 } else {
1158 const dir = cache.prefixes()[header.flags.prefix].handle;1209 const dir = cache.prefixes()[prefix].handle;
1159 const handle = try dir.openFile(io, header.path(), .{ .mode = .read_only });1210 const sub_path = filePath(m.contents.items, file_offset);
1211 const handle = try dir.openFile(io, sub_path, .{ .mode = .read_only });
1160 defer handle.close(io);1212 defer handle.close(io);
1161 try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only);1213 try populateFile(m, header, need_stat, handle, options.contents, metadata_only);
1162 },1214 },
1163 }1215 }
1164 }1216 }
...@@ -1175,7 +1227,7 @@ pub const Manifest = struct {...@@ -1175,7 +1227,7 @@ pub const Manifest = struct {
11751227
1176 if (need_stat) {1228 if (need_stat) {
1177 const stat = try handle.stat(io);1229 const stat = try handle.stat(io);
1178 try file.setStat(m, stat);1230 try file.setStat(m, .init(stat));
1179 }1231 }
1180 if (metadata_only) return;1232 if (metadata_only) return;
1181 if (contents) |bytes| {1233 if (contents) |bytes| {
...@@ -1201,7 +1253,7 @@ pub const Manifest = struct {...@@ -1201,7 +1253,7 @@ pub const Manifest = struct {
12011253
1202 if (need_stat) {1254 if (need_stat) {
1203 const stat = try handle.stat(io);1255 const stat = try handle.stat(io);
1204 try file.setStat(m, stat);1256 try file.setStat(m, .init(stat));
1205 }1257 }
1206 if (metadata_only) return;1258 if (metadata_only) return;
1207 if (contents) |bytes| {1259 if (contents) |bytes| {
...@@ -1238,28 +1290,26 @@ pub const Manifest = struct {...@@ -1238,28 +1290,26 @@ pub const Manifest = struct {
1238 defer resolve_buf.deinit(gpa);1290 defer resolve_buf.deinit(gpa);
12391291
1240 var it: DepTokenizer = .{ .bytes = dep_file_contents };1292 var it: DepTokenizer = .{ .bytes = dep_file_contents };
1241 while (it.next()) |token| {1293 while (it.next()) |token| switch (token) {
1242 switch (token) {1294 // We don't care about targets, we only want the prereqs
1243 // We don't care about targets, we only want the prereqs1295 // Clang is invoked in single-source mode but other programs may not
1244 // Clang is invoked in single-source mode but other programs may not1296 .target, .target_must_resolve => {},
1245 .target, .target_must_resolve => {},1297 .prereq => |file_path| if (self.manifest_file == null) {
1246 .prereq => |file_path| if (self.manifest_file == null) {1298 _ = try self.addInputPath(.initCwd(file_path), .{});
1247 _ = try self.addInputPath(.initCwd(file_path), .{});1299 } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(file_path) } }),
1248 } else try self.addPathPost(file_path),1300 .prereq_must_resolve => {
1249 .prereq_must_resolve => {1301 resolve_buf.clearRetainingCapacity();
1250 resolve_buf.clearRetainingCapacity();1302 try token.resolve(gpa, &resolve_buf);
1251 try token.resolve(gpa, &resolve_buf);1303 if (self.manifest_file == null) {
1252 if (self.manifest_file == null) {1304 _ = try self.addInputPath(.initCwd(resolve_buf.items), .{});
1253 _ = try self.addInputPath(.initCwd(resolve_buf.items), .{});1305 } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(resolve_buf.items) } });
1254 } else try self.addPathPost(resolve_buf.items);1306 },
1255 },1307 else => |err| {
1256 else => |err| {1308 try err.printError(gpa, &error_buf);
1257 try err.printError(gpa, &error_buf);1309 log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items });
1258 log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items });1310 return error.InvalidDepFile;
1259 return error.InvalidDepFile;1311 },
1260 },1312 };
1261 }
1262 }
1263 }1313 }
12641314
1265 /// Returns a binary hash of the inputs.1315 /// Returns a binary hash of the inputs.
...@@ -1368,6 +1418,10 @@ pub const Manifest = struct {...@@ -1368,6 +1418,10 @@ pub const Manifest = struct {
1368 pub fn takeFiles(m: *Manifest) SelfContainedFiles {1418 pub fn takeFiles(m: *Manifest) SelfContainedFiles {
1369 defer m.files = .empty;1419 defer m.files = .empty;
1370 defer m.contents = .empty;1420 defer m.contents = .empty;
1421 return borrowFiles(m);
1422 }
1423
1424 pub fn borrowFiles(m: *const Manifest) SelfContainedFiles {
1371 return .{1425 return .{
1372 .files = m.files,1426 .files = m.files,
1373 .contents = m.contents,1427 .contents = m.contents,
...@@ -1488,7 +1542,7 @@ pub const Manifest = struct {...@@ -1488,7 +1542,7 @@ pub const Manifest = struct {
1488 while (true) {1542 while (true) {
1489 const entries = entry_buffer[0..try reader.read(io, &entry_buffer)];1543 const entries = entry_buffer[0..try reader.read(io, &entry_buffer)];
1490 for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| {1544 for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| {
1491 off.* = contents.items.len;1545 off.* = @intCast(contents.items.len);
1492 // As an optimization, make the reservation also count the duplication1546 // As an optimization, make the reservation also count the duplication
1493 // of the contents buffer that will be required after sorting.1547 // of the contents buffer that will be required after sorting.
1494 try contents.ensureUnusedCapacity(gpa, (contents.items.len + entry.name.len + 2 - contents_start) * 2);1548 try contents.ensureUnusedCapacity(gpa, (contents.items.len + entry.name.len + 2 - contents_start) * 2);
lib/std/zig.zig+6-6
...@@ -1837,12 +1837,12 @@ pub fn buildExeSubprocess(...@@ -1837,12 +1837,12 @@ pub fn buildExeSubprocess(
1837 var it = mem.splitScalar(u8, body, 0);1837 var it = mem.splitScalar(u8, body, 0);
1838 while (it.next()) |prefixed_path| {1838 while (it.next()) |prefixed_path| {
1839 const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1));1839 const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1));
1840 const sub_path = try gpa.dupe(u8, prefixed_path[1..]);1840 const sub_path = prefixed_path[1..];
1841 var keep = false;1841 man.addPathPost(.{
1842 defer if (!keep) gpa.free(sub_path);1842 .path = .{ .prefixed = .{
1843 keep = man.addPrefixedPathPost(.{1843 .prefix = @intCast(@backingInt(prefix)),
1844 .prefix = @backingInt(prefix),1844 .sub_path = sub_path,
1845 .sub_path = sub_path,1845 } },
1846 }) catch |err| switch (err) {1846 }) catch |err| switch (err) {
1847 error.Canceled, error.OutOfMemory => |e| return e,1847 error.Canceled, error.OutOfMemory => |e| return e,
1848 else => |e| {1848 else => |e| {