authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-10-18 22:49:16+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-10-19 18:30:35+01:00
log067f8dcb375aae6cfe1fdc66cf70372067382c49
tree85e48e2b6fe252631585f6be608f88704262243f
parent19c4660c77130e78503e2e7e9e4ad2ec8e0985d6
signature Commit is signed but in an unrecognized format.

std.Build.Cache: support depending on nonexistent file/directory

4 files changed, 119 insertions(+), 67 deletions(-)

lib/compiler/Maker.zig+7-1
......@@ -2336,6 +2336,7 @@ fn cacheCatOne(input_hex_digest: []const u8, contents: []const u8, writer: *Io.W
23362336 try file_obj.field("digest", @as([]const u8, &hex_digest), .{});
23372337 if (file.flags.is_directory) try file_obj.field("directory", true, .{});
23382338 if (file.flags.metadata_only) try file_obj.field("metadata", true, .{});
2339 if (!file.flags.exists) try file_obj.field("exists", false, .{});
23392340 try file_obj.field("prefix", file.flags.prefix, .{});
23402341 try file_obj.field("path", path, .{});
23412342 try file_obj.end();
......@@ -3370,7 +3371,12 @@ fn printStepStatus(maker: *Maker, step_index: Configuration.Step.Index, stderr:
33703371 try writer.writeAll(" invalid_manifest");
33713372 try stderr.setColor(.green);
33723373 },
3373 .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |path, tag| {
3374 .path_created,
3375 .path_deleted,
3376 .directory_status_changed,
3377 .metadata_changed,
3378 .contents_changed,
3379 => |path, tag| {
33743380 try stderr.setColor(.blue);
33753381 try writer.print(" {t}: {s}", .{ tag, path });
33763382 try stderr.setColor(.green);
lib/compiler/Maker/Step.zig+15-2
......@@ -84,6 +84,7 @@ const ResultCacheStatus = union(enum) {
8484 incomplete_manifest,
8585 invalid_manifest,
8686 path_deleted: [*:0]const u8,
87 path_created: [*:0]const u8,
8788 directory_status_changed: [*:0]const u8,
8889 metadata_changed: [*:0]const u8,
8990 contents_changed: [*:0]const u8,
......@@ -815,7 +816,13 @@ fn setResultCacheStatus(s: *Step, man: *const Cache.Manifest, status: Cache.Mani
815816 .hit => .hit,
816817 .incomplete_manifest => .incomplete_manifest,
817818 .invalid_manifest => .invalid_manifest,
818 inline .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |off, tag| @unionInit(
819
820 inline .path_created,
821 .path_deleted,
822 .directory_status_changed,
823 .metadata_changed,
824 .contents_changed,
825 => |off, tag| @unionInit(
819826 ResultCacheStatus,
820827 @tagName(tag),
821828 try gpa.dupeSentinel(u8, off.path(man.contents.items), 0),
......@@ -827,7 +834,13 @@ fn setResultCacheStatus(s: *Step, man: *const Cache.Manifest, status: Cache.Mani
827834fn clearResultCacheStatus(s: *Step, gpa: Allocator) void {
828835 switch (s.result_cache_status) {
829836 .hit, .incomplete_manifest, .invalid_manifest => {},
830 .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |path| {
837
838 .path_created,
839 .path_deleted,
840 .directory_status_changed,
841 .metadata_changed,
842 .contents_changed,
843 => |path| {
831844 gpa.free(std.mem.sliceTo(path, 0));
832845 },
833846 }
lib/std/Build.zig+7-3
......@@ -2643,9 +2643,11 @@ pub fn systemIntegrationOption(
26432643
26442644/// Indicates that the build.zig logic depends on a particular file's contents.
26452645///
2646/// If the file is created, deleted, or has its contents changed, the configure
2647/// phase will be repeated. If the inode or mtime change, but the file contents
2648/// remain the same, it will not cause the configure logic to be repeated.
2646/// The file must exist when this function is called.
2647///
2648/// If the file is deleted, or has its contents changed, the configure phase
2649/// will be repeated. If the inode or mtime change, but the file contents remain
2650/// the same, it will not cause the configure logic to be repeated.
26492651///
26502652/// This is an alternative to `Graph.poisonCache` that avoids making every invocation
26512653/// of `zig build` into a cache miss.
......@@ -2695,6 +2697,8 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void {
26952697
26962698/// Indicates that the build.zig logic depends on a particular directory's entries.
26972699///
2700/// The directory must exist when this function is called.
2701///
26982702/// This is an alternative to `Graph.poisonCache` that avoids making every invocation
26992703/// of `zig build` into a cache miss.
27002704///
lib/std/Build/Cache.zig+90-61
......@@ -56,7 +56,7 @@ pub fn prefixes(cache: *const Cache) []const Directory {
5656 return cache.prefixes_buffer[0..cache.prefixes_len];
5757}
5858
59const PrefixIndex = u6;
59const PrefixIndex = u3;
6060
6161const PrefixedPath = struct {
6262 prefix: PrefixIndex,
......@@ -453,7 +453,9 @@ pub const Manifest = struct {
453453 pub const Flags = packed struct(u8) {
454454 is_directory: bool,
455455 metadata_only: bool,
456 exists: bool,
456457 prefix: PrefixIndex,
458 _: u2 = 0,
457459 };
458460
459461 /// Prefixes path names in encoded directory contents. Starts numbering
......@@ -576,6 +578,7 @@ pub const Manifest = struct {
576578
577579 /// Returns true if the stat was changed. Updates the `file` with the new stat value.
578580 fn setStatChanged(file: *File, m: *Manifest, s: Stat) Io.Cancelable!bool {
581 assert(file.flags.exists);
579582 if (s.size == file.size and
580583 s.mtime.nanoseconds == file.mtime and
581584 s.inode == file.inode)
......@@ -693,12 +696,12 @@ pub const Manifest = struct {
693696 };
694697
695698 /// Add a file or directory path as a dependency of process being cached.
696 /// When `hit` is called, the contents will be checked to ensure
699 /// When `check` is called, the contents will be checked to ensure
697700 /// that it matches the contents from previous times.
698701 ///
699702 /// The contents of the input file may be requested and subsequently
700703 /// obtained via methods of the returned `InputPath.Index` after calling
701 /// `hit`.
704 /// `check`.
702705 ///
703706 /// Contents of a directory are considered to be the sorted list of file
704707 /// names of direct entries, separated by null byte. Each file name is
......@@ -726,6 +729,7 @@ pub const Manifest = struct {
726729 .prefix = new_prefix,
727730 .is_directory = is_directory,
728731 .metadata_only = options.metadata_only,
732 .exists = true, // modified later if necessary
729733 };
730734
731735 const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ .contents = m.contents.items });
......@@ -841,6 +845,7 @@ pub const Manifest = struct {
841845 hit,
842846 incomplete_manifest,
843847 invalid_manifest,
848 path_created: File.Offset,
844849 path_deleted: File.Offset,
845850 directory_status_changed: File.Offset,
846851 metadata_changed: File.Offset,
......@@ -853,7 +858,7 @@ pub const Manifest = struct {
853858 pub fn format(this: @This(), w: *Io.Writer) Io.Writer.Error!void {
854859 switch (this.check_result) {
855860 .hit, .incomplete_manifest, .invalid_manifest => return w.writeAll(@tagName(this.check_result)),
856 .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |off| {
861 .path_created, .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |off| {
857862 const path = off.path(this.manifest.contents.items);
858863 return w.print("{t} {s}", .{ this.check_result, path });
859864 },
......@@ -1192,38 +1197,36 @@ pub const Manifest = struct {
11921197 };
11931198 if (!input_path.have_stat and stat_path_ok) {
11941199 // Since this is an input file, FileNotFound counts as a failure, not a cache miss.
1195 const actual_stat = switch (input_path.getHandle(input_file)) {
1196 .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) {
1197 error.Canceled => |e| return e,
1198 else => |e| return fail(diag, .{ .file_stat = .{
1199 .file_offset = file_off,
1200 .err = e,
1201 } }),
1202 },
1203 .file => |opened_file| opened_file.stat(io) catch |err| switch (err) {
1204 error.Canceled => |e| return e,
1205 else => |e| return fail(diag, .{ .file_stat = .{
1206 .file_offset = file_off,
1207 .err = e,
1208 } }),
1209 },
1210 .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) {
1211 error.Canceled => |e| return e,
1212 else => |e| return fail(diag, .{ .file_stat = .{
1213 .file_offset = file_off,
1214 .err = e,
1215 } }),
1216 },
1200 const stat_result = switch (input_path.getHandle(input_file)) {
1201 .none => parent_dir.statFile(io, file_path, .{}),
1202 .file => |opened_file| opened_file.stat(io),
1203 .dir => |opened_dir| opened_dir.stat(io),
12171204 };
1218 const actual_is_directory = actual_stat.kind == .directory;
1219 if (actual_is_directory != input_file.flags.is_directory) {
1220 // Since this is an input file, this is a failure, not a cache miss.
1221 return fail(diag, .{ .file_stat = .{
1205 if (stat_result) |actual_stat| {
1206 const actual_is_directory = actual_stat.kind == .directory;
1207 if (actual_is_directory != input_file.flags.is_directory) {
1208 // Since this is an input file, this is a failure, not a cache miss.
1209 return fail(diag, .{ .file_stat = .{
1210 .file_offset = file_off,
1211 .err = if (actual_is_directory) error.IsDir else error.NotDir,
1212 } });
1213 }
1214 try input_file.setStat(m, .init(actual_stat));
1215 } else |err| switch (err) {
1216 error.Canceled => |e| return e,
1217
1218 error.FileNotFound => |e| if (input_file.flags.metadata_only) {
1219 input_file.flags.exists = false;
1220 } else return fail(diag, .{ .file_stat = .{
12221221 .file_offset = file_off,
1223 .err = if (actual_is_directory) error.IsDir else error.NotDir,
1224 } });
1222 .err = e,
1223 } }),
1224
1225 else => |e| return fail(diag, .{ .file_stat = .{
1226 .file_offset = file_off,
1227 .err = e,
1228 } }),
12251229 }
1226 try input_file.setStat(m, .init(actual_stat));
12271230 }
12281231
12291232 if (input_file.flags.metadata_only) {
......@@ -1369,6 +1372,9 @@ pub const Manifest = struct {
13691372 // space to compute the digest in case stat changed, in order to compare with `disk_file.digest`.
13701373
13711374 if (input_path.have_stat) {
1375 if (!disk_file.flags.exists) {
1376 return .{ .path_created = file_off };
1377 }
13721378 const changed = try disk_file.setStatChanged(m, input_file.stat());
13731379 if (!changed) return .hit;
13741380 }
......@@ -1376,29 +1382,28 @@ pub const Manifest = struct {
13761382 if (disk_file.flags.metadata_only) {
13771383 if (!input_path.have_stat) {
13781384 // Since this is an input file, FileNotFound counts as a failure, not a cache miss.
1379 const actual_stat = switch (input_path.getHandle(input_file)) {
1380 .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) {
1381 error.Canceled => |e| return e,
1382 else => |e| return fail(diag, .{ .file_stat = .{
1383 .file_offset = file_off,
1384 .err = e,
1385 } }),
1386 },
1387 .file => |opened_file| opened_file.stat(io) catch |err| switch (err) {
1388 error.Canceled => |e| return e,
1389 else => |e| return fail(diag, .{ .file_stat = .{
1390 .file_offset = file_off,
1391 .err = e,
1392 } }),
1393 },
1394 .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) {
1395 error.Canceled => |e| return e,
1396 else => |e| return fail(diag, .{ .file_stat = .{
1385 const stat_result = switch (input_path.getHandle(input_file)) {
1386 .none => parent_dir.statFile(io, file_path, .{}),
1387 .file => |opened_file| opened_file.stat(io),
1388 .dir => |opened_dir| opened_dir.stat(io),
1389 };
1390 const actual_stat = stat_result catch |err| switch (err) {
1391 error.Canceled => |e| return e,
1392 error.FileNotFound => |e| switch (disk_file.flags.exists) {
1393 false => return .hit,
1394 true => return fail(diag, .{ .file_stat = .{
13971395 .file_offset = file_off,
13981396 .err = e,
13991397 } }),
14001398 },
1399 else => |e| return fail(diag, .{ .file_stat = .{
1400 .file_offset = file_off,
1401 .err = e,
1402 } }),
14011403 };
1404 if (!disk_file.flags.exists) {
1405 return .{ .path_created = file_off };
1406 }
14021407 // In the other case (have_stat=true), this check is redundant since we already asserted the flags are
14031408 // identical, which contains the prefix.
14041409 const actual_is_directory = actual_stat.kind == .directory;
......@@ -1559,7 +1564,10 @@ pub const Manifest = struct {
15591564
15601565 if (file.flags.metadata_only) {
15611566 const actual_stat = parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) {
1562 error.FileNotFound => return .{ .path_deleted = file_off },
1567 error.FileNotFound => switch (file.flags.exists) {
1568 false => return .hit,
1569 true => return .{ .path_deleted = file_off },
1570 },
15631571 error.Canceled => |e| return e,
15641572 else => |e| return fail(diag, .{ .file_stat = .{
15651573 .file_offset = file_off,
......@@ -1567,6 +1575,10 @@ pub const Manifest = struct {
15671575 } }),
15681576 };
15691577
1578 if (!file.flags.exists) {
1579 return .{ .path_created = file_off };
1580 }
1581
15701582 const actual_is_directory = actual_stat.kind == .directory;
15711583 if (actual_is_directory != file.flags.is_directory) return .{ .directory_status_changed = file_off };
15721584
......@@ -1576,6 +1588,8 @@ pub const Manifest = struct {
15761588 return .hit;
15771589 }
15781590
1591 assert(file.flags.exists); // file not existing can only be tracked with metadata_only
1592
15791593 if (file.flags.is_directory) {
15801594 const opened_dir = parent_dir.openDir(io, file_path, .{
15811595 .iterate = true,
......@@ -1774,6 +1788,7 @@ pub const Manifest = struct {
17741788 .prefix = new_prefix,
17751789 .is_directory = is_directory,
17761790 .metadata_only = options.metadata_only,
1791 .exists = true, // modified later if necessary
17771792 };
17781793
17791794 const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ .contents = m.contents.items });
......@@ -1817,18 +1832,25 @@ pub const Manifest = struct {
18171832 } else {
18181833 const dir = cache.prefixes()[prefix].handle;
18191834 const sub_path = file_offset.path(m.contents.items);
1820 const handle = dir.openDir(io, sub_path, .{
1835 if (dir.openDir(io, sub_path, .{
18211836 .access_sub_paths = false,
18221837 .iterate = true,
1823 }) catch |err| switch (err) {
1838 })) |handle| {
1839 defer handle.close(io);
1840 try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic);
1841 } else |err| switch (err) {
18241842 error.Canceled => |e| return e,
1843 error.FileNotFound => |e| if (options.metadata_only) {
1844 header.flags.exists = false;
1845 } else {
1846 if (options.diagnostic) |d| d.* = .{ .open_dir = e };
1847 return error.FileSystemFailure;
1848 },
18251849 else => |e| {
18261850 if (options.diagnostic) |d| d.* = .{ .open_dir = e };
18271851 return error.FileSystemFailure;
18281852 },
1829 };
1830 defer handle.close(io);
1831 try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic);
1853 }
18321854 },
18331855
18341856 .file => |opt_handle| if (opt_handle) |handle| {
......@@ -1836,15 +1858,22 @@ pub const Manifest = struct {
18361858 } else {
18371859 const dir = cache.prefixes()[prefix].handle;
18381860 const sub_path = file_offset.path(m.contents.items);
1839 const handle = dir.openFile(io, sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
1861 if (dir.openFile(io, sub_path, .{ .mode = .read_only })) |handle| {
1862 defer handle.close(io);
1863 try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic);
1864 } else |err| switch (err) {
18401865 error.Canceled => |e| return e,
1866 error.FileNotFound => |e| if (options.metadata_only) {
1867 header.flags.exists = false;
1868 } else {
1869 if (options.diagnostic) |d| d.* = .{ .open_file = e };
1870 return error.FileSystemFailure;
1871 },
18411872 else => |e| {
18421873 if (options.diagnostic) |d| d.* = .{ .open_file = e };
18431874 return error.FileSystemFailure;
18441875 },
1845 };
1846 defer handle.close(io);
1847 try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic);
1876 }
18481877 },
18491878 }
18501879 }