From ec7805b8670e7411cb847fc44b1ce2b6fe37b5ec Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 23 Aug 2026 21:42:17 -0700 Subject: [PATCH 01/49] WIP: rework std.Build.Cache - implement std.Build functions: - `dependOnDirectoryMetadata` - `dependOnDirectoryContents` - `dependOnFileMetadata` - `dependOnFileContents` - switch cache manifest to binary format. saves about 25% of file size, which is not a ton, but this is in the really hot path, so any amount we can reduce file system cache pressure is going to help a lot. - also means saving and loading is simpler - just copying bytes in and out of memory plus a tiny bit of validation. - batch cache manifest memory allocations together. one big allocation for the entire manifest file contents, with the files hash map only tracking starting offsets for each entry, and the input files as a simple array list with the extra information needed for input files (so they can be lazily loaded only on cache miss) - remove the manifest file size limit - input files that have requested content to be loaded is loaded together into one big allocation rather than separately tracked. - each file entry in the cache now tracks two additional bits: - kind: file or directory - tracking mode: contents or metadata - when tracking a directory, "contents" are considered to be the sorted list of file names contained in the directory (non recursive) - rename "hit" to "check" and make it return enum instead of bool - merge most "add...File" function variants together into one that has an "options bag" parameter. - take advantage of async I/O when checking for cache hit to compute hashes concurrently. Care is taken to never cancel useful work, and never wait for unnecessary work to complete. bonus: - build.zig: use std.log for printing log messages --- build.zig | 21 +- lib/std/Build.zig | 36 +- lib/std/Build/Cache.zig | 1174 +++++++++++++++---------------- lib/std/Build/Configuration.zig | 5 +- src/Zcu/PerThread.zig | 11 +- test/src/Cases.zig | 2 +- 6 files changed, 621 insertions(+), 628 deletions(-) diff --git a/build.zig b/build.zig index 94283dffd11afcf9f126f2418d21081710be4361..8c18ea4841bcd00e19239273014f3e11699781f8 100644 --- a/build.zig +++ b/build.zig @@ -263,8 +263,8 @@ pub fn build(b: *std.Build) !void { const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git."); const version_slice = if (opt_version_string) |version| version else v: { if (!std.process.can_spawn) { - std.debug.print("error: version info cannot be retrieved from git. Zig version must be provided using -Dversion-string\n", .{}); - std.process.exit(1); + std.log.info("version info can be provided explicitly via \"-Dversion-string\"", .{}); + std.process.fatal("version info cannot be retrieved from git", .{}); } // Ensure git version changes get picked up. @@ -310,8 +310,9 @@ pub fn build(b: *std.Build) !void { 0 => { // Tagged release version (e.g. 0.10.0). if (!mem.eql(u8, git_describe, version_string)) { - std.debug.print("Zig version '{s}' does not match Git tag '{s}'\n", .{ version_string, git_describe }); - std.process.exit(1); + std.process.fatal("zig version {q} does not match Git tag {q}", .{ + version_string, git_describe, + }); } break :v version_string; }, @@ -324,13 +325,14 @@ pub fn build(b: *std.Build) !void { const ancestor_ver = try std.SemanticVersion.parse(tagged_ancestor); if (zig_version.order(ancestor_ver) != .gt) { - std.debug.print("Zig version '{f}' must be greater than tagged ancestor '{f}'\n", .{ zig_version, ancestor_ver }); - std.process.exit(1); + std.process.fatal("zig version {f} must be greater than tagged ancestor {qf}", .{ + zig_version, ancestor_ver, + }); } // Check that the commit hash is prefixed with a 'g' (a Git convention). if (commit_id.len < 1 or commit_id[0] != 'g') { - std.debug.print("Unexpected `git describe` output: {s}\n", .{git_describe}); + std.log.warn("unexpected \"git describe\" output: {s}", .{git_describe}); break :v version_string; } @@ -338,7 +340,7 @@ pub fn build(b: *std.Build) !void { break :v b.fmt("{s}-dev.{s}+{s}", .{ version_string, commit_height, commit_id[1..] }); }, else => { - std.debug.print("Unexpected `git describe` output: {s}\n", .{git_describe}); + std.log.warn("unexpected \"git describe\" output: {s}", .{git_describe}); break :v version_string; }, } @@ -354,7 +356,8 @@ pub fn build(b: *std.Build) !void { const file_contents = cwd.readFileAlloc(io, config_h_path, arena, .limited(max_config_h_bytes)) catch unreachable; break :blk parseConfigH(b, file_contents); } else { - std.log.warn("config.h could not be located automatically. Consider providing it explicitly via \"-Dconfig_h\"", .{}); + std.log.warn("config.h could not be located automatically", .{}); + std.log.info("config.h can be provided explicitly via \"-Dconfig_h\"", .{}); break :blk null; } }; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 55271b4a6a4e2977093675e15b82cf40fe54e8cd..f0be0461b9450c0cf4221a9c48493a9c60c70345 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -2636,6 +2636,35 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void { /// This is an alternative to `Graph.poisonCache` that avoids making every invocation /// of `zig build` into a cache miss. /// +/// If any file is created, deleted, or renamed in this directory, leaving the +/// directory in a different state than last configuration with respect to +/// existence and naming of entries, the configure phase will be repeated. +/// +/// Only a subset of `LazyPath` are supported: +/// - Relative to cwd +/// - Relative to any package root +/// - Relative to zig cache or zig installation +/// +/// If the directory would be inside one of the search prefixes, then the dependency +/// cannot be tracked; `Graph.poisonCache` must be used instead. +/// +/// Not recursive. +pub fn dependOnDirectoryContents(b: *Build, lazy_path: LazyPath) void { + validateConfigureDependency(lazy_path); + const graph = b.graph; + graph.configure_dependencies.append(graph.arena, .{ + .lazy_path = lazy_path.dupe(graph), + .is_directory = true, + .metadata_only = false, + }) catch @panic("OOM"); +} + +/// Indicates that the build.zig logic depends on a particular directory's last +/// modification date. +/// +/// This is an alternative to `Graph.poisonCache` that avoids making every invocation +/// of `zig build` into a cache miss. +/// /// If any file is created, deleted, or renamed in this directory, the /// configure phase will be repeated. /// @@ -2646,12 +2675,15 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void { /// /// If the directory would be inside one of the search prefixes, then the dependency /// cannot be tracked; `Graph.poisonCache` must be used instead. -pub fn dependOnDirectory(b: *Build, lazy_path: LazyPath) void { +/// +/// Not recursive. +pub fn dependOnDirectoryMetadata(b: *Build, lazy_path: LazyPath) void { validateConfigureDependency(lazy_path); const graph = b.graph; graph.configure_dependencies.append(graph.arena, .{ .lazy_path = lazy_path.dupe(graph), - .mode = .directory, + .is_directory = true, + .metadata_only = true, }) catch @panic("OOM"); } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 94051c51658796081d413cc18133a97c93814d55..a644aabe7d4a9be06c1e64d0de9ce8f8273c8467 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -62,7 +62,7 @@ pub const PrefixedPath = struct { sub_path: []const u8, fn eql(a: PrefixedPath, b: PrefixedPath) bool { - return a.prefix == b.prefix and std.mem.eql(u8, a.sub_path, b.sub_path); + return a.prefix == b.prefix and mem.eql(u8, a.sub_path, b.sub_path); } fn hash(pp: PrefixedPath) u32 { @@ -118,7 +118,7 @@ fn getPrefixSubpath(gpa: Allocator, cwd: []const u8, prefix: []const u8, path: [ return error.NotASubPath; } const first_component = component_iterator.first(); - if (first_component != null and std.mem.eql(u8, first_component.?.name, "..")) { + if (first_component != null and mem.eql(u8, first_component.?.name, "..")) { return error.NotASubPath; } return relative; @@ -130,10 +130,6 @@ pub const hex_digest_len = bin_digest_len * 2; pub const BinDigest = [bin_digest_len]u8; pub const HexDigest = [hex_digest_len]u8; -/// This is currently just an arbitrary non-empty string that can't match another manifest line. -const manifest_header = "0"; -pub const manifest_file_size_max = 100 * 1024 * 1024; - /// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it /// provides enough collision resistance for the Manifest use cases, while being one of our /// fastest options right now. @@ -149,50 +145,6 @@ pub const hasher_init: Hasher = Hasher.init(&.{ 0x77, 0xd6, 0xf0, 0x60, }); -pub const File = struct { - prefixed_path: PrefixedPath, - max_file_size: ?usize, - /// Populated if the user calls `addOpenedFile`. - /// The handle is not owned here. - handle: ?Io.File, - stat: Stat, - bin_digest: BinDigest, - contents: ?[]const u8, - - pub const Stat = struct { - inode: Io.File.INode, - size: u64, - mtime: Io.Timestamp, - - pub fn fromFs(fs_stat: Io.File.Stat) Stat { - return .{ - .inode = fs_stat.inode, - .size = fs_stat.size, - .mtime = fs_stat.mtime, - }; - } - }; - - pub fn deinit(self: *File, gpa: Allocator) void { - gpa.free(self.prefixed_path.sub_path); - if (self.contents) |contents| { - gpa.free(contents); - self.contents = null; - } - self.* = undefined; - } - - pub fn updateMaxSize(file: *File, new_max_size: ?usize) void { - const new = new_max_size orelse return; - file.max_file_size = if (file.max_file_size) |old| @max(old, new) else new; - } - - pub fn updateHandle(file: *File, new_handle: ?Io.File) void { - const handle = new_handle orelse return; - file.handle = handle; - } -}; - pub const HashHelper = struct { hasher: Hasher = hasher_init, @@ -319,10 +271,15 @@ pub const Lock = struct { } }; +/// Format: a series of consecutive `Manifest.File`, followed by a final +/// terminating zero byte to distinguish empty manifest file from manifest with +/// zero files. pub const Manifest = struct { cache: *Cache, /// Current state for incremental hashing. hash: HashHelper, + hex_digest: HexDigest, + /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. manifest_file: ?Io.File, manifest_dirty: bool, /// Set this flag to true before calling hit() in order to indicate that @@ -335,12 +292,140 @@ pub const Manifest = struct { // order to obtain a problematic timestamp for the next call. Calls after that // will then use the same timestamp, to avoid unnecessary filesystem writes. want_refresh_timestamp: bool = true, - files: Files = .{}, - hex_digest: HexDigest, + /// Uses `Cache.gpa`. + files: Files = .empty, + /// Indexes line up with `files`, but only up until `hit` is called. Uses + /// `Cache.gpa`. + input_files: std.ArrayList(InputFile) = .empty, diagnostic: Diagnostic = .none, /// Keeps track of the last time we performed a file system write to observe /// what time the file system thinks it is, according to its own granularity. recent_problematic_timestamp: Io.Timestamp = .zero, + /// The entire manifest file contents, except for the final terminating + /// zero byte. However maintains always at least 1 unused capacity so the + /// final terminating byte can be added without allocation. Uses + /// `Cache.gpa`. + contents: std.ArrayList(u8) = .empty, + /// All contents from all `input_files` whose contents were requested, + /// concatenated. Total byte size will be less than `max_input_content_len` + /// otherwise an error is returned. + all_input_content: std.ArrayList(u8) = .empty, + max_input_content_len: usize = std.math.maxInt(u32), + + pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); + + /// Source files whose prefix and relative path are included when computing + /// the cache manifest digest. It's the information needed to lazily hash + /// the input files only when a cache miss occurs. + /// + /// `File.prefix`, `File.path`, and `File.mode` will be always populated, + /// but the other fields of `File` will be populated depending on the + /// fields of `InputFile`. + pub const InputFile = struct { + request_handle: bool, + have_handle: bool, + /// Determines whether `File.size`, `File.inode`, and `File.mtime` are populated. + have_stat: bool, + /// Determines whether `File.digest` is populated. + have_digest: bool, + contents: enum (usize) { + requested = std.math.maxInt(u32) - 1, + not_requested = std.math.maxInt(u32), + /// Byte offset index into `Manifest.all_input_content`. + _, + }, + /// `have_handle` determines whether this is populated. + handle: Io.File, + + /// Index into `Manifest.input_files`. + pub const Index = enum(u32) { + _, + }; + }; + + /// The data per tracked input file that is stored in the manifest file. + pub const File = extern struct { + size: u64, + inode: u64, + digest: BinDigest, + /// Nanoseconds. + mtime: i64, + /// Starting with this field and continuing into the path, excluding the null byte, + /// is the string that is hashed for the manifest digest. + flags: Flags, + /// Terminated by zero byte, then followed by padding until 8-byte aligned. + path_start: [0]u8, + + pub const Flags = packed struct (u8) { + is_directory: bool, + metadata_only: bool, + prefix: u6, + }; + + /// Byte index within `Manifest.contents` where the entry starts. + pub const Offset = enum(u32) { + _, + + pub fn get(offset: Offset, m: *const Manifest) *File { + return @ptrCast(m.contents.items[@backingInt(offset)..][0..@sizeOf(File)]); + } + + pub fn getFallible(offset: Offset, m: *const Manifest) error{EndOfStream}!*File { + if (@backingInt(offset) + @sizeOf(File) >= m.contents.len) return error.EndOfStream; + return get(offset, m); + } + }; + + pub const HashContext = struct { + manifest: *const Manifest, + + pub fn hash(this: @This(), off: Offset) u32 { + const file = off.get(this.manifest); + return @truncate(std.hash.Wyhash.hash(file.prefix, file.path())); + } + + pub fn eql(this: @This(), a_off: Offset, b_off: Offset, b_index: usize) bool { + _ = b_index; + const a = a_off.get(this.manifest); + const b = b_off.get(this.manifest); + return a.prefix == b.prefix and mem.eql(u8, a.path(), b.path()); + } + }; + + + pub fn path(file: *const File) [:0]const u8 { + return pathFallible(file) catch unreachable; + } + + pub fn pathFallible(file: *const File) error{EndOfStream}![:0]const u8 { + const ptr: [*]u8 = &file.path_start; + const len = mem.findScalar(u8, ptr, 0) orelse return error.EndOfStream; + return ptr[0..len :0]; + } + + fn manifestDigestHash(file: *const File, hasher: *Hasher) void { + const path_ptr: [*]u8 = &file.path_start; + const path_len = mem.findScalar(u8, path_ptr, 0).?; + comptime assert(@offsetOf(File, "path_start") - @offsetOf(File, "flags") == 1); + // Includes flags and sentinel. + const hash_string = (path_ptr - 1)[0..path_len + 2]; + hasher.update(hash_string); + } + + fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void { + file.size = stat.size; + file.inode = stat.inode; + file.mtime = stat.mtime; + + if (try m.isProblematicTimestamp(stat.mtime)) { + // The actual file has an unreliable timestamp; force it to be hashed. + file.stat.mtime = 0; + file.stat.inode = 0; + } + } + + }; + pub const Diagnostic = union(enum) { none, @@ -358,110 +443,118 @@ pub const Manifest = struct { }; }; - pub const Files = std.array_hash_map.Custom(File, void, FilesContext, false); - - pub const FilesContext = struct { - pub fn hash(fc: FilesContext, file: File) u32 { - _ = fc; - return file.prefixed_path.hash(); - } - - pub fn eql(fc: FilesContext, a: File, b: File, b_index: usize) bool { - _ = fc; - _ = b_index; - return a.prefixed_path.eql(b.prefixed_path); - } + pub const Stat = struct { + size: u64, + inode: Io.File.INode, + mtime: Io.Timestamp, }; - const FilesAdapter = struct { - pub fn eql(context: @This(), a: PrefixedPath, b: File, b_index: usize) bool { - _ = context; - _ = b_index; - return a.eql(b.prefixed_path); - } + pub const AddInputFileOptions = struct { + handle: ?Io.File = null, + stat: ?Stat = null, + request_handle: bool = false, + request_contents: bool = false, + is_directory: bool = false, + metadata_only: bool = false, - pub fn hash(context: @This(), key: PrefixedPath) u32 { - _ = context; - return key.hash(); - } }; + pub const AddInputFileError = error { + /// The same file path has been added to the cache manifest both as a + /// directory and as a normal file, making the intended caching + /// behavior ambiguous. + IsDirectoryAmbiguous, + } || Allocator.Error; + /// Add a file as a dependency of process being cached. When `hit` is /// called, the file's contents will be checked to ensure that it matches /// the contents from previous times. /// - /// Max file size will be used to determine the amount of space the file contents - /// are allowed to take up in memory. If max_file_size is null, then the contents - /// will not be loaded into memory. - /// - /// Returns the index of the entry in the `files` array list. You can use it - /// to access the contents of the file after calling `hit()` like so: - /// - /// ``` - /// var file_contents = cache_hash.files.keys()[file_index].contents.?; - /// ``` - pub fn addFilePath(m: *Manifest, file_path: Path, max_file_size: ?usize) !usize { - return addOpenedFile(m, file_path, null, max_file_size); - } - - /// Same as `addFilePath` except the file has already been opened. - pub fn addOpenedFile(m: *Manifest, path: Path, handle: ?Io.File, max_file_size: ?usize) !usize { + /// The contents of the input file may be requested and subsequently + /// obtained via methods of the returned `InputFile.Index` after calling + /// `hit`. + pub fn addInputFile(m: *Manifest, path: Path, options: AddInputFileOptions) Allocator.Error!InputFile.Index { const gpa = m.cache.gpa; try m.files.ensureUnusedCapacity(gpa, 1); - const resolved_path = try std.fs.path.resolve(gpa, &.{ - path.root_dir.path orelse ".", - path.subPathOrDot(), + try m.input_files.ensureUnusedCapacity(gpa, 1); + + const prev_contents_len = m.contents.items.len; + const header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))); + errdefer m.contents.shrinkRetainingCapacity(prev_contents_len); + + header.* = .{ + .flags = .{ + .prefix = try m.cache.findAppendPrefixedPath(&m.contents, path), + .is_directory = options.is_directory, + .metadata_only = options.metadata_only, + }, + .size = undefined, + .inode = undefined, + .mtime = undefined, + .digest = undefined, + }; + assert(m.contents.items.len % @alignOf(File) == 0); + + const gop = try m.files.getOrPutAssumeCapacityContext(@fromBackingInt(prev_contents_len), .{ + .manifest = m, }); - errdefer gpa.free(resolved_path); - const prefixed_path = try m.cache.findPrefixResolved(resolved_path); - return addFileInner(m, prefixed_path, handle, max_file_size); - } - - fn addFileInner(self: *Manifest, prefixed_path: PrefixedPath, handle: ?Io.File, max_file_size: ?usize) usize { - const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{}); if (gop.found_existing) { - self.cache.gpa.free(prefixed_path.sub_path); - gop.key_ptr.updateMaxSize(max_file_size); - gop.key_ptr.updateHandle(handle); - return gop.index; + m.contents.shrinkRetainingCapacity(prev_contents_len); + const existing_input_file = &m.input_files.items[gop.index]; + if (options.handle) |handle| { + existing_input_file.handle = handle; + existing_input_file.have_handle = true; + } + if (options.request_contents) switch (existing_input_file.contents) { + .requested, .not_requested => existing_input_file.contents = .requested, + _ => {}, + }; + const existing_header = &m.files.keys()[gop.index]; + if (options.stat) |stat| { + existing_input_file.have_stat = true; + existing_header.size = stat.size; + existing_header.inode = stat.inode; + existing_header.mtime = stat.mtime; + } + if (existing_header.flags.is_directory != options.is_directory) + return error.IsDirectoryAmbiguous; + if (!options.metadata_only) + existing_header.flags.metadata_only = false; + } else { + m.input_files.appendAssumeCapacity(.{ + .request_handle = options.request_handle, + .have_handle = options.handle != null, + .handle = if (options.handle) |handle| handle else undefined, + .contents = if (options.request_contents) .requested else .not_requested, + .have_digest = false, + .have_stat = options.stat != null, + }); + assert(m.input_files.items.len - 1 == gop.index); + if (options.stat) |stat| { + header.size = stat.size; + header.inode = stat.inode; + header.mtime = stat.mtime; + } } - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .contents = null, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - .handle = handle, - }; - - self.hash.add(prefixed_path.prefix); - self.hash.addBytes(prefixed_path.sub_path); - - return gop.index; - } - - pub fn addOptionalFilePath(self: *Manifest, optional_file_path: ?Path) !void { - self.hash.add(optional_file_path != null); - const file_path = optional_file_path orelse return; - _ = try self.addFilePath(file_path, null); + return @fromBackingInt(gop.index); } - pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { - assert(self.manifest_file == null); - return self.addDepFileMaybePost(dir, dep_file_sub_path); + pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputFileOptions) Allocator.Error!void { + m.hash.add(opt_path != null); + _ = try addInputFile(m, opt_path orelse return, options); } - pub const HitError = error{ + pub const CheckError = error{ /// Unable to check the cache for a reason that has been recorded into /// the `diagnostic` field. CacheCheckFailed, /// A cache manifest file exists however it could not be parsed. InvalidFormat, - OutOfMemory, - Canceled, - }; + } || Allocator.Error || Io.Cancelable; - /// Check the cache to see if the input exists in it. If it exists, returns `true`. + pub const CheckStatus = enum { hit, miss }; + + /// Check the cache to see if the input exists in it. /// A hex encoding of its hash is available by calling `final`. /// /// This function will also acquire an exclusive lock to the manifest file. This means @@ -473,50 +566,48 @@ pub const Manifest = struct { /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn hit(man: *Manifest, parent_progress_node: std.Progress.Node) HitError!bool { + pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) CheckError!CheckStatus { const node = parent_progress_node.start("Reusing Cache Artifacts", 0); defer node.end(); - return hitInner(man); + return checkProgressless(man); } - pub fn hitInner(self: *Manifest) HitError!bool { - assert(self.manifest_file == null); + pub fn checkProgressless(man: *Manifest) CheckError!CheckStatus { + assert(man.manifest_file == null); - self.diagnostic = .none; + for (man.files.keys()[0..man.input_files.items.len]) |file_off| { + file_off.get(man).manifestDigestHash(&man.hash.hasher); + } - const ext = ".txt"; - var manifest_file_path: [hex_digest_len + ext.len]u8 = undefined; + man.diagnostic = .none; var bin_digest: BinDigest = undefined; - self.hash.hasher.final(&bin_digest); + man.hash.hasher.final(&bin_digest); + man.hex_digest = binToHex(bin_digest); - self.hex_digest = binToHex(bin_digest); - - @memcpy(manifest_file_path[0..self.hex_digest.len], &self.hex_digest); - manifest_file_path[hex_digest_len..][0..ext.len].* = ext.*; - - const io = self.cache.io; + const manifest_file_path = &man.hex_digest; + const io = man.cache.io; // We'll try to open the cache with an exclusive lock, but if that would block // and `want_shared_lock` is set, a shared lock might be sufficient, so we'll // open with a shared lock instead. while (true) { - if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{ + if (man.cache.manifest_dir.createFile(io, manifest_file_path, .{ .read = true, .truncate = false, .lock = .exclusive, - .lock_nonblocking = self.want_shared_lock, + .lock_nonblocking = man.want_shared_lock, })) |manifest_file| { - self.manifest_file = manifest_file; - self.have_exclusive_lock = true; + man.manifest_file = manifest_file; + man.have_exclusive_lock = true; break; } else |err| switch (err) { error.WouldBlock => { - self.manifest_file = self.cache.manifest_dir.openFile(io, &manifest_file_path, .{ + man.manifest_file = man.cache.manifest_dir.openFile(io, manifest_file_path, .{ .mode = .read_write, .lock = .shared, }) catch |e| { - self.diagnostic = .{ .manifest_create = e }; + man.diagnostic = .{ .manifest_create = e }; return error.CacheCheckFailed; }; break; @@ -532,315 +623,262 @@ pub const Manifest = struct { // failure was a race, or ENOENT, indicating deletion of // the directory of our open handle. if (!builtin.os.tag.isDarwin()) { - self.diagnostic = .{ .manifest_create = error.FileNotFound }; + man.diagnostic = .{ .manifest_create = error.FileNotFound }; return error.CacheCheckFailed; } - if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{ + if (man.cache.manifest_dir.createFile(io, manifest_file_path, .{ .read = true, .truncate = false, .lock = .exclusive, - .lock_nonblocking = self.want_shared_lock, + .lock_nonblocking = man.want_shared_lock, .exclusive = true, })) |manifest_file| { - self.manifest_file = manifest_file; - self.have_exclusive_lock = true; + man.manifest_file = manifest_file; + man.have_exclusive_lock = true; break; } else |excl_err| switch (excl_err) { error.WouldBlock, error.PathAlreadyExists => continue, error.FileNotFound => { - self.diagnostic = .{ .manifest_create = error.FileNotFound }; + man.diagnostic = .{ .manifest_create = error.FileNotFound }; return error.CacheCheckFailed; }, error.Canceled => |e| return e, else => |e| { - self.diagnostic = .{ .manifest_create = e }; + man.diagnostic = .{ .manifest_create = e }; return error.CacheCheckFailed; }, } }, error.Canceled => |e| return e, else => |e| { - self.diagnostic = .{ .manifest_create = e }; + man.diagnostic = .{ .manifest_create = e }; return error.CacheCheckFailed; }, } } - self.want_refresh_timestamp = true; - - const input_file_count = self.files.entries.len; + man.want_refresh_timestamp = true; // We're going to construct a second hash. Its input will begin with the digest we've // already computed (`bin_digest`), and then it'll have the digests of each input file, // including "post" files (see `addFilePost`). If this is a hit, we learn the set of "post" // files from the manifest on disk. If this is a miss, we'll learn those from future calls - // to `addFilePost` etc. As such, the state of `self.hash.hasher` after this function + // to `addFilePost` etc. As such, the state of `man.hash.hasher` after this function // depends on whether this is a hit or a miss. // - // If we return `true` indicating a cache hit, then `self.hash.hasher` must already include + // If we return `CacheStatus.hit`, then `man.hash.hasher` must already include // the digests of the "post" files, so the caller can call `final`. Otherwise, on a cache - // miss, `self.hash.hasher` will include the digests of all non-"post" files -- that is, + // miss, `man.hash.hasher` will include the digests of all non-"post" files -- that is, // the ones we've already been told about. The rest will be discovered through calls to // `addFilePost` etc, which will update the hasher. After all files are added, the user can // use `final`, and will at some point `writeManifest` the file list to disk. - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); + man.hash.hasher = hasher_init; + man.hash.hasher.update(&bin_digest); hit: { - const file_digests_populated: usize = digests: { - switch (try self.hitWithCurrentLock()) { + digests: { + switch (try man.checkLocked()) { .hit => break :hit, - .miss => |m| if (!try self.upgradeToExclusiveLock()) { - break :digests m.file_digests_populated; - }, + .miss => if (!try man.upgradeToExclusiveLock()) break :digests, } // We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone // else might have modified the digest, so we need to check again before deciding to miss. - // Before trying again, we must reset `self.hash.hasher` and `self.files`. + // Before trying again, we must reset `man.hash.hasher` and `man.files`. // This is basically just the first half of `unhit`. - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - while (self.files.count() != input_file_count) { - var file = self.files.pop().?; - file.key.deinit(self.cache.gpa); - } - switch (try self.hitWithCurrentLock()) { + man.hash.hasher = hasher_init; + man.hash.hasher.update(&bin_digest); + man.shrinkFilesToInput(); + switch (try man.checkLocked()) { .hit => break :hit, - .miss => |m| break :digests m.file_digests_populated, + .miss => break :digests, } - }; + } - // This is a guaranteed cache miss. We're almost ready to return `false`, but there's a - // little bookkeeping to do first. The first `file_digests_populated` entries in `files` - // have their `bin_digest` populated; there may be some left in `input_file_count` which - // we'll need to populate ourselves. Other than that, this is basically `unhit`. - self.manifest_dirty = true; - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - while (self.files.count() != input_file_count) { - var file = self.files.pop().?; - file.key.deinit(self.cache.gpa); - } - for (self.files.keys(), 0..) |*file, idx| { - if (idx < file_digests_populated) { - // `bin_digest` is already populated by `hitWithCurrentLock`, so we can use it directly. - self.hash.hasher.update(&file.bin_digest); - } else { - self.populateFileHash(file) catch |err| { - self.diagnostic = .{ .file_hash = .{ - .file_index = idx, - .err = err, - } }; - return error.CacheCheckFailed; - }; - } - } - return false; + // Cache miss. `checkLocked` guarantees that all input files have their digests populated + // unless it returns an error. + man.manifest_dirty = true; + // All input file digests are already populated by `checkLocked`, so we can call `unhit` directly. + unhit(man, &bin_digest); + return .miss; } - if (self.want_shared_lock) { - self.downgradeToSharedLock() catch |err| { - self.diagnostic = .{ .manifest_lock = err }; + if (man.want_shared_lock) { + man.downgradeToSharedLock() catch |err| { + man.diagnostic = .{ .manifest_lock = err }; return error.CacheCheckFailed; }; } - return true; + return .hit; + } + + fn shrinkFilesToInput(m: *Manifest) void { + if (m.files.count() <= m.input_files.items.len) return; + const off = m.files.keys()[m.input_files.items.len]; + m.contents.shrinkRetainingCapacity(@backingInt(off)); + assert(m.contents.len % @alignOf(File) == 0); + m.files.shrinkRetainingCapacity(m.input_files.items.len); } /// Assumes that `self.hash.hasher` has been updated only with the original digest and that /// `self.files` contains only the original input files. - fn hitWithCurrentLock(self: *Manifest) HitError!union(enum) { - hit, - miss: struct { - file_digests_populated: usize, - }, - } { - const gpa = self.cache.gpa; - const io = self.cache.io; - const input_file_count = self.files.entries.len; - var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded - var manifest_reader = self.manifest_file.?.reader(io, &tiny_buffer); // Reads positionally from zero. - const limit: std.Io.Limit = .limited(manifest_file_size_max); - const file_contents = manifest_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { + fn checkLocked(m: *Manifest) CheckError!CheckStatus { + const gpa = m.cache.gpa; + const io = m.cache.io; + + var manifest_reader = m.manifest_file.?.reader(io, &.{}); // Reads positionally from zero. + m.contents.clearRetainingCapacity(); + manifest_reader.interface.appendRemainingUnlimited(gpa, &m.contents) catch |err| switch (err) { error.OutOfMemory => |e| return e, - error.StreamTooLong => return error.OutOfMemory, error.ReadFailed => { - self.diagnostic = .{ .manifest_read = manifest_reader.err.? }; + m.diagnostic = .{ .manifest_read = manifest_reader.err.? }; return error.CacheCheckFailed; }, }; - defer gpa.free(file_contents); - - var any_file_changed = false; - var line_iter = mem.tokenizeScalar(u8, file_contents, '\n'); - var idx: usize = 0; - const header_valid = valid: { - const line = line_iter.next() orelse break :valid false; - break :valid std.mem.eql(u8, line, manifest_header); + + // Guess number of files based on manifest contents len to reduce allocations. + try m.files.ensureUnusedCapacity(gpa, m.contents.len / (@sizeOf(File) + 32)); + + var file_index: usize = 0; + var off: usize = 0; + + // This group we always want to compute the hash digests, even on a cache miss. + var input_group: Io.Group = .init; + defer input_group.cancel(io); + + // This group we would like to cancel as soon as a cache miss is discovered. + const PostResult = union(enum) { + checkFile: CheckFileResult, }; - if (!header_valid) { - return .{ .miss = .{ .file_digests_populated = 0 } }; + var post_select_buffer: [10]PostResult = undefined; + var post_select: Io.Select(PostResult) = .init(&post_select_buffer); + var post_select_remaining: usize = 0; + defer post_select.cancel(io); + + while (off + 1 < m.contents.len) { + const file_off: File.Offset = @fromBackingInt(off); + const file = try File.getFallible(file_off, m); + if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; + const path = try file.pathFallible(); + if (path.len == 0) return error.InvalidFormat; + + if (file_index < m.input_files.items.len) { + if (m.files.keys()[file_index] != file_off) return error.InvalidFormat; + + input_group.async(io, checkFile, .{m.cache, file, path}); + } else { + try m.files.put(gpa, file_off); + + post_select.async(.checkFile, checkFile, .{m.cache, file, path}); + post_select_remaining += 1; + } + + file_index += 1; + off += @sizeOf(File) + path.len + 1; } - while (line_iter.next()) |line| { - defer idx += 1; - - var iter = mem.tokenizeScalar(u8, line, ' '); - const size = iter.next() orelse return error.InvalidFormat; - const inode = iter.next() orelse return error.InvalidFormat; - const mtime_nsec_str = iter.next() orelse return error.InvalidFormat; - const digest_str = iter.next() orelse return error.InvalidFormat; - const prefix_str = iter.next() orelse return error.InvalidFormat; - const file_path = iter.rest(); - - const stat_size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; - const stat_inode = fmt.parseInt(Io.File.INode, inode, 10) catch return error.InvalidFormat; - const stat_mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; - const file_bin_digest = b: { - if (digest_str.len != hex_digest_len) return error.InvalidFormat; - var bd: BinDigest = undefined; - _ = fmt.hexToBytes(&bd, digest_str) catch return error.InvalidFormat; - break :b bd; - }; - const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat; - if (prefix >= self.cache.prefixes_len) return error.InvalidFormat; - - if (file_path.len == 0) return error.InvalidFormat; - - const cache_hash_file = f: { - const prefixed_path: PrefixedPath = .{ - .prefix = prefix, - .sub_path = file_path, // expires with file_contents - }; - if (idx < input_file_count) { - const file = &self.files.keys()[idx]; - if (!file.prefixed_path.eql(prefixed_path)) - return error.InvalidFormat; - - file.stat = .{ - .size = stat_size, - .inode = stat_inode, - .mtime = .{ .nanoseconds = stat_mtime }, - }; - file.bin_digest = file_bin_digest; - break :f file; - } - const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = self.files.pop(); - if (!gop.found_existing) { - gop.key_ptr.* = .{ - .prefixed_path = .{ - .prefix = prefix, - .sub_path = try gpa.dupe(u8, file_path), - }, - .contents = null, - .max_file_size = null, - .handle = null, - .stat = .{ - .size = stat_size, - .inode = stat_inode, - .mtime = .{ .nanoseconds = stat_mtime }, - }, - .bin_digest = file_bin_digest, - }; - } - break :f gop.key_ptr; - }; + // Final terminating zero byte to distinguish empty manifest file from + // manifest with zero files. + const file_valid = off + 1 == m.contents.len and m.contents[off] == 0; + if (!file_valid or file_index < m.input_files.items.len) { + try input_group.await(io); + return .miss; + } - const pp = cache_hash_file.prefixed_path; - const dir = self.cache.prefixes()[pp.prefix].handle; - const this_file = dir.openFile(io, pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => { - // Every digest before this one has been populated successfully. - return .{ .miss = .{ .file_digests_populated = idx } }; - }, - error.Canceled => |e| return e, - else => |e| { - self.diagnostic = .{ .file_open = .{ - .file_index = idx, - .err = e, - } }; - return error.CacheCheckFailed; + // Don't track the trailing zero byte in contents. + m.contents.len -= 1; + + var post_await_buffer: [10]PostResult = undefined; + while (post_select_remaining > 0) { + const n = try post_select.awaitMany(&post_await_buffer, 1); + post_select_remaining -= n; + for (post_await_buffer[0..n]) |u| switch (u) { + .checkFile => |result| switch (result) { + .hit => continue, + .miss => { + post_select.cancelDiscard(); + try input_group.await(io); + return .miss; + }, + .fail => |diagnostic| { + m.diagnostic = diagnostic; + return error.CacheCheckFailed; + }, }, }; - defer this_file.close(io); - - const actual_stat = this_file.stat(io) catch |err| { - self.diagnostic = .{ .file_stat = .{ - .file_index = idx, - .err = err, - } }; - return error.CacheCheckFailed; - }; - const size_match = actual_stat.size == cache_hash_file.stat.size; - const mtime_match = actual_stat.mtime.nanoseconds == cache_hash_file.stat.mtime.nanoseconds; - const inode_match = actual_stat.inode == cache_hash_file.stat.inode; - - if (!size_match or !mtime_match or !inode_match) { - cache_hash_file.stat = .{ - .size = actual_stat.size, - .mtime = actual_stat.mtime, - .inode = actual_stat.inode, - }; - - if (try self.isProblematicTimestamp(cache_hash_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - cache_hash_file.stat.mtime = .zero; - cache_hash_file.stat.inode = 0; - } - - var actual_digest: BinDigest = undefined; - hashFile(io, this_file, &actual_digest) catch |err| { - self.diagnostic = .{ .file_read = .{ - .file_index = idx, - .err = err, - } }; - return error.CacheCheckFailed; - }; - - if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { - cache_hash_file.bin_digest = actual_digest; - // keep going until we have the input file digests - any_file_changed = true; - } - } + } - if (!any_file_changed) { - self.hash.hasher.update(&cache_hash_file.bin_digest); - } + try input_group.await(io); + + for (m.files.keys()) |file_off| { + m.hash.hasher.update(&file_off.get(m).digest); } - // If the manifest was somehow missing one of our input files, or if any file hash has changed, - // then this is a cache miss. However, we have successfully populated some or all of the file - // digests. - if (any_file_changed or idx < input_file_count) { - return .{ .miss = .{ .file_digests_populated = idx } }; + return .hit; + } + + const CheckFileResult = union(enum) { + hit, + miss, + fail: Diagnostic, + }; + + /// Runs concurrently with other `checkFile`. + fn checkFile(cache: *const Cache, file: *File, file_path: [:0]const u8) Io.Cancelable!CheckFileResult { + const io = cache.io; + const dir = cache.prefixes()[file.flags.prefix].handle; + + const this_file = dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => return .miss, + error.Canceled => |e| return e, + else => |e| return .{ .fail = .{ .file_open = .{ + .file_index = file_index, + .err = e, + } }}, + }; + defer this_file.close(io); + + const actual_stat = this_file.stat(io) catch |err| return .{ .fail = .{ .file_stat = .{ + .file_index = file_index, + .err = err, + } }}; + const size_match = actual_stat.size == file.size; + const mtime_match = actual_stat.mtime.nanoseconds == file.mtime; + const inode_match = actual_stat.inode == file.inode; + + if (!size_match or !mtime_match or !inode_match) { + try file.setStat(actual_stat); + + var actual_digest: BinDigest = undefined; + hashFile(io, this_file, &actual_digest) catch |err| return .{ .fail = .{ .file_read = .{ + .file_index = file_index, + .err = err, + } }}; + + if (!mem.eql(u8, &file.digest, &actual_digest)) { + file.digest = actual_digest; + return .miss; + } } return .hit; } - /// Reset `self.hash.hasher` to the state it should be in after `hit` returns `false`. + /// Reset `man.hash.hasher` to the state it should be in after `hit` returns `CheckStatus.miss`. /// The hasher contains the original input digest, and all original input file digests (i.e. /// not including post files). - /// Assumes that `bin_digest` is populated for all files up to `input_file_count`. As such, - /// this is not necessarily safe to call within `hit`. - pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void { + /// + /// Assumes that `bin_digest` is populated for all input files. + pub fn unhit(man: *Manifest, bin_digest: BinDigest) void { // Reset the hash. - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - - // Remove files not in the initial hash. - while (self.files.count() != input_file_count) { - var file = self.files.pop().?; - file.key.deinit(self.cache.gpa); - } - - for (self.files.keys()) |file| { - self.hash.hasher.update(&file.bin_digest); + man.hash.hasher = hasher_init; + man.hash.hasher.update(&bin_digest); + man.shrinkFilesToInput(); + for (man.files.keys()) |off| { + const file = off.get(man); + man.hash.hasher.update(&file.digest); } } @@ -887,203 +925,133 @@ pub const Manifest = struct { return timestamp.nanoseconds >= man.recent_problematic_timestamp.nanoseconds; } - fn populateFileHash(self: *Manifest, ch_file: *File) !void { - const io = self.cache.io; + pub const AddFilePostOptions = struct { + handle: union(enum) { + file: ?Io.File, + dir: ?Io.Dir, + } = .{ .file = null }, + stat: ?Stat = null, + contents: ?[]const u8 = null, + metadata_only: bool = false, + }; - if (ch_file.handle) |handle| { - return populateFileHashHandle(self, ch_file, handle); - } else { - const pp = ch_file.prefixed_path; - const dir = self.cache.prefixes()[pp.prefix].handle; - const handle = try dir.openFile(io, pp.sub_path, .{}); - defer handle.close(io); - return populateFileHashHandle(self, ch_file, handle); - } - } + pub const AddFilePostError = error { + /// The same file path has been added to the cache manifest both as a + /// directory and as a normal file, making the intended caching + /// behavior ambiguous. + IsDirectoryAmbiguous, + } || Allocator.Error; - fn populateFileHashHandle(self: *Manifest, ch_file: *File, io_file: Io.File) !void { - const io = self.cache.io; - const gpa = self.cache.gpa; + /// Add a file as a dependency of process being cached, after cache miss + /// occurs. + pub fn addFilePost(m: *Manifest, path: Path, options: AddFilePostOptions) AddFilePostError!void { + assert(m.manifest_file != null); + const cache = m.cache; + const gpa = cache.gpa; + const io = cache.io; + const is_directory = options.handle == .dir; - const actual_stat = try io_file.stat(io); - ch_file.stat = .{ - .size = actual_stat.size, - .mtime = actual_stat.mtime, - .inode = actual_stat.inode, + try m.files.ensureUnusedCapacity(gpa, 1); + + const prev_contents_len = m.contents.items.len; + const new_header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))); + errdefer m.contents.shrinkRetainingCapacity(prev_contents_len); + + new_header.* = .{ + .flags = .{ + .prefix = try cache.findAppendPrefixedPath(&m.contents, path), + .is_directory = is_directory, + .metadata_only = options.metadata_only, + }, + .size = undefined, + .inode = undefined, + .mtime = undefined, + .digest = @splat(0), }; + assert(m.contents.items.len % @alignOf(File) == 0); - if (try self.isProblematicTimestamp(ch_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - ch_file.stat.mtime = .zero; - ch_file.stat.inode = 0; - } + const gop = m.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{ + .manifest = m, + }); + m.files.lockPointers(); + defer m.files.unlockPointers(); - if (ch_file.max_file_size) |max_file_size| { - if (ch_file.stat.size > max_file_size) return error.FileTooBig; + const header = if (gop.found_existing) h: { + m.contents.shrinkRetainingCapacity(prev_contents_len); + const existing_off = gop.key_ptr.*; + const header = existing_off.get(m); + if (header.flags.is_directory != is_directory) + return error.IsDirectoryAmbiguous; + if (!options.metadata_only) + header.flags.metadata_only = false; + break :h header; + } else new_header; - // Hash while reading from disk, to keep the contents in the cpu - // cache while doing hashing. - const contents = try gpa.alloc(u8, @intCast(ch_file.stat.size)); - errdefer gpa.free(contents); - - var hasher = hasher_init; - var off: usize = 0; - while (true) { - const bytes_read = try io_file.readPositional(io, &.{contents[off..]}, off); - if (bytes_read == 0) break; - hasher.update(contents[off..][0..bytes_read]); - off += bytes_read; + if (options.stat) |stat| { + try header.setStat(m, stat); + if (header.metadata_only) { + return; + } else if (options.contents) |contents| { + var hasher = hasher_init; + hasher.update(contents); + hasher.final(&header.digest); + return; } - hasher.final(&ch_file.bin_digest); - - ch_file.contents = contents; - } else { - try hashFile(io, io_file, &ch_file.bin_digest); } - self.hash.hasher.update(&ch_file.bin_digest); - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. This is useful for processes that don't know all the files that - /// are depended on ahead of time. For example, a source file that can import other files - /// will need to be recompiled if the imported file is changed. - pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 { - assert(self.manifest_file != null); - - const gpa = self.cache.gpa; - const prefixed_path = try self.cache.findPrefix(file_path); - errdefer gpa.free(prefixed_path.sub_path); - - const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = self.files.pop(); - - if (gop.found_existing) { - gpa.free(prefixed_path.sub_path); - return gop.key_ptr.contents.?; + const need_stat = options.stat == null; + + switch (options.handle) { + .dir => |opt_handle| if (opt_handle) |handle| { + try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); + } else { + const dir = cache.prefixes()[header.flags.prefix].handle; + const handle = try dir.openDir(io, header.path(), .{ .access_sub_paths = false, .iterate = true, }); + defer handle.close(io); + try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); + }, + + .file => |opt_handle| if (opt_handle) |handle| { + try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only); + } else { + const dir = cache.prefixes()[header.flags.prefix].handle; + const handle = try dir.openFile(io, header.path(), .{ .mode = .read_only }); + defer handle.close(io); + try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only); + }, } - - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - .handle = null, - }; - - self.files.lockPointers(); - defer self.files.unlockPointers(); - - try self.populateFileHash(gop.key_ptr); - return gop.key_ptr.contents.?; - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. - /// - /// This is useful for processes that don't know the all the files that are - /// depended on ahead of time. For example, a source file that can import - /// other files will need to be recompiled if the imported file is changed. - pub fn addFilePost(man: *Manifest, file_path: []const u8) !void { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; - const prefixed_path = try man.cache.findPrefix(file_path); - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try addPrefixedPathPost(man, prefixed_path); - } - - pub fn addPathPost(man: *Manifest, path: Path) !void { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; - const prefixed_path: PrefixedPath = try man.cache.findPrefixPath(path); - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try addPrefixedPathPost(man, prefixed_path); } - /// Low level function. `prefixed_path` references cloned memory. Returns - /// whether or not `prefixed_path.sub_path` should be kept. - pub fn addPrefixedPathPost(man: *Manifest, prefixed_path: PrefixedPath) !bool { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; + fn populateFile(m: *Manifest, file: *File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void { + const io = m.cache.io; - const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = man.files.pop(); - - if (gop.found_existing) return false; - - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = null, - .handle = null, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - }; - - man.files.lockPointers(); - defer man.files.unlockPointers(); - - try man.populateFileHash(gop.key_ptr); - return true; - } - - /// Like `addFilePost` but when the file contents have already been loaded from disk. - pub fn addFilePostContents( - man: *Manifest, - file_path: []const u8, - bytes: []const u8, - stat: File.Stat, - ) !void { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; - const prefixed_path = try man.cache.findPrefix(file_path); - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try addPrefixedPathPostContents(man, prefixed_path, bytes, stat); - } - - /// Low level function. `prefixed_path` references cloned memory. Returns - /// whether or not `prefixed_path.sub_path` should be kept. - pub fn addPrefixedPathPostContents( - man: *Manifest, - prefixed_path: PrefixedPath, - bytes: []const u8, - stat: File.Stat, - ) !bool { - const gpa = man.cache.gpa; - const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = man.files.pop(); - - if (gop.found_existing) return false; - - const new_file = gop.key_ptr; - - new_file.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = null, - .handle = null, - .stat = stat, - .bin_digest = undefined, - .contents = null, - }; - - if (try man.isProblematicTimestamp(new_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - new_file.stat.mtime = .zero; - new_file.stat.inode = 0; + if (need_stat) { + const stat = try handle.stat(io); + try file.setStat(m, stat); } - - { + if (metadata_only) return; + if (contents) |bytes| { var hasher = hasher_init; hasher.update(bytes); - hasher.final(&new_file.bin_digest); + hasher.final(&file.digest); + } else { + try hashFile(io, handle, &file.digest); } + } - man.hash.hasher.update(&new_file.bin_digest); - return true; + fn populateDirectory(m: *Manifest, file: *File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void { + _ = m; + _ = file; + _ = need_stat; + _ = handle; + _ = contents; + _ = metadata_only; + @panic("TODO"); + } + + pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { + assert(self.manifest_file == null); + return self.addDepFileMaybePost(dir, dep_file_sub_path); } pub fn addDepFilePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { @@ -1094,7 +1062,7 @@ pub const Manifest = struct { fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { const gpa = self.cache.gpa; const io = self.cache.io; - const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(manifest_file_size_max)); + const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(file_size_max)); defer gpa.free(dep_file_contents); var error_buf: std.ArrayList(u8) = .empty; @@ -1151,39 +1119,24 @@ pub const Manifest = struct { /// If `want_shared_lock` is true, this function automatically downgrades the /// lock from exclusive to shared. - pub fn writeManifest(self: *Manifest) !void { - assert(self.have_exclusive_lock); - const io = self.cache.io; - const manifest_file = self.manifest_file.?; - if (self.manifest_dirty) { - self.manifest_dirty = false; + pub fn writeManifest(m: *Manifest) !void { + assert(m.have_exclusive_lock); + const io = m.cache.io; + const manifest_file = m.manifest_file.?; + if (m.manifest_dirty) { - var buffer: [4000]u8 = undefined; - var fw = manifest_file.writer(io, &buffer); - writeDirtyManifestToStream(self, &fw) catch |err| switch (err) { - error.WriteFailed => return fw.err.?, - else => |e| return e, - }; - } + m.contents.appendAssumeCapacity(0); + defer _ = m.contents.pop().?; + + try manifest_file.setLength(io, m.contents.items.len); + try manifest_file.writePositionalAll(io, m.contents.items, 0); - if (self.want_shared_lock) { - try self.downgradeToSharedLock(); + m.manifest_dirty = false; } - } - fn writeDirtyManifestToStream(self: *Manifest, fw: *Io.File.Writer) !void { - try fw.interface.writeAll(manifest_header ++ "\n"); - for (self.files.keys()) |file| { - try fw.interface.print("{d} {d} {d} {x} {d} {s}\n", .{ - file.stat.size, - file.stat.inode, - file.stat.mtime, - &file.bin_digest, - file.prefixed_path.prefix, - file.prefixed_path.sub_path, - }); + if (m.want_shared_lock) { + try m.downgradeToSharedLock(); } - try fw.end(); } fn downgradeToSharedLock(self: *Manifest) !void { @@ -1275,33 +1228,40 @@ pub const Manifest = struct { pub fn populateOtherManifest(man: *Manifest, other: *Manifest, prefix_map: [5]u8) Allocator.Error!void { const gpa = other.cache.gpa; + assert(other.manifest_file != null); assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len); assert(man.cache.prefixes_len == 5); - for (man.files.keys()) |file| { - const prefixed_path: PrefixedPath = .{ - .prefix = prefix_map[file.prefixed_path.prefix], - .sub_path = try gpa.dupe(u8, file.prefixed_path.sub_path), - }; - errdefer gpa.free(prefixed_path.sub_path); - - const gop = try other.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = other.files.pop(); + + const orig_files_len = other.files.count(); + const orig_contents_len = other.contents.items.len; + errdefer { + other.files.shrinkRetainingCapacity(orig_files_len); + other.contents.shrinkRetainingCapacity(orig_contents_len); + } + + for (man.files.keys(), 0..) |off, file_index| { + try other.files.ensureUnusedCapacity(gpa, 1); + + const next_off = if (file_index < man.files.count()) + @backingInt(man.files.keys()[file_index + 1]) + else + man.contents.items.len; + + const copy_bytes = man.contents.items[@backingInt(off)..next_off]; + const prev_contents_len = other.contents.items.len; + try other.contents.appendSlice(gpa, copy_bytes); + + const gop = other.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{ + .manifest = other, + }); if (gop.found_existing) { - gpa.free(prefixed_path.sub_path); + other.contents.shrinkRetainingCapacity(prev_contents_len); continue; } - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = file.max_file_size, - .handle = file.handle, - .stat = file.stat, - .bin_digest = file.bin_digest, - .contents = null, - }; - - other.hash.hasher.update(&gop.key_ptr.bin_digest); + const other_file = File.get(@fromBackingInt(prev_contents_len)); + other_file.prefix = prefix_map[other_file.prefix]; } } }; diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index e216a272e95bd69e462d494123bfcf3c7083669c..542d7863abcb1881581000beb08129d0b6909a95 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1873,12 +1873,11 @@ pub const PathDep = extern struct { pkg: Package.OptionalIndex, pub const Flags = packed struct(u32) { - mode: Mode, + is_directory: bool, + metadata_only: bool, base: LazyPath.Relative.Base, _: u16 = 0, }; - - pub const Mode = enum(u8) { directory, contents, metadata }; }; pub const InstallDestDir = enum(u32) { diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index d63b4778c1d5c075237f3fc9d62112fc9916974a..78cf664f71afd94efe6e9044d93d93b1dc1487d8 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -348,11 +348,8 @@ pub fn update( } } fn workerUpdateBuiltinFile(comp: *Compilation, file: *Zcu.File) void { - Builtin.updateFileOnDisk(file, comp) catch |err| comp.lockAndSetMiscFailure( - .write_builtin_zig, - "unable to write '{f}': {s}", - .{ file.path.fmt(comp), @errorName(err) }, - ); + Builtin.updateFileOnDisk(file, comp) catch |err| + comp.lockAndSetMiscFailure(.write_builtin_zig, "unable to write {qf}: {t}", .{ file.path.fmt(comp), err }); } fn workerUpdateFile( comp: *Compilation, @@ -371,7 +368,9 @@ fn workerUpdateFile( const active = comp.zcu.?.activate(tid); defer active.deactivate(); active.pt.updateFile(file_index, file) catch |err| { - active.pt.reportRetryableFileError(file_index, "unable to load '{s}': {s}", .{ std.fs.path.basename(file.path.sub_path), @errorName(err) }) catch |oom| switch (oom) { + active.pt.reportRetryableFileError(file_index, "unable to load {q}: {t}", .{ + std.fs.path.basename(file.path.sub_path), err, + }) catch |oom| switch (oom) { error.OutOfMemory => { comp.mutex.lockUncancelable(io); defer comp.mutex.unlock(io); diff --git a/test/src/Cases.zig b/test/src/Cases.zig index d04bc01972bbb8b0fe09885471971fc5b5f1e8a0..e9ce39718a8340b17e2c4b07afd7499f0861da74 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -319,7 +319,7 @@ pub fn addCompile( pub fn addFromDir(ctx: *Cases, dir: Io.Dir, path_from_root: []const u8, b: *std.Build) void { var current_file: []const u8 = "none"; ctx.addFromDirInner(dir, path_from_root, ¤t_file, b) catch |err| { - std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}\n", .{ + std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}", .{ current_file, err, }); }; -- 2.54.0 From 9ad1a6be17b50c6f54e121ef9e5104f6f3272f8b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 24 Aug 2026 20:08:32 -0700 Subject: [PATCH 02/49] std.Build.Cache: implement is_directory and metadata_only --- lib/std/Build/Cache.zig | 412 ++++++++++++++++++++++++++++++---------- 1 file changed, 316 insertions(+), 96 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index a644aabe7d4a9be06c1e64d0de9ce8f8273c8467..a3069f48dc21adaa1f985c7fcb1d7b5c591e3fae 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -49,7 +49,6 @@ pub fn obtain(cache: *Cache) Manifest { .hash = cache.hash, .manifest_file = null, .manifest_dirty = false, - .hex_digest = undefined, }; } @@ -251,7 +250,7 @@ pub const HashHelper = struct { pub fn binToHex(bin_digest: BinDigest) HexDigest { var out_digest: HexDigest = undefined; - var w: std.Io.Writer = .fixed(&out_digest); + var w: Io.Writer = .fixed(&out_digest); w.printHex(&bin_digest, .lower) catch unreachable; return out_digest; } @@ -278,7 +277,6 @@ pub const Manifest = struct { cache: *Cache, /// Current state for incremental hashing. hash: HashHelper, - hex_digest: HexDigest, /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. manifest_file: ?Io.File, manifest_dirty: bool, @@ -309,9 +307,34 @@ pub const Manifest = struct { /// All contents from all `input_files` whose contents were requested, /// concatenated. Total byte size will be less than `max_input_content_len` /// otherwise an error is returned. + /// + /// Data is invalidated when `addFilePost` is called. all_input_content: std.ArrayList(u8) = .empty, max_input_content_len: usize = std.math.maxInt(u32), + /// State that exists only during check. + pub const Check = struct { + /// Protects `Manifest.diagnostic` from data races. + diagnostic_lock: bool = false, + status: Status = .hit, + + pub const Status = enum { hit, miss }; + + pub const Error = error{ + /// Unable to check the cache for a reason that has been recorded into + /// the `diagnostic` field. + CacheCheckFailed, + /// A cache manifest file exists however it could not be parsed. + InvalidFormat, + } || Allocator.Error || Io.Cancelable; + + fn fail(c: *Check, m: *Manifest, diagnostic: Diagnostic) void { + if (!@atomicRmw(bool, &c.diagnostic_lock, .Xchg, true, .unordered)) { + m.diagnostic = diagnostic; + } + } + }; + pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); /// Source files whose prefix and relative path are included when computing @@ -328,7 +351,7 @@ pub const Manifest = struct { have_stat: bool, /// Determines whether `File.digest` is populated. have_digest: bool, - contents: enum (usize) { + contents: enum(usize) { requested = std.math.maxInt(u32) - 1, not_requested = std.math.maxInt(u32), /// Byte offset index into `Manifest.all_input_content`. @@ -356,12 +379,29 @@ pub const Manifest = struct { /// Terminated by zero byte, then followed by padding until 8-byte aligned. path_start: [0]u8, - pub const Flags = packed struct (u8) { + pub const Flags = packed struct(u8) { is_directory: bool, metadata_only: bool, prefix: u6, }; + /// Prefixes path names in encoded directory contents. Starts numbering + /// at `1` so that null byte can be used unambiguously as entry + /// separator. + pub const Kind = enum(u8) { + file = 1, + directory = 2, + other = 3, + + pub fn fromStat(kind: Io.File.Kind) @This() { + return switch (kind) { + .file => .file, + .directory => .directory, + else => .other, + }; + } + }; + /// Byte index within `Manifest.contents` where the entry starts. pub const Offset = enum(u32) { _, @@ -392,7 +432,6 @@ pub const Manifest = struct { } }; - pub fn path(file: *const File) [:0]const u8 { return pathFallible(file) catch unreachable; } @@ -408,7 +447,7 @@ pub const Manifest = struct { const path_len = mem.findScalar(u8, path_ptr, 0).?; comptime assert(@offsetOf(File, "path_start") - @offsetOf(File, "flags") == 1); // Includes flags and sentinel. - const hash_string = (path_ptr - 1)[0..path_len + 2]; + const hash_string = (path_ptr - 1)[0 .. path_len + 2]; hasher.update(hash_string); } @@ -424,9 +463,20 @@ pub const Manifest = struct { } } + /// Returns true if the stat was changed. Updates the `file` with the new stat value. + fn setStatChanged(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!bool { + if (stat.size == file.size and + stat.mtime.nanoseconds == file.mtime and + stat.inode == file.inode) + { + return false; + } else { + setStat(file, m, stat); + return true; + } + } }; - pub const Diagnostic = union(enum) { none, manifest_create: Io.File.OpenError, @@ -438,7 +488,7 @@ pub const Manifest = struct { file_hash: FileOp, pub const FileOp = struct { - file_index: usize, + file_offset: File.Offset, err: anyerror, }; }; @@ -450,16 +500,24 @@ pub const Manifest = struct { }; pub const AddInputFileOptions = struct { + /// If `is_directory` is true, this handle must be opened with + /// iteration capability. handle: ?Io.File = null, stat: ?Stat = null, request_handle: bool = false, + /// Can request file or directory contents depending on `is_directory`. request_contents: bool = false, + /// Contents of a directory are considered to be the sorted list of + /// file names of direct entries, separated by null byte. Each file name + /// is prefixed by `Io.File.Kind` byte, +1 so that the zero tag is + /// not aliased by the entry separator. is_directory: bool = false, + /// Content hashing skipped; any difference in metadata implies cache + /// miss. metadata_only: bool = false, - }; - pub const AddInputFileError = error { + pub const AddInputFileError = error{ /// The same file path has been added to the cache manifest both as a /// directory and as a normal file, making the intended caching /// behavior ambiguous. @@ -544,16 +602,6 @@ pub const Manifest = struct { _ = try addInputFile(m, opt_path orelse return, options); } - pub const CheckError = error{ - /// Unable to check the cache for a reason that has been recorded into - /// the `diagnostic` field. - CacheCheckFailed, - /// A cache manifest file exists however it could not be parsed. - InvalidFormat, - } || Allocator.Error || Io.Cancelable; - - pub const CheckStatus = enum { hit, miss }; - /// Check the cache to see if the input exists in it. /// A hex encoding of its hash is available by calling `final`. /// @@ -566,13 +614,13 @@ pub const Manifest = struct { /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) CheckError!CheckStatus { + pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) Check.Error!Check.Status { const node = parent_progress_node.start("Reusing Cache Artifacts", 0); defer node.end(); return checkProgressless(man); } - pub fn checkProgressless(man: *Manifest) CheckError!CheckStatus { + pub fn checkProgressless(man: *Manifest) Check.Error!Check.Status { assert(man.manifest_file == null); for (man.files.keys()[0..man.input_files.items.len]) |file_off| { @@ -583,9 +631,8 @@ pub const Manifest = struct { var bin_digest: BinDigest = undefined; man.hash.hasher.final(&bin_digest); - man.hex_digest = binToHex(bin_digest); - - const manifest_file_path = &man.hex_digest; + const hex_digest = binToHex(bin_digest); + const manifest_file_path = &hex_digest; const io = man.cache.io; // We'll try to open the cache with an exclusive lock, but if that would block @@ -724,7 +771,7 @@ pub const Manifest = struct { /// Assumes that `self.hash.hasher` has been updated only with the original digest and that /// `self.files` contains only the original input files. - fn checkLocked(m: *Manifest) CheckError!CheckStatus { + fn checkLocked(m: *Manifest) Check.Error!Check.Status { const gpa = m.cache.gpa; const io = m.cache.io; @@ -750,11 +797,12 @@ pub const Manifest = struct { // This group we would like to cancel as soon as a cache miss is discovered. const PostResult = union(enum) { - checkFile: CheckFileResult, + checkFile: Check.Status, }; var post_select_buffer: [10]PostResult = undefined; var post_select: Io.Select(PostResult) = .init(&post_select_buffer); var post_select_remaining: usize = 0; + var c: Check = .{}; defer post_select.cancel(io); while (off + 1 < m.contents.len) { @@ -767,11 +815,11 @@ pub const Manifest = struct { if (file_index < m.input_files.items.len) { if (m.files.keys()[file_index] != file_off) return error.InvalidFormat; - input_group.async(io, checkFile, .{m.cache, file, path}); + input_group.async(io, checkInputFile, .{ m, &c, file_off, path }); } else { try m.files.put(gpa, file_off); - post_select.async(.checkFile, checkFile, .{m.cache, file, path}); + post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path }); post_select_remaining += 1; } @@ -794,6 +842,18 @@ pub const Manifest = struct { while (post_select_remaining > 0) { const n = try post_select.awaitMany(&post_await_buffer, 1); post_select_remaining -= n; + + // Detect if input group already had a miss. In this case we still wait + // for those digests to be updated, but cancel the non input group. + switch (@atomicLoad(Check.Status, &c.status, .unordered)) { + .miss => { + post_select.cancelDiscard(); + try input_group.await(io); + return .miss; + }, + .hit => continue, + } + for (post_await_buffer[0..n]) |u| switch (u) { .checkFile => |result| switch (result) { .hit => continue, @@ -811,6 +871,7 @@ pub const Manifest = struct { } try input_group.await(io); + if (c.status == .miss) return .miss; for (m.files.keys()) |file_off| { m.hash.hasher.update(&file_off.get(m).digest); @@ -819,54 +880,121 @@ pub const Manifest = struct { return .hit; } - const CheckFileResult = union(enum) { - hit, - miss, - fail: Diagnostic, - }; + fn checkInputFile(m: *Manifest, c: *Check, file_off: File.Offset, file_path: [:0]const u8) Io.Cancelable!void { + // TODO use already open handle + // TODO use already provided stat + // TODO implement request_handle + // TODO implement request_contents + switch (try checkFile(m, c, file_off, file_path)) { + .hit => return, + .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered), + } + } /// Runs concurrently with other `checkFile`. - fn checkFile(cache: *const Cache, file: *File, file_path: [:0]const u8) Io.Cancelable!CheckFileResult { + fn checkFile( + m: *Manifest, + c: *Check, + file_off: File.Offset, + file_path: [:0]const u8, + ) Io.Cancelable!Check.Status { + const file = file_off.get(m); + const cache = m.cache; + const gpa = cache.gpa; const io = cache.io; - const dir = cache.prefixes()[file.flags.prefix].handle; + const parent_dir = cache.prefixes()[file.flags.prefix].handle; - const this_file = dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => return .miss, - error.Canceled => |e| return e, - else => |e| return .{ .fail = .{ .file_open = .{ - .file_index = file_index, - .err = e, - } }}, - }; - defer this_file.close(io); + if (file.flags.metadata_only) { + const actual_stat = parent_dir.statFile() catch |err| switch (err) { + error.FileNotFound => return .miss, + error.Canceled => |e| return e, + else => |e| return c.fail(m, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; - const actual_stat = this_file.stat(io) catch |err| return .{ .fail = .{ .file_stat = .{ - .file_index = file_index, - .err = err, - } }}; - const size_match = actual_stat.size == file.size; - const mtime_match = actual_stat.mtime.nanoseconds == file.mtime; - const inode_match = actual_stat.inode == file.inode; + const actual_is_directory = actual_stat.kind == .directory; + if (actual_is_directory != file.flags.is_directory) return .miss; - if (!size_match or !mtime_match or !inode_match) { - try file.setStat(actual_stat); + if (try file.setStatChanged(m, actual_stat)) return .miss; - var actual_digest: BinDigest = undefined; - hashFile(io, this_file, &actual_digest) catch |err| return .{ .fail = .{ .file_read = .{ - .file_index = file_index, - .err = err, - } }}; + return .hit; + } - if (!mem.eql(u8, &file.digest, &actual_digest)) { - file.digest = actual_digest; - return .miss; + if (file.flags.is_directory) { + const opened_dir = parent_dir.openDir(io, file_path, .{ + .iterate = true, + .access_sub_paths = false, + }) catch |err| switch (err) { + error.FileNotFound, error.NotDir => return .miss, + error.Canceled => |e| return e, + else => |e| return c.fail(m, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer opened_dir.close(io); + + const actual_stat = opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return c.fail(m, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + if (try file.setStatChanged(m, actual_stat)) { + const prev_digest: BinDigest = file.digest; + var contents: std.ArrayList(u8) = .empty; + defer contents.deinit(gpa); + hashDir(gpa, io, opened_dir, &file.digest, &contents) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return c.fail(m, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + + if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss; } + return .hit; + } + + const opened_file = parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound, error.IsDir => return .miss, + error.Canceled => |e| return e, + else => |e| return c.fail(m, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer opened_file.close(io); + + const actual_stat = opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return c.fail(m, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + + if (try file.setStatChanged(m, actual_stat)) { + const prev_digest: BinDigest = file.digest; + hashFile(io, opened_file, &file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return c.fail(m, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + + if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss; } return .hit; } - /// Reset `man.hash.hasher` to the state it should be in after `hit` returns `CheckStatus.miss`. + /// Reset `man.hash.hasher` to the state it should be in after `hit` returns `Check.Status.miss`. /// The hasher contains the original input digest, and all original input file digests (i.e. /// not including post files). /// @@ -931,16 +1059,18 @@ pub const Manifest = struct { dir: ?Io.Dir, } = .{ .file = null }, stat: ?Stat = null, + /// If it is a directory, there is a special encoding required for contents, which + /// is null-separated sorted entries, each one prefixed with `File.Kind`. contents: ?[]const u8 = null, metadata_only: bool = false, }; - pub const AddFilePostError = error { + pub const AddFilePostError = error{ /// The same file path has been added to the cache manifest both as a /// directory and as a normal file, making the intended caching /// behavior ambiguous. IsDirectoryAmbiguous, - } || Allocator.Error; + } || Io.Cancelable || Allocator.Error; /// Add a file as a dependency of process being cached, after cache miss /// occurs. @@ -1006,7 +1136,10 @@ pub const Manifest = struct { try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); } else { const dir = cache.prefixes()[header.flags.prefix].handle; - const handle = try dir.openDir(io, header.path(), .{ .access_sub_paths = false, .iterate = true, }); + const handle = try dir.openDir(io, header.path(), .{ + .access_sub_paths = false, + .iterate = true, + }); defer handle.close(io); try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); }, @@ -1022,7 +1155,14 @@ pub const Manifest = struct { } } - fn populateFile(m: *Manifest, file: *File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void { + fn populateFile( + m: *Manifest, + file: *File, + need_stat: bool, + handle: Io.File, + contents: ?[]const u8, + metadata_only: bool, + ) !void { const io = m.cache.io; if (need_stat) { @@ -1039,14 +1179,32 @@ pub const Manifest = struct { } } - fn populateDirectory(m: *Manifest, file: *File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void { - _ = m; - _ = file; - _ = need_stat; - _ = handle; - _ = contents; - _ = metadata_only; - @panic("TODO"); + fn populateDirectory( + m: *Manifest, + file: *File, + need_stat: bool, + handle: Io.Dir, + contents: ?[]const u8, + metadata_only: bool, + ) !void { + const cache = m.cache; + const io = cache.io; + const gpa = cache.gpa; + + if (need_stat) { + const stat = try handle.stat(io); + try file.setStat(m, stat); + } + if (metadata_only) return; + if (contents) |bytes| { + var hasher = hasher_init; + hasher.update(bytes); + hasher.final(&file.digest); + } else { + const prev_contents_len = m.all_input_content.items.len; + defer m.all_input_content.shrinkRetainingCapacity(prev_contents_len); + try hashDir(gpa, io, handle, &file.digest, &m.all_input_content); + } } pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { @@ -1062,7 +1220,7 @@ pub const Manifest = struct { fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { const gpa = self.cache.gpa; const io = self.cache.io; - const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(file_size_max)); + const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .unlimited); defer gpa.free(dep_file_contents); var error_buf: std.ArrayList(u8) = .empty; @@ -1124,7 +1282,6 @@ pub const Manifest = struct { const io = m.cache.io; const manifest_file = m.manifest_file.?; if (m.manifest_dirty) { - m.contents.appendAssumeCapacity(0); defer _ = m.contents.pop().?; @@ -1264,22 +1421,85 @@ pub const Manifest = struct { other_file.prefix = prefix_map[other_file.prefix]; } } + + fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void { + var buffer: [2048]u8 = undefined; + var hasher = hasher_init; + var offset: u64 = 0; + while (true) { + const n = try file.readPositional(io, &.{&buffer}, offset); + if (n == 0) break; + hasher.update(buffer[0..n]); + offset += n; + } + hasher.final(bin_digest); + } + + const HashDirError = Io.Dir.Reader.Error || Allocator.Error; + + /// Appends the sorted, encoded directory entries to `contents`. + fn hashDir( + gpa: Allocator, + io: Io, + dir: Io.Dir, + bin_digest: *[Hasher.mac_length]u8, + contents: *std.ArrayList(u8), + ) HashDirError!void { + var buffer: [@max(2048, Io.Dir.Reader.min_buffer_len)]u8 align(@alignOf(usize)) = undefined; + var reader: Io.Dir.Reader = .init(dir, &buffer); + var entry_buffer: [16]Io.Dir.Entry = undefined; + + const contents_start = contents.items.len; + errdefer contents.shrinkRetainingCapacity(contents_start); + + // Each index points into `contents`. + var entries_list: std.ArrayList(u32) = .empty; + defer entries_list.deinit(gpa); + + while (true) { + const entries = entry_buffer[0..try reader.read(io, &entry_buffer)]; + for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| { + off.* = contents.items.len; + // As an optimization, make the reservation also count the duplication + // of the contents buffer that will be required after sorting. + try contents.ensureUnusedCapacity(gpa, (contents.items.len + entry.name.len + 2 - contents_start) * 2); + contents.appendAssumeCapacity(@backingInt(Manifest.File.Kind.fromStat(entry.kind))); + contents.appendSliceAssumeCapacity(entry.name); + contents.appendAssumeCapacity(0); + } + } + + const Sort = struct { + contents: []const u8, + pub fn lessThan(this: @This(), lhs: u32, rhs: u32) bool { + return mem.lessThanZ(u8, this.contents[lhs + 1 ..], this.contents[rhs + 1 ..]); // +1 for kind byte + } + }; + mem.sortUnstable(u32, entries_list.items, @as(Sort, .{ .contents = contents.items }), Sort.lessThan); + + // Duplicate the contents such that we may refer to it while creating a + // sorted copy in the original position (at contents_start). We will then + // offset all the entries_list offsets by contents len when reading from the unsorted copy. + const contents_len = contents.items.len - contents_start; + @memcpy(contents.addManyAsSliceAssumeCapacity(contents_len), contents.items[contents_start..][0..contents_len]); + + var new_offset: usize = contents_start; + for (entries_list.items) |wrong_offset| { + const offset = wrong_offset + contents_len; + // Includes the kind prefix which we also want to copy. + const entry: [*:0]const u8 = @ptrCast(contents.items[offset..]); + new_offset += mem.copySentinel(u8, 0, contents.items[new_offset..], entry); + } + assert(new_offset == contents_start + contents_len); + contents.shrinkRetainingCapacity(contents_start + contents_len); + + var hasher = hasher_init; + hasher.update(contents.items[contents_start..][0..contents_len]); + hasher.final(bin_digest); + } }; -fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void { - var buffer: [2048]u8 = undefined; - var hasher = hasher_init; - var offset: u64 = 0; - while (true) { - const n = try file.readPositional(io, &.{&buffer}, offset); - if (n == 0) break; - hasher.update(buffer[0..n]); - offset += n; - } - hasher.final(bin_digest); -} - -// Create/Write a file, close it, then grab its stat.mtime timestamp. +/// Create/Write a file, close it, then grab its stat.mtime timestamp. fn testGetCurrentFileTimestamp(io: Io, dir: Io.Dir) !Io.Timestamp { const test_out_file = "test-filetimestamp.tmp"; @@ -1312,7 +1532,7 @@ test "cache file and then recall it" { // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } var digest1: HexDigest = undefined; @@ -1382,7 +1602,7 @@ test "check that changing a file makes cache fail" { // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } var digest1: HexDigest = undefined; @@ -1508,7 +1728,7 @@ test "Manifest with files added after initial hash work" { // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } var digest1: HexDigest = undefined; @@ -1560,7 +1780,7 @@ test "Manifest with files added after initial hash work" { // Wait for file timestamps to tick const initial_time2 = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time2.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } { -- 2.54.0 From 872e8c8f7daeed748e330a4e6105542b2526afe7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 15:02:32 -0700 Subject: [PATCH 03/49] std.Build.Cache: fix some compilation errors from rewrite --- lib/compiler/Maker.zig | 20 +- lib/compiler/Maker/Step.zig | 36 ++- lib/compiler/Maker/Step/ObjCopy.zig | 2 +- lib/compiler/Maker/Step/Options.zig | 2 +- lib/compiler/Maker/Step/Run.zig | 10 +- lib/compiler/Maker/Step/WriteFile.zig | 4 +- lib/std/Build/Cache.zig | 330 +++++++++++++++----------- lib/std/Build/Configuration.zig | 2 +- lib/std/Io/Reader.zig | 1 + src/Compilation.zig | 28 +-- src/libs/freebsd.zig | 8 +- src/libs/glibc.zig | 8 +- src/libs/mingw.zig | 4 +- src/libs/netbsd.zig | 8 +- src/libs/openbsd.zig | 8 +- src/link/MachO.zig | 2 +- src/main.zig | 2 +- 17 files changed, 269 insertions(+), 206 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 6038cad96d84e824b062749abcc63d44625a0451..d5cde77ae631f5cad9a297429a5b045a4046e610 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -702,8 +702,8 @@ pub fn main(init: process.Init.Minimal) !void { configure: while (true) { // Set of files that, if modified, imply that recompiling and rerunning // configurer is needed. - var configure_source_files: Cache.Manifest.Files = .empty; - defer Cache.Manifest.freeFiles(gpa, &configure_source_files); + var configure_source_files: Cache.Manifest.SelfContainedFiles = .empty; + defer configure_source_files.deinit(gpa); // If this fails, we can still start the server and wait for user // to request a rebuild. If it returns error.FailedButCacheIntact @@ -1043,7 +1043,7 @@ const ConfigureOptions = struct { fetch_only: bool, print_configuration: PrintConfiguration, forks: []Fork, - src_files: *Cache.Manifest.Files, + src_files: *Cache.Manifest.SelfContainedFiles, }; fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { @@ -1398,7 +1398,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { defer compile_prog_node.end(); if (config_man) |man| { - if (try man.hit(compile_prog_node)) { + if (.hit == try man.check(compile_prog_node)) { const digest = man.final(); const path: Path = .{ .root_dir = graph.local_cache_root, @@ -1497,11 +1497,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { } if (config_man) |man| for (configuration.path_deps) |path_dep| { - switch (path_dep.flags.mode) { - .directory => {}, // TODO - .contents => try man.addPathPost(try confPathDepToCachePath(arena, graph, &configuration, path_dep)), - .metadata => {}, // TODO - } + const path = try confPathDepToCachePath(arena, graph, &configuration, path_dep); + try man.addPathPost(path, .{ + .handle = if (path_dep.flags.is_directory) .{ .dir = null } else .{ .file = null }, + .metadata_only = path_dep.flags.metadata_only, + }); }; // If it is poisoned, there is no point in moving it to cached @@ -2220,7 +2220,7 @@ fn resolveTopLevelSteps(maker: *Maker, step_names: []const []const u8) ![]const fn prepare( maker: *Maker, step_indices: []const Configuration.Step.Index, - configure_source_files: *const Cache.Manifest.Files, + configure_source_files: *const Cache.Manifest.SelfContainedFiles, ) !void { const gpa = maker.gpa; const graph = maker.graph; diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 0fd82226cce06989c0acea46f4f22e4dcd6d6e58..690afd2e0fb403fad83d0c2f9b0a747ffed9a088 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -726,8 +726,9 @@ pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.T /// Prefer `cacheHitWatched` unless you already added watch inputs /// separately from using the cache system. pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { - s.result_cached = man.hit(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err); - return s.result_cached; + const hit = .hit == (man.check(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err)); + s.result_cached = hit; + return hit; } /// Clears previous watch inputs, if any, and then populates watch inputs from @@ -735,32 +736,29 @@ pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_n /// /// Must be accompanied with `writeManifestAndWatch`. pub fn cacheHitWatched(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { - const is_hit = man.hit(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err); - s.result_cached = is_hit; + const hit = .hit == (man.check(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err)); + s.result_cached = hit; // The above call to hit() populates the manifest with files, so in case of // a hit, we need to populate watch inputs. - if (is_hit) try setWatchInputsFromManifest(s, maker, man); - return is_hit; + if (hit) try setWatchInputsFromManifest(s, maker, man); + return hit; } fn failWithCacheError( s: *Step, maker: *Maker, man: *const Cache.Manifest, - err: Cache.Manifest.HitError, + err: Cache.Manifest.Check.Error, ) error{ OutOfMemory, Canceled, MakeFailed } { switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail(maker, "failed checking cache: {t} {t}", .{ - man.diagnostic, e, - }), + .manifest_create, .manifest_read, .manifest_lock => |e| { + return s.fail(maker, "failed checking cache: {t} {t}", .{ man.diagnostic, e }); + }, .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - return s.fail(maker, "failed checking cache: {s}{c}{s} {t} {t}", .{ - prefix, Dir.path.sep, pp.sub_path, man.diagnostic, op.err, - }); + const path = op.path(man); + return s.fail(maker, "failed checking cache: {f} {t} {t}", .{ path, man.diagnostic, op.err }); }, }, error.OutOfMemory, error.Canceled => |e| return e, @@ -795,17 +793,17 @@ pub fn setWatchInputsFromManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) pub fn setWatchInputsFromManifestFiles( s: *Step, maker: *Maker, - files: *const Cache.Manifest.Files, + scf: *const Cache.Manifest.SelfContainedFiles, prefixes: []const Cache.Directory, ) !void { const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena clearWatchInputs(s, maker); - for (files.keys()) |file| { + for (scf.files.keys()) |file_offset| { // The file path data is freed when the cache manifest is cleaned up at the end of `make`. - const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); + const sub_path = try arena.dupe(u8, scf.path(file_offset)); try addWatchInputFromPath(s, maker, .{ - .root_dir = prefixes[file.prefixed_path.prefix], + .root_dir = prefixes[file_offset.get(scf.contents.items).flags.prefix], .sub_path = Dir.path.dirname(sub_path) orelse "", }, Dir.path.basename(sub_path)); } diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 49424d4b35daea41bb6a485e1fb2a37acfe5e468..7924b21c683866a3536bf72e2bf9b04f3850f3c5 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -34,7 +34,7 @@ pub fn make( defer man.deinit(); const input_path = try maker.resolveLazyPath(arena, input_lazy_path, step_index); - _ = try man.addFilePath(input_path, null); + _ = try man.addInputPath(input_path, .{}); man.hash.addOptionalBytes(only_section); man.hash.addOptionalBytes(opt_basename); man.hash.addOptionalBytes(opt_debug_basename); diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index 7c31f144d59a20f70f1e6550320b8f8c05237276..e0812402e1b976b98cc9e15f42d77afd04c4f326 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -42,7 +42,7 @@ pub fn make( const lazy_path = arg.path.get(conf); try step.addWatchInput(maker, arena, lazy_path); const arg_path = try maker.resolveLazyPath(arena, lazy_path, step_index); - _ = try man.addFilePath(arg_path, null); + _ = try man.addInputPath(arg_path, .{}); try args_bytes.print(arena, "pub const {f}: []const u8 = \"{f}\";\n", .{ std.zig.fmtId(name), arg_path.fmtEscapeString(), }); diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index c8219543c2891c24ee5003f76a81efdf91f0d5a1..04b5764d04a191ace28403be0737635f047019fb 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -97,7 +97,7 @@ pub fn make( man.hash.add(arg.flags.make_absolute); man.hash.addBytesZ(prefix); man.hash.addBytesZ(suffix); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .path_directory => { const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; @@ -135,7 +135,7 @@ pub fn make( argv_list.appendAssumeCapacity(result.written()); man.hash.addBytesZ(prefix); man.hash.addBytesZ(suffix); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .artifact => { const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; @@ -155,7 +155,7 @@ pub fn make( man.hash.add(arg.flags.make_absolute); man.hash.addBytesZ(prefix); man.hash.addBytesZ(suffix); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .output_file, .output_directory => { const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; @@ -211,7 +211,7 @@ pub fn make( }, .lazy_path => |lazy_path| { const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .none => {}, } @@ -240,7 +240,7 @@ pub fn make( for (conf_run.file_inputs.slice) |lazy_path| { const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); } if (conf_run.cwd.value) |lazy_path| { diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index 65b19bdbae8137e94b407fded33c31af71805bc2..afea7de96ea7d9598cd2d4f2e55febc40cbad0d1 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -55,7 +55,7 @@ pub fn make( man.hash.addBytes(copy.sub_path.slice(conf)); const src_lazy_path = copy.src_file.get(conf); const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); - _ = try man.addFilePath(source_path, null); + _ = try man.addInputPath(source_path, .{}); try step.addWatchInput(maker, arena, src_lazy_path); } @@ -96,7 +96,7 @@ pub fn make( }, .file => { const entry_path = try src_dir_path.join(arena, entry.path); - _ = try man.addFilePath(entry_path, null); + _ = try man.addInputPath(entry_path, .{}); total_items += 1; }, else => continue, diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index a3069f48dc21adaa1f985c7fcb1d7b5c591e3fae..da90f0954d2e0e11db5ec9323721f2226ed89c01 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -294,7 +294,7 @@ pub const Manifest = struct { files: Files = .empty, /// Indexes line up with `files`, but only up until `hit` is called. Uses /// `Cache.gpa`. - input_files: std.ArrayList(InputFile) = .empty, + input_paths: std.ArrayList(InputPath) = .empty, diagnostic: Diagnostic = .none, /// Keeps track of the last time we performed a file system write to observe /// what time the file system thinks it is, according to its own granularity. @@ -304,11 +304,11 @@ pub const Manifest = struct { /// final terminating byte can be added without allocation. Uses /// `Cache.gpa`. contents: std.ArrayList(u8) = .empty, - /// All contents from all `input_files` whose contents were requested, + /// All contents from all `input_paths` whose contents were requested, /// concatenated. Total byte size will be less than `max_input_content_len` /// otherwise an error is returned. /// - /// Data is invalidated when `addFilePost` is called. + /// Data is invalidated when `addPathPost` is called. all_input_content: std.ArrayList(u8) = .empty, max_input_content_len: usize = std.math.maxInt(u32), @@ -328,23 +328,24 @@ pub const Manifest = struct { InvalidFormat, } || Allocator.Error || Io.Cancelable; - fn fail(c: *Check, m: *Manifest, diagnostic: Diagnostic) void { - if (!@atomicRmw(bool, &c.diagnostic_lock, .Xchg, true, .unordered)) { + fn fail(c: *Check, m: *Manifest, diagnostic: Diagnostic) error{CacheCheckFailed} { + if (!@atomicRmw(bool, &c.diagnostic_lock, .Xchg, true, .monotonic)) { m.diagnostic = diagnostic; } + return error.CacheCheckFailed; } }; pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); - /// Source files whose prefix and relative path are included when computing - /// the cache manifest digest. It's the information needed to lazily hash - /// the input files only when a cache miss occurs. + /// Source files and directories whose prefix and relative path are + /// included when computing the cache manifest digest. It's the information + /// needed to lazily hash the input files only when a cache miss occurs. /// /// `File.prefix`, `File.path`, and `File.mode` will be always populated, /// but the other fields of `File` will be populated depending on the - /// fields of `InputFile`. - pub const InputFile = struct { + /// fields of `InputPath`. + pub const InputPath = struct { request_handle: bool, have_handle: bool, /// Determines whether `File.size`, `File.inode`, and `File.mtime` are populated. @@ -360,7 +361,7 @@ pub const Manifest = struct { /// `have_handle` determines whether this is populated. handle: Io.File, - /// Index into `Manifest.input_files`. + /// Index into `Manifest.input_paths`. pub const Index = enum(u32) { _, }; @@ -406,13 +407,13 @@ pub const Manifest = struct { pub const Offset = enum(u32) { _, - pub fn get(offset: Offset, m: *const Manifest) *File { - return @ptrCast(m.contents.items[@backingInt(offset)..][0..@sizeOf(File)]); + pub fn get(offset: Offset, contents: []u8) *File { + return @ptrCast(@alignCast(contents.items[@backingInt(offset)..][0..@sizeOf(File)])); } - pub fn getFallible(offset: Offset, m: *const Manifest) error{EndOfStream}!*File { - if (@backingInt(offset) + @sizeOf(File) >= m.contents.len) return error.EndOfStream; - return get(offset, m); + pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File { + if (@backingInt(offset) + @sizeOf(File) >= contents.items.len) return error.InvalidFormat; + return get(offset, contents); } }; @@ -432,25 +433,6 @@ pub const Manifest = struct { } }; - pub fn path(file: *const File) [:0]const u8 { - return pathFallible(file) catch unreachable; - } - - pub fn pathFallible(file: *const File) error{EndOfStream}![:0]const u8 { - const ptr: [*]u8 = &file.path_start; - const len = mem.findScalar(u8, ptr, 0) orelse return error.EndOfStream; - return ptr[0..len :0]; - } - - fn manifestDigestHash(file: *const File, hasher: *Hasher) void { - const path_ptr: [*]u8 = &file.path_start; - const path_len = mem.findScalar(u8, path_ptr, 0).?; - comptime assert(@offsetOf(File, "path_start") - @offsetOf(File, "flags") == 1); - // Includes flags and sentinel. - const hash_string = (path_ptr - 1)[0 .. path_len + 2]; - hasher.update(hash_string); - } - fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void { file.size = stat.size; file.inode = stat.inode; @@ -490,6 +472,16 @@ pub const Manifest = struct { pub const FileOp = struct { file_offset: File.Offset, err: anyerror, + + /// Returned `Path` references `Manifest.contents`. + pub fn path(fo: FileOp, manifest: *const Manifest) Path { + const contents = manifest.contents.items; + const prefix = fo.file_offset.get(contents).flags.prefix; + return .{ + .root_dir = manifest.cache.prefixes()[prefix], + .sub_path = filePath(contents, fo.file_offset), + }; + } }; }; @@ -499,42 +491,49 @@ pub const Manifest = struct { mtime: Io.Timestamp, }; - pub const AddInputFileOptions = struct { - /// If `is_directory` is true, this handle must be opened with - /// iteration capability. - handle: ?Io.File = null, + pub const PathHandle = union(enum) { + file: ?Io.File, + /// If provided, this handle must be opened with iteration capability. + dir: ?Io.Dir, + }; + + pub const AddInputPathOptions = struct { + handle: PathHandle = .{ .file = null }, stat: ?Stat = null, request_handle: bool = false, - /// Can request file or directory contents depending on `is_directory`. + /// Can request file or directory contents depending on `handle`. request_contents: bool = false, - /// Contents of a directory are considered to be the sorted list of - /// file names of direct entries, separated by null byte. Each file name - /// is prefixed by `Io.File.Kind` byte, +1 so that the zero tag is - /// not aliased by the entry separator. - is_directory: bool = false, /// Content hashing skipped; any difference in metadata implies cache /// miss. metadata_only: bool = false, }; - pub const AddInputFileError = error{ + pub const AddInputPathError = error{ /// The same file path has been added to the cache manifest both as a /// directory and as a normal file, making the intended caching /// behavior ambiguous. IsDirectoryAmbiguous, } || Allocator.Error; - /// Add a file as a dependency of process being cached. When `hit` is - /// called, the file's contents will be checked to ensure that it matches - /// the contents from previous times. + /// Add a file or directory path as a dependency of process being cached. + /// When `hit` is called, the contents will be checked to ensure + /// that it matches the contents from previous times. /// /// The contents of the input file may be requested and subsequently - /// obtained via methods of the returned `InputFile.Index` after calling + /// obtained via methods of the returned `InputPath.Index` after calling /// `hit`. - pub fn addInputFile(m: *Manifest, path: Path, options: AddInputFileOptions) Allocator.Error!InputFile.Index { + /// + /// Contents of a directory are considered to be the sorted list of file + /// names of direct entries, separated by null byte. Each file name is + /// prefixed by `Io.File.Kind` byte, +1 so that the zero tag is not aliased + /// by the entry separator. + /// + /// See also: + /// * `addPathPost` + pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) AddInputPathError!InputPath.Index { const gpa = m.cache.gpa; try m.files.ensureUnusedCapacity(gpa, 1); - try m.input_files.ensureUnusedCapacity(gpa, 1); + try m.input_paths.ensureUnusedCapacity(gpa, 1); const prev_contents_len = m.contents.items.len; const header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))); @@ -558,7 +557,7 @@ pub const Manifest = struct { }); if (gop.found_existing) { m.contents.shrinkRetainingCapacity(prev_contents_len); - const existing_input_file = &m.input_files.items[gop.index]; + const existing_input_file = &m.input_paths.items[gop.index]; if (options.handle) |handle| { existing_input_file.handle = handle; existing_input_file.have_handle = true; @@ -579,7 +578,7 @@ pub const Manifest = struct { if (!options.metadata_only) existing_header.flags.metadata_only = false; } else { - m.input_files.appendAssumeCapacity(.{ + m.input_paths.appendAssumeCapacity(.{ .request_handle = options.request_handle, .have_handle = options.handle != null, .handle = if (options.handle) |handle| handle else undefined, @@ -587,7 +586,7 @@ pub const Manifest = struct { .have_digest = false, .have_stat = options.stat != null, }); - assert(m.input_files.items.len - 1 == gop.index); + assert(m.input_paths.items.len - 1 == gop.index); if (options.stat) |stat| { header.size = stat.size; header.inode = stat.inode; @@ -597,9 +596,9 @@ pub const Manifest = struct { return @fromBackingInt(gop.index); } - pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputFileOptions) Allocator.Error!void { + pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { m.hash.add(opt_path != null); - _ = try addInputFile(m, opt_path orelse return, options); + _ = try addInputPath(m, opt_path orelse return, options); } /// Check the cache to see if the input exists in it. @@ -623,8 +622,8 @@ pub const Manifest = struct { pub fn checkProgressless(man: *Manifest) Check.Error!Check.Status { assert(man.manifest_file == null); - for (man.files.keys()[0..man.input_files.items.len]) |file_off| { - file_off.get(man).manifestDigestHash(&man.hash.hasher); + for (man.files.keys()[0..man.input_paths.items.len]) |file_off| { + man.digestHash(file_off, &man.hash.hasher); } man.diagnostic = .none; @@ -709,16 +708,16 @@ pub const Manifest = struct { // We're going to construct a second hash. Its input will begin with the digest we've // already computed (`bin_digest`), and then it'll have the digests of each input file, - // including "post" files (see `addFilePost`). If this is a hit, we learn the set of "post" + // including "post" files (see `addPathPost`). If this is a hit, we learn the set of "post" // files from the manifest on disk. If this is a miss, we'll learn those from future calls - // to `addFilePost` etc. As such, the state of `man.hash.hasher` after this function + // to `addPathPost` etc. As such, the state of `man.hash.hasher` after this function // depends on whether this is a hit or a miss. // // If we return `CacheStatus.hit`, then `man.hash.hasher` must already include // the digests of the "post" files, so the caller can call `final`. Otherwise, on a cache // miss, `man.hash.hasher` will include the digests of all non-"post" files -- that is, // the ones we've already been told about. The rest will be discovered through calls to - // `addFilePost` etc, which will update the hasher. After all files are added, the user can + // `addPathPost` etc, which will update the hasher. After all files are added, the user can // use `final`, and will at some point `writeManifest` the file list to disk. man.hash.hasher = hasher_init; @@ -762,11 +761,11 @@ pub const Manifest = struct { } fn shrinkFilesToInput(m: *Manifest) void { - if (m.files.count() <= m.input_files.items.len) return; - const off = m.files.keys()[m.input_files.items.len]; + if (m.files.count() <= m.input_paths.items.len) return; + const off = m.files.keys()[m.input_paths.items.len]; m.contents.shrinkRetainingCapacity(@backingInt(off)); - assert(m.contents.len % @alignOf(File) == 0); - m.files.shrinkRetainingCapacity(m.input_files.items.len); + assert(m.contents.items.len % @alignOf(File) == 0); + m.files.shrinkRetainingCapacity(m.input_paths.items.len); } /// Assumes that `self.hash.hasher` has been updated only with the original digest and that @@ -784,59 +783,70 @@ pub const Manifest = struct { return error.CacheCheckFailed; }, }; + const contents = m.contents.items; - // Guess number of files based on manifest contents len to reduce allocations. - try m.files.ensureUnusedCapacity(gpa, m.contents.len / (@sizeOf(File) + 32)); + var off: u32 = 0; + var c: Check = .{}; - var file_index: usize = 0; - var off: usize = 0; - - // This group we always want to compute the hash digests, even on a cache miss. + // This group we always want to compute the hash digests, even on a + // cache miss, because they will be used in the manifest digest. var input_group: Io.Group = .init; defer input_group.cancel(io); + // First the input files section, which must match our input files, + // otherwise it's invalid format. + for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| { + if (off + 1 >= contents.len) return error.InvalidFormat; + const file_off: File.Offset = @fromBackingInt(off); + const file = try file_off.getFallible(contents); + if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; + const path = try filePathFallible(contents, file_off); + if (path.len == 0) return error.InvalidFormat; + if (input_file_off != file_off) return error.InvalidFormat; + + input_group.async(io, checkInputFile, .{ m, &c, file_off, path, input_path }); + + off = @intCast(@as(usize, off) + @sizeOf(File) + path.len + 1); + } + + // Guess number of files based on manifest contents len to reduce allocations. + // This is not an upper bound; subsequent insertions may potentially allocate. + try m.files.ensureUnusedCapacity(gpa, contents.len / (@sizeOf(File) + 32)); + // This group we would like to cancel as soon as a cache miss is discovered. const PostResult = union(enum) { checkFile: Check.Status, }; var post_select_buffer: [10]PostResult = undefined; - var post_select: Io.Select(PostResult) = .init(&post_select_buffer); + var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer); var post_select_remaining: usize = 0; - var c: Check = .{}; - defer post_select.cancel(io); + defer post_select.cancelDiscard(); - while (off + 1 < m.contents.len) { + while (off + 1 < contents.len) { const file_off: File.Offset = @fromBackingInt(off); - const file = try File.getFallible(file_off, m); + const file = try file_off.getFallible(m); if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; - const path = try file.pathFallible(); + const path = try filePathFallible(contents, file_off); if (path.len == 0) return error.InvalidFormat; - if (file_index < m.input_files.items.len) { - if (m.files.keys()[file_index] != file_off) return error.InvalidFormat; + try m.files.put(gpa, file_off, {}); - input_group.async(io, checkInputFile, .{ m, &c, file_off, path }); - } else { - try m.files.put(gpa, file_off); + post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path }); + post_select_remaining += 1; - post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path }); - post_select_remaining += 1; - } - - file_index += 1; - off += @sizeOf(File) + path.len + 1; + off = @intCast(@as(usize, off) + @sizeOf(File) + path.len + 1); } // Final terminating zero byte to distinguish empty manifest file from // manifest with zero files. - const file_valid = off + 1 == m.contents.len and m.contents[off] == 0; - if (!file_valid or file_index < m.input_files.items.len) { + const file_valid = off + 1 == contents.len and contents[off] == 0; + if (!file_valid) { try input_group.await(io); return .miss; } // Don't track the trailing zero byte in contents. - m.contents.len -= 1; + m.contents.items.len -= 1; var post_await_buffer: [10]PostResult = undefined; while (post_select_remaining > 0) { @@ -880,11 +890,17 @@ pub const Manifest = struct { return .hit; } - fn checkInputFile(m: *Manifest, c: *Check, file_off: File.Offset, file_path: [:0]const u8) Io.Cancelable!void { - // TODO use already open handle - // TODO use already provided stat - // TODO implement request_handle - // TODO implement request_contents + fn checkInputFile( + m: *Manifest, + c: *Check, + file_off: File.Offset, + file_path: [:0]const u8, + input_path: *InputPath, + ) Io.Cancelable!void { + if (input_path.have_handle) @panic("TODO"); + if (input_path.have_stat) @panic("TODO"); + if (input_path.contents != .not_requested) @panic("TODO"); + if (input_path.request_handle) @panic("TODO"); switch (try checkFile(m, c, file_off, file_path)) { .hit => return, .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered), @@ -897,7 +913,7 @@ pub const Manifest = struct { c: *Check, file_off: File.Offset, file_path: [:0]const u8, - ) Io.Cancelable!Check.Status { + ) error{ Canceled, CacheCheckFailed }!Check.Status { const file = file_off.get(m); const cache = m.cache; const gpa = cache.gpa; @@ -905,7 +921,7 @@ pub const Manifest = struct { const parent_dir = cache.prefixes()[file.flags.prefix].handle; if (file.flags.metadata_only) { - const actual_stat = parent_dir.statFile() catch |err| switch (err) { + const actual_stat = parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { error.FileNotFound => return .miss, error.Canceled => |e| return e, else => |e| return c.fail(m, .{ .file_stat = .{ @@ -999,10 +1015,10 @@ pub const Manifest = struct { /// not including post files). /// /// Assumes that `bin_digest` is populated for all input files. - pub fn unhit(man: *Manifest, bin_digest: BinDigest) void { + pub fn unhit(man: *Manifest, bin_digest: *const BinDigest) void { // Reset the hash. man.hash.hasher = hasher_init; - man.hash.hasher.update(&bin_digest); + man.hash.hasher.update(bin_digest); man.shrinkFilesToInput(); for (man.files.keys()) |off| { const file = off.get(man); @@ -1053,11 +1069,8 @@ pub const Manifest = struct { return timestamp.nanoseconds >= man.recent_problematic_timestamp.nanoseconds; } - pub const AddFilePostOptions = struct { - handle: union(enum) { - file: ?Io.File, - dir: ?Io.Dir, - } = .{ .file = null }, + pub const AddPathPostOptions = struct { + handle: PathHandle = .{ .file = null }, stat: ?Stat = null, /// If it is a directory, there is a special encoding required for contents, which /// is null-separated sorted entries, each one prefixed with `File.Kind`. @@ -1065,7 +1078,7 @@ pub const Manifest = struct { metadata_only: bool = false, }; - pub const AddFilePostError = error{ + pub const AddPathPostError = error{ /// The same file path has been added to the cache manifest both as a /// directory and as a normal file, making the intended caching /// behavior ambiguous. @@ -1074,7 +1087,10 @@ pub const Manifest = struct { /// Add a file as a dependency of process being cached, after cache miss /// occurs. - pub fn addFilePost(m: *Manifest, path: Path, options: AddFilePostOptions) AddFilePostError!void { + /// + /// See also: + /// * `addInputPath` + pub fn addPathPost(m: *Manifest, path: Path, options: AddPathPostOptions) AddPathPostError!void { assert(m.manifest_file != null); const cache = m.cache; const gpa = cache.gpa; @@ -1236,14 +1252,14 @@ pub const Manifest = struct { // Clang is invoked in single-source mode but other programs may not .target, .target_must_resolve => {}, .prereq => |file_path| if (self.manifest_file == null) { - _ = try self.addFilePath(.initCwd(file_path), null); - } else try self.addFilePost(file_path), + _ = try self.addInputPath(.initCwd(file_path), .{}); + } else try self.addPathPost(file_path), .prereq_must_resolve => { resolve_buf.clearRetainingCapacity(); try token.resolve(gpa, &resolve_buf); if (self.manifest_file == null) { - _ = try self.addFilePath(.initCwd(resolve_buf.items), null); - } else try self.addFilePost(resolve_buf.items); + _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); + } else try self.addPathPost(resolve_buf.items); }, else => |err| { try err.printError(gpa, &error_buf); @@ -1336,25 +1352,45 @@ pub const Manifest = struct { return .{ .manifest_file = self.manifest_file.? }; } - pub fn takeFiles(man: *Manifest) Files { - defer man.files = .empty; - return man.files; - } - - pub fn freeFiles(gpa: Allocator, files: *Files) void { - for (files.keys()) |*file| file.deinit(gpa); - files.deinit(gpa); + pub const SelfContainedFiles = struct { + /// References memory inside `contents`. + files: Files, + contents: std.ArrayList(u8), + + pub const empty: @This() = .{ + .files = .empty, + .contents = .empty, + }; + + pub fn deinit(scf: *SelfContainedFiles, gpa: Allocator) void { + scf.files.deinit(gpa); + scf.contents.deinit(gpa); + scf.* = undefined; + } + + pub fn path(scf: *const SelfContainedFiles, file_offset: File.Offset) [:0]const u8 { + return filePath(scf.contents.items, file_offset); + } + }; + + pub fn takeFiles(m: *Manifest) SelfContainedFiles { + defer m.files = .empty; + defer m.contents = .empty; + return .{ + .files = m.files, + .contents = m.contents, + }; } /// Releases the manifest file and frees any memory the Manifest was using. /// `Manifest.hit` must be called first. /// /// Don't forget to call `writeManifest` before this! - pub fn deinit(man: *Manifest) void { - const io = man.cache.io; - const gpa = man.cache.gpa; + pub fn deinit(m: *Manifest) void { + const io = m.cache.io; + const gpa = m.cache.gpa; - if (man.manifest_file) |file| { + if (m.manifest_file) |file| { if (builtin.os.tag == .windows) { // See Lock.release for why this is required on Windows file.unlock(io); @@ -1362,8 +1398,9 @@ pub const Manifest = struct { file.close(io); } - freeFiles(gpa, &man.files); - man.* = undefined; + m.files.deinit(gpa); + m.contents.deinit(gpa); + m.* = undefined; } pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void { @@ -1497,6 +1534,25 @@ pub const Manifest = struct { hasher.update(contents.items[contents_start..][0..contents_len]); hasher.final(bin_digest); } + + fn digestHash(m: *const Manifest, off: File.Offset, hasher: *Hasher) void { + const contents = m.contents.items; + const flags_off = @offsetOf(File, "flags"); + comptime assert(@offsetOf(File, "path_start") - flags_off == 1); + const hash_start = @backingInt(off) + flags_off; + const hash_end = mem.findScalarPos(u8, contents, hash_start, 0).?; + hasher.update(contents[hash_start..hash_end]); + } + + fn filePathFallible(contents: []const u8, off: File.Offset) error{InvalidFormat}![:0]const u8 { + const path_start = @backingInt(off) + @offsetOf(File, "path_start"); + const path_end = mem.findScalarPos(u8, contents, path_start, 0) orelse return error.InvalidFormat; + return contents[path_start..path_end :0]; + } + + fn filePath(contents: []const u8, off: File.Offset) [:0]const u8 { + return filePathFallible(contents, off) catch unreachable; + } }; /// Create/Write a file, close it, then grab its stat.mtime timestamp. @@ -1555,7 +1611,7 @@ test "cache file and then recall it" { ch.hash.add(true); ch.hash.add(@as(u16, 1234)); ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file), null); + _ = try ch.addInputPath(.initCwd(temp_file), .{}); // There should be nothing in the cache try testing.expectEqual(false, try ch.hit(.none)); @@ -1570,7 +1626,7 @@ test "cache file and then recall it" { ch.hash.add(true); ch.hash.add(@as(u16, 1234)); ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file), null); + _ = try ch.addInputPath(.initCwd(temp_file), .{}); // Cache hit! We just "built" the same file try testing.expect(try ch.hit(.none)); @@ -1623,7 +1679,7 @@ test "check that changing a file makes cache fail" { defer ch.deinit(); ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFilePath(.initCwd(temp_file), 100); + const temp_file_idx = try ch.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); // There should be nothing in the cache try testing.expectEqual(false, try ch.hit(.none)); @@ -1642,7 +1698,7 @@ test "check that changing a file makes cache fail" { defer ch.deinit(); ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFilePath(.initCwd(temp_file), 100); + const temp_file_idx = try ch.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); // A file that we depend on has been updated, so the cache should not contain an entry for it try testing.expectEqual(false, try ch.hit(.none)); @@ -1689,7 +1745,7 @@ test "no file inputs" { man.hash.addBytes("1234"); // There should be nothing in the cache - try testing.expectEqual(false, try man.hit(.none)); + try testing.expectEqual(false, try man.check(.none)); digest1 = man.final(); @@ -1701,7 +1757,7 @@ test "no file inputs" { man.hash.addBytes("1234"); - try testing.expect(try man.hit(.none)); + try testing.expect(try man.check(.none)); digest2 = man.final(); try testing.expectEqual(false, man.have_exclusive_lock); } @@ -1750,12 +1806,12 @@ test "Manifest with files added after initial hash work" { defer ch.deinit(); ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file1), null); + _ = try ch.addInputPath(.initCwd(temp_file1), .{}); // There should be nothing in the cache try testing.expectEqual(false, try ch.hit(.none)); - _ = try ch.addFilePost(temp_file2); + _ = try ch.addPathPost(temp_file2); digest1 = ch.final(); try ch.writeManifest(); @@ -1765,7 +1821,7 @@ test "Manifest with files added after initial hash work" { defer ch.deinit(); ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file1), null); + _ = try ch.addInputPath(.initCwd(temp_file1), .{}); try testing.expect(try ch.hit(.none)); digest2 = ch.final(); @@ -1788,12 +1844,12 @@ test "Manifest with files added after initial hash work" { defer ch.deinit(); ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file1), null); + _ = try ch.addInputPath(.initCwd(temp_file1), .{}); // A file that we depend on has been updated, so the cache should not contain an entry for it try testing.expectEqual(false, try ch.hit(.none)); - _ = try ch.addFilePost(temp_file2); + _ = try ch.addPathPost(temp_file2); digest3 = ch.final(); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 542d7863abcb1881581000beb08129d0b6909a95..f32fb26ec5ba4a2b6f1cb4bf52638cbb55c034a7 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1876,7 +1876,7 @@ pub const PathDep = extern struct { is_directory: bool, metadata_only: bool, base: LazyPath.Relative.Base, - _: u16 = 0, + _: u22 = 0, }; }; diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 372a7890b734638cf9fd7d4a8f99ca5a29c13c01..947f01694bcf6a1dea849bb7bbadef92c3f57096 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -397,6 +397,7 @@ pub fn appendRemainingAligned( pub const UnlimitedAllocError = Allocator.Error || ShortError; pub fn appendRemainingUnlimited(r: *Reader, gpa: Allocator, list: *ArrayList(u8)) UnlimitedAllocError!void { + list.pointer_stability.assertUnlocked(); var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.allocatedSlice()); a.writer.end = list.items.len; list.* = .empty; diff --git a/src/Compilation.zig b/src/Compilation.zig index 1663f740c6e14d370c4e8c4da8865cdf125223a1..af532d6c0a253a46650ebd35fb2b939d69d5e107 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1341,19 +1341,19 @@ pub const cache_helpers = struct { } } - pub fn hashCSource(self: *Cache.Manifest, c_source: CSourceFile) !void { - _ = try self.addFilePath(.initCwd(c_source.src_path), null); + pub fn hashCSource(man: *Cache.Manifest, c_source: CSourceFile) !void { + _ = try man.addInputPath(.initCwd(c_source.src_path), .{}); // Hash the extra flags, with special care to call addFile for file parameters. // TODO this logic can likely be improved by utilizing clang_options_data.zig. const file_args = [_][]const u8{"-include"}; var arg_i: usize = 0; while (arg_i < c_source.extra_flags.len) : (arg_i += 1) { const arg = c_source.extra_flags[arg_i]; - self.hash.addBytes(arg); + man.hash.addBytes(arg); for (file_args) |file_arg| { if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) { arg_i += 1; - _ = try self.addFilePath(.initCwd(c_source.extra_flags[arg_i]), null); + _ = try man.addInputPath(.initCwd(c_source.extra_flags[arg_i]), .{}); } } } @@ -2835,7 +2835,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE man.want_shared_lock = false; } - const is_hit = man.hit(main_progress_node) catch |err| switch (err) { + const is_hit = man.check(main_progress_node) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, @@ -3410,7 +3410,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes try link.hashInputs(man, comp.link_inputs); for (comp.c_objects.items) |c_object| { - _ = try man.addFilePath(.initCwd(c_object.src.src_path), null); + _ = try man.addInputPath(.initCwd(c_object.src.src_path), .{}); man.hash.addOptional(c_object.src.ext); man.hash.addListOfBytes(c_object.src.extra_flags); } @@ -3418,11 +3418,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes for (comp.win32_resources.items) |win32_resource| { switch (win32_resource.src) { .rc => |rc_src| { - _ = try man.addFilePath(.initCwd(rc_src.src_path), null); + _ = try man.addInputPath(.initCwd(rc_src.src_path), .{}); man.hash.addListOfBytes(rc_src.extra_flags); }, .manifest => |manifest_path| { - _ = try man.addFilePath(.initCwd(manifest_path), null); + _ = try man.addInputPath(.initCwd(manifest_path), .{}); }, } } @@ -5556,7 +5556,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr const target = comp.getTarget(); assert(target.ofmt != .c); const o_ext = target.ofmt.fileExt(target.cpu.arch); - const digest = if (!comp.disable_c_depfile and try man.hit(child_progress_node)) man.final() else blk: { + const digest = if (!comp.disable_c_depfile and try man.check(child_progress_node)) man.final() else blk: { var argv: std.array_list.Managed([]const u8) = .init(gpa); defer argv.deinit(); @@ -5817,7 +5817,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr } // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. - if (comp.disable_c_depfile) _ = try man.hit(child_progress_node); + if (comp.disable_c_depfile) _ = try man.check(child_progress_node); // Rename into place. const digest = man.final(); @@ -5900,12 +5900,12 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // the XML data as a RT_MANIFEST resource. This means we can skip preprocessing, // include paths, CLI options, etc. if (win32_resource.src == .manifest) { - _ = try man.addFilePath(.initCwd(src_path), null); + _ = try man.addInputPath(.initCwd(src_path), .{}); const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); - const digest = if (try man.hit(child_progress_node)) man.final() else blk: { + const digest = if (try man.check(child_progress_node)) man.final() else blk: { // The digest only depends on the .manifest file, so we can // get the digest now and write the .res directly to the cache const digest = man.final(); @@ -5993,12 +5993,12 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // We now know that we're compiling an .rc file const rc_src = win32_resource.src.rc; - _ = try man.addFilePath(.initCwd(rc_src.src_path), null); + _ = try man.addInputPath(.initCwd(rc_src.src_path), .{}); man.hash.addListOfBytes(rc_src.extra_flags); const rc_basename_noext = src_basename[0 .. src_basename.len - fs.path.extension(src_basename).len]; - const digest = if (try man.hit(child_progress_node)) man.final() else blk: { + const digest = if (try man.check(child_progress_node)) man.final() else blk: { var zig_cache_tmp_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, "tmp", .{}); defer zig_cache_tmp_dir.close(io); diff --git a/src/libs/freebsd.zig b/src/libs/freebsd.zig index d7b53387347e27ab502c13282dc99f8c75763df4..bda03f2787e4ff60e19e920ee3a43ab246366193 100644 --- a/src/libs/freebsd.zig +++ b/src/libs/freebsd.zig @@ -458,12 +458,14 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_os_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); + }, .{ + .request_contents = true, + }); - if (try man.hit(prog_node)) { + if (try man.check(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/libs/glibc.zig b/src/libs/glibc.zig index ebc36af06304aa28b760477167ce0b04570abac3..67dd70eae78487ab5473c06a63bab226b91c6205 100644 --- a/src/libs/glibc.zig +++ b/src/libs/glibc.zig @@ -698,12 +698,14 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); + }, .{ + .request_contents = true, + }); - if (try man.hit(prog_node)) { + if (try man.check(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index a91c24291d0a395b189712c4d8f199bf13cd57d0..63c1e5ad16ccf7a745256242f528203d5a5cbdcf 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -246,12 +246,12 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P var man = cache.obtain(); defer man.deinit(); - _ = try man.addFilePath(def_file_path, null); + _ = try man.addInputPath(def_file_path, .{}); const final_lib_basename = try std.fmt.allocPrint(gpa, "{s}.lib", .{lib_name}); errdefer gpa.free(final_lib_basename); - const is_hit = man.hit(prog_node) catch |err| switch (err) { + const is_hit = man.check(prog_node) catch |err| switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, .manifest_create, .manifest_read, .manifest_lock => |e| { diff --git a/src/libs/netbsd.zig b/src/libs/netbsd.zig index b0fd269b22279b860426781e41a21e5297989c75..8cafe51563f5d6e32971fb77035f4f7e54d65409 100644 --- a/src/libs/netbsd.zig +++ b/src/libs/netbsd.zig @@ -406,12 +406,14 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); + }, .{ + .request_contents = true, + }); - if (try man.hit(prog_node)) { + if (try man.check(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/libs/openbsd.zig b/src/libs/openbsd.zig index ba65dd46943a10fc8e61f2216e6762debf549e4a..dea2c5c3350afcced2f7c8c005a1ceb24a102f8d 100644 --- a/src/libs/openbsd.zig +++ b/src/libs/openbsd.zig @@ -327,12 +327,14 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); + }, .{ + .request_contents = true, + }); - if (try man.hit(prog_node)) { + if (try man.check(prog_node)) { const digest = man.final(); return queueSharedObjects(comp, .{ diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 72e34b943e1d565e6c3360cfd017b4d9c7ba3f5e..c32301b826482026993bfb7e8763357cd57dbf06 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -155,7 +155,7 @@ pub fn hashAddFrameworks(man: *Cache.Manifest, hm: []const Framework) !void { for (hm) |value| { man.hash.add(value.needed); man.hash.add(value.weak); - _ = try man.addFilePath(value.path, null); + _ = try man.addInputPath(value.path, .{}); } } diff --git a/src/main.zig b/src/main.zig index a2218a480f8f38b30c4963402e8bfe413df2b8bd..dc691fd8a6ed15dc099dd3d60484af285329c836 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4876,7 +4876,7 @@ fn cmdTranslateC( Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| fatal("unable to process {q}: {t}", .{ c_source_file.src_path, err }); - const result: Compilation.TranslateCResult = if (try man.hit(prog_node)) .{ + const result: Compilation.TranslateCResult = if (try man.check(prog_node)) .{ .digest = man.finalBin(), .cache_hit = true, .errors = std.zig.ErrorBundle.empty, -- 2.54.0 From 0b561847c15b3c90c4af8e1084c681fc1ecb0068 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 15:04:36 -0700 Subject: [PATCH 04/49] std.Build.Cache: make IsDirectoryAmbiguous an assertion not error --- lib/std/Build/Cache.zig | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index da90f0954d2e0e11db5ec9323721f2226ed89c01..067e44bc4c04d7499068f1a3270be6ea1d76d140 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -508,13 +508,6 @@ pub const Manifest = struct { metadata_only: bool = false, }; - pub const AddInputPathError = error{ - /// The same file path has been added to the cache manifest both as a - /// directory and as a normal file, making the intended caching - /// behavior ambiguous. - IsDirectoryAmbiguous, - } || Allocator.Error; - /// Add a file or directory path as a dependency of process being cached. /// When `hit` is called, the contents will be checked to ensure /// that it matches the contents from previous times. @@ -530,7 +523,7 @@ pub const Manifest = struct { /// /// See also: /// * `addPathPost` - pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) AddInputPathError!InputPath.Index { + pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { const gpa = m.cache.gpa; try m.files.ensureUnusedCapacity(gpa, 1); try m.input_paths.ensureUnusedCapacity(gpa, 1); @@ -573,8 +566,10 @@ pub const Manifest = struct { existing_header.inode = stat.inode; existing_header.mtime = stat.mtime; } - if (existing_header.flags.is_directory != options.is_directory) - return error.IsDirectoryAmbiguous; + // If it trips, the same file path has been added to the cache + // manifest both as a directory and as a normal file, making the + // intended caching behavior ambiguous. + assert(existing_header.flags.is_directory == options.is_directory); if (!options.metadata_only) existing_header.flags.metadata_only = false; } else { @@ -1078,12 +1073,7 @@ pub const Manifest = struct { metadata_only: bool = false, }; - pub const AddPathPostError = error{ - /// The same file path has been added to the cache manifest both as a - /// directory and as a normal file, making the intended caching - /// behavior ambiguous. - IsDirectoryAmbiguous, - } || Io.Cancelable || Allocator.Error; + pub const AddPathPostError = Io.Cancelable || Allocator.Error; /// Add a file as a dependency of process being cached, after cache miss /// occurs. @@ -1126,8 +1116,10 @@ pub const Manifest = struct { m.contents.shrinkRetainingCapacity(prev_contents_len); const existing_off = gop.key_ptr.*; const header = existing_off.get(m); - if (header.flags.is_directory != is_directory) - return error.IsDirectoryAmbiguous; + // If it trips, the same file path has been added to the cache + // manifest both as a directory and as a normal file, making the + // intended caching behavior ambiguous. + assert(header.flags.is_directory == is_directory); if (!options.metadata_only) header.flags.metadata_only = false; break :h header; -- 2.54.0 From 11d3e2515dd22f7bdf6f7c8021332006684ae8e9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 17:57:55 -0700 Subject: [PATCH 05/49] std.ArrayList: add "see also" doc comment --- lib/std/array_list.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index 7b6a004603a2d557dd2af751f322536dfa23b1a6..7c34f58dba4d4151e1df3426eeccbbc1a7064b16 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -776,6 +776,9 @@ pub fn Aligned(comptime T: type, comptime alignment: ?mem.Alignment) type { /// /// Asserts what the capacity is equal to the length. /// Never invalidates element pointers. + /// + /// See also: + /// * `shrinkToLen` pub fn toOwnedSliceAssert(self: *Self) Slice { assert(self.items.len == self.capacity); const items = self.items; -- 2.54.0 From d3231ad09025f14c87018447f607df027d17d993 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 18:34:48 -0700 Subject: [PATCH 06/49] std.Build.Cache: get it compiling again --- lib/compiler/Maker.zig | 3 +- lib/compiler/Maker/Step.zig | 2 +- lib/std/Build/Cache.zig | 362 +++++++++++++++++++++--------------- lib/std/zig.zig | 12 +- 4 files changed, 217 insertions(+), 162 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index d5cde77ae631f5cad9a297429a5b045a4046e610..97760f7d15cd2f4e58858ba2a4494b6f0cd53740 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1498,7 +1498,8 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { if (config_man) |man| for (configuration.path_deps) |path_dep| { const path = try confPathDepToCachePath(arena, graph, &configuration, path_dep); - try man.addPathPost(path, .{ + try man.addPathPost(.{ + .path = .{ .unresolved = path }, .handle = if (path_dep.flags.is_directory) .{ .dir = null } else .{ .file = null }, .metadata_only = path_dep.flags.metadata_only, }); diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 690afd2e0fb403fad83d0c2f9b0a747ffed9a088..b1e89364fb050e5ba397d40f8de6c43966dedb04 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -787,7 +787,7 @@ pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !voi } pub fn setWatchInputsFromManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { - return setWatchInputsFromManifestFiles(s, maker, &man.files, man.cache.prefixes()); + return setWatchInputsFromManifestFiles(s, maker, &man.borrowFiles(), man.cache.prefixes()); } pub fn setWatchInputsFromManifestFiles( diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 067e44bc4c04d7499068f1a3270be6ea1d76d140..8a3a80007e09e1f7b257c30f9a26720f0bf288d3 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -56,71 +56,60 @@ pub fn prefixes(cache: *const Cache) []const Directory { return cache.prefixes_buffer[0..cache.prefixes_len]; } -pub const PrefixedPath = struct { - prefix: u8, +const PrefixIndex = u6; + +const PrefixedPath = struct { + prefix: PrefixIndex, sub_path: []const u8, - - fn eql(a: PrefixedPath, b: PrefixedPath) bool { - return a.prefix == b.prefix and mem.eql(u8, a.sub_path, b.sub_path); - } - - fn hash(pp: PrefixedPath) u32 { - return @truncate(std.hash.Wyhash.hash(pp.prefix, pp.sub_path)); - } }; -fn findPrefixPath(cache: *const Cache, path: Path) !PrefixedPath { - const gpa = cache.gpa; - const resolved_path = try std.fs.path.resolve(gpa, &.{ - cache.cwd, path.root_dir.path orelse ".", path.subPathOrDot(), - }); - errdefer gpa.free(resolved_path); - return findPrefixResolved(cache, resolved_path); +fn appendPrefixedPath(cache: *const Cache, contents: *std.ArrayList(u8), prefixed_path: PrefixedPath) !PrefixIndex { + const end = contents.items.len + prefixed_path.sub_path.len; + const needed_alignment = @alignOf(Manifest.File) - (end % @alignOf(Manifest.File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + try contents.ensureTotalCapacity(cache.gpa, end + needed_alignment); + contents.appendSliceAssumeCapacity(prefixed_path.sub_path); + contents.appendNTimesAssumeCapacity(0, needed_alignment); + return prefixed_path.prefix; } -fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath { - const gpa = cache.gpa; - const resolved_path = try std.fs.path.resolve(gpa, &.{file_path}); - errdefer gpa.free(resolved_path); - return findPrefixResolved(cache, resolved_path); -} - -/// Takes ownership of `resolved_path` on success. -fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath { +fn resolveAppendPath(cache: *const Cache, contents: *std.ArrayList(u8), path: Path) !PrefixIndex { const gpa = cache.gpa; const cwd = cache.cwd; + const path_start = contents.items.len; + + const resolved_path = try std.fs.path.resolveAlloc(gpa, &.{ + path.root_dir.path orelse cwd, + path.subPathOrDot(), + }); + defer gpa.free(resolved_path); + for (cache.prefixes(), 0..) |prefix, i| { - const p = prefix.path orelse continue; - const sub_path = getPrefixSubpath(gpa, cwd, p, resolved_path) catch |err| switch (err) { - error.NotASubPath => continue, - else => |e| return e, - }; - // Free the resolved path since we're not going to return it - gpa.free(resolved_path); - return .{ - .prefix = @intCast(i), - .sub_path = sub_path, - }; - } + const pp = prefix.path orelse continue; + contents.shrinkRetainingCapacity(path_start); + try std.fs.path.relativeAppend(gpa, contents, cwd, null, pp, resolved_path); + const relative = contents.items[path_start..]; - return .{ - .prefix = 0, - .sub_path = resolved_path, - }; -} + var component_iterator: std.fs.path.NativeComponentIterator = .init(relative); + if (component_iterator.root() != null) continue; + const first_component = component_iterator.first(); + if (first_component != null and mem.eql(u8, first_component.?.name, "..")) continue; -fn getPrefixSubpath(gpa: Allocator, cwd: []const u8, prefix: []const u8, path: []u8) ![]u8 { - const relative = try std.fs.path.relative(gpa, cwd, null, prefix, path); - errdefer gpa.free(relative); - var component_iterator: std.fs.path.NativeComponentIterator = .init(relative); - if (component_iterator.root() != null) { - return error.NotASubPath; - } - const first_component = component_iterator.first(); - if (first_component != null and mem.eql(u8, first_component.?.name, "..")) { - return error.NotASubPath; + const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + try contents.appendNTimes(gpa, 0, needed_alignment); + + return @intCast(i); } - return relative; + + contents.shrinkRetainingCapacity(path_start); + try contents.appendSlice(gpa, resolved_path); + + const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + try contents.appendNTimes(gpa, 0, needed_alignment); + + return 0; } /// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 @@ -359,7 +348,10 @@ pub const Manifest = struct { _, }, /// `have_handle` determines whether this is populated. - handle: Io.File, + handle: union { + file: Io.File, + dir: Io.Dir, + }, /// Index into `Manifest.input_paths`. pub const Index = enum(u32) { @@ -383,7 +375,7 @@ pub const Manifest = struct { pub const Flags = packed struct(u8) { is_directory: bool, metadata_only: bool, - prefix: u6, + prefix: PrefixIndex, }; /// Prefixes path names in encoded directory contents. Starts numbering @@ -408,40 +400,49 @@ pub const Manifest = struct { _, pub fn get(offset: Offset, contents: []u8) *File { - return @ptrCast(@alignCast(contents.items[@backingInt(offset)..][0..@sizeOf(File)])); + return @constCast(getConst(offset, contents)); + } + + pub fn getConst(offset: Offset, contents: []const u8) *const File { + return @ptrCast(@alignCast(contents[@backingInt(offset)..][0..@sizeOf(File)])); } pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File { - if (@backingInt(offset) + @sizeOf(File) >= contents.items.len) return error.InvalidFormat; + if (@backingInt(offset) + @sizeOf(File) >= contents.len) return error.InvalidFormat; return get(offset, contents); } }; + /// Intentionally matches if the files are different only by flags other than prefix. pub const HashContext = struct { - manifest: *const Manifest, + contents: []const u8, pub fn hash(this: @This(), off: Offset) u32 { - const file = off.get(this.manifest); - return @truncate(std.hash.Wyhash.hash(file.prefix, file.path())); + const file_prefix = off.getConst(this.contents).flags.prefix; + const file_path = filePath(this.contents, off); + return @truncate(std.hash.Wyhash.hash(file_prefix, file_path)); } pub fn eql(this: @This(), a_off: Offset, b_off: Offset, b_index: usize) bool { _ = b_index; - const a = a_off.get(this.manifest); - const b = b_off.get(this.manifest); - return a.prefix == b.prefix and mem.eql(u8, a.path(), b.path()); + const a_prefix = a_off.getConst(this.contents).flags.prefix; + const b_prefix = b_off.getConst(this.contents).flags.prefix; + if (a_prefix != b_prefix) return false; + const a_path = filePath(this.contents, a_off); + const b_path = filePath(this.contents, b_off); + return mem.eql(u8, a_path, b_path); } }; fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void { file.size = stat.size; file.inode = stat.inode; - file.mtime = stat.mtime; + file.mtime = @intCast(stat.mtime.toNanoseconds()); if (try m.isProblematicTimestamp(stat.mtime)) { // The actual file has an unreliable timestamp; force it to be hashed. - file.stat.mtime = 0; - file.stat.inode = 0; + file.mtime = 0; + file.inode = 0; } } @@ -453,7 +454,7 @@ pub const Manifest = struct { { return false; } else { - setStat(file, m, stat); + try setStat(file, m, stat); return true; } } @@ -489,12 +490,31 @@ pub const Manifest = struct { size: u64, inode: Io.File.INode, mtime: Io.Timestamp, + + pub fn init(other: Io.File.Stat) Stat { + return .{ + .size = other.size, + .inode = other.inode, + .mtime = other.mtime, + }; + } }; pub const PathHandle = union(enum) { file: ?Io.File, /// If provided, this handle must be opened with iteration capability. dir: ?Io.Dir, + + pub fn isDirectory(this: @This()) bool { + return this == .dir; + } + + pub fn have(this: @This()) bool { + return switch (this) { + .file => |opt_file| opt_file != null, + .dir => |opt_dir| opt_dir != null, + }; + } }; pub const AddInputPathOptions = struct { @@ -524,59 +544,73 @@ pub const Manifest = struct { /// See also: /// * `addPathPost` pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { - const gpa = m.cache.gpa; - try m.files.ensureUnusedCapacity(gpa, 1); + const cache = m.cache; + const gpa = cache.gpa; + try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); try m.input_paths.ensureUnusedCapacity(gpa, 1); const prev_contents_len = m.contents.items.len; - const header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))); + const header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)))); errdefer m.contents.shrinkRetainingCapacity(prev_contents_len); header.* = .{ .flags = .{ - .prefix = try m.cache.findAppendPrefixedPath(&m.contents, path), - .is_directory = options.is_directory, + .prefix = try cache.resolveAppendPath(&m.contents, path), + .is_directory = options.handle.isDirectory(), .metadata_only = options.metadata_only, }, .size = undefined, .inode = undefined, .mtime = undefined, .digest = undefined, + .path_start = .{}, }; - assert(m.contents.items.len % @alignOf(File) == 0); + assert(mem.isAligned(m.contents.items.len, @alignOf(File))); - const gop = try m.files.getOrPutAssumeCapacityContext(@fromBackingInt(prev_contents_len), .{ - .manifest = m, + const gop = m.files.getOrPutAssumeCapacityContext(@fromBackingInt(@intCast(prev_contents_len)), .{ + .contents = m.contents.items, }); + m.files.lockPointers(); + defer m.files.unlockPointers(); + if (gop.found_existing) { m.contents.shrinkRetainingCapacity(prev_contents_len); const existing_input_file = &m.input_paths.items[gop.index]; - if (options.handle) |handle| { - existing_input_file.handle = handle; - existing_input_file.have_handle = true; + switch (options.handle) { + .file => |opt_file| if (opt_file) |file| { + existing_input_file.handle = .{ .file = file }; + existing_input_file.have_handle = true; + }, + .dir => |opt_dir| if (opt_dir) |dir| { + existing_input_file.handle = .{ .dir = dir }; + existing_input_file.have_handle = true; + }, } if (options.request_contents) switch (existing_input_file.contents) { .requested, .not_requested => existing_input_file.contents = .requested, _ => {}, }; - const existing_header = &m.files.keys()[gop.index]; + const existing_header = m.files.keys()[gop.index].get(m.contents.items); if (options.stat) |stat| { existing_input_file.have_stat = true; existing_header.size = stat.size; existing_header.inode = stat.inode; - existing_header.mtime = stat.mtime; + existing_header.mtime = @intCast(stat.mtime.toNanoseconds()); } // If it trips, the same file path has been added to the cache // manifest both as a directory and as a normal file, making the // intended caching behavior ambiguous. - assert(existing_header.flags.is_directory == options.is_directory); + assert(existing_header.flags.is_directory == options.handle.isDirectory()); if (!options.metadata_only) existing_header.flags.metadata_only = false; } else { m.input_paths.appendAssumeCapacity(.{ .request_handle = options.request_handle, - .have_handle = options.handle != null, - .handle = if (options.handle) |handle| handle else undefined, + .have_handle = options.handle.have(), + .handle = switch (options.handle) { + .file => |opt_file| if (opt_file) |file| .{ .file = file } else undefined, + .dir => |opt_dir| if (opt_dir) |dir| .{ .dir = dir } else undefined, + }, .contents = if (options.request_contents) .requested else .not_requested, .have_digest = false, .have_stat = options.stat != null, @@ -585,10 +619,10 @@ pub const Manifest = struct { if (options.stat) |stat| { header.size = stat.size; header.inode = stat.inode; - header.mtime = stat.mtime; + header.mtime = @intCast(stat.mtime.toNanoseconds()); } } - return @fromBackingInt(gop.index); + return @fromBackingInt(@intCast(gop.index)); } pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { @@ -759,8 +793,8 @@ pub const Manifest = struct { if (m.files.count() <= m.input_paths.items.len) return; const off = m.files.keys()[m.input_paths.items.len]; m.contents.shrinkRetainingCapacity(@backingInt(off)); - assert(m.contents.items.len % @alignOf(File) == 0); - m.files.shrinkRetainingCapacity(m.input_paths.items.len); + assert(mem.isAligned(m.contents.items.len, @alignOf(File))); + m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items }); } /// Assumes that `self.hash.hasher` has been updated only with the original digest and that @@ -806,11 +840,13 @@ pub const Manifest = struct { // Guess number of files based on manifest contents len to reduce allocations. // This is not an upper bound; subsequent insertions may potentially allocate. - try m.files.ensureUnusedCapacity(gpa, contents.len / (@sizeOf(File) + 32)); + try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{ + .contents = contents, + }); // This group we would like to cancel as soon as a cache miss is discovered. const PostResult = union(enum) { - checkFile: Check.Status, + checkFile: CheckFileError!Check.Status, }; var post_select_buffer: [10]PostResult = undefined; var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer); @@ -819,12 +855,12 @@ pub const Manifest = struct { while (off + 1 < contents.len) { const file_off: File.Offset = @fromBackingInt(off); - const file = try file_off.getFallible(m); + const file = try file_off.getFallible(contents); if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; const path = try filePathFallible(contents, file_off); if (path.len == 0) return error.InvalidFormat; - try m.files.put(gpa, file_off, {}); + try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path }); post_select_remaining += 1; @@ -867,19 +903,19 @@ pub const Manifest = struct { try input_group.await(io); return .miss; }, - .fail => |diagnostic| { - m.diagnostic = diagnostic; - return error.CacheCheckFailed; - }, }, }; } try input_group.await(io); if (c.status == .miss) return .miss; + if (m.diagnostic != .none) return error.CacheCheckFailed; + + // Needed due to the length mutation above. + const refreshed_contents = m.contents.items; for (m.files.keys()) |file_off| { - m.hash.hasher.update(&file_off.get(m).digest); + m.hash.hasher.update(&file_off.get(refreshed_contents).digest); } return .hit; @@ -896,20 +932,25 @@ pub const Manifest = struct { if (input_path.have_stat) @panic("TODO"); if (input_path.contents != .not_requested) @panic("TODO"); if (input_path.request_handle) @panic("TODO"); - switch (try checkFile(m, c, file_off, file_path)) { + if (checkFile(m, c, file_off, file_path)) |status| switch (status) { .hit => return, .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered), + } else |err| switch (err) { + error.CacheCheckFailed => assert(m.diagnostic != .none), + else => |e| return e, } } + const CheckFileError = error{ Canceled, CacheCheckFailed }; + /// Runs concurrently with other `checkFile`. fn checkFile( m: *Manifest, c: *Check, file_off: File.Offset, file_path: [:0]const u8, - ) error{ Canceled, CacheCheckFailed }!Check.Status { - const file = file_off.get(m); + ) CheckFileError!Check.Status { + const file = file_off.get(m.contents.items); const cache = m.cache; const gpa = cache.gpa; const io = cache.io; @@ -928,7 +969,7 @@ pub const Manifest = struct { const actual_is_directory = actual_stat.kind == .directory; if (actual_is_directory != file.flags.is_directory) return .miss; - if (try file.setStatChanged(m, actual_stat)) return .miss; + if (try file.setStatChanged(m, .init(actual_stat))) return .miss; return .hit; } @@ -954,7 +995,7 @@ pub const Manifest = struct { .err = e, } }), }; - if (try file.setStatChanged(m, actual_stat)) { + if (try file.setStatChanged(m, .init(actual_stat))) { const prev_digest: BinDigest = file.digest; var contents: std.ArrayList(u8) = .empty; defer contents.deinit(gpa); @@ -989,7 +1030,7 @@ pub const Manifest = struct { } }), }; - if (try file.setStatChanged(m, actual_stat)) { + if (try file.setStatChanged(m, .init(actual_stat))) { const prev_digest: BinDigest = file.digest; hashFile(io, opened_file, &file.digest) catch |err| switch (err) { error.Canceled => |e| return e, @@ -1015,8 +1056,9 @@ pub const Manifest = struct { man.hash.hasher = hasher_init; man.hash.hasher.update(bin_digest); man.shrinkFilesToInput(); + const contents = man.contents.items; for (man.files.keys()) |off| { - const file = off.get(man); + const file = off.get(contents); man.hash.hasher.update(&file.digest); } } @@ -1065,6 +1107,10 @@ pub const Manifest = struct { } pub const AddPathPostOptions = struct { + path: union(enum) { + unresolved: Path, + prefixed: PrefixedPath, + }, handle: PathHandle = .{ .file = null }, stat: ?Stat = null, /// If it is a directory, there is a special encoding required for contents, which @@ -1073,29 +1119,30 @@ pub const Manifest = struct { metadata_only: bool = false, }; - pub const AddPathPostError = Io.Cancelable || Allocator.Error; - /// Add a file as a dependency of process being cached, after cache miss /// occurs. /// /// See also: /// * `addInputPath` - pub fn addPathPost(m: *Manifest, path: Path, options: AddPathPostOptions) AddPathPostError!void { + pub fn addPathPost(m: *Manifest, options: AddPathPostOptions) !void { assert(m.manifest_file != null); const cache = m.cache; const gpa = cache.gpa; const io = cache.io; const is_directory = options.handle == .dir; - try m.files.ensureUnusedCapacity(gpa, 1); + try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); - const prev_contents_len = m.contents.items.len; - const new_header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))); - errdefer m.contents.shrinkRetainingCapacity(prev_contents_len); + const new_file_offset: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); + const new_header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)))); + errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); new_header.* = .{ .flags = .{ - .prefix = try cache.findAppendPrefixedPath(&m.contents, path), + .prefix = switch (options.path) { + .unresolved => |unresolved| try cache.resolveAppendPath(&m.contents, unresolved), + .prefixed => |prefixed| try cache.appendPrefixedPath(&m.contents, prefixed), + }, .is_directory = is_directory, .metadata_only = options.metadata_only, }, @@ -1103,31 +1150,32 @@ pub const Manifest = struct { .inode = undefined, .mtime = undefined, .digest = @splat(0), + .path_start = .{}, }; - assert(m.contents.items.len % @alignOf(File) == 0); + assert(mem.isAligned(m.contents.items.len, @alignOf(File))); - const gop = m.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{ - .manifest = m, + const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ + .contents = m.contents.items, }); m.files.lockPointers(); defer m.files.unlockPointers(); - const header = if (gop.found_existing) h: { - m.contents.shrinkRetainingCapacity(prev_contents_len); + const header, const file_offset = if (gop.found_existing) h: { + m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); const existing_off = gop.key_ptr.*; - const header = existing_off.get(m); + const header = existing_off.get(m.contents.items); // If it trips, the same file path has been added to the cache // manifest both as a directory and as a normal file, making the // intended caching behavior ambiguous. assert(header.flags.is_directory == is_directory); if (!options.metadata_only) header.flags.metadata_only = false; - break :h header; - } else new_header; + break :h .{ header, existing_off }; + } else .{ new_header, new_file_offset }; if (options.stat) |stat| { try header.setStat(m, stat); - if (header.metadata_only) { + if (header.flags.metadata_only) { return; } else if (options.contents) |contents| { var hasher = hasher_init; @@ -1138,27 +1186,31 @@ pub const Manifest = struct { } const need_stat = options.stat == null; + const metadata_only = header.flags.metadata_only; + const prefix = header.flags.prefix; switch (options.handle) { .dir => |opt_handle| if (opt_handle) |handle| { - try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); + try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only); } else { - const dir = cache.prefixes()[header.flags.prefix].handle; - const handle = try dir.openDir(io, header.path(), .{ + const dir = cache.prefixes()[prefix].handle; + const sub_path = filePath(m.contents.items, file_offset); + const handle = try dir.openDir(io, sub_path, .{ .access_sub_paths = false, .iterate = true, }); defer handle.close(io); - try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); + try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only); }, .file => |opt_handle| if (opt_handle) |handle| { - try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only); + try populateFile(m, header, need_stat, handle, options.contents, metadata_only); } else { - const dir = cache.prefixes()[header.flags.prefix].handle; - const handle = try dir.openFile(io, header.path(), .{ .mode = .read_only }); + const dir = cache.prefixes()[prefix].handle; + const sub_path = filePath(m.contents.items, file_offset); + const handle = try dir.openFile(io, sub_path, .{ .mode = .read_only }); defer handle.close(io); - try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only); + try populateFile(m, header, need_stat, handle, options.contents, metadata_only); }, } } @@ -1175,7 +1227,7 @@ pub const Manifest = struct { if (need_stat) { const stat = try handle.stat(io); - try file.setStat(m, stat); + try file.setStat(m, .init(stat)); } if (metadata_only) return; if (contents) |bytes| { @@ -1201,7 +1253,7 @@ pub const Manifest = struct { if (need_stat) { const stat = try handle.stat(io); - try file.setStat(m, stat); + try file.setStat(m, .init(stat)); } if (metadata_only) return; if (contents) |bytes| { @@ -1238,28 +1290,26 @@ pub const Manifest = struct { defer resolve_buf.deinit(gpa); var it: DepTokenizer = .{ .bytes = dep_file_contents }; - while (it.next()) |token| { - switch (token) { - // We don't care about targets, we only want the prereqs - // Clang is invoked in single-source mode but other programs may not - .target, .target_must_resolve => {}, - .prereq => |file_path| if (self.manifest_file == null) { - _ = try self.addInputPath(.initCwd(file_path), .{}); - } else try self.addPathPost(file_path), - .prereq_must_resolve => { - resolve_buf.clearRetainingCapacity(); - try token.resolve(gpa, &resolve_buf); - if (self.manifest_file == null) { - _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); - } else try self.addPathPost(resolve_buf.items); - }, - else => |err| { - try err.printError(gpa, &error_buf); - log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items }); - return error.InvalidDepFile; - }, - } - } + while (it.next()) |token| switch (token) { + // We don't care about targets, we only want the prereqs + // Clang is invoked in single-source mode but other programs may not + .target, .target_must_resolve => {}, + .prereq => |file_path| if (self.manifest_file == null) { + _ = try self.addInputPath(.initCwd(file_path), .{}); + } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(file_path) } }), + .prereq_must_resolve => { + resolve_buf.clearRetainingCapacity(); + try token.resolve(gpa, &resolve_buf); + if (self.manifest_file == null) { + _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); + } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(resolve_buf.items) } }); + }, + else => |err| { + try err.printError(gpa, &error_buf); + log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items }); + return error.InvalidDepFile; + }, + }; } /// Returns a binary hash of the inputs. @@ -1368,6 +1418,10 @@ pub const Manifest = struct { pub fn takeFiles(m: *Manifest) SelfContainedFiles { defer m.files = .empty; defer m.contents = .empty; + return borrowFiles(m); + } + + pub fn borrowFiles(m: *const Manifest) SelfContainedFiles { return .{ .files = m.files, .contents = m.contents, @@ -1488,7 +1542,7 @@ pub const Manifest = struct { while (true) { const entries = entry_buffer[0..try reader.read(io, &entry_buffer)]; for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| { - off.* = contents.items.len; + off.* = @intCast(contents.items.len); // As an optimization, make the reservation also count the duplication // of the contents buffer that will be required after sorting. try contents.ensureUnusedCapacity(gpa, (contents.items.len + entry.name.len + 2 - contents_start) * 2); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 46556a225263153d2b404f283d2ae80d1a64eee1..882f0385c16073229f71382b0b796f93b7058915 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1838,12 +1838,12 @@ pub fn buildExeSubprocess( var it = mem.splitScalar(u8, body, 0); while (it.next()) |prefixed_path| { const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1)); - const sub_path = try gpa.dupe(u8, prefixed_path[1..]); - var keep = false; - defer if (!keep) gpa.free(sub_path); - keep = man.addPrefixedPathPost(.{ - .prefix = @backingInt(prefix), - .sub_path = sub_path, + const sub_path = prefixed_path[1..]; + man.addPathPost(.{ + .path = .{ .prefixed = .{ + .prefix = @intCast(@backingInt(prefix)), + .sub_path = sub_path, + } }, }) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, else => |e| { -- 2.54.0 From d142f63f89e967ed0a1bab33dce13fab33aa0410 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 19:19:24 -0700 Subject: [PATCH 07/49] std.Build.Cache: fix adding files no longer adding unwanted padding bytes --- lib/std/Build.zig | 12 ++++-- lib/std/Build/Cache.zig | 73 ++++++++++++++++--------------------- lib/std/Build/Serialize.zig | 3 +- 3 files changed, 41 insertions(+), 47 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f0be0461b9450c0cf4221a9c48493a9c60c70345..47993f66d4edb8905c6c497288825c103e80ba9a 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -58,7 +58,8 @@ available_deps: AvailableDeps, pub const ConfigureDependency = struct { lazy_path: LazyPath, - mode: std.Build.Configuration.PathDep.Mode, + is_directory: bool, + metadata_only: bool, }; pub const ReleaseMode = enum { @@ -102,7 +103,8 @@ pub const Graph = struct { /// Populated by calling one of: /// * `dependOnFileContents` /// * `dependOnFileMetadata` - /// * `dependOnDirectory` + /// * `dependOnDirectoryContents` + /// * `dependOnDirectoryMetadata` configure_dependencies: ArrayList(ConfigureDependency) = .empty, /// If the cache is poisoned means that the **configure logic** had side @@ -2602,7 +2604,8 @@ pub fn dependOnFileContents(b: *Build, lazy_path: LazyPath) void { const graph = b.graph; graph.configure_dependencies.append(graph.arena, .{ .lazy_path = lazy_path.dupe(graph), - .mode = .contents, + .is_directory = false, + .metadata_only = false, }) catch @panic("OOM"); } @@ -2627,7 +2630,8 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void { const graph = b.graph; graph.configure_dependencies.append(graph.arena, .{ .lazy_path = lazy_path.dupe(graph), - .mode = .metadata, + .is_directory = false, + .metadata_only = true, }) catch @panic("OOM"); } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 8a3a80007e09e1f7b257c30f9a26720f0bf288d3..46cb67f21e4c38dbf8bc877b2b9e3ff5ef6f42b7 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -363,9 +363,9 @@ pub const Manifest = struct { pub const File = extern struct { size: u64, inode: u64, - digest: BinDigest, /// Nanoseconds. mtime: i64, + digest: BinDigest, /// Starting with this field and continuing into the path, excluding the null byte, /// is the string that is hashed for the manifest digest. flags: Flags, @@ -546,35 +546,29 @@ pub const Manifest = struct { pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { const cache = m.cache; const gpa = cache.gpa; + const is_directory = options.handle.isDirectory(); + try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); try m.input_paths.ensureUnusedCapacity(gpa, 1); - const prev_contents_len = m.contents.items.len; - const header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)))); - errdefer m.contents.shrinkRetainingCapacity(prev_contents_len); + const new_file_offset: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); + try m.contents.appendNTimes(gpa, 0, @offsetOf(File, "path_start")); + errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); - header.* = .{ - .flags = .{ - .prefix = try cache.resolveAppendPath(&m.contents, path), - .is_directory = options.handle.isDirectory(), - .metadata_only = options.metadata_only, - }, - .size = undefined, - .inode = undefined, - .mtime = undefined, - .digest = undefined, - .path_start = .{}, - }; + const new_prefix = try cache.resolveAppendPath(&m.contents, path); assert(mem.isAligned(m.contents.items.len, @alignOf(File))); + new_file_offset.get(m.contents.items).flags = .{ + .prefix = new_prefix, + .is_directory = is_directory, + .metadata_only = options.metadata_only, + }; - const gop = m.files.getOrPutAssumeCapacityContext(@fromBackingInt(@intCast(prev_contents_len)), .{ - .contents = m.contents.items, - }); + const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ .contents = m.contents.items }); m.files.lockPointers(); defer m.files.unlockPointers(); if (gop.found_existing) { - m.contents.shrinkRetainingCapacity(prev_contents_len); + m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); const existing_input_file = &m.input_paths.items[gop.index]; switch (options.handle) { .file => |opt_file| if (opt_file) |file| { @@ -600,7 +594,7 @@ pub const Manifest = struct { // If it trips, the same file path has been added to the cache // manifest both as a directory and as a normal file, making the // intended caching behavior ambiguous. - assert(existing_header.flags.is_directory == options.handle.isDirectory()); + assert(existing_header.flags.is_directory == is_directory); if (!options.metadata_only) existing_header.flags.metadata_only = false; } else { @@ -617,6 +611,7 @@ pub const Manifest = struct { }); assert(m.input_paths.items.len - 1 == gop.index); if (options.stat) |stat| { + const header = new_file_offset.get(m.contents.items); header.size = stat.size; header.inode = stat.inode; header.mtime = @intCast(stat.mtime.toNanoseconds()); @@ -1134,33 +1129,25 @@ pub const Manifest = struct { try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); const new_file_offset: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); - const new_header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)))); + try m.contents.appendNTimes(gpa, 0, @offsetOf(File, "path_start")); errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); - new_header.* = .{ - .flags = .{ - .prefix = switch (options.path) { - .unresolved => |unresolved| try cache.resolveAppendPath(&m.contents, unresolved), - .prefixed => |prefixed| try cache.appendPrefixedPath(&m.contents, prefixed), - }, - .is_directory = is_directory, - .metadata_only = options.metadata_only, - }, - .size = undefined, - .inode = undefined, - .mtime = undefined, - .digest = @splat(0), - .path_start = .{}, + const new_prefix = switch (options.path) { + .unresolved => |unresolved| try cache.resolveAppendPath(&m.contents, unresolved), + .prefixed => |prefixed| try cache.appendPrefixedPath(&m.contents, prefixed), }; assert(mem.isAligned(m.contents.items.len, @alignOf(File))); + new_file_offset.get(m.contents.items).flags = .{ + .prefix = new_prefix, + .is_directory = is_directory, + .metadata_only = options.metadata_only, + }; - const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ - .contents = m.contents.items, - }); + const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ .contents = m.contents.items }); m.files.lockPointers(); defer m.files.unlockPointers(); - const header, const file_offset = if (gop.found_existing) h: { + const file_offset = if (gop.found_existing) h: { m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); const existing_off = gop.key_ptr.*; const header = existing_off.get(m.contents.items); @@ -1170,8 +1157,10 @@ pub const Manifest = struct { assert(header.flags.is_directory == is_directory); if (!options.metadata_only) header.flags.metadata_only = false; - break :h .{ header, existing_off }; - } else .{ new_header, new_file_offset }; + break :h existing_off; + } else new_file_offset; + + const header = file_offset.get(m.contents.items); if (options.stat) |stat| { try header.setStat(m, stat); diff --git a/lib/std/Build/Serialize.zig b/lib/std/Build/Serialize.zig index 7582df0a43f1ef3264da6e7332c4fab8bf87d108..eafc8e0ab7063ac42f80183e125f1c1b6c11358a 100644 --- a/lib/std/Build/Serialize.zig +++ b/lib/std/Build/Serialize.zig @@ -34,7 +34,8 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi .cwd_relative => .cwd, .relative => |r| r.base, }, - .mode = src.mode, + .is_directory = src.is_directory, + .metadata_only = src.metadata_only, }, .sub = switch (src.lazy_path) { .src_path => |sp| try wc.addString(sp.sub_path), -- 2.54.0 From 48d6b1aa4a67758b66bbbcd49a6933ef6341cf87 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 19:21:06 -0700 Subject: [PATCH 08/49] std.Build.Cache: rename addPathPost to addDiscoveredPath --- lib/compiler/Maker.zig | 2 +- lib/std/Build/Cache.zig | 20 ++++++++++---------- lib/std/zig.zig | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 97760f7d15cd2f4e58858ba2a4494b6f0cd53740..76fb47ac7546d620a2c9c7ba5253ee5c0aa84ff1 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1498,7 +1498,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { if (config_man) |man| for (configuration.path_deps) |path_dep| { const path = try confPathDepToCachePath(arena, graph, &configuration, path_dep); - try man.addPathPost(.{ + try man.addDiscoveredPath(.{ .path = .{ .unresolved = path }, .handle = if (path_dep.flags.is_directory) .{ .dir = null } else .{ .file = null }, .metadata_only = path_dep.flags.metadata_only, diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 46cb67f21e4c38dbf8bc877b2b9e3ff5ef6f42b7..4930da00e23e6b146b3f2d74ec717a6607eaeebe 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -297,7 +297,7 @@ pub const Manifest = struct { /// concatenated. Total byte size will be less than `max_input_content_len` /// otherwise an error is returned. /// - /// Data is invalidated when `addPathPost` is called. + /// Data is invalidated when `addDiscoveredPath` is called. all_input_content: std.ArrayList(u8) = .empty, max_input_content_len: usize = std.math.maxInt(u32), @@ -542,7 +542,7 @@ pub const Manifest = struct { /// by the entry separator. /// /// See also: - /// * `addPathPost` + /// * `addDiscoveredPath` pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { const cache = m.cache; const gpa = cache.gpa; @@ -732,16 +732,16 @@ pub const Manifest = struct { // We're going to construct a second hash. Its input will begin with the digest we've // already computed (`bin_digest`), and then it'll have the digests of each input file, - // including "post" files (see `addPathPost`). If this is a hit, we learn the set of "post" + // including "post" files (see `addDiscoveredPath`). If this is a hit, we learn the set of "post" // files from the manifest on disk. If this is a miss, we'll learn those from future calls - // to `addPathPost` etc. As such, the state of `man.hash.hasher` after this function + // to `addDiscoveredPath` etc. As such, the state of `man.hash.hasher` after this function // depends on whether this is a hit or a miss. // // If we return `CacheStatus.hit`, then `man.hash.hasher` must already include // the digests of the "post" files, so the caller can call `final`. Otherwise, on a cache // miss, `man.hash.hasher` will include the digests of all non-"post" files -- that is, // the ones we've already been told about. The rest will be discovered through calls to - // `addPathPost` etc, which will update the hasher. After all files are added, the user can + // `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user can // use `final`, and will at some point `writeManifest` the file list to disk. man.hash.hasher = hasher_init; @@ -1119,7 +1119,7 @@ pub const Manifest = struct { /// /// See also: /// * `addInputPath` - pub fn addPathPost(m: *Manifest, options: AddPathPostOptions) !void { + pub fn addDiscoveredPath(m: *Manifest, options: AddPathPostOptions) !void { assert(m.manifest_file != null); const cache = m.cache; const gpa = cache.gpa; @@ -1285,13 +1285,13 @@ pub const Manifest = struct { .target, .target_must_resolve => {}, .prereq => |file_path| if (self.manifest_file == null) { _ = try self.addInputPath(.initCwd(file_path), .{}); - } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(file_path) } }), + } else try self.addDiscoveredPath(.{ .path = .{ .unresolved = .initCwd(file_path) } }), .prereq_must_resolve => { resolve_buf.clearRetainingCapacity(); try token.resolve(gpa, &resolve_buf); if (self.manifest_file == null) { _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); - } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(resolve_buf.items) } }); + } else try self.addDiscoveredPath(.{ .path = .{ .unresolved = .initCwd(resolve_buf.items) } }); }, else => |err| { try err.printError(gpa, &error_buf); @@ -1846,7 +1846,7 @@ test "Manifest with files added after initial hash work" { // There should be nothing in the cache try testing.expectEqual(false, try ch.hit(.none)); - _ = try ch.addPathPost(temp_file2); + _ = try ch.addDiscoveredPath(temp_file2); digest1 = ch.final(); try ch.writeManifest(); @@ -1884,7 +1884,7 @@ test "Manifest with files added after initial hash work" { // A file that we depend on has been updated, so the cache should not contain an entry for it try testing.expectEqual(false, try ch.hit(.none)); - _ = try ch.addPathPost(temp_file2); + _ = try ch.addDiscoveredPath(temp_file2); digest3 = ch.final(); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 882f0385c16073229f71382b0b796f93b7058915..15218e43c432ad51d2210142b02f2a05cdbc3973 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1839,7 +1839,7 @@ pub fn buildExeSubprocess( while (it.next()) |prefixed_path| { const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1)); const sub_path = prefixed_path[1..]; - man.addPathPost(.{ + man.addDiscoveredPath(.{ .path = .{ .prefixed = .{ .prefix = @intCast(@backingInt(prefix)), .sub_path = sub_path, -- 2.54.0 From 4dbe872dfed8ef46b986a7eebe4bc7475d2c5db6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 19:47:08 -0700 Subject: [PATCH 09/49] std.Build.Cache: fix calculation of File size when reading manifest --- lib/std/Build/Cache.zig | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 4930da00e23e6b146b3f2d74ec717a6607eaeebe..84c7fb5a4e1718993a09af3bbec65654d27a3661 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -409,6 +409,7 @@ pub const Manifest = struct { pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File { if (@backingInt(offset) + @sizeOf(File) >= contents.len) return error.InvalidFormat; + if (!mem.isAligned(@backingInt(offset), @alignOf(File))) return error.InvalidFormat; return get(offset, contents); } }; @@ -458,6 +459,14 @@ pub const Manifest = struct { return true; } } + + /// `path_len` does not include the null byte. + fn sizeOf(path_len: usize) usize { + const end = @offsetOf(File, "path_start") + path_len; + const needed_alignment = @alignOf(File) - (end % @alignOf(File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + return end + needed_alignment; + } }; pub const Diagnostic = union(enum) { @@ -809,7 +818,7 @@ pub const Manifest = struct { }; const contents = m.contents.items; - var off: u32 = 0; + var off: usize = 0; var c: Check = .{}; // This group we always want to compute the hash digests, even on a @@ -821,7 +830,7 @@ pub const Manifest = struct { // otherwise it's invalid format. for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| { if (off + 1 >= contents.len) return error.InvalidFormat; - const file_off: File.Offset = @fromBackingInt(off); + const file_off: File.Offset = @fromBackingInt(@intCast(off)); const file = try file_off.getFallible(contents); if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; const path = try filePathFallible(contents, file_off); @@ -830,7 +839,7 @@ pub const Manifest = struct { input_group.async(io, checkInputFile, .{ m, &c, file_off, path, input_path }); - off = @intCast(@as(usize, off) + @sizeOf(File) + path.len + 1); + off += File.sizeOf(path.len); } // Guess number of files based on manifest contents len to reduce allocations. @@ -849,7 +858,7 @@ pub const Manifest = struct { defer post_select.cancelDiscard(); while (off + 1 < contents.len) { - const file_off: File.Offset = @fromBackingInt(off); + const file_off: File.Offset = @fromBackingInt(@intCast(off)); const file = try file_off.getFallible(contents); if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; const path = try filePathFallible(contents, file_off); @@ -860,7 +869,7 @@ pub const Manifest = struct { post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path }); post_select_remaining += 1; - off = @intCast(@as(usize, off) + @sizeOf(File) + path.len + 1); + off += File.sizeOf(path.len); } // Final terminating zero byte to distinguish empty manifest file from -- 2.54.0 From 2a82193af49b514c2c784746abd22fdec7c31500 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 20:12:52 -0700 Subject: [PATCH 10/49] std.Build.Cache: fix deadlock from using Io.Select Io.Select.async has a fatal flaw: if the queue size isn't big enough to hold the result of an eager async call, then it deadlocks. The workaround in this thread is using concurrent() which should be reverted because it will spawn too many threads. --- lib/std/Build/Cache.zig | 43 +++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 84c7fb5a4e1718993a09af3bbec65654d27a3661..16d2e3a1c5c46061592f95fb121b63f446270b76 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -795,10 +795,12 @@ pub const Manifest = struct { fn shrinkFilesToInput(m: *Manifest) void { if (m.files.count() <= m.input_paths.items.len) return; + // Reads from files hash map whose data is destroyed on the next line. const off = m.files.keys()[m.input_paths.items.len]; - m.contents.shrinkRetainingCapacity(@backingInt(off)); - assert(mem.isAligned(m.contents.items.len, @alignOf(File))); + // Reads from the unshrunken contents whose data is destroyed on the next line. m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items }); + m.contents.shrinkRetainingCapacity(@backingInt(off)); + assert(mem.isAligned(m.contents.items.len, @alignOf(File))); } /// Assumes that `self.hash.hasher` has been updated only with the original digest and that @@ -852,7 +854,7 @@ pub const Manifest = struct { const PostResult = union(enum) { checkFile: CheckFileError!Check.Status, }; - var post_select_buffer: [10]PostResult = undefined; + var post_select_buffer: [16]PostResult = undefined; var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer); var post_select_remaining: usize = 0; defer post_select.cancelDiscard(); @@ -866,8 +868,33 @@ pub const Manifest = struct { try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); - post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path }); - post_select_remaining += 1; + // In order to call async here we would need to ensure `post_select_buffer` + // has capacity for as many elements as files being checked. Since we don't want + // to dynamically allocate that buffer, we use concurrent + fallback here. + if (post_select.concurrent(.checkFile, checkFile, .{ m, &c, file_off, path })) |_| { + post_select_remaining += 1; + } else |err| switch (err) { + error.ConcurrencyUnavailable => { + // Detect if input group already had a miss. In this case we still wait + // for those digests to be updated, but cancel the non input group. + switch (@atomicLoad(Check.Status, &c.status, .unordered)) { + .miss => { + post_select.cancelDiscard(); + try input_group.await(io); + return .miss; + }, + .hit => {}, + } + switch (try checkFile(m, &c, file_off, path)) { + .hit => continue, + .miss => { + post_select.cancelDiscard(); + try input_group.await(io); + return .miss; + }, + } + }, + } off += File.sizeOf(path.len); } @@ -883,7 +910,7 @@ pub const Manifest = struct { // Don't track the trailing zero byte in contents. m.contents.items.len -= 1; - var post_await_buffer: [10]PostResult = undefined; + var post_await_buffer: [16]PostResult = undefined; while (post_select_remaining > 0) { const n = try post_select.awaitMany(&post_await_buffer, 1); post_select_remaining -= n; @@ -896,11 +923,11 @@ pub const Manifest = struct { try input_group.await(io); return .miss; }, - .hit => continue, + .hit => {}, } for (post_await_buffer[0..n]) |u| switch (u) { - .checkFile => |result| switch (result) { + .checkFile => |result| switch (try result) { .hit => continue, .miss => { post_select.cancelDiscard(); -- 2.54.0 From 7eca05812af81a4ed64fcf86288923679b146c03 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2026 20:37:25 -0700 Subject: [PATCH 11/49] build.zig: update API usage --- test/src/Cases.zig | 2 +- test/tests.zig | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/src/Cases.zig b/test/src/Cases.zig index e9ce39718a8340b17e2c4b07afd7499f0861da74..5dbff266e78f9b3f585bdbd87b99ac8e2f27f09f 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -348,7 +348,7 @@ fn addFromDirInner( try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path)); }, .directory => { - b.dependOnDirectory(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); + b.dependOnDirectoryContents(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); }, else => continue, } diff --git a/test/tests.zig b/test/tests.zig index 18c9fe113a8d607caa374aaca7cad68386707e40..645152b8457384367c4fa7a7933c5ca2034ad56d 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -3339,7 +3339,7 @@ pub fn addCases( var cases = @import("src/Cases.zig").init(gpa, arena, io); - b.dependOnDirectory(b.path("test/cases")); + b.dependOnDirectoryContents(b.path("test/cases")); var dir = try b.root.openDir(io, "test/cases", .{ .iterate = true }); defer dir.close(io); @@ -3401,7 +3401,7 @@ pub fn addIncrementalTests( }), }); - b.dependOnDirectory(b.path("test/incremental")); + b.dependOnDirectoryContents(b.path("test/incremental")); var dir = try b.root.openDir(io, "test/incremental", .{ .iterate = true }); defer dir.close(io); @@ -3417,7 +3417,7 @@ pub fn addIncrementalTests( switch (entry.kind) { .file => {}, .directory => { - b.dependOnDirectory(b.path(b.pathJoin(&.{ "test", "incremental", entry.path }))); + b.dependOnDirectoryContents(b.path(b.pathJoin(&.{ "test", "incremental", entry.path }))); }, else => continue, } -- 2.54.0 From f7bdb939e14c247e3dfe111cb04c08da6ec7cc85 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 4 Sep 2026 19:09:38 -0700 Subject: [PATCH 12/49] add cache-cat subcommand for printing .zig-cache/h/* files to zon format --- lib/compiler/Maker.zig | 98 ++++++++++++++++++++++++++++++++++++++++- lib/std/Build/Cache.zig | 11 +++-- src/main.zig | 10 +++-- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 76fb47ac7546d620a2c9c7ba5253ee5c0aa84ff1..738b381762fd53482e241d9c3fd7125308890191 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -199,12 +199,13 @@ pub fn main(init: process.Init.Minimal) !void { .random_seed = parseRandomSeed(seed_arg), }; - const cmd = stringToEnum(enum { libc, init, fetch, build }, cmd_name) orelse + const cmd = stringToEnum(enum { libc, init, fetch, build, @"cache-cat" }, cmd_name) orelse fatal("bad command name: {q}", .{cmd_name}); switch (cmd) { .libc => return cmdLibC(gpa, &graph, args[arg_i..]), .init => return cmdInit(gpa, &graph, args[arg_i..]), .fetch => return cmdFetch(gpa, &graph, args[arg_i..]), + .@"cache-cat" => return cmdCacheCat(gpa, &graph, args[arg_i..]), .build => {}, } @@ -1880,6 +1881,101 @@ fn cmdFetch(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { return process.cleanExit(io); } +fn cmdCacheCat(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { + const io = graph.io; + + var arg_i: usize = 0; + var contents: std.ArrayList(u8) = .empty; + defer contents.deinit(gpa); + + while (nextArg(args, &arg_i)) |arg| { + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try Io.File.stdout().writeStreamingAll(io, usage_cache_cat); + return process.cleanExit(io); + } else { + fatal("unrecognized parameter: {q}", .{arg}); + } + } else { + var file = Dir.cwd().openFile(io, arg, .{}) catch |err| fatal("opening {q} failed: {t}", .{ arg, err }); + defer file.close(io); + + var manifest_reader = file.reader(io, &.{}); // Reads positionally from zero. + contents.clearRetainingCapacity(); + manifest_reader.interface.appendRemainingUnlimited(gpa, &contents) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => switch (manifest_reader.err.?) { + error.Canceled => |e| return e, + else => |e| fatal("reading from {q} failed: {t}", .{ arg, e }), + }, + }; + const hex_digest = Dir.path.basename(arg); + cacheCatOne(hex_digest, contents.items, initStdoutWriter(io)) catch |err| switch (err) { + error.WriteFailed => fatal("writing to stdout failed: {t}", .{stdout_writer_allocation.err.?}), + else => |e| fatal("parsing {q} failed: {t}", .{ arg, e }), + }; + try stdout_writer_allocation.flush(); + } + } +} + +fn cacheCatOne(input_hex_digest: []const u8, contents: []const u8, writer: *Io.Writer) !void { + var bin_digest: Cache.BinDigest = undefined; + _ = try fmt.hexToBytes(&bin_digest, input_hex_digest); + + var hh: Cache.HashHelper = .{}; + hh.hasher.update(&bin_digest); + + var serializer: std.zon.Serializer = .{ .writer = writer }; + var top_level = try serializer.beginStruct(.{}); + try top_level.field("input_hash", input_hex_digest, .{}); + var files_tuple = try top_level.beginTupleField("files", .{}); + var off: usize = 0; + while (off + 1 < contents.len) { + const file_off: Cache.Manifest.File.Offset = @fromBackingInt(@intCast(off)); + const file = try file_off.getFallibleConst(contents); + const path = try Cache.Manifest.filePathFallible(contents, file_off); + if (path.len == 0) return error.InvalidFormat; + + var file_obj = try files_tuple.beginStructField(.{ .whitespace_style = .{ .wrap = false } }); + try file_obj.field("size", file.size, .{}); + try file_obj.field("inode", file.inode, .{}); + try file_obj.field("mtime", file.mtime, .{}); + const hex_digest = Cache.binToHex(file.digest); + try file_obj.field("digest", @as([]const u8, &hex_digest), .{}); + if (file.flags.is_directory) try file_obj.field("directory", true, .{}); + if (file.flags.metadata_only) try file_obj.field("metadata", true, .{}); + try file_obj.field("prefix", file.flags.prefix, .{}); + try file_obj.field("path", path, .{}); + try file_obj.end(); + + hh.hasher.update(&file.digest); + + off += Cache.Manifest.File.sizeOf(path.len); + } + + try files_tuple.end(); + + var discovered_bin_digest: Cache.BinDigest = undefined; + hh.hasher.final(&discovered_bin_digest); + const discovered_hex_digest = Cache.binToHex(discovered_bin_digest); + try top_level.field("discovered_hash", @as([]const u8, &discovered_hex_digest), .{}); + + try top_level.end(); + try writer.writeByte('\n'); +} + +const usage_cache_cat = + \\Usage: zig cache-cat + \\ + \\ Prints .zig-cache/h/* manifest files in text form. + \\ + \\Options: + \\ -h, --help Print this help and exit + \\ + \\ +; + const usage_fetch = \\Usage: zig fetch [options] \\Usage: zig fetch [options] diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 16d2e3a1c5c46061592f95fb121b63f446270b76..ab03895f423fa0bfd5808b2209d5b0e23b2bb3c9 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -408,9 +408,14 @@ pub const Manifest = struct { } pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File { + // TODO make @constCast support in-memory coercion across error unions and optionals + return @constCast(try getFallibleConst(offset, contents)); + } + + pub fn getFallibleConst(offset: Offset, contents: []const u8) error{InvalidFormat}!*const File { if (@backingInt(offset) + @sizeOf(File) >= contents.len) return error.InvalidFormat; if (!mem.isAligned(@backingInt(offset), @alignOf(File))) return error.InvalidFormat; - return get(offset, contents); + return getConst(offset, contents); } }; @@ -461,7 +466,7 @@ pub const Manifest = struct { } /// `path_len` does not include the null byte. - fn sizeOf(path_len: usize) usize { + pub fn sizeOf(path_len: usize) usize { const end = @offsetOf(File, "path_start") + path_len; const needed_alignment = @alignOf(File) - (end % @alignOf(File)); assert(needed_alignment >= 1); // Always need at least a null byte. @@ -1615,7 +1620,7 @@ pub const Manifest = struct { hasher.update(contents[hash_start..hash_end]); } - fn filePathFallible(contents: []const u8, off: File.Offset) error{InvalidFormat}![:0]const u8 { + pub fn filePathFallible(contents: []const u8, off: File.Offset) error{InvalidFormat}![:0]const u8 { const path_start = @backingInt(off) + @offsetOf(File, "path_start"); const path_end = mem.findScalarPos(u8, contents, path_start, 0) orelse return error.InvalidFormat; return contents[path_start..path_end :0]; diff --git a/src/main.zig b/src/main.zig index dc691fd8a6ed15dc099dd3d60484af285329c836..164c2f15ee081a7e0ac22dacad8659d1e9d8f596 100644 --- a/src/main.zig +++ b/src/main.zig @@ -107,6 +107,7 @@ const normal_usage = \\ \\ env Print lib path, std path, cache directory, and version \\ help Print this help and exit + \\ cache-cat Print a zig-cache manifest file as zon \\ std View standard library documentation in a browser \\ libc Display native libc paths file or validate one \\ targets List available compilation targets @@ -242,6 +243,10 @@ const Cmd = enum { ar, build, + @"cache-cat", + fetch, + init, + libc, clang, @"-cc1", @@ -258,10 +263,7 @@ const Cmd = enum { fmt, objcopy, objdump, - fetch, - libc, std, - init, targets, version, env, @@ -351,7 +353,7 @@ fn mainArgs( dev.check(.ar_command); return process.exit(try llvmArMain(arena, args)); }, - .build, .fetch, .init, .libc => { + .build, .fetch, .init, .libc, .@"cache-cat" => { return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "maker", .root_src_path = "Maker.zig", -- 2.54.0 From 5b55509e092ef0a5b922f517f5599234778b2f00 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 4 Sep 2026 19:59:47 -0700 Subject: [PATCH 13/49] Maker: use quotes when reporting error with file path --- lib/compiler/Maker.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 738b381762fd53482e241d9c3fd7125308890191..9a3d72714bf56443572ecb4a52679926a4ab5dc6 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1565,10 +1565,10 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { const configuration = c: { var file = configuration_path.root_dir.handle.openFile(io, configuration_path.sub_path, .{}) catch |err| - fatal("failed to open configuration file {f}: {t}", .{ configuration_path, err }); + fatal("failed to open configuration file {qf}: {t}", .{ configuration_path, err }); defer file.close(io); break :c Configuration.loadFile(arena, io, file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ configuration_path, err }); + fatal("failed to load configuration file {qf}: {t}", .{ configuration_path, err }); }; // Technically if the configuration is marked as poisoned, we could // already delete the file now, but we leave it around in case the -- 2.54.0 From b42297432262725cc276ed1989563453199c94ba Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 4 Sep 2026 20:08:35 -0700 Subject: [PATCH 14/49] std.Build.Cache: add missing hash-computing logic --- lib/std/Build/Cache.zig | 86 +++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index ab03895f423fa0bfd5808b2209d5b0e23b2bb3c9..a0aef6a1058721adba1a9f6ff4285b409004586c 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -365,6 +365,8 @@ pub const Manifest = struct { inode: u64, /// Nanoseconds. mtime: i64, + /// To simplify the hashing logic, this value is computed from size, inode, and mtime + /// in `hashFromMetadata` in the case that `Flags.metadata_only` is `true`. digest: BinDigest, /// Starting with this field and continuing into the path, excluding the null byte, /// is the string that is hashed for the manifest digest. @@ -440,6 +442,14 @@ pub const Manifest = struct { } }; + fn hashFromMetadata(file: *File) void { + var hasher = hasher_init; + hasher.update(mem.asBytes(&file.size)); + hasher.update(mem.asBytes(&file.inode)); + hasher.update(mem.asBytes(&file.mtime)); + hasher.final(&file.digest); + } + fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void { file.size = stat.size; file.inode = stat.inode; @@ -660,16 +670,19 @@ pub const Manifest = struct { pub fn checkProgressless(man: *Manifest) Check.Error!Check.Status { assert(man.manifest_file == null); + // This is *not* hashing the contents of the input files. It is the + // flags (including prefix) and path only. for (man.files.keys()[0..man.input_paths.items.len]) |file_off| { - man.digestHash(file_off, &man.hash.hasher); + man.hashFlagsAndPath(file_off, &man.hash.hasher); } man.diagnostic = .none; - var bin_digest: BinDigest = undefined; - man.hash.hasher.final(&bin_digest); - const hex_digest = binToHex(bin_digest); - const manifest_file_path = &hex_digest; + var input_digest: BinDigest = undefined; + man.hash.hasher.final(&input_digest); + const input_hex_digest = binToHex(input_digest); + + const manifest_file_path = &input_hex_digest; const io = man.cache.io; // We'll try to open the cache with an exclusive lock, but if that would block @@ -744,22 +757,20 @@ pub const Manifest = struct { man.want_refresh_timestamp = true; - // We're going to construct a second hash. Its input will begin with the digest we've - // already computed (`bin_digest`), and then it'll have the digests of each input file, - // including "post" files (see `addDiscoveredPath`). If this is a hit, we learn the set of "post" - // files from the manifest on disk. If this is a miss, we'll learn those from future calls - // to `addDiscoveredPath` etc. As such, the state of `man.hash.hasher` after this function - // depends on whether this is a hit or a miss. + // We're going to construct a second hash. Its input will begin with the digest we've already computed + // (`input_digest`), and then it'll have the digests of each input file, including discovered files (see + // `addDiscoveredPath`). If this is a hit, we learn the set of discovered files from the manifest on disk. If + // this is a miss, we'll learn those from future calls to `addDiscoveredPath` etc. As such, the state of + // `man.hash.hasher` after this function depends on whether this is a hit or a miss. // - // If we return `CacheStatus.hit`, then `man.hash.hasher` must already include - // the digests of the "post" files, so the caller can call `final`. Otherwise, on a cache - // miss, `man.hash.hasher` will include the digests of all non-"post" files -- that is, - // the ones we've already been told about. The rest will be discovered through calls to - // `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user can - // use `final`, and will at some point `writeManifest` the file list to disk. + // If we return `CacheStatus.hit`, then `man.hash.hasher` must already include the digests of the discovered + // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests + // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered + // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user + // can use `final`, and will at some point `writeManifest` the file list to disk. man.hash.hasher = hasher_init; - man.hash.hasher.update(&bin_digest); + man.hash.hasher.update(&input_digest); hit: { digests: { @@ -772,7 +783,7 @@ pub const Manifest = struct { // Before trying again, we must reset `man.hash.hasher` and `man.files`. // This is basically just the first half of `unhit`. man.hash.hasher = hasher_init; - man.hash.hasher.update(&bin_digest); + man.hash.hasher.update(&input_digest); man.shrinkFilesToInput(); switch (try man.checkLocked()) { .hit => break :hit, @@ -784,7 +795,7 @@ pub const Manifest = struct { // unless it returns an error. man.manifest_dirty = true; // All input file digests are already populated by `checkLocked`, so we can call `unhit` directly. - unhit(man, &bin_digest); + unhit(man, &input_digest); return .miss; } @@ -1005,9 +1016,9 @@ pub const Manifest = struct { const actual_is_directory = actual_stat.kind == .directory; if (actual_is_directory != file.flags.is_directory) return .miss; - if (try file.setStatChanged(m, .init(actual_stat))) return .miss; - - return .hit; + const changed = try file.setStatChanged(m, .init(actual_stat)); + file.hashFromMetadata(); + return if (changed) .miss else .hit; } if (file.flags.is_directory) { @@ -1082,15 +1093,15 @@ pub const Manifest = struct { return .hit; } - /// Reset `man.hash.hasher` to the state it should be in after `hit` returns `Check.Status.miss`. + /// Reset `man.hash.hasher` to the state it should be in after `check` returns `Check.Status.miss`. /// The hasher contains the original input digest, and all original input file digests (i.e. - /// not including post files). + /// not including discovered files). /// - /// Assumes that `bin_digest` is populated for all input files. - pub fn unhit(man: *Manifest, bin_digest: *const BinDigest) void { + /// Assumes that `digest` is populated for all input files. + pub fn unhit(man: *Manifest, input_digest: *const BinDigest) void { // Reset the hash. man.hash.hasher = hasher_init; - man.hash.hasher.update(bin_digest); + man.hash.hasher.update(input_digest); man.shrinkFilesToInput(); const contents = man.contents.items; for (man.files.keys()) |off| { @@ -1206,13 +1217,14 @@ pub const Manifest = struct { if (options.stat) |stat| { try header.setStat(m, stat); if (header.flags.metadata_only) { - return; + header.hashFromMetadata(); } else if (options.contents) |contents| { var hasher = hasher_init; hasher.update(contents); hasher.final(&header.digest); - return; } + m.hash.hasher.update(&header.digest); + return; } const need_stat = options.stat == null; @@ -1243,6 +1255,8 @@ pub const Manifest = struct { try populateFile(m, header, need_stat, handle, options.contents, metadata_only); }, } + + m.hash.hasher.update(&header.digest); } fn populateFile( @@ -1259,7 +1273,10 @@ pub const Manifest = struct { const stat = try handle.stat(io); try file.setStat(m, .init(stat)); } - if (metadata_only) return; + if (metadata_only) { + file.hashFromMetadata(); + return; + } if (contents) |bytes| { var hasher = hasher_init; hasher.update(bytes); @@ -1285,7 +1302,10 @@ pub const Manifest = struct { const stat = try handle.stat(io); try file.setStat(m, .init(stat)); } - if (metadata_only) return; + if (metadata_only) { + file.hashFromMetadata(); + return; + } if (contents) |bytes| { var hasher = hasher_init; hasher.update(bytes); @@ -1611,7 +1631,7 @@ pub const Manifest = struct { hasher.final(bin_digest); } - fn digestHash(m: *const Manifest, off: File.Offset, hasher: *Hasher) void { + fn hashFlagsAndPath(m: *const Manifest, off: File.Offset, hasher: *Hasher) void { const contents = m.contents.items; const flags_off = @offsetOf(File, "flags"); comptime assert(@offsetOf(File, "path_start") - flags_off == 1); -- 2.54.0 From 28e360fb617d8b5ebd0bd46d2d3a918590f55410 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 4 Sep 2026 21:03:03 -0700 Subject: [PATCH 15/49] std.Build.Cache: eliminate unbounded concurrency would be nice if Io.Select.async had a variant where it returned the eager result rather than trying to put it in the queue so that the queue buffer doesn't need to be dynamically allocated --- lib/std/Build/Cache.zig | 71 ++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index a0aef6a1058721adba1a9f6ff4285b409004586c..417f309200cd63ff0457fbbec51d957b7d0a72cc 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -866,15 +866,7 @@ pub const Manifest = struct { .contents = contents, }); - // This group we would like to cancel as soon as a cache miss is discovered. - const PostResult = union(enum) { - checkFile: CheckFileError!Check.Status, - }; - var post_select_buffer: [16]PostResult = undefined; - var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer); - var post_select_remaining: usize = 0; - defer post_select.cancelDiscard(); - + // Validate and count discovered files. while (off + 1 < contents.len) { const file_off: File.Offset = @fromBackingInt(@intCast(off)); const file = try file_off.getFallible(contents); @@ -884,34 +876,6 @@ pub const Manifest = struct { try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); - // In order to call async here we would need to ensure `post_select_buffer` - // has capacity for as many elements as files being checked. Since we don't want - // to dynamically allocate that buffer, we use concurrent + fallback here. - if (post_select.concurrent(.checkFile, checkFile, .{ m, &c, file_off, path })) |_| { - post_select_remaining += 1; - } else |err| switch (err) { - error.ConcurrencyUnavailable => { - // Detect if input group already had a miss. In this case we still wait - // for those digests to be updated, but cancel the non input group. - switch (@atomicLoad(Check.Status, &c.status, .unordered)) { - .miss => { - post_select.cancelDiscard(); - try input_group.await(io); - return .miss; - }, - .hit => {}, - } - switch (try checkFile(m, &c, file_off, path)) { - .hit => continue, - .miss => { - post_select.cancelDiscard(); - try input_group.await(io); - return .miss; - }, - } - }, - } - off += File.sizeOf(path.len); } @@ -925,6 +889,36 @@ pub const Manifest = struct { // Don't track the trailing zero byte in contents. m.contents.items.len -= 1; + // Needed due to the length mutation above. + const refreshed_contents = m.contents.items; + + // This group we would like to cancel as soon as a cache miss is discovered. + const PostResult = union(enum) { + checkFile: CheckFileError!Check.Status, + }; + // In order to call async in the loop we need to ensure this buffer has capacity for as many elements as files + // being checked, otherwise a deadlock could occur since writing to the queue is waiting on the same task as + // would read from it. + const post_select_buffer = try gpa.alloc(PostResult, m.files.count() - m.input_paths.items.len); + defer gpa.free(post_select_buffer); + + var post_select: Io.Select(PostResult) = .init(io, post_select_buffer); + defer post_select.cancelDiscard(); + + var post_select_remaining: usize = 0; + for (m.files.keys()[m.input_paths.items.len..]) |file_off| { + post_select.async(.checkFile, checkFile, .{ m, &c, file_off, filePath(refreshed_contents, file_off) }); + post_select_remaining += 1; + // In case the async checkFile runs eagerly. + switch (@atomicLoad(Check.Status, &c.status, .unordered)) { + .miss => { + post_select.cancelDiscard(); + try input_group.await(io); + return .miss; + }, + .hit => {}, + } + } var post_await_buffer: [16]PostResult = undefined; while (post_select_remaining > 0) { @@ -958,9 +952,6 @@ pub const Manifest = struct { if (c.status == .miss) return .miss; if (m.diagnostic != .none) return error.CacheCheckFailed; - // Needed due to the length mutation above. - const refreshed_contents = m.contents.items; - for (m.files.keys()) |file_off| { m.hash.hasher.update(&file_off.get(refreshed_contents).digest); } -- 2.54.0 From 2c605872fc4fb5f4df99647ad7f4fbe7f65612a5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 7 Sep 2026 14:37:32 -0700 Subject: [PATCH 16/49] std.Build.Cache: non-async checking Empirically this went from 29% slower than status quo on my machine to 15% faster, for `zig build` on the `zig init` project. --- lib/std/Build/Cache.zig | 102 +++++++--------------------------------- 1 file changed, 18 insertions(+), 84 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 417f309200cd63ff0457fbbec51d957b7d0a72cc..161da9b7534e31ef1655870867bed7761c87bcd3 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -839,13 +839,10 @@ pub const Manifest = struct { var off: usize = 0; var c: Check = .{}; - // This group we always want to compute the hash digests, even on a - // cache miss, because they will be used in the manifest digest. - var input_group: Io.Group = .init; - defer input_group.cancel(io); + // We always want to compute the input file hash digests, even on a cache miss, because they will be + // used in the manifest digest. - // First the input files section, which must match our input files, - // otherwise it's invalid format. + // First the input files section, which must match our input files, otherwise it's invalid format. for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| { if (off + 1 >= contents.len) return error.InvalidFormat; const file_off: File.Offset = @fromBackingInt(@intCast(off)); @@ -855,11 +852,14 @@ pub const Manifest = struct { if (path.len == 0) return error.InvalidFormat; if (input_file_off != file_off) return error.InvalidFormat; - input_group.async(io, checkInputFile, .{ m, &c, file_off, path, input_path }); + try checkInputFile(m, &c, file_off, path, input_path); off += File.sizeOf(path.len); } + if (c.status == .miss) return .miss; + if (m.diagnostic != .none) return error.CacheCheckFailed; + // Guess number of files based on manifest contents len to reduce allocations. // This is not an upper bound; subsequent insertions may potentially allocate. try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{ @@ -875,6 +875,10 @@ pub const Manifest = struct { if (path.len == 0) return error.InvalidFormat; try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); + switch (try checkFile(m, &c, file_off, filePath(contents, file_off))) { + .hit => {}, + .miss => return .miss, + } off += File.sizeOf(path.len); } @@ -882,79 +886,12 @@ pub const Manifest = struct { // Final terminating zero byte to distinguish empty manifest file from // manifest with zero files. const file_valid = off + 1 == contents.len and contents[off] == 0; - if (!file_valid) { - try input_group.await(io); - return .miss; - } + if (!file_valid) return .miss; + + for (m.files.keys()) |file_off| m.hash.hasher.update(&file_off.get(contents).digest); // Don't track the trailing zero byte in contents. m.contents.items.len -= 1; - // Needed due to the length mutation above. - const refreshed_contents = m.contents.items; - - // This group we would like to cancel as soon as a cache miss is discovered. - const PostResult = union(enum) { - checkFile: CheckFileError!Check.Status, - }; - // In order to call async in the loop we need to ensure this buffer has capacity for as many elements as files - // being checked, otherwise a deadlock could occur since writing to the queue is waiting on the same task as - // would read from it. - const post_select_buffer = try gpa.alloc(PostResult, m.files.count() - m.input_paths.items.len); - defer gpa.free(post_select_buffer); - - var post_select: Io.Select(PostResult) = .init(io, post_select_buffer); - defer post_select.cancelDiscard(); - - var post_select_remaining: usize = 0; - for (m.files.keys()[m.input_paths.items.len..]) |file_off| { - post_select.async(.checkFile, checkFile, .{ m, &c, file_off, filePath(refreshed_contents, file_off) }); - post_select_remaining += 1; - // In case the async checkFile runs eagerly. - switch (@atomicLoad(Check.Status, &c.status, .unordered)) { - .miss => { - post_select.cancelDiscard(); - try input_group.await(io); - return .miss; - }, - .hit => {}, - } - } - - var post_await_buffer: [16]PostResult = undefined; - while (post_select_remaining > 0) { - const n = try post_select.awaitMany(&post_await_buffer, 1); - post_select_remaining -= n; - - // Detect if input group already had a miss. In this case we still wait - // for those digests to be updated, but cancel the non input group. - switch (@atomicLoad(Check.Status, &c.status, .unordered)) { - .miss => { - post_select.cancelDiscard(); - try input_group.await(io); - return .miss; - }, - .hit => {}, - } - - for (post_await_buffer[0..n]) |u| switch (u) { - .checkFile => |result| switch (try result) { - .hit => continue, - .miss => { - post_select.cancelDiscard(); - try input_group.await(io); - return .miss; - }, - }, - }; - } - - try input_group.await(io); - if (c.status == .miss) return .miss; - if (m.diagnostic != .none) return error.CacheCheckFailed; - - for (m.files.keys()) |file_off| { - m.hash.hasher.update(&file_off.get(refreshed_contents).digest); - } return .hit; } @@ -965,17 +902,14 @@ pub const Manifest = struct { file_off: File.Offset, file_path: [:0]const u8, input_path: *InputPath, - ) Io.Cancelable!void { + ) CheckFileError!void { if (input_path.have_handle) @panic("TODO"); if (input_path.have_stat) @panic("TODO"); if (input_path.contents != .not_requested) @panic("TODO"); if (input_path.request_handle) @panic("TODO"); - if (checkFile(m, c, file_off, file_path)) |status| switch (status) { - .hit => return, - .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered), - } else |err| switch (err) { - error.CacheCheckFailed => assert(m.diagnostic != .none), - else => |e| return e, + switch (try checkFile(m, c, file_off, file_path)) { + .hit => {}, + .miss => c.status = .miss, } } -- 2.54.0 From 5b57dca62d8094d58a21e5458b5095febce47dc5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 7 Sep 2026 18:15:29 -0700 Subject: [PATCH 17/49] std.build.Cache: fix regressions from the branch --- lib/std/Build/Cache.zig | 79 ++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 161da9b7534e31ef1655870867bed7761c87bcd3..1081bdbf8ef35c73798b1ca29db22eb90b671396 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -67,7 +67,8 @@ fn appendPrefixedPath(cache: *const Cache, contents: *std.ArrayList(u8), prefixe const end = contents.items.len + prefixed_path.sub_path.len; const needed_alignment = @alignOf(Manifest.File) - (end % @alignOf(Manifest.File)); assert(needed_alignment >= 1); // Always need at least a null byte. - try contents.ensureTotalCapacity(cache.gpa, end + needed_alignment); + // Extra +1 here is due to the requirement to keep at least 1 unused capacity in `Manifest.contents` at all times. + try contents.ensureTotalCapacity(cache.gpa, end + needed_alignment + 1); contents.appendSliceAssumeCapacity(prefixed_path.sub_path); contents.appendNTimesAssumeCapacity(0, needed_alignment); return prefixed_path.prefix; @@ -97,7 +98,10 @@ fn resolveAppendPath(cache: *const Cache, contents: *std.ArrayList(u8), path: Pa const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); assert(needed_alignment >= 1); // Always need at least a null byte. - try contents.appendNTimes(gpa, 0, needed_alignment); + // Extra +1 here is due to the requirement to keep at least 1 unused capacity in `Manifest.contents` at all + // times. + try contents.ensureUnusedCapacity(gpa, needed_alignment + 1); + contents.appendNTimesAssumeCapacity(0, needed_alignment); return @intCast(i); } @@ -107,7 +111,9 @@ fn resolveAppendPath(cache: *const Cache, contents: *std.ArrayList(u8), path: Pa const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); assert(needed_alignment >= 1); // Always need at least a null byte. - try contents.appendNTimes(gpa, 0, needed_alignment); + // Extra +1 here is due to the requirement to keep at least 1 unused capacity in `Manifest.contents` at all times. + try contents.ensureUnusedCapacity(gpa, needed_alignment + 1); + contents.appendNTimesAssumeCapacity(0, needed_alignment); return 0; } @@ -127,10 +133,10 @@ pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); /// Refresh this with new random bytes when the manifest /// format is modified in a non-backwards-compatible way. pub const hasher_init: Hasher = Hasher.init(&.{ - 0x33, 0x52, 0xa2, 0x84, - 0xcf, 0x17, 0x56, 0x57, - 0x01, 0xbb, 0xcd, 0xe4, - 0x77, 0xd6, 0xf0, 0x60, + 0x02, 0xe9, 0xfa, 0xfe, + 0xe0, 0x95, 0x81, 0x55, + 0xf5, 0x5a, 0x15, 0xcb, + 0x4a, 0xf4, 0x00, 0x09, }); pub const HashHelper = struct { @@ -487,7 +493,8 @@ pub const Manifest = struct { pub const Diagnostic = union(enum) { none, manifest_create: Io.File.OpenError, - manifest_read: Io.File.Reader.Error, + manifest_stat: Io.File.StatError, + manifest_read: Io.File.ReadPositionalError, manifest_lock: Io.File.LockError, file_open: FileOp, file_stat: FileOp, @@ -644,7 +651,7 @@ pub const Manifest = struct { return @fromBackingInt(@intCast(gop.index)); } - pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { + pub fn addInputPathOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { m.hash.add(opt_path != null); _ = try addInputPath(m, opt_path orelse return, options); } @@ -824,24 +831,43 @@ pub const Manifest = struct { fn checkLocked(m: *Manifest) Check.Error!Check.Status { const gpa = m.cache.gpa; const io = m.cache.io; + const manifest_file = m.manifest_file.?; - var manifest_reader = m.manifest_file.?.reader(io, &.{}); // Reads positionally from zero. - m.contents.clearRetainingCapacity(); - manifest_reader.interface.appendRemainingUnlimited(gpa, &m.contents) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => { - m.diagnostic = .{ .manifest_read = manifest_reader.err.? }; + const manifest_size = if (manifest_file.stat(io)) |stat| stat.size else |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + m.diagnostic = .{ .manifest_stat = e }; return error.CacheCheckFailed; }, }; - const contents = m.contents.items; - - var off: usize = 0; - var c: Check = .{}; // We always want to compute the input file hash digests, even on a cache miss, because they will be // used in the manifest digest. + if (manifest_size == 0 or manifest_size < m.contents.items.len) { + // Don't clobber our input files by overwriting contents since it's a cache miss. + try m.contents.ensureUnusedCapacity(gpa, 1); + return .miss; + } + + // Already supposedly includes the extra null byte that we have to maintain. + try m.contents.resize(gpa, manifest_size); + + { + const n = manifest_file.readPositionalAll(io, m.contents.items, 0) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + m.diagnostic = .{ .manifest_read = e }; + return error.CacheCheckFailed; + }, + }; + m.contents.shrinkRetainingCapacity(n); + } + const contents = m.contents.items; + + var off: usize = 0; + var c: Check = .{}; + // First the input files section, which must match our input files, otherwise it's invalid format. for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| { if (off + 1 >= contents.len) return error.InvalidFormat; @@ -1466,7 +1492,7 @@ pub const Manifest = struct { const prev_contents_len = other.contents.items.len; try other.contents.appendSlice(gpa, copy_bytes); - const gop = other.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{ + const gop = other.files.getOrPutAssumeCapacityContext(@fromBackingInt(prev_contents_len), .{ .manifest = other, }); @@ -1514,7 +1540,7 @@ pub const Manifest = struct { var entries_list: std.ArrayList(u32) = .empty; defer entries_list.deinit(gpa); - while (true) { + while (reader.state != .finished) { const entries = entry_buffer[0..try reader.read(io, &entry_buffer)]; for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| { off.* = @intCast(contents.items.len); @@ -1528,12 +1554,15 @@ pub const Manifest = struct { } const Sort = struct { - contents: []const u8, + contents: [*:0]const u8, pub fn lessThan(this: @This(), lhs: u32, rhs: u32) bool { - return mem.lessThanZ(u8, this.contents[lhs + 1 ..], this.contents[rhs + 1 ..]); // +1 for kind byte + // This comparison includes the kind byte. + return mem.lessThanZ(u8, this.contents + lhs, this.contents + rhs); } }; - mem.sortUnstable(u32, entries_list.items, @as(Sort, .{ .contents = contents.items }), Sort.lessThan); + mem.sortUnstable(u32, entries_list.items, @as(Sort, .{ + .contents = @ptrCast(contents.items.ptr), + }), Sort.lessThan); // Duplicate the contents such that we may refer to it while creating a // sorted copy in the original position (at contents_start). We will then @@ -1546,7 +1575,7 @@ pub const Manifest = struct { const offset = wrong_offset + contents_len; // Includes the kind prefix which we also want to copy. const entry: [*:0]const u8 = @ptrCast(contents.items[offset..]); - new_offset += mem.copySentinel(u8, 0, contents.items[new_offset..], entry); + new_offset += mem.copySentinelInclusive(u8, 0, contents.items[new_offset..], entry); } assert(new_offset == contents_start + contents_len); contents.shrinkRetainingCapacity(contents_start + contents_len); -- 2.54.0 From 0334f14a13c0d61c00a767e87499292d13ef2af1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 7 Sep 2026 18:18:54 -0700 Subject: [PATCH 18/49] compiler: update to new std.Build.Cache API --- lib/compiler/Maker.zig | 2 +- lib/compiler/Maker/Step.zig | 2 +- lib/std/Build/Cache.zig | 129 +++++++++++++++++++++++++--------- lib/std/zig.zig | 2 +- src/Compilation.zig | 100 ++++++++++---------------- src/Zcu.zig | 4 +- src/Zcu/PerThread.zig | 38 +++++----- src/libs/mingw.zig | 12 ++-- src/link.zig | 12 +++- src/link/Elf.zig | 4 +- src/link/Elf/SharedObject.zig | 2 +- src/main.zig | 2 +- 12 files changed, 175 insertions(+), 134 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 9a3d72714bf56443572ecb4a52679926a4ab5dc6..9e7d1b01e47b034279d83577eeeabb3cf24ebfc7 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1500,7 +1500,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { if (config_man) |man| for (configuration.path_deps) |path_dep| { const path = try confPathDepToCachePath(arena, graph, &configuration, path_dep); try man.addDiscoveredPath(.{ - .path = .{ .unresolved = path }, + .discovered_path = .{ .unresolved = path }, .handle = if (path_dep.flags.is_directory) .{ .dir = null } else .{ .file = null }, .metadata_only = path_dep.flags.metadata_only, }); diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index b1e89364fb050e5ba397d40f8de6c43966dedb04..25ea943f39d61909415e80741832b547b7f60334 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -753,7 +753,7 @@ fn failWithCacheError( switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| { + .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { return s.fail(maker, "failed checking cache: {t} {t}", .{ man.diagnostic, e }); }, .file_open, .file_stat, .file_read, .file_hash => |op| { diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 1081bdbf8ef35c73798b1ca29db22eb90b671396..8481117fcc40788a334fa80bc17d045c6b5d23bd 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1104,25 +1104,45 @@ pub const Manifest = struct { return timestamp.nanoseconds >= man.recent_problematic_timestamp.nanoseconds; } - pub const AddPathPostOptions = struct { - path: union(enum) { - unresolved: Path, - prefixed: PrefixedPath, - }, + pub const AddDiscoveredPathOptions = struct { + discovered_path: DiscoveredPath, handle: PathHandle = .{ .file = null }, stat: ?Stat = null, /// If it is a directory, there is a special encoding required for contents, which /// is null-separated sorted entries, each one prefixed with `File.Kind`. contents: ?[]const u8 = null, metadata_only: bool = false, + /// Populated if and only if `error.FileSystemFailure` is returned from `addDiscoveredPath`. + diagnostic: ?*AddDiscoveredPathDiagnostic = null, }; + pub const DiscoveredPath = union(enum) { + unresolved: Path, + prefixed: PrefixedPath, + }; + + pub const AddDiscoveredPathDiagnostic = union(enum) { + none, + open_file: Io.File.OpenError, + open_dir: Io.Dir.OpenError, + stat_file: Io.File.StatError, + stat_dir: Io.Dir.StatError, + read_file: Io.File.ReadPositionalError, + read_dir: Io.Dir.Reader.Error, + }; + + pub const AddDiscoveredPathError = error{ + /// If this is returned, diagnostic will be populated. + /// If contents and stat are both provided, this is unreachable. + FileSystemFailure, + } || Allocator.Error || Io.Cancelable; + /// Add a file as a dependency of process being cached, after cache miss /// occurs. /// /// See also: /// * `addInputPath` - pub fn addDiscoveredPath(m: *Manifest, options: AddPathPostOptions) !void { + pub fn addDiscoveredPath(m: *Manifest, options: AddDiscoveredPathOptions) AddDiscoveredPathError!void { assert(m.manifest_file != null); const cache = m.cache; const gpa = cache.gpa; @@ -1135,7 +1155,7 @@ pub const Manifest = struct { try m.contents.appendNTimes(gpa, 0, @offsetOf(File, "path_start")); errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); - const new_prefix = switch (options.path) { + const new_prefix = switch (options.discovered_path) { .unresolved => |unresolved| try cache.resolveAppendPath(&m.contents, unresolved), .prefixed => |prefixed| try cache.appendPrefixedPath(&m.contents, prefixed), }; @@ -1184,26 +1204,38 @@ pub const Manifest = struct { switch (options.handle) { .dir => |opt_handle| if (opt_handle) |handle| { - try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only); + try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); } else { const dir = cache.prefixes()[prefix].handle; const sub_path = filePath(m.contents.items, file_offset); - const handle = try dir.openDir(io, sub_path, .{ + const handle = dir.openDir(io, sub_path, .{ .access_sub_paths = false, .iterate = true, - }); + }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (options.diagnostic) |d| d.* = .{ .open_dir = e }; + return error.FileSystemFailure; + }, + }; defer handle.close(io); - try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only); + try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); }, .file => |opt_handle| if (opt_handle) |handle| { - try populateFile(m, header, need_stat, handle, options.contents, metadata_only); + try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); } else { const dir = cache.prefixes()[prefix].handle; const sub_path = filePath(m.contents.items, file_offset); - const handle = try dir.openFile(io, sub_path, .{ .mode = .read_only }); + const handle = dir.openFile(io, sub_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (options.diagnostic) |d| d.* = .{ .open_file = e }; + return error.FileSystemFailure; + }, + }; defer handle.close(io); - try populateFile(m, header, need_stat, handle, options.contents, metadata_only); + try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); }, } @@ -1217,11 +1249,18 @@ pub const Manifest = struct { handle: Io.File, contents: ?[]const u8, metadata_only: bool, - ) !void { + diagnostic: ?*AddDiscoveredPathDiagnostic, + ) AddDiscoveredPathError!void { const io = m.cache.io; if (need_stat) { - const stat = try handle.stat(io); + const stat = handle.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .stat_file = e }; + return error.FileSystemFailure; + }, + }; try file.setStat(m, .init(stat)); } if (metadata_only) { @@ -1233,7 +1272,13 @@ pub const Manifest = struct { hasher.update(bytes); hasher.final(&file.digest); } else { - try hashFile(io, handle, &file.digest); + hashFile(io, handle, &file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .read_file = e }; + return error.FileSystemFailure; + }, + }; } } @@ -1244,13 +1289,20 @@ pub const Manifest = struct { handle: Io.Dir, contents: ?[]const u8, metadata_only: bool, - ) !void { + diagnostic: ?*AddDiscoveredPathDiagnostic, + ) AddDiscoveredPathError!void { const cache = m.cache; const io = cache.io; const gpa = cache.gpa; if (need_stat) { - const stat = try handle.stat(io); + const stat = handle.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .stat_dir = e }; + return error.FileSystemFailure; + }, + }; try file.setStat(m, .init(stat)); } if (metadata_only) { @@ -1264,7 +1316,13 @@ pub const Manifest = struct { } else { const prev_contents_len = m.all_input_content.items.len; defer m.all_input_content.shrinkRetainingCapacity(prev_contents_len); - try hashDir(gpa, io, handle, &file.digest, &m.all_input_content); + hashDir(gpa, io, handle, &file.digest, &m.all_input_content) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .read_dir = e }; + return error.FileSystemFailure; + }, + }; } } @@ -1297,13 +1355,15 @@ pub const Manifest = struct { .target, .target_must_resolve => {}, .prereq => |file_path| if (self.manifest_file == null) { _ = try self.addInputPath(.initCwd(file_path), .{}); - } else try self.addDiscoveredPath(.{ .path = .{ .unresolved = .initCwd(file_path) } }), + } else try self.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(file_path) } }), .prereq_must_resolve => { resolve_buf.clearRetainingCapacity(); try token.resolve(gpa, &resolve_buf); if (self.manifest_file == null) { _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); - } else try self.addDiscoveredPath(.{ .path = .{ .unresolved = .initCwd(resolve_buf.items) } }); + } else try self.addDiscoveredPath(.{ + .discovered_path = .{ .unresolved = .initCwd(resolve_buf.items) }, + }); }, else => |err| { try err.printError(gpa, &error_buf); @@ -1456,10 +1516,12 @@ pub const Manifest = struct { const gpa = man.cache.gpa; const files = man.files.keys(); if (files.len > 0) { + const contents = man.contents.items; for (files) |file| { - try buf.ensureUnusedCapacity(gpa, file.prefixed_path.sub_path.len + 2); - buf.appendAssumeCapacity(file.prefixed_path.prefix + 1); - buf.appendSliceAssumeCapacity(file.prefixed_path.sub_path); + const file_path = filePath(contents, file); + try buf.ensureUnusedCapacity(gpa, file_path.len + 2); + buf.appendAssumeCapacity(@as(u8, file.get(contents).flags.prefix) + 1); + buf.appendSliceAssumeCapacity(file_path); buf.appendAssumeCapacity(0); } // The null byte is a separator, not a terminator. @@ -1476,12 +1538,12 @@ pub const Manifest = struct { const orig_files_len = other.files.count(); const orig_contents_len = other.contents.items.len; errdefer { - other.files.shrinkRetainingCapacity(orig_files_len); + other.files.shrinkRetainingCapacityContext(orig_files_len, .{ .contents = other.contents.items }); other.contents.shrinkRetainingCapacity(orig_contents_len); } for (man.files.keys(), 0..) |off, file_index| { - try other.files.ensureUnusedCapacity(gpa, 1); + try other.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = other.contents.items }); const next_off = if (file_index < man.files.count()) @backingInt(man.files.keys()[file_index + 1]) @@ -1489,20 +1551,21 @@ pub const Manifest = struct { man.contents.items.len; const copy_bytes = man.contents.items[@backingInt(off)..next_off]; - const prev_contents_len = other.contents.items.len; + const prev_contents_len: File.Offset = @fromBackingInt(@intCast(other.contents.items.len)); try other.contents.appendSlice(gpa, copy_bytes); - const gop = other.files.getOrPutAssumeCapacityContext(@fromBackingInt(prev_contents_len), .{ - .manifest = other, + const gop = other.files.getOrPutAssumeCapacityContext(prev_contents_len, .{ + .contents = other.contents.items, }); if (gop.found_existing) { - other.contents.shrinkRetainingCapacity(prev_contents_len); + other.contents.shrinkRetainingCapacity(@backingInt(prev_contents_len)); continue; } - const other_file = File.get(@fromBackingInt(prev_contents_len)); - other_file.prefix = prefix_map[other_file.prefix]; + // Flags are already copied but the prefix is supposed to be filtered by `prefix_map`. + const other_file = prev_contents_len.get(other.contents.items); + other_file.flags.prefix = @intCast(prefix_map[other_file.flags.prefix]); } } diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 15218e43c432ad51d2210142b02f2a05cdbc3973..57bda0bbfd85ad057dd45dae7b482f226012b2ae 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1840,7 +1840,7 @@ pub fn buildExeSubprocess( const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1)); const sub_path = prefixed_path[1..]; man.addDiscoveredPath(.{ - .path = .{ .prefixed = .{ + .discovered_path = .{ .prefixed = .{ .prefix = @intCast(@backingInt(prefix)), .sub_path = sub_path, } }, diff --git a/src/Compilation.zig b/src/Compilation.zig index af532d6c0a253a46650ebd35fb2b939d69d5e107..38229017566600e927fb5dfe2d51f9491bce1d2f 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -701,39 +701,12 @@ pub const Path = struct { }; } - pub fn addToCacheManifestPostHit(p: Path, man: *Cache.Manifest, dirs: *const std.zig.Directories) !void { - comptime assert(0 == @backingInt(std.zig.Server.Message.PathPrefix.cwd)); - comptime assert(1 == @backingInt(std.zig.Server.Message.PathPrefix.zig_lib)); - comptime assert(2 == @backingInt(std.zig.Server.Message.PathPrefix.local_cache)); - comptime assert(3 == @backingInt(std.zig.Server.Message.PathPrefix.global_cache)); - comptime assert(4 == @backingInt(std.zig.Server.Message.PathPrefix.build_root)); - comptime assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == 5); - const gpa = man.cache.gpa; - const prefixed_path: Cache.PrefixedPath = .{ - .prefix = switch (p.root) { - .none => { - const path = try p.toAbsolute(dirs, gpa); - defer gpa.free(path); - return man.addFilePost(path); - }, - .zig_lib => 1, - .local_cache => 2, - .global_cache => 3, - .build_root => 4, - }, - .sub_path = try gpa.dupe(u8, p.sub_path), - }; - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try man.addPrefixedPathPost(prefixed_path); - } - - pub fn addToCacheManifestPostHitContents( + pub fn addToCacheManifestAsDiscovered( p: Path, man: *Cache.Manifest, dirs: *const std.zig.Directories, - bytes: []const u8, - stat: Cache.File.Stat, + contents: ?[]const u8, + stat: ?Cache.Manifest.Stat, ) !void { comptime assert(0 == @backingInt(std.zig.Server.Message.PathPrefix.cwd)); comptime assert(1 == @backingInt(std.zig.Server.Message.PathPrefix.zig_lib)); @@ -741,24 +714,29 @@ pub const Path = struct { comptime assert(3 == @backingInt(std.zig.Server.Message.PathPrefix.global_cache)); comptime assert(4 == @backingInt(std.zig.Server.Message.PathPrefix.build_root)); comptime assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == 5); - const gpa = man.cache.gpa; - const prefixed_path: Cache.PrefixedPath = .{ - .prefix = switch (p.root) { - .none => { - const path = try p.toAbsolute(dirs, gpa); - defer gpa.free(path); - return man.addFilePostContents(path, bytes, stat); + try man.addDiscoveredPath(.{ + .discovered_path = .{ .prefixed = .{ + .prefix = switch (p.root) { + .none => { + const gpa = man.cache.gpa; + const path = try p.toAbsolute(dirs, gpa); + defer gpa.free(path); + return man.addDiscoveredPath(.{ + .discovered_path = .{ .unresolved = .initCwd(path) }, + .contents = contents, + .stat = stat, + }); + }, + .zig_lib => 1, + .local_cache => 2, + .global_cache => 3, + .build_root => 4, }, - .zig_lib => 1, - .local_cache => 2, - .global_cache => 3, - .build_root => 4, - }, - .sub_path = try gpa.dupe(u8, p.sub_path), - }; - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try man.addPrefixedPathPostContents(prefixed_path, bytes, stat); + .sub_path = p.sub_path, + } }, + .contents = contents, + .stat = stat, + }); } }; @@ -2835,20 +2813,18 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE man.want_shared_lock = false; } - const is_hit = man.check(main_progress_node) catch |err| switch (err) { + const status = man.check(main_progress_node) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return comp.setMiscFailure( + .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| return comp.setMiscFailure( .check_whole_cache, "failed to check cache: {t} {t}", .{ man.diagnostic, e }, ), .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix]; - return comp.setMiscFailure(.check_whole_cache, "failed to check cache: {f}{s} {t} {t}", .{ - prefix, pp.sub_path, man.diagnostic, op.err, + return comp.setMiscFailure(.check_whole_cache, "failed to check cache: {f} {t} {t}", .{ + op.path(&man), man.diagnostic, op.err, }); }, }, @@ -2858,7 +2834,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE .{}, ), }; - if (is_hit and !ignore_hit) { + if (status == .hit and !ignore_hit) { // In this case the cache hit contains the full set of file system inputs. Nice! if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); if (comp.parent_whole_cache) |pwc| { @@ -3454,8 +3430,8 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes const opts = comp.cache_use.whole.lf_open_opts; - try man.addOptionalFilePath(opts.linker_script); - try man.addOptionalFilePath(opts.version_script); + try man.addInputPathOptional(opts.linker_script, .{}); + try man.addInputPathOptional(opts.version_script, .{}); man.hash.add(opts.allow_undefined_version); man.hash.addOptional(opts.enable_new_dtags); @@ -3514,7 +3490,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes // Mach-O specific stuff try link.File.MachO.hashAddFrameworks(man, opts.frameworks); - try man.addOptionalFilePath(opts.entitlements); + try man.addInputPathOptional(opts.entitlements, .{}); man.hash.addOptional(opts.pagezero_size); man.hash.addOptional(opts.headerpad_size); man.hash.add(opts.headerpad_max_install_names); @@ -5905,7 +5881,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); - const digest = if (try man.check(child_progress_node)) man.final() else blk: { + const digest = if (.hit == try man.check(child_progress_node)) man.final() else blk: { // The digest only depends on the .manifest file, so we can // get the digest now and write the .res directly to the cache const digest = man.final(); @@ -5998,7 +5974,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename_noext = src_basename[0 .. src_basename.len - fs.path.extension(src_basename).len]; - const digest = if (try man.check(child_progress_node)) man.final() else blk: { + const digest = if (.hit == try man.check(child_progress_node)) man.final() else blk: { var zig_cache_tmp_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, "tmp", .{}); defer zig_cache_tmp_dir.close(io); @@ -6055,12 +6031,14 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 return comp.failWin32Resource(win32_resource, "depfile from zig rc has unexpected format", .{}); } const dep_file_path = element.string; - try man.addFilePost(dep_file_path); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(dep_file_path) } }); switch (comp.cache_use) { .whole => |whole| if (whole.cache_manifest) |whole_cache_manifest| { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try whole_cache_manifest.addFilePost(dep_file_path); + try whole_cache_manifest.addDiscoveredPath(.{ + .discovered_path = .{ .unresolved = .initCwd(dep_file_path) }, + }); }, .incremental, .none => {}, } diff --git a/src/Zcu.zig b/src/Zcu.zig index d6f9da974052860224583dd47cd3b6e44c90d056..56118073660df5a55f77517a459fe5f666c1d442 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -974,7 +974,7 @@ pub const File = struct { success, }, /// Whether this is populated depends on `status`. - stat: Cache.File.Stat, + stat: Cache.Manifest.Stat, /// Whether this file is the generated file of a "builtin" module. This matters because those /// files are generated and stored in-nemory rather than being read off-disk. The rest of the @@ -1246,7 +1246,7 @@ pub const EmbedFile = struct { val: InternPool.Index, /// If this is `null` and `val` is `.none`, the file has never been loaded. err: ?(Io.File.OpenError || Io.File.StatError || Io.File.Reader.Error || error{UnexpectedEof}), - stat: Cache.File.Stat, + stat: Cache.Manifest.Stat, pub const Index = enum(u32) { _, diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 78cf664f71afd94efe6e9044d93d93b1dc1487d8..f0b1764f463dbe9d085e0fca931d20286e99255f 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -224,11 +224,12 @@ pub fn update( const result = res: { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - if (file.source) |source| { - break :res file.path.addToCacheManifestPostHitContents(man, &comp.dirs, source, file.stat); - } else { - break :res file.path.addToCacheManifestPostHit(man, &comp.dirs); - } + break :res file.path.addToCacheManifestAsDiscovered( + man, + &comp.dirs, + file.source, + if (file.source != null) file.stat else null, + ); }; result catch |err| switch (err) { error.OutOfMemory => |e| return e, @@ -2835,26 +2836,21 @@ fn updateBuiltinModule(pt: Zcu.PerThread, opts: Builtin) Allocator.Error!struct assert(mem.eql(u8, import_path, "std")); // the single import is of 'std' } - Builtin.updateFileOnDisk(file, comp) catch |err| comp.setMiscFailure( - .write_builtin_zig, - "unable to write '{f}': {s}", - .{ file.path.fmt(comp), @errorName(err) }, - ); + Builtin.updateFileOnDisk(file, comp) catch |err| + comp.setMiscFailure(.write_builtin_zig, "unable to write {qf}: {t}", .{ file.path.fmt(comp), err }); return .{ .file = file_index, .module_root = mod, }; } +pub const EmbedFileError = error{ImportOutsideModulePath} || Allocator.Error || Io.Cancelable; + pub fn embedFile( pt: Zcu.PerThread, cur_file: *Zcu.File, import_string: []const u8, -) error{ - OutOfMemory, - Canceled, - ImportOutsideModulePath, -}!Zcu.EmbedFile.Index { +) EmbedFileError!Zcu.EmbedFile.Index { const zcu = pt.zcu; const gpa = zcu.gpa; @@ -2937,7 +2933,7 @@ fn updateEmbedFileInner( }; defer file.close(io); - const stat: Cache.File.Stat = .fromFs(try file.stat(io)); + const stat: Cache.Manifest.Stat = .init(try file.stat(io)); if (ef.val != .none) { const old_stat = ef.stat; @@ -2994,10 +2990,7 @@ fn updateEmbedFileInner( } /// Assumes that `path` is allocated into `gpa`. Takes ownership of `path` on success. -fn newEmbedFile( - pt: Zcu.PerThread, - path: Compilation.Path, -) !*Zcu.EmbedFile { +fn newEmbedFile(pt: Zcu.PerThread, path: Compilation.Path) EmbedFileError!*Zcu.EmbedFile { const zcu = pt.zcu; const comp = zcu.comp; const io = comp.io; @@ -3034,7 +3027,10 @@ fn newEmbedFile( try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try path.addToCacheManifestPostHitContents(man, &comp.dirs, contents, new_file.stat); + path.addToCacheManifestAsDiscovered(man, &comp.dirs, contents, new_file.stat) catch |err| switch (err) { + error.FileSystemFailure => unreachable, // contents and stat are both provided + else => |e| return e, + }; } return new_file; diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index 63c1e5ad16ccf7a745256242f528203d5a5cbdcf..f0d0ef2990089fd1d6ac7ef5844678d1950f7b3c 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -251,18 +251,16 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P const final_lib_basename = try std.fmt.allocPrint(gpa, "{s}.lib", .{lib_name}); errdefer gpa.free(final_lib_basename); - const is_hit = man.check(prog_node) catch |err| switch (err) { + const status = man.check(prog_node) catch |err| switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| { + .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { comp.setMiscFailure(.windows_import_lib, "checking cache failed: {t} {t}", .{ man.diagnostic, e }); return error.AlreadyReported; }, .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix]; - comp.setMiscFailure(.windows_import_lib, "checking cache failed: {f}{s} {t} {t}", .{ - prefix, pp.sub_path, man.diagnostic, op.err, + comp.setMiscFailure(.windows_import_lib, "checking cache failed: {f} {t} {t}", .{ + op.path(&man), man.diagnostic, op.err, }); return error.AlreadyReported; }, @@ -273,7 +271,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P return error.AlreadyReported; }, }; - if (is_hit) { + if (status == .hit) { const digest = man.final(); const sub_path = try std.fs.path.join(gpa, &.{ "o", &digest, final_lib_basename }); errdefer gpa.free(sub_path); diff --git a/src/link.zig b/src/link.zig index babee6e563e072fd863c3622be6107fdac6a35fd..df68e0216d3f20ab09837c55f849461c30fbd5a4 100644 --- a/src/link.zig +++ b/src/link.zig @@ -1925,15 +1925,21 @@ pub fn hashInputs(man: *Cache.Manifest, link_inputs: []const Input) !void { man.hash.add(@as(@typeInfo(Input).@"union".tag_type.?, link_input)); switch (link_input) { .object, .archive => |obj| { - _ = try man.addOpenedFile(obj.path, obj.file, null); + _ = try man.addInputPath(obj.path, .{ + .handle = .{ .file = obj.file }, + }); man.hash.add(obj.must_link); man.hash.add(obj.hidden); }, .res => |res| { - _ = try man.addOpenedFile(res.path, res.file, null); + _ = try man.addInputPath(res.path, .{ + .handle = .{ .file = res.file }, + }); }, .dso => |dso| { - _ = try man.addOpenedFile(dso.path, dso.file, null); + _ = try man.addInputPath(dso.path, .{ + .handle = .{ .file = dso.file }, + }); man.hash.add(dso.needed); man.hash.add(dso.weak); man.hash.add(dso.reexport); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 416eab37e05651c42cfedb4e025bca62b9cac016..f04d7582bd6dac07669861f0d920c86efa6d6d1c 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1120,7 +1120,7 @@ fn parseDso( const handle = dso.file; - const stat = Stat.fromFs(try handle.stat(io)); + const stat: Stat = .init(try handle.stat(io)); var header = try SharedObject.parseHeader(gpa, io, diags, dso.path, handle, stat, target); defer header.deinit(gpa); @@ -4407,7 +4407,7 @@ const mem = std.mem; const Allocator = std.mem.Allocator; const Hash = std.hash.Wyhash; const Path = std.Build.Cache.Path; -const Stat = std.Build.Cache.File.Stat; +const Stat = std.Build.Cache.Manifest.Stat; const codegen = @import("../codegen.zig"); const eh_frame = @import("Elf/eh_frame.zig"); diff --git a/src/link/Elf/SharedObject.zig b/src/link/Elf/SharedObject.zig index 052093dce3acdf01c893ec6af73368f55523d234..3030a45bff3806d93b1af418d6eaa49a7a649558 100644 --- a/src/link/Elf/SharedObject.zig +++ b/src/link/Elf/SharedObject.zig @@ -7,7 +7,7 @@ const elf = std.elf; const log = std.log.scoped(.elf); const mem = std.mem; const Path = std.Build.Cache.Path; -const Stat = std.Build.Cache.File.Stat; +const Stat = std.Build.Cache.Manifest.Stat; const Allocator = mem.Allocator; const Elf = @import("../Elf.zig"); diff --git a/src/main.zig b/src/main.zig index 164c2f15ee081a7e0ac22dacad8659d1e9d8f596..bcffbfee9176c9c5fb050927d147a5e39332788b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4878,7 +4878,7 @@ fn cmdTranslateC( Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| fatal("unable to process {q}: {t}", .{ c_source_file.src_path, err }); - const result: Compilation.TranslateCResult = if (try man.check(prog_node)) .{ + const result: Compilation.TranslateCResult = if (.hit == try man.check(prog_node)) .{ .digest = man.finalBin(), .cache_hit = true, .errors = std.zig.ErrorBundle.empty, -- 2.54.0 From 31b86e53aaa73c6acf522b4a193f600b56a07262 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Sep 2026 19:25:07 -0700 Subject: [PATCH 19/49] std: update to new std.Build.Cache API --- lib/std/Build/Cache.zig | 102 +++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 8481117fcc40788a334fa80bc17d045c6b5d23bd..58d6aeac03f4beabadc1b7e665987fa712199e01 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -275,10 +275,9 @@ pub const Manifest = struct { /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. manifest_file: ?Io.File, manifest_dirty: bool, - /// Set this flag to true before calling hit() in order to indicate that - /// upon a cache hit, the code using the cache will not modify the files - /// within the cache directory. This allows multiple processes to utilize - /// the same cache directory at the same time. + /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code + /// using the cache will not modify the files within the cache directory. This allows multiple processes + /// to utilize the same cache directory at the same time. want_shared_lock: bool = true, have_exclusive_lock: bool = false, // Indicate that we want isProblematicTimestamp to perform a filesystem write in @@ -347,12 +346,23 @@ pub const Manifest = struct { have_stat: bool, /// Determines whether `File.digest` is populated. have_digest: bool, - contents: enum(usize) { - requested = std.math.maxInt(u32) - 1, - not_requested = std.math.maxInt(u32), - /// Byte offset index into `Manifest.all_input_content`. - _, - }, + contents: packed union(u64) { + status: enum(u64) { + unrequested = std.math.maxInt(u64) - 1, + requested = std.math.maxInt(u64), + _, + pub fn populated(this: @This()) bool { + return switch (this) { + .unrequested, .requested => false, + _ => true, + }; + } + }, + populated: packed struct(u64) { + off: u32, + len: u32, + }, + } align(@alignOf(u32)), /// `have_handle` determines whether this is populated. handle: union { file: Io.File, @@ -362,6 +372,20 @@ pub const Manifest = struct { /// Index into `Manifest.input_paths`. pub const Index = enum(u32) { _, + + pub fn get(i: @This(), manifest: *const Manifest) InputPath { + return manifest.input_paths.items[@backingInt(i)]; + } + + pub fn contents(i: @This(), manifest: *const Manifest) []const u8 { + const input_path = get(i, manifest); + const populated = switch (input_path.contents.status) { + .unrequested => unreachable, // Must request contents before calling `check`. + .requested => unreachable, // Must call `check` first. + _ => input_path.contents.populated, + }; + return manifest.all_input_content.items[populated.off..][0..populated.len]; + } }; }; @@ -611,8 +635,8 @@ pub const Manifest = struct { existing_input_file.have_handle = true; }, } - if (options.request_contents) switch (existing_input_file.contents) { - .requested, .not_requested => existing_input_file.contents = .requested, + if (options.request_contents) switch (existing_input_file.contents.status) { + .requested, .unrequested => existing_input_file.contents = .{ .status = .requested }, _ => {}, }; const existing_header = m.files.keys()[gop.index].get(m.contents.items); @@ -636,7 +660,7 @@ pub const Manifest = struct { .file => |opt_file| if (opt_file) |file| .{ .file = file } else undefined, .dir => |opt_dir| if (opt_dir) |dir| .{ .dir = dir } else undefined, }, - .contents = if (options.request_contents) .requested else .not_requested, + .contents = .{ .status = if (options.request_contents) .requested else .unrequested }, .have_digest = false, .have_stat = options.stat != null, }); @@ -931,7 +955,7 @@ pub const Manifest = struct { ) CheckFileError!void { if (input_path.have_handle) @panic("TODO"); if (input_path.have_stat) @panic("TODO"); - if (input_path.contents != .not_requested) @panic("TODO"); + if (input_path.contents.status != .unrequested) @panic("TODO"); if (input_path.request_handle) @panic("TODO"); switch (try checkFile(m, c, file_off, file_path)) { .hit => {}, @@ -1727,7 +1751,7 @@ test "cache file and then recall it" { _ = try ch.addInputPath(.initCwd(temp_file), .{}); // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit(.none)); + try testing.expectEqual(.miss, try ch.check(.none)); digest1 = ch.final(); try ch.writeManifest(); @@ -1742,7 +1766,7 @@ test "cache file and then recall it" { _ = try ch.addInputPath(.initCwd(temp_file), .{}); // Cache hit! We just "built" the same file - try testing.expect(try ch.hit(.none)); + try testing.expectEqual(.hit, try ch.check(.none)); digest2 = ch.final(); try testing.expectEqual(false, ch.have_exclusive_lock); @@ -1788,40 +1812,40 @@ test "check that changing a file makes cache fail" { defer cache.manifest_dir.close(io); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); + man.hash.addBytes("1234"); + const temp_file_idx = try man.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit(.none)); + try testing.expectEqual(.miss, try man.check(.none)); - try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.keys()[temp_file_idx].contents.?)); + try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); - digest1 = ch.final(); + digest1 = man.final(); - try ch.writeManifest(); + try man.writeManifest(); } try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = updated_temp_file_contents }); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); + man.hash.addBytes("1234"); + const temp_file_idx = try man.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit(.none)); + try testing.expectEqual(.miss, try man.check(.none)); // The cache system does not keep the contents of re-hashed input files. - try testing.expect(ch.files.keys()[temp_file_idx].contents == null); + try testing.expectEqual(false, temp_file_idx.get(&man).contents.status.populated()); - digest2 = ch.final(); + digest2 = man.final(); - try ch.writeManifest(); + try man.writeManifest(); } try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); @@ -1858,7 +1882,7 @@ test "no file inputs" { man.hash.addBytes("1234"); // There should be nothing in the cache - try testing.expectEqual(false, try man.check(.none)); + try testing.expectEqual(.miss, try man.check(.none)); digest1 = man.final(); @@ -1870,7 +1894,7 @@ test "no file inputs" { man.hash.addBytes("1234"); - try testing.expect(try man.check(.none)); + try testing.expectEqual(.hit, try man.check(.none)); digest2 = man.final(); try testing.expectEqual(false, man.have_exclusive_lock); } @@ -1922,9 +1946,9 @@ test "Manifest with files added after initial hash work" { _ = try ch.addInputPath(.initCwd(temp_file1), .{}); // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit(.none)); + try testing.expectEqual(.miss, try ch.check(.none)); - _ = try ch.addDiscoveredPath(temp_file2); + _ = try ch.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); digest1 = ch.final(); try ch.writeManifest(); @@ -1936,7 +1960,7 @@ test "Manifest with files added after initial hash work" { ch.hash.addBytes("1234"); _ = try ch.addInputPath(.initCwd(temp_file1), .{}); - try testing.expect(try ch.hit(.none)); + try testing.expect(.hit == try ch.check(.none)); digest2 = ch.final(); try testing.expectEqual(false, ch.have_exclusive_lock); @@ -1960,9 +1984,9 @@ test "Manifest with files added after initial hash work" { _ = try ch.addInputPath(.initCwd(temp_file1), .{}); // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit(.none)); + try testing.expectEqual(.miss, try ch.check(.none)); - _ = try ch.addDiscoveredPath(temp_file2); + _ = try ch.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); digest3 = ch.final(); -- 2.54.0 From 3f092f8c286b2462cda2922120dd732cd558c91c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Sep 2026 19:31:43 -0700 Subject: [PATCH 20/49] std.Build.Cache: add missing resource deallocation --- lib/std/Build/Cache.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 58d6aeac03f4beabadc1b7e665987fa712199e01..874a511e9e030b852e84ebabc8ee115329a4c9cf 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1531,6 +1531,8 @@ pub const Manifest = struct { } m.files.deinit(gpa); m.contents.deinit(gpa); + m.input_paths.deinit(gpa); + m.all_input_content.deinit(gpa); m.* = undefined; } -- 2.54.0 From a24b4758f4cabe7c690a5707f2fe2fda913b549c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 9 Sep 2026 20:25:37 -0700 Subject: [PATCH 21/49] std.Build.Cache: rework the core logic again - implement the logic for populating remaining input file digests when a miss occurs while checking one of the input files - make `Manifest.check` return not just "miss" but also explain why it was a miss and which file caused it - fix fundamental design flaw which clobbered input file digest, path, and stat information that was needed to compare against on disk manifest entries - implement the logic for when the Manifest API user provides an already available handle, stat, or contents, and when they request the handle or contents. - rename `Manifest.writeManifest` to `Manifest.finalize` - rename `Manifest.manifest_dirty` to `Manifest.dirty` --- lib/std/Build/Cache.zig | 723 ++++++++++++++++++++++++++++++---------- 1 file changed, 544 insertions(+), 179 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 874a511e9e030b852e84ebabc8ee115329a4c9cf..64c740790b85ab9a0daa9d8d36a290b68130e433 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -48,7 +48,7 @@ pub fn obtain(cache: *Cache) Manifest { .cache = cache, .hash = cache.hash, .manifest_file = null, - .manifest_dirty = false, + .dirty = false, }; } @@ -274,7 +274,8 @@ pub const Manifest = struct { hash: HashHelper, /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. manifest_file: ?Io.File, - manifest_dirty: bool, + /// `finalize` has not been called yet. + dirty: bool, /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code /// using the cache will not modify the files within the cache directory. This allows multiple processes /// to utilize the same cache directory at the same time. @@ -306,30 +307,6 @@ pub const Manifest = struct { all_input_content: std.ArrayList(u8) = .empty, max_input_content_len: usize = std.math.maxInt(u32), - /// State that exists only during check. - pub const Check = struct { - /// Protects `Manifest.diagnostic` from data races. - diagnostic_lock: bool = false, - status: Status = .hit, - - pub const Status = enum { hit, miss }; - - pub const Error = error{ - /// Unable to check the cache for a reason that has been recorded into - /// the `diagnostic` field. - CacheCheckFailed, - /// A cache manifest file exists however it could not be parsed. - InvalidFormat, - } || Allocator.Error || Io.Cancelable; - - fn fail(c: *Check, m: *Manifest, diagnostic: Diagnostic) error{CacheCheckFailed} { - if (!@atomicRmw(bool, &c.diagnostic_lock, .Xchg, true, .monotonic)) { - m.diagnostic = diagnostic; - } - return error.CacheCheckFailed; - } - }; - pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); /// Source files and directories whose prefix and relative path are @@ -340,7 +317,9 @@ pub const Manifest = struct { /// but the other fields of `File` will be populated depending on the /// fields of `InputPath`. pub const InputPath = struct { + /// Determines whether `check` will keep the handle open or close it. request_handle: bool, + /// Determines whether `handle` is populated. have_handle: bool, /// Determines whether `File.size`, `File.inode`, and `File.mtime` are populated. have_stat: bool, @@ -362,6 +341,13 @@ pub const Manifest = struct { off: u32, len: u32, }, + + pub fn get(this: @This(), all_contents: []const u8) ?[]const u8 { + return if (this.status.populated()) + all_contents[this.status.populated.off..][0..this.status.populated.len] + else + null; + } } align(@alignOf(u32)), /// `have_handle` determines whether this is populated. handle: union { @@ -369,6 +355,12 @@ pub const Manifest = struct { dir: Io.Dir, }, + pub const Handle = union(enum) { + none, + file: Io.File, + dir: Io.Dir, + }; + /// Index into `Manifest.input_paths`. pub const Index = enum(u32) { _, @@ -387,6 +379,13 @@ pub const Manifest = struct { return manifest.all_input_content.items[populated.off..][0..populated.len]; } }; + + pub fn getHandle(this: @This(), file: *const File) Handle { + return if (!this.have_handle) .none else switch (file.flags.is_directory) { + false => .{ .file = this.handle.file }, + true => .{ .dir = this.handle.dir }, + }; + } }; /// The data per tracked input file that is stored in the manifest file. @@ -449,6 +448,24 @@ pub const Manifest = struct { if (!mem.isAligned(@backingInt(offset), @alignOf(File))) return error.InvalidFormat; return getConst(offset, contents); } + + pub fn flagsAndPath(off: File.Offset, contents: []const u8) [:0]const u8 { + const flags_off = @offsetOf(File, "flags"); + comptime assert(@offsetOf(File, "path_start") - flags_off == 1); + const hash_start = @backingInt(off) + flags_off; + const hash_end = mem.findScalarPos(u8, contents, hash_start, 0).?; + return contents[hash_start..hash_end]; + } + + pub fn pathFallible(off: File.Offset, contents: []const u8) error{InvalidFormat}![:0]const u8 { + const path_start = @backingInt(off) + @offsetOf(File, "path_start"); + const path_end = mem.findScalarPos(u8, contents, path_start, 0) orelse return error.InvalidFormat; + return contents[path_start..path_end :0]; + } + + pub fn path(off: File.Offset, contents: []const u8) [:0]const u8 { + return pathFallible(contents, off) catch unreachable; + } }; /// Intentionally matches if the files are different only by flags other than prefix. @@ -457,7 +474,7 @@ pub const Manifest = struct { pub fn hash(this: @This(), off: Offset) u32 { const file_prefix = off.getConst(this.contents).flags.prefix; - const file_path = filePath(this.contents, off); + const file_path = off.path(this.contents); return @truncate(std.hash.Wyhash.hash(file_prefix, file_path)); } @@ -466,8 +483,8 @@ pub const Manifest = struct { const a_prefix = a_off.getConst(this.contents).flags.prefix; const b_prefix = b_off.getConst(this.contents).flags.prefix; if (a_prefix != b_prefix) return false; - const a_path = filePath(this.contents, a_off); - const b_path = filePath(this.contents, b_off); + const a_path = a_off.path(this.contents); + const b_path = b_off.path(this.contents); return mem.eql(u8, a_path, b_path); } }; @@ -480,12 +497,24 @@ pub const Manifest = struct { hasher.final(&file.digest); } - fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void { - file.size = stat.size; - file.inode = stat.inode; - file.mtime = @intCast(stat.mtime.toNanoseconds()); + fn stat(file: *const File) Stat { + return .{ + .size = file.size, + .inode = file.inode, + .mtime = file.mtime, + }; + } - if (try m.isProblematicTimestamp(stat.mtime)) { + fn setStatUnchecked(file: *File, s: Stat) void { + file.size = s.size; + file.inode = s.inode; + file.mtime = @intCast(s.mtime.toNanoseconds()); + } + + fn setStat(file: *File, m: *Manifest, s: Stat) Io.Cancelable!void { + setStatUnchecked(file, s); + + if (try m.isProblematicTimestamp(s.mtime)) { // The actual file has an unreliable timestamp; force it to be hashed. file.mtime = 0; file.inode = 0; @@ -493,14 +522,14 @@ pub const Manifest = struct { } /// Returns true if the stat was changed. Updates the `file` with the new stat value. - fn setStatChanged(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!bool { - if (stat.size == file.size and - stat.mtime.nanoseconds == file.mtime and - stat.inode == file.inode) + fn setStatChanged(file: *File, m: *Manifest, s: Stat) Io.Cancelable!bool { + if (s.size == file.size and + s.mtime.nanoseconds == file.mtime and + s.inode == file.inode) { return false; } else { - try setStat(file, m, stat); + try setStat(file, m, s); return true; } } @@ -535,7 +564,7 @@ pub const Manifest = struct { const prefix = fo.file_offset.get(contents).flags.prefix; return .{ .root_dir = manifest.cache.prefixes()[prefix], - .sub_path = filePath(contents, fo.file_offset), + .sub_path = fo.file_offset.path(contents), }; } }; @@ -680,6 +709,21 @@ pub const Manifest = struct { _ = try addInputPath(m, opt_path orelse return, options); } + pub const CheckResult = union(enum) { + hit, + incomplete_manifest, + path_deleted: File.Offset, + directory_status_changed: File.Offset, + metadata_changed: File.Offset, + contents_changed: File.Offset, + }; + + pub const CheckError = error{ + /// Unable to check the cache for a reason that has been recorded into + /// the `diagnostic` field. + CacheCheckFailed, + } || Allocator.Error || Io.Cancelable; + /// Check the cache to see if the input exists in it. /// A hex encoding of its hash is available by calling `final`. /// @@ -692,13 +736,13 @@ pub const Manifest = struct { /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) Check.Error!Check.Status { + pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) CheckError!CheckResult { const node = parent_progress_node.start("Reusing Cache Artifacts", 0); defer node.end(); return checkProgressless(man); } - pub fn checkProgressless(man: *Manifest) Check.Error!Check.Status { + pub fn checkProgressless(man: *Manifest) CheckError!CheckResult { assert(man.manifest_file == null); // This is *not* hashing the contents of the input files. It is the @@ -798,16 +842,18 @@ pub const Manifest = struct { // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user - // can use `final`, and will at some point `writeManifest` the file list to disk. + // can use `final`, and will at some point `finalize` the file list to disk. man.hash.hasher = hasher_init; man.hash.hasher.update(&input_digest); hit: { - digests: { - switch (try man.checkLocked()) { - .hit => break :hit, - .miss => if (!try man.upgradeToExclusiveLock()) break :digests, + const miss_result = miss: { + const result = try man.checkLocked(); + if (result == .hit) { + break :hit; + } else if (!try man.upgradeToExclusiveLock()) { + break :miss result; } // We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone // else might have modified the digest, so we need to check again before deciding to miss. @@ -816,18 +862,16 @@ pub const Manifest = struct { man.hash.hasher = hasher_init; man.hash.hasher.update(&input_digest); man.shrinkFilesToInput(); - switch (try man.checkLocked()) { - .hit => break :hit, - .miss => break :digests, - } - } + const refreshed_result = try man.checkLocked(); + if (refreshed_result == .hit) break :hit; + break :miss refreshed_result; + }; - // Cache miss. `checkLocked` guarantees that all input files have their digests populated - // unless it returns an error. - man.manifest_dirty = true; - // All input file digests are already populated by `checkLocked`, so we can call `unhit` directly. + // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a + // cache miss, which is needed because they will be used in the manifest digest. + man.dirty = true; unhit(man, &input_digest); - return .miss; + return miss_result; } if (man.want_shared_lock) { @@ -852,127 +896,454 @@ pub const Manifest = struct { /// Assumes that `self.hash.hasher` has been updated only with the original digest and that /// `self.files` contains only the original input files. - fn checkLocked(m: *Manifest) Check.Error!Check.Status { + fn checkLocked(m: *Manifest) CheckError!CheckResult { const gpa = m.cache.gpa; const io = m.cache.io; const manifest_file = m.manifest_file.?; const manifest_size = if (manifest_file.stat(io)) |stat| stat.size else |err| switch (err) { error.Canceled => |e| return e, - else => |e| { - m.diagnostic = .{ .manifest_stat = e }; - return error.CacheCheckFailed; - }, + else => |e| return fail(&m.diagnostic, .{ .manifest_stat = e }), }; - // We always want to compute the input file hash digests, even on a cache miss, because they will be - // used in the manifest digest. - if (manifest_size == 0 or manifest_size < m.contents.items.len) { - // Don't clobber our input files by overwriting contents since it's a cache miss. + // Manifest file was never finalized. try m.contents.ensureUnusedCapacity(gpa, 1); - return .miss; + return .incomplete_manifest; } - // Already supposedly includes the extra null byte that we have to maintain. - try m.contents.resize(gpa, manifest_size); - + // We must not clobber existing `Manifest.contents` because it possibly contains prepopulated stat and digest + // information which we must compare with the file system. Since non input files are typically greater in number + // than input files, the strategy here is to move the contents to make room for entirety of the manifest + // contents on disk, and then later truncate the contents array. + // + // The file on disk already supposedly includes the extra null byte that we have to maintain. + const input_contents_len = m.contents.len; + // After this resize, every return statement needs to modify m.contents length. + try m.contents.resize(gpa, input_contents_len + manifest_size); + errdefer m.contents.shrinkRetainingCapacity(input_contents_len); + @memcpy(m.contents.items[manifest_size..][0..input_contents_len], m.contents.items[0..input_contents_len]); { const n = manifest_file.readPositionalAll(io, m.contents.items, 0) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| { - m.diagnostic = .{ .manifest_read = e }; - return error.CacheCheckFailed; - }, + else => |e| return fail(&m.diagnostic, .{ .manifest_read = e }), }; - m.contents.shrinkRetainingCapacity(n); + if (n != manifest_size) return checkLockedMiss(m, input_contents_len, .incomplete_manifest); } - const contents = m.contents.items; + const disk_contents = m.contents.items[0..manifest_size]; + const input_contents = m.contents.items[manifest_size..input_contents_len]; + // First the input files section, which must match our input files, otherwise it's invalid or incomplete. var off: usize = 0; - var c: Check = .{}; + for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len], 0..) |*input_path, file_off, i| { + if (@backingInt(file_off) + 1 >= disk_contents.len) + return missInput(m, i, input_contents_len, manifest_size, .incomplete_manifest); + if (!mem.eql(u8, file_off.flagsAndPath(disk_contents), file_off.flagsAndPath(input_contents))) + return missInput(m, i, input_contents_len, manifest_size, .invalid_manifest); - // First the input files section, which must match our input files, otherwise it's invalid format. - for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len]) |*input_path, input_file_off| { - if (off + 1 >= contents.len) return error.InvalidFormat; - const file_off: File.Offset = @fromBackingInt(@intCast(off)); - const file = try file_off.getFallible(contents); - if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; - const path = try filePathFallible(contents, file_off); - if (path.len == 0) return error.InvalidFormat; - if (input_file_off != file_off) return error.InvalidFormat; - - try checkInputFile(m, &c, file_off, path, input_path); - - off += File.sizeOf(path.len); + const result = try checkInputPath(m, file_off, input_path, disk_contents, input_contents); + if (result != .hit) return missInput(m, i + 1, input_contents_len, manifest_size, result); + off = @backingInt(file_off); } - if (c.status == .miss) return .miss; - if (m.diagnostic != .none) return error.CacheCheckFailed; - // Guess number of files based on manifest contents len to reduce allocations. // This is not an upper bound; subsequent insertions may potentially allocate. - try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{ - .contents = contents, + try m.files.ensureUnusedCapacityContext(gpa, disk_contents.len / (@sizeOf(File) + 32), .{ + .contents = disk_contents, }); - // Validate and count discovered files. - while (off + 1 < contents.len) { + // Validate and check discovered files. + while (off + 1 < disk_contents.len) { const file_off: File.Offset = @fromBackingInt(@intCast(off)); - const file = try file_off.getFallible(contents); - if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; - const path = try filePathFallible(contents, file_off); - if (path.len == 0) return error.InvalidFormat; + const file = try file_off.getFallible(disk_contents); + if (file.flags.prefix >= m.cache.prefixes_len) + return checkLockedMiss(m, input_contents_len, .invalid_manifest); + const path = try file_off.pathFallible(disk_contents); + if (path.len == 0) return checkLockedMiss(m, input_contents_len, .invalid_manifest); - try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); - switch (try checkFile(m, &c, file_off, filePath(contents, file_off))) { - .hit => {}, - .miss => return .miss, - } + try m.files.putContext(gpa, file_off, {}, .{ .contents = disk_contents }); + const result = try checkDiscoveredPath(file_off, file_off.path(disk_contents)); + if (result != .hit) return checkLockedMiss(m, input_contents_len, result); off += File.sizeOf(path.len); } // Final terminating zero byte to distinguish empty manifest file from // manifest with zero files. - const file_valid = off + 1 == contents.len and contents[off] == 0; - if (!file_valid) return .miss; + const file_valid = off + 1 == disk_contents.len and disk_contents[off] == 0; + if (!file_valid) return checkLockedMiss(m, input_contents_len, .incomplete_manifest); - for (m.files.keys()) |file_off| m.hash.hasher.update(&file_off.get(contents).digest); - - // Don't track the trailing zero byte in contents. - m.contents.items.len -= 1; + for (m.files.keys()) |file_off| m.hash.hasher.update(&file_off.get(disk_contents).digest); + // Since it's a cache hit, we accept the input file contents from disk and discard the other copy. + // Furthermore, don't track the trailing zero byte in contents. + m.contents.shrinkRetainingCapacity(manifest_size - 1); return .hit; } - fn checkInputFile( + /// Restore input file contents to prepare for cache miss workflow. We want to keep everything prior to `off` to + /// avoid redundant computation of digest and stat. + /// + /// Furthermore, this function is responsible for populating the remaining input file digests and metadata after the + /// missed one. + fn missInput( + m: *Manifest, + next_file_index: usize, + input_contents_len: usize, + manifest_size: usize, + result: CheckResult, + ) CheckError!CheckResult { + const file_offs = m.files.keys(); + const off = if (file_offs.len - next_file_index == 0) input_contents_len else file_offs[next_file_index]; + const copy_len = input_contents_len - off; + @memcpy( + m.contents.items[off..][0..copy_len], + m.contents.items[manifest_size + off ..][0..copy_len], + ); + m.contents.shrinkRetainingCapacity(input_contents_len); + // And now iterate over remaining input files and populate the missing input file hashes. + const contents = m.contents.items; + for (file_offs[next_file_index..], m.input_paths.items) |input_file_off, *input_path| { + assert(@backingInt(input_file_off) == off); + try populateInputPath(m, input_file_off, input_path, contents); + } + return result; + } + + fn checkLockedMiss(m: *Manifest, input_contents_len: usize, result: CheckResult) CheckResult { + m.contents.shrinkRetainingCapacity(input_contents_len); + return result; + } + + /// Similar to `checkInputPath` and `checkDiscoveredPath` but does not compare metadata or digest against file + /// system. Only ensures that metadata and digest are available. + fn populateInputPath( m: *Manifest, - c: *Check, file_off: File.Offset, - file_path: [:0]const u8, input_path: *InputPath, - ) CheckFileError!void { - if (input_path.have_handle) @panic("TODO"); - if (input_path.have_stat) @panic("TODO"); - if (input_path.contents.status != .unrequested) @panic("TODO"); - if (input_path.request_handle) @panic("TODO"); - switch (try checkFile(m, c, file_off, file_path)) { - .hit => {}, - .miss => c.status = .miss, + contents: []const u8, + ) CheckError!void { + const cache = m.cache; + const io = cache.io; + const input_file = file_off.get(contents); + const parent_dir = cache.prefixes()[input_file.flags.prefix].handle; + const file_path = file_off.path(contents); + + if (input_path.have_digest) return; + + const stat_path_ok = input_path.contents.status.populated() or input_file.flags.metadata_only; + if (!input_path.have_stat and stat_path_ok) { + // Since this is an input file, FileNotFound counts as a failure, not a cache miss. + const actual_stat = switch (input_path.getHandle(input_file)) { + .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .file => |opened_file| opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + }; + const actual_is_directory = actual_stat.kind == .directory; + if (actual_is_directory != input_file.flags.is_directory) { + // Since this is an input file, this is a failure, not a cache miss. + return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = if (actual_is_directory) error.IsDir else error.NotDir, + } }); + } + try input_file.setStat(m, .init(actual_stat)); + } + + if (input_file.flags.metadata_only) { + assert(stat_path_ok); + input_file.hashFromMetadata(); + return; + } + + if (input_path.contents.get()) |file_or_dir_contents| { + assert(stat_path_ok); + var hasher = hasher_init; + hasher.update(file_or_dir_contents); + hasher.final(&input_file.digest); + return; + } + + if (input_file.flags.is_directory) { + const opened_dir = if (input_path.have_handle) + input_path.handle.dir + else + parent_dir.openDir(io, file_path, .{ + .iterate = true, + .access_sub_paths = false, + }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer if (!input_path.request_handle) opened_dir.close(io) else { + input_path.handle = .{ .dir = opened_dir }; + input_path.have_handle = true; + }; + + if (!input_path.have_stat) { + assert(!stat_path_ok); + const actual_stat = opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + try input_file.setStat(m, .init(actual_stat)); + } + + const dir_contents_start = m.all_input_content.items.len; + const gpa = cache.gpa; + hashDir(gpa, io, opened_dir, &input_file.digest, &m.all_input_content) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer if (input_path.contents.status == .requested) { + input_path.contents = .{ .populated = .{ + .off = @intCast(dir_contents_start), + .len = @intCast(m.all_input_content.items.len - dir_contents_start), + } }; + } else m.all_input_content.shrinkRetainingCapacity(dir_contents_start); + } else { + const opened_file = if (input_path.have_handle) + input_path.handle.file + else + parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer if (!input_path.request_handle) opened_file.close(io) else { + input_path.handle = .{ .file = opened_file }; + input_path.have_handle = true; + }; + + if (!input_path.have_stat) { + assert(!stat_path_ok); + const actual_stat = opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + try input_file.setStat(m, .init(actual_stat)); + } + + hashFile(io, opened_file, &input_file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + + if (input_path.contents.status == .requested) @panic("TODO"); } } - const CheckFileError = error{ Canceled, CacheCheckFailed }; + /// Upon return, disk_contents digest and stat will be populated regardless of whether hit or miss occurs. + fn checkInputPath( + m: *Manifest, + file_off: File.Offset, + input_path: *InputPath, + disk_contents: []const u8, + input_contents: []const u8, + ) CheckError!CheckResult { + const cache = m.cache; + const io = cache.io; + const disk_file = file_off.get(disk_contents); + const input_file = file_off.get(input_contents); + const parent_dir = cache.prefixes()[disk_file.flags.prefix].handle; + const file_path = file_off.path(disk_contents); + + assert(disk_file.flags == input_file.flags); + + if (input_path.have_digest) { + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) + return .hit; + } + + if (input_path.have_stat) { + const changed = try disk_file.setStatChanged(m, input_file.stat); + if (!changed) return .hit; + } + + if (disk_file.flags.metadata_only) { + if (!input_path.have_stat) { + // Since this is an input file, FileNotFound counts as a failure, not a cache miss. + const actual_stat = switch (input_path.getHandle(input_file)) { + .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .file => |opened_file| opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + }; + // In the other case (have_stat=true), this check is redundant since we already asserted the flags are + // identical, which contains the prefix. + const actual_is_directory = actual_stat.kind == .directory; + if (actual_is_directory != disk_file.flags.is_directory) { + // Since this is an input file, this is a failure, not a cache miss. + return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = if (actual_is_directory) error.IsDir else error.NotDir, + } }); + } + const stat: Stat = .init(actual_stat); + const changed = try disk_file.setStatChanged(m, stat); + disk_file.hashFromMetadata(); + if (!changed) return .hit; + } + return .{ .metadata_changed = file_off }; + } + + if (input_path.contents.get()) |file_or_dir_contents| { + var hasher = hasher_init; + hasher.update(file_or_dir_contents); + hasher.final(&disk_file.digest); + + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; + return .{ .contents_changed = file_off }; + } + + if (disk_file.flags.is_directory) { + const opened_dir = if (input_path.have_handle) + input_path.handle.dir + else + parent_dir.openDir(io, file_path, .{ + .iterate = true, + .access_sub_paths = false, + }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer if (!input_path.request_handle) opened_dir.close(io) else { + input_path.handle = .{ .dir = opened_dir }; + input_path.have_handle = true; + }; + + if (!input_path.have_stat) { + const actual_stat = opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + if (!try disk_file.setStatChanged(m, .init(actual_stat))) return .hit; + } + + const dir_contents_start = m.all_input_content.items.len; + const gpa = cache.gpa; + hashDir(gpa, io, opened_dir, &disk_file.digest, &m.all_input_content) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer if (input_path.contents.status == .requested) { + input_path.contents = .{ .populated = .{ + .off = @intCast(dir_contents_start), + .len = @intCast(m.all_input_content.items.len - dir_contents_start), + } }; + } else m.all_input_content.shrinkRetainingCapacity(dir_contents_start); + + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; + return .{ .contents_changed = file_off }; + } else { + const opened_file = if (input_path.have_handle) + input_path.handle.file + else + parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer if (!input_path.request_handle) opened_file.close(io) else { + input_path.handle = .{ .file = opened_file }; + input_path.have_handle = true; + }; + + if (!input_path.have_stat) { + const actual_stat = opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + if (!try disk_file.setStatChanged(m, .init(actual_stat))) return .hit; + } + + hashFile(io, opened_file, &disk_file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + if (input_path.contents.status == .requested) @panic("TODO"); + + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; + return .{ .contents_changed = file_off }; + } + } - /// Runs concurrently with other `checkFile`. - fn checkFile( + fn checkDiscoveredPath( m: *Manifest, - c: *Check, file_off: File.Offset, file_path: [:0]const u8, - ) CheckFileError!Check.Status { - const file = file_off.get(m.contents.items); + contents: []const u8, + ) CheckError!CheckResult { + const file = file_off.get(contents); const cache = m.cache; const gpa = cache.gpa; const io = cache.io; @@ -980,20 +1351,21 @@ pub const Manifest = struct { if (file.flags.metadata_only) { const actual_stat = parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { - error.FileNotFound => return .miss, + error.FileNotFound => return .{ .path_deleted = file_off }, error.Canceled => |e| return e, - else => |e| return c.fail(m, .{ .file_stat = .{ + else => |e| return fail(m, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), }; const actual_is_directory = actual_stat.kind == .directory; - if (actual_is_directory != file.flags.is_directory) return .miss; + if (actual_is_directory != file.flags.is_directory) return .{ .directory_status_changed = file_off }; const changed = try file.setStatChanged(m, .init(actual_stat)); file.hashFromMetadata(); - return if (changed) .miss else .hit; + if (changed) return .{ .metadata_changed = file_off }; + return .hit; } if (file.flags.is_directory) { @@ -1001,9 +1373,10 @@ pub const Manifest = struct { .iterate = true, .access_sub_paths = false, }) catch |err| switch (err) { - error.FileNotFound, error.NotDir => return .miss, + error.FileNotFound => return .{ .path_deleted = file_off }, + error.NotDir => return .{ .directory_status_changed = file_off }, error.Canceled => |e| return e, - else => |e| return c.fail(m, .{ .file_open = .{ + else => |e| return fail(m, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1012,32 +1385,33 @@ pub const Manifest = struct { const actual_stat = opened_dir.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return c.fail(m, .{ .file_stat = .{ + else => |e| return fail(m, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), }; if (try file.setStatChanged(m, .init(actual_stat))) { const prev_digest: BinDigest = file.digest; - var contents: std.ArrayList(u8) = .empty; - defer contents.deinit(gpa); - hashDir(gpa, io, opened_dir, &file.digest, &contents) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| return c.fail(m, .{ .file_read = .{ + const dir_contents_start = m.all_input_content.items.len; + defer m.all_input_content.shrinkRetainingCapacity(dir_contents_start); + hashDir(gpa, io, opened_dir, &file.digest, &m.all_input_contents) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(m, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), }; - if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss; + if (!mem.eql(u8, &file.digest, &prev_digest)) return .{ .contents_changed = file_off }; } return .hit; } const opened_file = parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound, error.IsDir => return .miss, + error.FileNotFound => return .{ .path_deleted = file_off }, + error.IsDir => return .{ .directory_status_changed = file_off }, error.Canceled => |e| return e, - else => |e| return c.fail(m, .{ .file_open = .{ + else => |e| return fail(m, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1046,7 +1420,7 @@ pub const Manifest = struct { const actual_stat = opened_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return c.fail(m, .{ .file_stat = .{ + else => |e| return fail(m, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1056,19 +1430,25 @@ pub const Manifest = struct { const prev_digest: BinDigest = file.digest; hashFile(io, opened_file, &file.digest) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return c.fail(m, .{ .file_read = .{ + else => |e| return fail(m, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), }; - if (!mem.eql(u8, &file.digest, &prev_digest)) return .miss; + if (!mem.eql(u8, &file.digest, &prev_digest)) return .{ .contents_changed = file_off }; } return .hit; } - /// Reset `man.hash.hasher` to the state it should be in after `check` returns `Check.Status.miss`. + fn fail(ptr: *Diagnostic, d: Diagnostic) error{CacheCheckFailed} { + assert(ptr.* == .none); + ptr.* = d; + return error.CacheCheckFailed; + } + + /// Reset `man.hash.hasher` to the state it should be in after `check` returns `CheckResult.miss`. /// The hasher contains the original input digest, and all original input file digests (i.e. /// not including discovered files). /// @@ -1231,7 +1611,7 @@ pub const Manifest = struct { try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); } else { const dir = cache.prefixes()[prefix].handle; - const sub_path = filePath(m.contents.items, file_offset); + const sub_path = file_offset.path(m.contents.items); const handle = dir.openDir(io, sub_path, .{ .access_sub_paths = false, .iterate = true, @@ -1250,7 +1630,7 @@ pub const Manifest = struct { try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); } else { const dir = cache.prefixes()[prefix].handle; - const sub_path = filePath(m.contents.items, file_offset); + const sub_path = file_offset.path(m.contents.items); const handle = dir.openFile(io, sub_path, .{ .mode = .read_only }) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| { @@ -1420,18 +1800,18 @@ pub const Manifest = struct { /// If `want_shared_lock` is true, this function automatically downgrades the /// lock from exclusive to shared. - pub fn writeManifest(m: *Manifest) !void { + pub fn finalize(m: *Manifest) !void { assert(m.have_exclusive_lock); const io = m.cache.io; const manifest_file = m.manifest_file.?; - if (m.manifest_dirty) { + if (m.dirty) { m.contents.appendAssumeCapacity(0); defer _ = m.contents.pop().?; try manifest_file.setLength(io, m.contents.items.len); try manifest_file.writePositionalAll(io, m.contents.items, 0); - m.manifest_dirty = false; + m.dirty = false; } if (m.want_shared_lock) { @@ -1473,7 +1853,7 @@ pub const Manifest = struct { /// Obtain only the data needed to maintain a lock on the manifest file. /// The `Manifest` remains safe to deinit. /// - /// Don't forget to call `writeManifest` before this! + /// Don't forget to call `finalize` before this! pub fn toOwnedLock(self: *Manifest) Lock { defer self.manifest_file = null; return .{ .manifest_file = self.manifest_file.? }; @@ -1496,7 +1876,7 @@ pub const Manifest = struct { } pub fn path(scf: *const SelfContainedFiles, file_offset: File.Offset) [:0]const u8 { - return filePath(scf.contents.items, file_offset); + return file_offset.path(scf.contents.items); } }; @@ -1516,7 +1896,7 @@ pub const Manifest = struct { /// Releases the manifest file and frees any memory the Manifest was using. /// `Manifest.hit` must be called first. /// - /// Don't forget to call `writeManifest` before this! + /// Don't forget to call `finalize` before this! pub fn deinit(m: *Manifest) void { const io = m.cache.io; const gpa = m.cache.gpa; @@ -1544,7 +1924,7 @@ pub const Manifest = struct { if (files.len > 0) { const contents = man.contents.items; for (files) |file| { - const file_path = filePath(contents, file); + const file_path = file.path(contents); try buf.ensureUnusedCapacity(gpa, file_path.len + 2); buf.appendAssumeCapacity(@as(u8, file.get(contents).flags.prefix) + 1); buf.appendSliceAssumeCapacity(file_path); @@ -1675,22 +2055,7 @@ pub const Manifest = struct { } fn hashFlagsAndPath(m: *const Manifest, off: File.Offset, hasher: *Hasher) void { - const contents = m.contents.items; - const flags_off = @offsetOf(File, "flags"); - comptime assert(@offsetOf(File, "path_start") - flags_off == 1); - const hash_start = @backingInt(off) + flags_off; - const hash_end = mem.findScalarPos(u8, contents, hash_start, 0).?; - hasher.update(contents[hash_start..hash_end]); - } - - pub fn filePathFallible(contents: []const u8, off: File.Offset) error{InvalidFormat}![:0]const u8 { - const path_start = @backingInt(off) + @offsetOf(File, "path_start"); - const path_end = mem.findScalarPos(u8, contents, path_start, 0) orelse return error.InvalidFormat; - return contents[path_start..path_end :0]; - } - - fn filePath(contents: []const u8, off: File.Offset) [:0]const u8 { - return filePathFallible(contents, off) catch unreachable; + hasher.update(off.flagsAndPath(m.contents.items)); } }; @@ -1756,7 +2121,7 @@ test "cache file and then recall it" { try testing.expectEqual(.miss, try ch.check(.none)); digest1 = ch.final(); - try ch.writeManifest(); + try ch.finalize(); } { var ch = cache.obtain(); @@ -1827,7 +2192,7 @@ test "check that changing a file makes cache fail" { digest1 = man.final(); - try man.writeManifest(); + try man.finalize(); } try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = updated_temp_file_contents }); @@ -1847,7 +2212,7 @@ test "check that changing a file makes cache fail" { digest2 = man.final(); - try man.writeManifest(); + try man.finalize(); } try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); @@ -1888,7 +2253,7 @@ test "no file inputs" { digest1 = man.final(); - try man.writeManifest(); + try man.finalize(); } { var man = cache.obtain(); @@ -1953,7 +2318,7 @@ test "Manifest with files added after initial hash work" { _ = try ch.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); digest1 = ch.final(); - try ch.writeManifest(); + try ch.finalize(); } { var ch = cache.obtain(); @@ -1992,7 +2357,7 @@ test "Manifest with files added after initial hash work" { digest3 = ch.final(); - try ch.writeManifest(); + try ch.finalize(); } try testing.expect(!mem.eql(u8, &digest1, &digest3)); -- 2.54.0 From b91a6dc5b1c99ec8308fd14224d9cd8c44785021 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 11 Sep 2026 19:08:30 -0700 Subject: [PATCH 22/49] update to newest std.Build.Cache API --- lib/compiler/Maker.zig | 4 +- lib/compiler/Maker/Step.zig | 15 +- lib/compiler/Maker/Step/ConfigHeader.zig | 2 +- lib/compiler/Maker/Step/ObjCopy.zig | 2 +- lib/compiler/Maker/Step/Options.zig | 2 +- lib/compiler/Maker/Step/Run.zig | 4 +- lib/compiler/Maker/Step/WriteFile.zig | 2 +- lib/std/Build/Cache.zig | 261 +++++++++++++---------- src/Compilation.zig | 20 +- src/libs/freebsd.zig | 4 +- src/libs/glibc.zig | 4 +- src/libs/mingw.zig | 5 +- src/libs/netbsd.zig | 4 +- src/libs/openbsd.zig | 4 +- src/main.zig | 2 +- 15 files changed, 184 insertions(+), 151 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 9e7d1b01e47b034279d83577eeeabb3cf24ebfc7..118bcffb8dcfcc6dab6f37dcbf6678b716344594 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1543,7 +1543,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { config_tmp_path, final_path, e, }); }; - man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); + man.finalize() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); options.src_files.* = man.takeFiles(); break :cp .{ final_path, man.toOwnedLock() }; } @@ -1934,7 +1934,7 @@ fn cacheCatOne(input_hex_digest: []const u8, contents: []const u8, writer: *Io.W while (off + 1 < contents.len) { const file_off: Cache.Manifest.File.Offset = @fromBackingInt(@intCast(off)); const file = try file_off.getFallibleConst(contents); - const path = try Cache.Manifest.filePathFallible(contents, file_off); + const path = try file_off.pathFallible(contents); if (path.len == 0) return error.InvalidFormat; var file_obj = try files_tuple.beginStructField(.{ .whitespace_style = .{ .wrap = false } }); diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 25ea943f39d61909415e80741832b547b7f60334..c3ff8962b0d5cd940a766c7b4e6aef9ed7faeb9c 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -734,7 +734,7 @@ pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_n /// Clears previous watch inputs, if any, and then populates watch inputs from /// the full set of files picked up by the cache manifest. /// -/// Must be accompanied with `writeManifestAndWatch`. +/// Must be accompanied with `finalizeManifestAndWatch`. pub fn cacheHitWatched(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { const hit = .hit == (man.check(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err)); s.result_cached = hit; @@ -748,7 +748,7 @@ fn failWithCacheError( s: *Step, maker: *Maker, man: *const Cache.Manifest, - err: Cache.Manifest.Check.Error, + err: Cache.Manifest.CheckError, ) error{ OutOfMemory, Canceled, MakeFailed } { switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { @@ -762,15 +762,14 @@ fn failWithCacheError( }, }, error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => return s.fail(maker, "failed checking cache: invalid manifest file format", .{}), } } -/// Prefer `writeManifestAndWatch` unless you already added watch inputs +/// Prefer `finalizeManifestAndWatch` unless you already added watch inputs /// separately from using the cache system. -pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { +pub fn finalizeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { if (s.test_results.isSuccess()) { - man.writeManifest() catch |err| switch (err) { + man.finalize() catch |err| switch (err) { error.Canceled => |e| return e, else => |e| try s.addError(maker, "failed writing cache manifest: {t}", .{e}), }; @@ -781,8 +780,8 @@ pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { /// the full set of files picked up by the cache manifest. /// /// Must be accompanied with `cacheHitWatched`. -pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { - try writeManifest(s, maker, man); +pub fn finalizeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { + try finalizeManifest(s, maker, man); try setWatchInputsFromManifest(s, maker, man); } diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index 20152955fe469b751648032e54152a7aa2b53f67..4ea9444d25b0dc6995ecfc43a3901105ce087be9 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -169,7 +169,7 @@ pub fn make( .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), }; - try step.writeManifest(maker, &man); + try step.finalizeManifest(maker, &man); } fn ensureAllValuesUsed( diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 7924b21c683866a3536bf72e2bf9b04f3850f3c5..db045d5f51e46e080e080d4fa434cff40e6ce1cd 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -165,7 +165,7 @@ pub fn make( maker.generatedPath(conf_oc.output_file).* = dest_path; - step.writeManifest(maker, &man) catch |err| switch (err) { + step.finalizeManifest(maker, &man) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| try step.addError(maker, "failed writing cache manifest: {t}", .{e}), }; diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index e0812402e1b976b98cc9e15f42d77afd04c4f326..ef932c7dae0bbf38284d38eebd53f1f1997b7b2f 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -95,7 +95,7 @@ pub fn make( }, }; - try step.writeManifestAndWatch(maker, &man); + try step.finalizeManifestAndWatch(maker, &man); maker.generatedPath(conf_options.generated_file).* = out_path; } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 04b5764d04a191ace28403be0737635f047019fb..f8b58d215997479660975b7b15de0c92c4e625ac 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -275,7 +275,7 @@ pub fn make( try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); try populateGeneratedPathsCreateDirs(arena, run_index, maker, output_dir_path, output_placeholders.items, argv_list.items); try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); - if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); + if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man); return; } @@ -344,7 +344,7 @@ pub fn make( }; } - if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); + if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man); try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index afea7de96ea7d9598cd2d4f2e55febc40cbad0d1..5f7bad0e44e7ce45fc04d30dd943a478db731690 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -122,7 +122,7 @@ pub fn make( progress_node.setEstimatedTotalItems(total_items); try operate(maker, step_index, open_dir_cache, out_path, progress_node); - try step.writeManifest(maker, &man); + try step.finalizeManifest(maker, &man); maker.generatedPath(conf_wf.generated_directory).* = out_path; }, diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 64c740790b85ab9a0daa9d8d36a290b68130e433..e44efe4c59a0b329c8e908576f02238694579e30 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -325,30 +325,59 @@ pub const Manifest = struct { have_stat: bool, /// Determines whether `File.digest` is populated. have_digest: bool, - contents: packed union(u64) { - status: enum(u64) { - unrequested = std.math.maxInt(u64) - 1, - requested = std.math.maxInt(u64), - _, - pub fn populated(this: @This()) bool { - return switch (this) { - .unrequested, .requested => false, - _ => true, - }; - } - }, - populated: packed struct(u64) { + contents: struct { + pop_off: enum(u32) { unpopulated = std.math.maxInt(u32), _ }, + req_len: enum(u32) { unrequested = std.math.maxInt(u32) - 1, requested = std.math.maxInt(u32), _ }, + + pub const requested: @This() = .{ + .pop_off = .unpopulated, + .req_len = .requested, + }; + + pub const unrequested: @This() = .{ + .pop_off = .unpopulated, + .req_len = .unrequested, + }; + + pub fn populated(p: Populated) @This() { + return .{ + .pop_off = @fromBackingInt(p.off), + .req_len = @fromBackingInt(p.len), + }; + } + + pub const Unwrapped = union(enum) { + unrequested, + requested, + populated: Populated, + }; + + pub const Populated = struct { off: u32, len: u32, - }, + }; + + pub fn unwrap(this: @This()) Unwrapped { + return switch (this.pop_off) { + .unpopulated => switch (this.req_len) { + .unrequested => .unrequested, + .requested => .requested, + _ => unreachable, + }, + _ => .{ .populated = .{ + .off = @backingInt(this.pop_off), + .len = @backingInt(this.req_len), + } }, + }; + } pub fn get(this: @This(), all_contents: []const u8) ?[]const u8 { - return if (this.status.populated()) - all_contents[this.status.populated.off..][0..this.status.populated.len] - else - null; + return switch (this.unwrap()) { + .unrequested, .requested => null, + .populated => |p| all_contents[p.off..][0..p.len], + }; } - } align(@alignOf(u32)), + }, /// `have_handle` determines whether this is populated. handle: union { file: Io.File, @@ -369,14 +398,13 @@ pub const Manifest = struct { return manifest.input_paths.items[@backingInt(i)]; } + pub fn offset(i: @This(), manifest: *const Manifest) File.Offset { + return manifest.files.keys()[@backingInt(i)]; + } + pub fn contents(i: @This(), manifest: *const Manifest) []const u8 { const input_path = get(i, manifest); - const populated = switch (input_path.contents.status) { - .unrequested => unreachable, // Must request contents before calling `check`. - .requested => unreachable, // Must call `check` first. - _ => input_path.contents.populated, - }; - return manifest.all_input_content.items[populated.off..][0..populated.len]; + return input_path.contents.get(manifest.all_input_content.items).?; } }; @@ -454,7 +482,7 @@ pub const Manifest = struct { comptime assert(@offsetOf(File, "path_start") - flags_off == 1); const hash_start = @backingInt(off) + flags_off; const hash_end = mem.findScalarPos(u8, contents, hash_start, 0).?; - return contents[hash_start..hash_end]; + return contents[hash_start..hash_end :0]; } pub fn pathFallible(off: File.Offset, contents: []const u8) error{InvalidFormat}![:0]const u8 { @@ -464,7 +492,7 @@ pub const Manifest = struct { } pub fn path(off: File.Offset, contents: []const u8) [:0]const u8 { - return pathFallible(contents, off) catch unreachable; + return pathFallible(off, contents) catch unreachable; } }; @@ -501,7 +529,7 @@ pub const Manifest = struct { return .{ .size = file.size, .inode = file.inode, - .mtime = file.mtime, + .mtime = .fromNanoseconds(file.mtime), }; } @@ -664,9 +692,9 @@ pub const Manifest = struct { existing_input_file.have_handle = true; }, } - if (options.request_contents) switch (existing_input_file.contents.status) { - .requested, .unrequested => existing_input_file.contents = .{ .status = .requested }, - _ => {}, + if (options.request_contents) switch (existing_input_file.contents.unwrap()) { + .requested, .unrequested => existing_input_file.contents = .requested, + .populated => {}, }; const existing_header = m.files.keys()[gop.index].get(m.contents.items); if (options.stat) |stat| { @@ -689,7 +717,7 @@ pub const Manifest = struct { .file => |opt_file| if (opt_file) |file| .{ .file = file } else undefined, .dir => |opt_dir| if (opt_dir) |dir| .{ .dir = dir } else undefined, }, - .contents = .{ .status = if (options.request_contents) .requested else .unrequested }, + .contents = if (options.request_contents) .requested else .unrequested, .have_digest = false, .have_stat = options.stat != null, }); @@ -712,6 +740,7 @@ pub const Manifest = struct { pub const CheckResult = union(enum) { hit, incomplete_manifest, + invalid_manifest, path_deleted: File.Offset, directory_status_changed: File.Offset, metadata_changed: File.Offset, @@ -858,7 +887,7 @@ pub const Manifest = struct { // We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone // else might have modified the digest, so we need to check again before deciding to miss. // Before trying again, we must reset `man.hash.hasher` and `man.files`. - // This is basically just the first half of `unhit`. + // This is basically just the first half of `undoCheck`. man.hash.hasher = hasher_init; man.hash.hasher.update(&input_digest); man.shrinkFilesToInput(); @@ -870,7 +899,7 @@ pub const Manifest = struct { // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a // cache miss, which is needed because they will be used in the manifest digest. man.dirty = true; - unhit(man, &input_digest); + undoCheck(man, &input_digest); return miss_result; } @@ -918,7 +947,7 @@ pub const Manifest = struct { // contents on disk, and then later truncate the contents array. // // The file on disk already supposedly includes the extra null byte that we have to maintain. - const input_contents_len = m.contents.len; + const input_contents_len = m.contents.items.len; // After this resize, every return statement needs to modify m.contents length. try m.contents.resize(gpa, input_contents_len + manifest_size); errdefer m.contents.shrinkRetainingCapacity(input_contents_len); @@ -931,7 +960,7 @@ pub const Manifest = struct { if (n != manifest_size) return checkLockedMiss(m, input_contents_len, .incomplete_manifest); } const disk_contents = m.contents.items[0..manifest_size]; - const input_contents = m.contents.items[manifest_size..input_contents_len]; + const input_contents = m.contents.items[manifest_size..][0..input_contents_len]; // First the input files section, which must match our input files, otherwise it's invalid or incomplete. var off: usize = 0; @@ -955,14 +984,16 @@ pub const Manifest = struct { // Validate and check discovered files. while (off + 1 < disk_contents.len) { const file_off: File.Offset = @fromBackingInt(@intCast(off)); - const file = try file_off.getFallible(disk_contents); + const file = file_off.getFallible(disk_contents) catch + return checkLockedMiss(m, input_contents_len, .invalid_manifest); if (file.flags.prefix >= m.cache.prefixes_len) return checkLockedMiss(m, input_contents_len, .invalid_manifest); - const path = try file_off.pathFallible(disk_contents); + const path = file_off.pathFallible(disk_contents) catch + return checkLockedMiss(m, input_contents_len, .invalid_manifest); if (path.len == 0) return checkLockedMiss(m, input_contents_len, .invalid_manifest); try m.files.putContext(gpa, file_off, {}, .{ .contents = disk_contents }); - const result = try checkDiscoveredPath(file_off, file_off.path(disk_contents)); + const result = try checkDiscoveredPath(m, file_off, disk_contents); if (result != .hit) return checkLockedMiss(m, input_contents_len, result); off += File.sizeOf(path.len); @@ -994,7 +1025,10 @@ pub const Manifest = struct { result: CheckResult, ) CheckError!CheckResult { const file_offs = m.files.keys(); - const off = if (file_offs.len - next_file_index == 0) input_contents_len else file_offs[next_file_index]; + const off = if (file_offs.len - next_file_index == 0) + input_contents_len + else + @backingInt(file_offs[next_file_index]); const copy_len = input_contents_len - off; @memcpy( m.contents.items[off..][0..copy_len], @@ -1021,7 +1055,7 @@ pub const Manifest = struct { m: *Manifest, file_off: File.Offset, input_path: *InputPath, - contents: []const u8, + contents: []u8, ) CheckError!void { const cache = m.cache; const io = cache.io; @@ -1031,7 +1065,10 @@ pub const Manifest = struct { if (input_path.have_digest) return; - const stat_path_ok = input_path.contents.status.populated() or input_file.flags.metadata_only; + const stat_path_ok = switch (input_path.contents.unwrap()) { + .populated => true, + .unrequested, .requested => input_file.flags.metadata_only, + }; if (!input_path.have_stat and stat_path_ok) { // Since this is an input file, FileNotFound counts as a failure, not a cache miss. const actual_stat = switch (input_path.getHandle(input_file)) { @@ -1074,7 +1111,7 @@ pub const Manifest = struct { return; } - if (input_path.contents.get()) |file_or_dir_contents| { + if (input_path.contents.get(m.all_input_content.items)) |file_or_dir_contents| { assert(stat_path_ok); var hasher = hasher_init; hasher.update(file_or_dir_contents); @@ -1122,12 +1159,14 @@ pub const Manifest = struct { .err = e, } }), }; - defer if (input_path.contents.status == .requested) { - input_path.contents = .{ .populated = .{ + defer switch (input_path.contents.unwrap()) { + .requested => input_path.contents = .populated(.{ .off = @intCast(dir_contents_start), .len = @intCast(m.all_input_content.items.len - dir_contents_start), - } }; - } else m.all_input_content.shrinkRetainingCapacity(dir_contents_start); + }), + .unrequested => m.all_input_content.shrinkRetainingCapacity(dir_contents_start), + .populated => unreachable, + }; } else { const opened_file = if (input_path.have_handle) input_path.handle.file @@ -1164,7 +1203,11 @@ pub const Manifest = struct { } }), }; - if (input_path.contents.status == .requested) @panic("TODO"); + switch (input_path.contents.unwrap()) { + .requested => @panic("TODO"), + .unrequested => {}, + .populated => unreachable, + } } } @@ -1173,8 +1216,8 @@ pub const Manifest = struct { m: *Manifest, file_off: File.Offset, input_path: *InputPath, - disk_contents: []const u8, - input_contents: []const u8, + disk_contents: []u8, + input_contents: []u8, ) CheckError!CheckResult { const cache = m.cache; const io = cache.io; @@ -1191,7 +1234,7 @@ pub const Manifest = struct { } if (input_path.have_stat) { - const changed = try disk_file.setStatChanged(m, input_file.stat); + const changed = try disk_file.setStatChanged(m, input_file.stat()); if (!changed) return .hit; } @@ -1239,7 +1282,7 @@ pub const Manifest = struct { return .{ .metadata_changed = file_off }; } - if (input_path.contents.get()) |file_or_dir_contents| { + if (input_path.contents.get(m.all_input_content.items)) |file_or_dir_contents| { var hasher = hasher_init; hasher.update(file_or_dir_contents); hasher.final(&disk_file.digest); @@ -1287,12 +1330,14 @@ pub const Manifest = struct { .err = e, } }), }; - defer if (input_path.contents.status == .requested) { - input_path.contents = .{ .populated = .{ + defer switch (input_path.contents.unwrap()) { + .requested => input_path.contents = .populated(.{ .off = @intCast(dir_contents_start), .len = @intCast(m.all_input_content.items.len - dir_contents_start), - } }; - } else m.all_input_content.shrinkRetainingCapacity(dir_contents_start); + }), + .unrequested => m.all_input_content.shrinkRetainingCapacity(dir_contents_start), + .populated => unreachable, + }; if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; return .{ .contents_changed = file_off }; @@ -1330,30 +1375,30 @@ pub const Manifest = struct { .err = e, } }), }; - if (input_path.contents.status == .requested) @panic("TODO"); + switch (input_path.contents.unwrap()) { + .requested => @panic("TODO"), + .unrequested => {}, + .populated => unreachable, + } if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; return .{ .contents_changed = file_off }; } } - fn checkDiscoveredPath( - m: *Manifest, - file_off: File.Offset, - file_path: [:0]const u8, - contents: []const u8, - ) CheckError!CheckResult { + fn checkDiscoveredPath(m: *Manifest, file_off: File.Offset, contents: []u8) CheckError!CheckResult { const file = file_off.get(contents); const cache = m.cache; const gpa = cache.gpa; const io = cache.io; const parent_dir = cache.prefixes()[file.flags.prefix].handle; + const file_path = file_off.path(contents); if (file.flags.metadata_only) { const actual_stat = parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { error.FileNotFound => return .{ .path_deleted = file_off }, error.Canceled => |e| return e, - else => |e| return fail(m, .{ .file_stat = .{ + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1376,7 +1421,7 @@ pub const Manifest = struct { error.FileNotFound => return .{ .path_deleted = file_off }, error.NotDir => return .{ .directory_status_changed = file_off }, error.Canceled => |e| return e, - else => |e| return fail(m, .{ .file_open = .{ + else => |e| return fail(&m.diagnostic, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1385,7 +1430,7 @@ pub const Manifest = struct { const actual_stat = opened_dir.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(m, .{ .file_stat = .{ + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1394,9 +1439,9 @@ pub const Manifest = struct { const prev_digest: BinDigest = file.digest; const dir_contents_start = m.all_input_content.items.len; defer m.all_input_content.shrinkRetainingCapacity(dir_contents_start); - hashDir(gpa, io, opened_dir, &file.digest, &m.all_input_contents) catch |err| switch (err) { + hashDir(gpa, io, opened_dir, &file.digest, &m.all_input_content) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - else => |e| return fail(m, .{ .file_read = .{ + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1411,7 +1456,7 @@ pub const Manifest = struct { error.FileNotFound => return .{ .path_deleted = file_off }, error.IsDir => return .{ .directory_status_changed = file_off }, error.Canceled => |e| return e, - else => |e| return fail(m, .{ .file_open = .{ + else => |e| return fail(&m.diagnostic, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1420,7 +1465,7 @@ pub const Manifest = struct { const actual_stat = opened_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(m, .{ .file_stat = .{ + else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1430,7 +1475,7 @@ pub const Manifest = struct { const prev_digest: BinDigest = file.digest; hashFile(io, opened_file, &file.digest) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(m, .{ .file_read = .{ + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1453,7 +1498,7 @@ pub const Manifest = struct { /// not including discovered files). /// /// Assumes that `digest` is populated for all input files. - pub fn unhit(man: *Manifest, input_digest: *const BinDigest) void { + pub fn undoCheck(man: *Manifest, input_digest: *const BinDigest) void { // Reset the hash. man.hash.hasher = hasher_init; man.hash.hasher.update(input_digest); @@ -2117,8 +2162,7 @@ test "cache file and then recall it" { ch.hash.addBytes("1234"); _ = try ch.addInputPath(.initCwd(temp_file), .{}); - // There should be nothing in the cache - try testing.expectEqual(.miss, try ch.check(.none)); + try testing.expectEqual(.incomplete_manifest, try ch.check(.none)); digest1 = ch.final(); try ch.finalize(); @@ -2185,8 +2229,7 @@ test "check that changing a file makes cache fail" { man.hash.addBytes("1234"); const temp_file_idx = try man.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); - // There should be nothing in the cache - try testing.expectEqual(.miss, try man.check(.none)); + try testing.expectEqual(.incomplete_manifest, try man.check(.none)); try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); @@ -2204,11 +2247,13 @@ test "check that changing a file makes cache fail" { man.hash.addBytes("1234"); const temp_file_idx = try man.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); - // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(.miss, try man.check(.none)); + // The one input file changed. + try testing.expectEqual( + @as(Manifest.CheckResult, .{ .contents_changed = temp_file_idx.offset(&man) }), + try man.check(.none), + ); - // The cache system does not keep the contents of re-hashed input files. - try testing.expectEqual(false, temp_file_idx.get(&man).contents.status.populated()); + try testing.expectEqualStrings(updated_temp_file_contents, temp_file_idx.contents(&man)); digest2 = man.final(); @@ -2248,8 +2293,7 @@ test "no file inputs" { man.hash.addBytes("1234"); - // There should be nothing in the cache - try testing.expectEqual(.miss, try man.check(.none)); + try testing.expectEqual(.incomplete_manifest, try man.check(.none)); digest1 = man.final(); @@ -2306,31 +2350,30 @@ test "Manifest with files added after initial hash work" { defer cache.manifest_dir.close(io); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - _ = try ch.addInputPath(.initCwd(temp_file1), .{}); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.initCwd(temp_file1), .{}); - // There should be nothing in the cache - try testing.expectEqual(.miss, try ch.check(.none)); + try testing.expectEqual(.incomplete_manifest, try man.check(.none)); - _ = try ch.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); - digest1 = ch.final(); - try ch.finalize(); + digest1 = man.final(); + try man.finalize(); } { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - _ = try ch.addInputPath(.initCwd(temp_file1), .{}); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.initCwd(temp_file1), .{}); - try testing.expect(.hit == try ch.check(.none)); - digest2 = ch.final(); + try testing.expect(.hit == try man.check(.none)); + digest2 = man.final(); - try testing.expectEqual(false, ch.have_exclusive_lock); + try testing.expectEqual(false, man.have_exclusive_lock); } try testing.expect(mem.eql(u8, &digest1, &digest2)); @@ -2344,20 +2387,24 @@ test "Manifest with files added after initial hash work" { } { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - _ = try ch.addInputPath(.initCwd(temp_file1), .{}); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.initCwd(temp_file1), .{}); - // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(.miss, try ch.check(.none)); + switch (try man.check(.none)) { + .contents_changed => |off| { + try testing.expectEqualStrings(temp_file2, off.path(man.contents.items)); + }, + else => return error.TestFailed, + } - _ = try ch.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); - digest3 = ch.final(); + digest3 = man.final(); - try ch.finalize(); + try man.finalize(); } try testing.expect(!mem.eql(u8, &digest1, &digest3)); diff --git a/src/Compilation.zig b/src/Compilation.zig index 38229017566600e927fb5dfe2d51f9491bce1d2f..6e97e9ca1a50f9038160e65e2531dbd908846a19 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2856,7 +2856,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE log.debug("CacheMode.whole cache miss for {s}", .{comp.root_name}); if (ignore_hit) { - // Okay, now set this back so that `writeManifest` will downgrade our lock later. + // Okay, now set this back so that `Manifest.finalize` will downgrade our lock later. man.want_shared_lock = true; } @@ -3144,7 +3144,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } // Failure here only means an unnecessary cache miss. - man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); + man.finalize() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); assert(whole.lock == null); whole.lock = man.toOwnedLock(); @@ -5810,11 +5810,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr // possible we had a hit and the manifest is dirty, for example if the file mtime changed but // the contents were the same, we hit the cache but the manifest is dirty and we need to update // it to prevent doing a full file content comparison the next time around. - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ - c_object.src.src_path, @errorName(err), - }); - }; + man.finalize() catch |err| + log.warn("failed to write cache manifest when compiling {q}: {t}", .{ c_object.src.src_path, err }); } const o_basename = try std.fmt.allocPrint(arena, "{s}{s}", .{ o_basename_noext, o_ext }); @@ -5950,9 +5947,8 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 }; if (man.have_exclusive_lock) { - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ src_path, @errorName(err) }); - }; + man.finalize() catch |err| + log.warn("failed to write cache manifest when compiling {q}: {t}", .{ src_path, err }); } win32_resource.status = .{ @@ -6060,8 +6056,8 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // possible we had a hit and the manifest is dirty, for example if the file mtime changed but // the contents were the same, we hit the cache but the manifest is dirty and we need to update // it to prevent doing a full file content comparison the next time around. - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ rc_src.src_path, @errorName(err) }); + man.finalize() catch |err| { + log.warn("failed to write cache manifest when compiling {q}: {t}", .{ rc_src.src_path, err }); }; } diff --git a/src/libs/freebsd.zig b/src/libs/freebsd.zig index bda03f2787e4ff60e19e920ee3a43ab246366193..b02b3569af70d363eb73619f7bdd7cd514e0d1cc 100644 --- a/src/libs/freebsd.zig +++ b/src/libs/freebsd.zig @@ -982,9 +982,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for FreeBSD libc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for FreeBSD libc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/glibc.zig b/src/libs/glibc.zig index 67dd70eae78487ab5473c06a63bab226b91c6205..08b9214d6db5d64a8fef5dcf4dde164e9f7f9f12 100644 --- a/src/libs/glibc.zig +++ b/src/libs/glibc.zig @@ -1131,9 +1131,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for glibc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for glibc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index f0d0ef2990089fd1d6ac7ef5844678d1950f7b3c..f6593e64af7dc6f48ff2c450db77ce7c792c19eb 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -363,9 +363,8 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P try file_writer.interface.flush(); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for DLL import {s}.lib: {s}", .{ lib_name, @errorName(err) }); - }; + man.finalize() catch |err| + log.warn("failed to write cache manifest for DLL import {s}.lib: {t}", .{ lib_name, err }); comp.mutex.lockUncancelable(io); defer comp.mutex.unlock(io); diff --git a/src/libs/netbsd.zig b/src/libs/netbsd.zig index 8cafe51563f5d6e32971fb77035f4f7e54d65409..9d120d46ae2db878175c5ff5b84c8ebf7eaf7c9b 100644 --- a/src/libs/netbsd.zig +++ b/src/libs/netbsd.zig @@ -643,9 +643,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for NetBSD libc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for NetBSD libc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/openbsd.zig b/src/libs/openbsd.zig index dea2c5c3350afcced2f7c8c005a1ceb24a102f8d..e65fc524967c09300979897a0e5dd089e5fd8ad1 100644 --- a/src/libs/openbsd.zig +++ b/src/libs/openbsd.zig @@ -564,9 +564,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for OpenBSD libc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for OpenBSD libc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/main.zig b/src/main.zig index bcffbfee9176c9c5fb050927d147a5e39332788b..d7f6fe574c4c228166cf6fc90b00f46cd0cd6501 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4906,7 +4906,7 @@ fn cmdTranslateC( } } - man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + man.finalize() catch |err| warn("failed to write cache manifest: {t}", .{err}); break :result result; }; -- 2.54.0 From 30c3c666ab88702c5a684ea489a2867260eab22a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 11 Sep 2026 19:44:41 -0700 Subject: [PATCH 23/49] starting to fix the unit test failures --- lib/std/Build/Cache.zig | 30 +++++++++++++++++++++--------- lib/std/testing.zig | 3 +++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index e44efe4c59a0b329c8e908576f02238694579e30..3f4e51aaa16fdd485ae070e6a9289fe01a5f3524 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -418,18 +418,18 @@ pub const Manifest = struct { /// The data per tracked input file that is stored in the manifest file. pub const File = extern struct { - size: u64, - inode: u64, + size: u64 align(1), + inode: u64 align(1), /// Nanoseconds. - mtime: i64, + mtime: i64 align(1), /// To simplify the hashing logic, this value is computed from size, inode, and mtime /// in `hashFromMetadata` in the case that `Flags.metadata_only` is `true`. - digest: BinDigest, + digest: BinDigest align(1), /// Starting with this field and continuing into the path, excluding the null byte, /// is the string that is hashed for the manifest digest. - flags: Flags, + flags: Flags align(1), /// Terminated by zero byte, then followed by padding until 8-byte aligned. - path_start: [0]u8, + path_start: [0]u8 align(1), pub const Flags = packed struct(u8) { is_directory: bool, @@ -2133,6 +2133,11 @@ test "cache file and then recall it" { const temp_manifest_dir = "temp_manifest_dir"; try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = "Hello, world!\n" }); + const tmp_directory: Directory = .{ + .path = try std.fs.path.join(testing.allocator, &.{std.testing.TmpDir.parent_dir_path}), + .handle = tmp.dir, + }; + defer testing.allocator.free(tmp_directory.path.?); // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); @@ -2150,7 +2155,8 @@ test "cache file and then recall it" { .manifest_dir = try tmp.dir.createDirPathOpen(io, temp_manifest_dir, .{}), .cwd = cwd, }; - cache.addPrefix(.{ .path = null, .handle = tmp.dir }); + cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + cache.addPrefix(tmp_directory); defer cache.manifest_dir.close(io); { @@ -2160,7 +2166,10 @@ test "cache file and then recall it" { ch.hash.add(true); ch.hash.add(@as(u16, 1234)); ch.hash.addBytes("1234"); - _ = try ch.addInputPath(.initCwd(temp_file), .{}); + _ = try ch.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{}); try testing.expectEqual(.incomplete_manifest, try ch.check(.none)); @@ -2174,7 +2183,10 @@ test "cache file and then recall it" { ch.hash.add(true); ch.hash.add(@as(u16, 1234)); ch.hash.addBytes("1234"); - _ = try ch.addInputPath(.initCwd(temp_file), .{}); + _ = try ch.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{}); // Cache hit! We just "built" the same file try testing.expectEqual(.hit, try ch.check(.none)); diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 7c010cf4c5dd4c08e89604000f2a3d262f6f78ad..d7ee376afb152587681de2ea8875c2c653a72f2e 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -592,6 +592,9 @@ pub const TmpDir = struct { const random_bytes_count = 12; const sub_path_len = std.base64.url_safe.Encoder.calcSize(random_bytes_count); + /// Deprecated. + pub const parent_dir_path: []const u8 = ".zig-cache" ++ std.fs.path.sep_str ++ "tmp"; + pub fn cleanup(self: *TmpDir) void { self.dir.close(io); self.parent_dir.deleteTree(io, &self.sub_path) catch {}; -- 2.54.0 From 29531cc9c056775dda9960c59b6644e4559d14bd Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Fri, 11 Sep 2026 23:28:40 -0400 Subject: [PATCH 24/49] finish fixing one unit test --- lib/std/Build/Cache.zig | 57 ++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 3f4e51aaa16fdd485ae070e6a9289fe01a5f3524..3daa23cf6d46f9f368ffc02d16d00d1e739d65ab 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -938,7 +938,7 @@ pub const Manifest = struct { if (manifest_size == 0 or manifest_size < m.contents.items.len) { // Manifest file was never finalized. try m.contents.ensureUnusedCapacity(gpa, 1); - return .incomplete_manifest; + return missInput(m, 0, m.contents.items.len, manifest_size, .incomplete_manifest); } // We must not clobber existing `Manifest.contents` because it possibly contains prepopulated stat and digest @@ -1030,14 +1030,14 @@ pub const Manifest = struct { else @backingInt(file_offs[next_file_index]); const copy_len = input_contents_len - off; - @memcpy( + if (manifest_size > 0) @memcpy( m.contents.items[off..][0..copy_len], m.contents.items[manifest_size + off ..][0..copy_len], ); m.contents.shrinkRetainingCapacity(input_contents_len); // And now iterate over remaining input files and populate the missing input file hashes. const contents = m.contents.items; - for (file_offs[next_file_index..], m.input_paths.items) |input_file_off, *input_path| { + for (file_offs[next_file_index..], m.input_paths.items[next_file_index..]) |input_file_off, *input_path| { assert(@backingInt(input_file_off) == off); try populateInputPath(m, input_file_off, input_path, contents); } @@ -2214,6 +2214,11 @@ test "check that changing a file makes cache fail" { const updated_temp_file_contents = "Hello, world; but updated!\n"; try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = original_temp_file_contents }); + const tmp_directory: Directory = .{ + .path = try std.fs.path.join(testing.allocator, &.{std.testing.TmpDir.parent_dir_path}), + .handle = tmp.dir, + }; + defer testing.allocator.free(tmp_directory.path.?); // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); @@ -2231,7 +2236,8 @@ test "check that changing a file makes cache fail" { .manifest_dir = try tmp.dir.createDirPathOpen(io, temp_manifest_dir, .{}), .cwd = cwd, }; - cache.addPrefix(.{ .path = null, .handle = tmp.dir }); + cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + cache.addPrefix(tmp_directory); defer cache.manifest_dir.close(io); { @@ -2239,7 +2245,10 @@ test "check that changing a file makes cache fail" { defer man.deinit(); man.hash.addBytes("1234"); - const temp_file_idx = try man.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); + const temp_file_idx = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{ .request_contents = true }); try testing.expectEqual(.incomplete_manifest, try man.check(.none)); @@ -2257,7 +2266,10 @@ test "check that changing a file makes cache fail" { defer man.deinit(); man.hash.addBytes("1234"); - const temp_file_idx = try man.addInputPath(.initCwd(temp_file), .{ .request_contents = true }); + const temp_file_idx = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{ .request_contents = true }); // The one input file changed. try testing.expectEqual( @@ -2340,6 +2352,11 @@ test "Manifest with files added after initial hash work" { try tmp.dir.writeFile(io, .{ .sub_path = temp_file1, .data = "Hello, world!\n" }); try tmp.dir.writeFile(io, .{ .sub_path = temp_file2, .data = "Hello world the second!\n" }); + const tmp_directory: Directory = .{ + .path = try std.fs.path.join(testing.allocator, &.{std.testing.TmpDir.parent_dir_path}), + .handle = tmp.dir, + }; + defer testing.allocator.free(tmp_directory.path.?); // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); @@ -2358,7 +2375,8 @@ test "Manifest with files added after initial hash work" { .manifest_dir = try tmp.dir.createDirPathOpen(io, temp_manifest_dir, .{}), .cwd = cwd, }; - cache.addPrefix(.{ .path = null, .handle = tmp.dir }); + cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + cache.addPrefix(tmp_directory); defer cache.manifest_dir.close(io); { @@ -2366,11 +2384,17 @@ test "Manifest with files added after initial hash work" { defer man.deinit(); man.hash.addBytes("1234"); - _ = try man.addInputPath(.initCwd(temp_file1), .{}); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file1, + }, .{}); try testing.expectEqual(.incomplete_manifest, try man.check(.none)); - try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .{ + .root_dir = tmp_directory, + .sub_path = temp_file2, + } } }); digest1 = man.final(); try man.finalize(); @@ -2380,7 +2404,10 @@ test "Manifest with files added after initial hash work" { defer man.deinit(); man.hash.addBytes("1234"); - _ = try man.addInputPath(.initCwd(temp_file1), .{}); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file1, + }, .{}); try testing.expect(.hit == try man.check(.none)); digest2 = man.final(); @@ -2403,7 +2430,10 @@ test "Manifest with files added after initial hash work" { defer man.deinit(); man.hash.addBytes("1234"); - _ = try man.addInputPath(.initCwd(temp_file1), .{}); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file1, + }, .{}); switch (try man.check(.none)) { .contents_changed => |off| { @@ -2412,7 +2442,10 @@ test "Manifest with files added after initial hash work" { else => return error.TestFailed, } - try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(temp_file2) } }); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .{ + .root_dir = tmp_directory, + .sub_path = temp_file2, + } } }); digest3 = man.final(); -- 2.54.0 From 9c325be987a2192228439e9cb536aae1667efde8 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sat, 12 Sep 2026 20:58:52 -0400 Subject: [PATCH 25/49] This assertion cannot be true on later iterations. And tautologically true on the first iteration. --- lib/std/Build/Cache.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 3daa23cf6d46f9f368ffc02d16d00d1e739d65ab..f7896eac5341d5f263347e57c332043f90d15d91 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1038,7 +1038,6 @@ pub const Manifest = struct { // And now iterate over remaining input files and populate the missing input file hashes. const contents = m.contents.items; for (file_offs[next_file_index..], m.input_paths.items[next_file_index..]) |input_file_off, *input_path| { - assert(@backingInt(input_file_off) == off); try populateInputPath(m, input_file_off, input_path, contents); } return result; -- 2.54.0 From 43397f22513c5769213c74a62822366f4d548364 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 16:49:22 -0700 Subject: [PATCH 26/49] std.Build.Cache: implement populated requested contents --- lib/std/Build/Cache.zig | 88 ++++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index f7896eac5341d5f263347e57c332043f90d15d91..951214f83bef0e998778f8676fa6aa66bdf593a5 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1061,6 +1061,7 @@ pub const Manifest = struct { const input_file = file_off.get(contents); const parent_dir = cache.prefixes()[input_file.flags.prefix].handle; const file_path = file_off.path(contents); + const gpa = cache.gpa; if (input_path.have_digest) return; @@ -1150,7 +1151,6 @@ pub const Manifest = struct { } const dir_contents_start = m.all_input_content.items.len; - const gpa = cache.gpa; hashDir(gpa, io, opened_dir, &input_file.digest, &m.all_input_content) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, else => |e| return fail(&m.diagnostic, .{ .file_read = .{ @@ -1194,17 +1194,28 @@ pub const Manifest = struct { try input_file.setStat(m, .init(actual_stat)); } - hashFile(io, opened_file, &input_file.digest) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ - .file_offset = file_off, - .err = e, - } }), - }; - switch (input_path.contents.unwrap()) { - .requested => @panic("TODO"), - .unrequested => {}, + .requested => { + const start = m.all_input_content.items.len; + hashFileAppend(io, opened_file, &input_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + input_path.contents = .populated(.{ + .off = @intCast(start), + .len = @intCast(m.all_input_content.items.len - start), + }); + }, + .unrequested => hashFile(io, opened_file, &input_file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }, .populated => unreachable, } } @@ -1220,6 +1231,7 @@ pub const Manifest = struct { ) CheckError!CheckResult { const cache = m.cache; const io = cache.io; + const gpa = cache.gpa; const disk_file = file_off.get(disk_contents); const input_file = file_off.get(input_contents); const parent_dir = cache.prefixes()[disk_file.flags.prefix].handle; @@ -1321,7 +1333,6 @@ pub const Manifest = struct { } const dir_contents_start = m.all_input_content.items.len; - const gpa = cache.gpa; hashDir(gpa, io, opened_dir, &disk_file.digest, &m.all_input_content) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, else => |e| return fail(&m.diagnostic, .{ .file_read = .{ @@ -1367,16 +1378,28 @@ pub const Manifest = struct { if (!try disk_file.setStatChanged(m, .init(actual_stat))) return .hit; } - hashFile(io, opened_file, &disk_file.digest) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ - .file_offset = file_off, - .err = e, - } }), - }; switch (input_path.contents.unwrap()) { - .requested => @panic("TODO"), - .unrequested => {}, + .requested => { + const start = m.all_input_content.items.len; + hashFileAppend(io, opened_file, &input_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + input_path.contents = .populated(.{ + .off = @intCast(start), + .len = @intCast(m.all_input_content.items.len - start), + }); + }, + .unrequested => hashFile(io, opened_file, &disk_file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }, .populated => unreachable, } @@ -2032,6 +2055,27 @@ pub const Manifest = struct { hasher.final(bin_digest); } + fn hashFileAppend( + io: Io, + file: Io.File, + bin_digest: *[Hasher.mac_length]u8, + al: *std.ArrayList(u8), + gpa: Allocator, + ) (Io.File.ReadPositionalError || Allocator.Error)!void { + var hasher = hasher_init; + var offset: u64 = 0; + while (true) { + try al.ensureUnusedCapacity(gpa, 2048); + const buffer = al.unusedCapacitySlice(); + const n = try file.readPositional(io, &.{buffer}, offset); + if (n == 0) break; + hasher.update(buffer[0..n]); + offset += n; + al.items.len += n; + } + hasher.final(bin_digest); + } + const HashDirError = Io.Dir.Reader.Error || Allocator.Error; /// Appends the sorted, encoded directory entries to `contents`. @@ -2198,7 +2242,7 @@ test "cache file and then recall it" { } } -test "check that changing a file makes cache fail" { +test "check that changing a file causes cache miss" { const io = testing.io; var tmp = testing.tmpDir(.{}); -- 2.54.0 From 228178c151e078dc1bef38c2e20038cebe24a887 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 18:18:47 -0700 Subject: [PATCH 27/49] std.Build.Cache: simplify hashing logic --- lib/std/Build/Cache.zig | 79 +++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 951214f83bef0e998778f8676fa6aa66bdf593a5..354a4261620bf1d0041eb2a0088bb399665f8e04 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -873,9 +873,6 @@ pub const Manifest = struct { // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user // can use `final`, and will at some point `finalize` the file list to disk. - man.hash.hasher = hasher_init; - man.hash.hasher.update(&input_digest); - hit: { const miss_result = miss: { const result = try man.checkLocked(); @@ -884,12 +881,8 @@ pub const Manifest = struct { } else if (!try man.upgradeToExclusiveLock()) { break :miss result; } - // We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone - // else might have modified the digest, so we need to check again before deciding to miss. - // Before trying again, we must reset `man.hash.hasher` and `man.files`. - // This is basically just the first half of `undoCheck`. - man.hash.hasher = hasher_init; - man.hash.hasher.update(&input_digest); + // Missed with the shared lock, and upgraded to an exclusive lock. However, another process may have + // modified the cache directory, so we need to check again before deciding to miss. man.shrinkFilesToInput(); const refreshed_result = try man.checkLocked(); if (refreshed_result == .hit) break :hit; @@ -899,7 +892,8 @@ pub const Manifest = struct { // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a // cache miss, which is needed because they will be used in the manifest digest. man.dirty = true; - undoCheck(man, &input_digest); + man.shrinkFilesToInput(); + hashFiles(man, &input_digest); return miss_result; } @@ -910,9 +904,23 @@ pub const Manifest = struct { }; } + hashFiles(man, &input_digest); return .hit; } + fn hashFiles(m: *Manifest, input_digest: *BinDigest) void { + const contents = m.contents.items; + + m.hash.hasher = hasher_init; + m.hash.hasher.update(input_digest); + + for (m.files.keys(), 0..) |off, i| { + const file = off.get(contents); + log.warn("hashing {d} {s} = {x}", .{ i, off.path(contents), &file.digest }); + m.hash.hasher.update(&file.digest); + } + } + fn shrinkFilesToInput(m: *Manifest) void { if (m.files.count() <= m.input_paths.items.len) return; // Reads from files hash map whose data is destroyed on the next line. @@ -923,9 +931,9 @@ pub const Manifest = struct { assert(mem.isAligned(m.contents.items.len, @alignOf(File))); } - /// Assumes that `self.hash.hasher` has been updated only with the original digest and that - /// `self.files` contains only the original input files. + /// Does not observe or modify `self.hash.hasher`. Asserts `self.files` contains only the original input files. fn checkLocked(m: *Manifest) CheckError!CheckResult { + assert(m.files.count() == m.input_paths.items.len); const gpa = m.cache.gpa; const io = m.cache.io; const manifest_file = m.manifest_file.?; @@ -938,7 +946,7 @@ pub const Manifest = struct { if (manifest_size == 0 or manifest_size < m.contents.items.len) { // Manifest file was never finalized. try m.contents.ensureUnusedCapacity(gpa, 1); - return missInput(m, 0, m.contents.items.len, manifest_size, .incomplete_manifest); + return populateMissingInputFileHashes(m, 0, .incomplete_manifest); } // We must not clobber existing `Manifest.contents` because it possibly contains prepopulated stat and digest @@ -957,7 +965,7 @@ pub const Manifest = struct { error.Canceled => |e| return e, else => |e| return fail(&m.diagnostic, .{ .manifest_read = e }), }; - if (n != manifest_size) return checkLockedMiss(m, input_contents_len, .incomplete_manifest); + if (n != manifest_size) return missInput(m, 0, input_contents_len, manifest_size, .incomplete_manifest); } const disk_contents = m.contents.items[0..manifest_size]; const input_contents = m.contents.items[manifest_size..][0..input_contents_len]; @@ -1004,8 +1012,6 @@ pub const Manifest = struct { const file_valid = off + 1 == disk_contents.len and disk_contents[off] == 0; if (!file_valid) return checkLockedMiss(m, input_contents_len, .incomplete_manifest); - for (m.files.keys()) |file_off| m.hash.hasher.update(&file_off.get(disk_contents).digest); - // Since it's a cache hit, we accept the input file contents from disk and discard the other copy. // Furthermore, don't track the trailing zero byte in contents. m.contents.shrinkRetainingCapacity(manifest_size - 1); @@ -1035,7 +1041,15 @@ pub const Manifest = struct { m.contents.items[manifest_size + off ..][0..copy_len], ); m.contents.shrinkRetainingCapacity(input_contents_len); - // And now iterate over remaining input files and populate the missing input file hashes. + return populateMissingInputFileHashes(m, next_file_index, result); + } + + fn populateMissingInputFileHashes( + m: *Manifest, + next_file_index: usize, + result: CheckResult, + ) CheckError!CheckResult { + const file_offs = m.files.keys(); const contents = m.contents.items; for (file_offs[next_file_index..], m.input_paths.items[next_file_index..]) |input_file_off, *input_path| { try populateInputPath(m, input_file_off, input_path, contents); @@ -1515,23 +1529,6 @@ pub const Manifest = struct { return error.CacheCheckFailed; } - /// Reset `man.hash.hasher` to the state it should be in after `check` returns `CheckResult.miss`. - /// The hasher contains the original input digest, and all original input file digests (i.e. - /// not including discovered files). - /// - /// Assumes that `digest` is populated for all input files. - pub fn undoCheck(man: *Manifest, input_digest: *const BinDigest) void { - // Reset the hash. - man.hash.hasher = hasher_init; - man.hash.hasher.update(input_digest); - man.shrinkFilesToInput(); - const contents = man.contents.items; - for (man.files.keys()) |off| { - const file = off.get(contents); - man.hash.hasher.update(&file.digest); - } - } - fn isProblematicTimestamp(man: *Manifest, timestamp: Io.Timestamp) error{Canceled}!bool { const io = man.cache.io; @@ -1844,22 +1841,20 @@ pub const Manifest = struct { }; } - /// Returns a binary hash of the inputs. + /// Returns a binary hash of the inputs, input file contents, and discovered file contents. pub fn finalBin(self: *Manifest) BinDigest { assert(self.manifest_file != null); - // We don't close the manifest file yet, because we want to - // keep it locked until the API user is done using it. - // We also don't write out the manifest yet, because until - // cache_release is called we still might be working on creating - // the artifacts to cache. + // We don't close the manifest file yet, because we want to keep it locked until the API user is done using it. + // We also don't write out the manifest yet, because until `finalize` is called we still might be working on + // creating the artifacts to cache. var bin_digest: BinDigest = undefined; self.hash.hasher.final(&bin_digest); return bin_digest; } - /// Returns a hex encoded hash of the inputs. + /// Returns a hex encoded hash of the inputs, input file contents, and discovered file contents. pub fn final(self: *Manifest) HexDigest { const bin_digest = self.finalBin(); return binToHex(bin_digest); @@ -2327,7 +2322,7 @@ test "check that changing a file causes cache miss" { try man.finalize(); } - try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); + try testing.expect(!mem.eql(u8, &digest1, &digest2)); } } -- 2.54.0 From e38052663ca729764c1de7d5f28794d4509d8b79 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 18:26:52 -0700 Subject: [PATCH 28/49] std.Build.Cache: fix writing digest to wrong place --- lib/std/Build/Cache.zig | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 354a4261620bf1d0041eb2a0088bb399665f8e04..799e2900450689ff3f8534a4345d1b33be8a069e 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -914,9 +914,8 @@ pub const Manifest = struct { m.hash.hasher = hasher_init; m.hash.hasher.update(input_digest); - for (m.files.keys(), 0..) |off, i| { + for (m.files.keys()) |off| { const file = off.get(contents); - log.warn("hashing {d} {s} = {x}", .{ i, off.path(contents), &file.digest }); m.hash.hasher.update(&file.digest); } } @@ -1395,7 +1394,7 @@ pub const Manifest = struct { switch (input_path.contents.unwrap()) { .requested => { const start = m.all_input_content.items.len; - hashFileAppend(io, opened_file, &input_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { + hashFileAppend(io, opened_file, &disk_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, else => |e| return fail(&m.diagnostic, .{ .file_read = .{ .file_offset = file_off, -- 2.54.0 From d1b961c9ebb875b8e8c6fc082a1f9d5e6f175daa Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 19:26:12 -0700 Subject: [PATCH 29/49] std.Build.Cache: introduce State enum --- lib/std/Build/Cache.zig | 167 ++++++++++++++++++++++++++-------------- 1 file changed, 108 insertions(+), 59 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 799e2900450689ff3f8534a4345d1b33be8a069e..c6fd2c1c8364f6208c2111f265a507bf4145a777 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -48,7 +48,7 @@ pub fn obtain(cache: *Cache) Manifest { .cache = cache, .hash = cache.hash, .manifest_file = null, - .dirty = false, + .state = .input, }; } @@ -274,8 +274,7 @@ pub const Manifest = struct { hash: HashHelper, /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. manifest_file: ?Io.File, - /// `finalize` has not been called yet. - dirty: bool, + state: State, /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code /// using the cache will not modify the files within the cache directory. This allows multiple processes /// to utilize the same cache directory at the same time. @@ -309,6 +308,27 @@ pub const Manifest = struct { pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); + pub const State = enum { + /// Call `addInputPath`, `addInputPathOptional`, and methods of `Manifest.hash`. Once all input files and state + /// have been added, call `check`. + input, + /// `check` has already been called and a hit occurred. Call `hitDigest` to find cache artifact directory. + hit, + /// `hitDigest` has already been called. No further operations are allowed. + hit_digested, + /// `check` has already been called and a miss occurred. `missDigest` has not been called yet. Miss diagnostic + /// data may still be accessed. Call `addDiscoveredPath` to add discovered files and directories. + miss, + /// A miss occurred, `missDigest` has not been called yet. Call `addDiscoveredPath` to add more discovered files + /// and directories. Miss diagnostic data may no longer be accessed. + miss_discovered, + /// `missDigest` has been called, but `finalize` has not been called yet. + miss_digested, + /// The cache manifest has been written to disk such that the next check for the same inputs will yield a hit. + /// No further operations are allowed. + miss_finalized, + }; + /// Source files and directories whose prefix and relative path are /// included when computing the cache manifest digest. It's the information /// needed to lazily hash the input files only when a cache miss occurs. @@ -656,6 +676,7 @@ pub const Manifest = struct { /// See also: /// * `addDiscoveredPath` pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { + assert(m.state == .input); const cache = m.cache; const gpa = cache.gpa; const is_directory = options.handle.isDirectory(); @@ -733,6 +754,7 @@ pub const Manifest = struct { } pub fn addInputPathOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { + assert(m.state == .input); m.hash.add(opt_path != null); _ = try addInputPath(m, opt_path orelse return, options); } @@ -772,6 +794,7 @@ pub const Manifest = struct { } pub fn checkProgressless(man: *Manifest) CheckError!CheckResult { + assert(man.state == .input); assert(man.manifest_file == null); // This is *not* hashing the contents of the input files. It is the @@ -872,6 +895,8 @@ pub const Manifest = struct { // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user // can use `final`, and will at some point `finalize` the file list to disk. + man.hash.hasher = hasher_init; + man.hash.hasher.update(&input_digest); hit: { const miss_result = miss: { @@ -891,9 +916,7 @@ pub const Manifest = struct { // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a // cache miss, which is needed because they will be used in the manifest digest. - man.dirty = true; - man.shrinkFilesToInput(); - hashFiles(man, &input_digest); + man.state = .miss; return miss_result; } @@ -904,19 +927,22 @@ pub const Manifest = struct { }; } - hashFiles(man, &input_digest); + man.state = .hit; return .hit; } - fn hashFiles(m: *Manifest, input_digest: *BinDigest) void { - const contents = m.contents.items; - - m.hash.hasher = hasher_init; - m.hash.hasher.update(input_digest); - - for (m.files.keys()) |off| { - const file = off.get(contents); - m.hash.hasher.update(&file.digest); + fn transitionToMissDiscovered(m: *Manifest) void { + switch (m.state) { + .input => unreachable, + .hit => unreachable, + .hit_digested => unreachable, + .miss => { + m.shrinkFilesToInput(); + m.state = .miss_discovered; + }, + .miss_discovered => {}, + .miss_digested => unreachable, + .miss_finalized => unreachable, } } @@ -1611,6 +1637,7 @@ pub const Manifest = struct { /// * `addInputPath` pub fn addDiscoveredPath(m: *Manifest, options: AddDiscoveredPathOptions) AddDiscoveredPathError!void { assert(m.manifest_file != null); + transitionToMissDiscovered(m); const cache = m.cache; const gpa = cache.gpa; const io = cache.io; @@ -1705,8 +1732,6 @@ pub const Manifest = struct { try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); }, } - - m.hash.hasher.update(&header.digest); } fn populateFile( @@ -1840,39 +1865,63 @@ pub const Manifest = struct { }; } - /// Returns a binary hash of the inputs, input file contents, and discovered file contents. - pub fn finalBin(self: *Manifest) BinDigest { - assert(self.manifest_file != null); + /// See also `hitDigestHex`. + pub fn hitDigest(m: *Manifest) BinDigest { + assert(m.manifest_file != null); + assert(m.state == .hit); + m.state = .hit_digested; + return finalDigest(m); + } + pub fn hitDigestHex(m: *Manifest) HexDigest { + return binToHex(m.hitDigest()); + } + + /// See also `missDigestHex`. + pub fn missDigest(m: *Manifest) BinDigest { + assert(m.manifest_file != null); + transitionToMissDiscovered(m); + m.state = .miss_digested; + return finalDigest(m); + } + + pub fn missDigestHex(m: *Manifest) HexDigest { + return binToHex(m.missDigest()); + } + + fn finalDigest(m: *Manifest) BinDigest { + const contents = m.contents.items; + const hasher = &m.hash.hasher; + + for (m.files.keys()) |off| { + const file = off.get(contents); + hasher.update(&file.digest); + } // We don't close the manifest file yet, because we want to keep it locked until the API user is done using it. // We also don't write out the manifest yet, because until `finalize` is called we still might be working on // creating the artifacts to cache. - var bin_digest: BinDigest = undefined; - self.hash.hasher.final(&bin_digest); + hasher.final(&bin_digest); return bin_digest; } - /// Returns a hex encoded hash of the inputs, input file contents, and discovered file contents. - pub fn final(self: *Manifest) HexDigest { - const bin_digest = self.finalBin(); - return binToHex(bin_digest); - } - /// If `want_shared_lock` is true, this function automatically downgrades the /// lock from exclusive to shared. pub fn finalize(m: *Manifest) !void { + const io = m.cache.io; + const manifest_file = m.manifest_file.?; + + assert(m.state == .miss_digested); assert(m.have_exclusive_lock); - const io = m.cache.io; - const manifest_file = m.manifest_file.?; - if (m.dirty) { + + { m.contents.appendAssumeCapacity(0); defer _ = m.contents.pop().?; try manifest_file.setLength(io, m.contents.items.len); try manifest_file.writePositionalAll(io, m.contents.items, 0); - m.dirty = false; + m.state = .miss_finalized; } if (m.want_shared_lock) { @@ -2197,39 +2246,39 @@ test "cache file and then recall it" { defer cache.manifest_dir.close(io); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addInputPath(.{ + man.hash.add(true); + man.hash.add(@as(u16, 1234)); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.{ .root_dir = tmp_directory, .sub_path = temp_file, }, .{}); - try testing.expectEqual(.incomplete_manifest, try ch.check(.none)); + try testing.expectEqual(.incomplete_manifest, try man.check(.none)); - digest1 = ch.final(); - try ch.finalize(); + digest1 = man.missDigestHex(); + try man.finalize(); } { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addInputPath(.{ + man.hash.add(true); + man.hash.add(@as(u16, 1234)); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.{ .root_dir = tmp_directory, .sub_path = temp_file, }, .{}); // Cache hit! We just "built" the same file - try testing.expectEqual(.hit, try ch.check(.none)); - digest2 = ch.final(); + try testing.expectEqual(.hit, try man.check(.none)); + digest2 = man.hitDigestHex(); - try testing.expectEqual(false, ch.have_exclusive_lock); + try testing.expectEqual(false, man.have_exclusive_lock); } try testing.expectEqual(digest1, digest2); @@ -2291,7 +2340,7 @@ test "check that changing a file causes cache miss" { try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); - digest1 = man.final(); + digest1 = man.missDigestHex(); try man.finalize(); } @@ -2316,7 +2365,7 @@ test "check that changing a file causes cache miss" { try testing.expectEqualStrings(updated_temp_file_contents, temp_file_idx.contents(&man)); - digest2 = man.final(); + digest2 = man.missDigestHex(); try man.finalize(); } @@ -2356,7 +2405,7 @@ test "no file inputs" { try testing.expectEqual(.incomplete_manifest, try man.check(.none)); - digest1 = man.final(); + digest1 = man.missDigestHex(); try man.finalize(); } @@ -2367,14 +2416,14 @@ test "no file inputs" { man.hash.addBytes("1234"); try testing.expectEqual(.hit, try man.check(.none)); - digest2 = man.final(); + digest2 = man.hitDigestHex(); try testing.expectEqual(false, man.have_exclusive_lock); } try testing.expectEqual(digest1, digest2); } -test "Manifest with files added after initial hash work" { +test "Manifest with files added after initial hash" { const io = testing.io; var tmp = testing.tmpDir(.{}); @@ -2433,7 +2482,7 @@ test "Manifest with files added after initial hash work" { .sub_path = temp_file2, } } }); - digest1 = man.final(); + digest1 = man.missDigestHex(); try man.finalize(); } { @@ -2447,7 +2496,7 @@ test "Manifest with files added after initial hash work" { }, .{}); try testing.expect(.hit == try man.check(.none)); - digest2 = man.final(); + digest2 = man.hitDigestHex(); try testing.expectEqual(false, man.have_exclusive_lock); } @@ -2484,7 +2533,7 @@ test "Manifest with files added after initial hash work" { .sub_path = temp_file2, } } }); - digest3 = man.final(); + digest3 = man.missDigestHex(); try man.finalize(); } -- 2.54.0 From 92439e73d06dfda221043830c23cad836d8232a5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 19:32:44 -0700 Subject: [PATCH 30/49] std.Build.Cache: don't preemptively truncate the manifest --- lib/std/Build/Cache.zig | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index c6fd2c1c8364f6208c2111f265a507bf4145a777..a4864beb29c3926406868d315d146b8e32079af8 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1017,17 +1017,14 @@ pub const Manifest = struct { // Validate and check discovered files. while (off + 1 < disk_contents.len) { const file_off: File.Offset = @fromBackingInt(@intCast(off)); - const file = file_off.getFallible(disk_contents) catch - return checkLockedMiss(m, input_contents_len, .invalid_manifest); - if (file.flags.prefix >= m.cache.prefixes_len) - return checkLockedMiss(m, input_contents_len, .invalid_manifest); - const path = file_off.pathFallible(disk_contents) catch - return checkLockedMiss(m, input_contents_len, .invalid_manifest); - if (path.len == 0) return checkLockedMiss(m, input_contents_len, .invalid_manifest); + const file = file_off.getFallible(disk_contents) catch return .invalid_manifest; + if (file.flags.prefix >= m.cache.prefixes_len) return .invalid_manifest; + const path = file_off.pathFallible(disk_contents) catch return .invalid_manifest; + if (path.len == 0) return .invalid_manifest; try m.files.putContext(gpa, file_off, {}, .{ .contents = disk_contents }); const result = try checkDiscoveredPath(m, file_off, disk_contents); - if (result != .hit) return checkLockedMiss(m, input_contents_len, result); + if (result != .hit) return result; off += File.sizeOf(path.len); } @@ -1035,7 +1032,7 @@ pub const Manifest = struct { // Final terminating zero byte to distinguish empty manifest file from // manifest with zero files. const file_valid = off + 1 == disk_contents.len and disk_contents[off] == 0; - if (!file_valid) return checkLockedMiss(m, input_contents_len, .incomplete_manifest); + if (!file_valid) return .incomplete_manifest; // Since it's a cache hit, we accept the input file contents from disk and discard the other copy. // Furthermore, don't track the trailing zero byte in contents. @@ -1082,11 +1079,6 @@ pub const Manifest = struct { return result; } - fn checkLockedMiss(m: *Manifest, input_contents_len: usize, result: CheckResult) CheckResult { - m.contents.shrinkRetainingCapacity(input_contents_len); - return result; - } - /// Similar to `checkInputPath` and `checkDiscoveredPath` but does not compare metadata or digest against file /// system. Only ensures that metadata and digest are available. fn populateInputPath( -- 2.54.0 From 88f3896522fa5314fa4b19b06a2fd3675233ddc3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 19:46:08 -0700 Subject: [PATCH 31/49] std: update to new Cache API --- lib/compiler/Maker.zig | 4 ++-- lib/compiler/Maker/Step/ConfigHeader.zig | 4 ++-- lib/compiler/Maker/Step/ObjCopy.zig | 4 ++-- lib/compiler/Maker/Step/Options.zig | 4 ++-- lib/compiler/Maker/Step/Run.zig | 6 +++--- lib/compiler/Maker/Step/WriteFile.zig | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 118bcffb8dcfcc6dab6f37dcbf6678b716344594..d0bea06e27c69d07ac220e41c1add40e3b38c2ff 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1400,7 +1400,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { if (config_man) |man| { if (.hit == try man.check(compile_prog_node)) { - const digest = man.final(); + const digest = man.hitDigestHex(); const path: Path = .{ .root_dir = graph.local_cache_root, .sub_path = try arena.print("c/{s}", .{&digest}), @@ -1512,7 +1512,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { break :cp .{ config_tmp_path, null }; } else { const man = config_man.?; - const digest = man.final(); + const digest = man.missDigestHex(); const final_path: Path = .{ .root_dir = graph.local_cache_root, .sub_path = try arena.print("c/{s}", .{&digest}), diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index 4ea9444d25b0dc6995ecfc43a3901105ce087be9..a539e2463128445541037d2feab024faa673681f 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -137,7 +137,7 @@ pub fn make( man.hash.addBytes(output); if (try step.cacheHit(maker, &man, progress_node)) { - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_ch.generated_dir).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), @@ -145,7 +145,7 @@ pub fn make( return; } - const digest = man.final(); + const digest = man.missDigestHex(); // If output_path has directory parts, deal with them. Example: // output_dir is zig-cache/o/HASH diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index db045d5f51e46e080e080d4fa434cff40e6ce1cd..52e3f9c7d4f20e0dc9dcd6623b57e647b8a26ca2 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -48,7 +48,7 @@ pub fn make( if (try step.cacheHit(maker, &man, progress_node)) { // Cache hit, skip subprocess execution. - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_oc.output_file).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), @@ -67,7 +67,7 @@ pub fn make( // We don't find out more input files while executing objcopy so we can // already obtain the digest and use it directly as the output path. - const digest = man.final(); + const digest = man.missDigestHex(); const dest_path: Path = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index ef932c7dae0bbf38284d38eebd53f1f1997b7b2f..b9e676ef6542343afe8a018e3f3bdff240a18936 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -54,7 +54,7 @@ pub fn make( const basename = "options.zig"; if (try step.cacheHitWatched(maker, &man, progress_node)) { - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_options.generated_file).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), @@ -63,7 +63,7 @@ pub fn make( return; } - const digest = man.final(); + const digest = man.missDigestHex(); const out_path: Cache.Path = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index f8b58d215997479660975b7b15de0c92c4e625ac..2e750ccfb9f963ed8bb544a5d99af61bb45fb77a 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -261,7 +261,7 @@ pub fn make( if (!has_side_effects and try step.cacheHitWatched(maker, &man, progress_node)) { // Cache hit; skip running command. - const digest = man.final(); + const digest = man.hitDigestHex(); try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); step.result_cached = true; @@ -270,7 +270,7 @@ pub fn make( if (!any_dep_files) { // We already know the final output paths; use them directly. - const digest = if (has_side_effects) man.hash.final() else man.final(); + const digest = if (has_side_effects) man.hash.final() else man.missDigestHex(); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); try populateGeneratedPathsCreateDirs(arena, run_index, maker, output_dir_path, output_placeholders.items, argv_list.items); @@ -308,7 +308,7 @@ pub fn make( } } - const digest = if (has_side_effects) man.hash.final() else man.final(); + const digest = if (has_side_effects) man.hash.final() else man.missDigestHex(); const any_output = output_placeholders.items.len > 0 or conf_run.captured_stdout.value != null or conf_run.captured_stderr.value != null; diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index 5f7bad0e44e7ce45fc04d30dd943a478db731690..6b142a02338315221f403397db090178b096ab1a 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -105,7 +105,7 @@ pub fn make( } if (try step.cacheHit(maker, &man, progress_node)) { - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_wf.generated_directory).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), @@ -114,7 +114,7 @@ pub fn make( return; } - const digest = man.final(); + const digest = man.missDigestHex(); const out_path: Path = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), -- 2.54.0 From 79a7b7c0f033787215471beb9eab99264c1b7003 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 20:38:14 -0700 Subject: [PATCH 32/49] compiler: update to new std.Build.Cache API --- src/Compilation.zig | 25 ++++++++++--------------- src/libs/mingw.zig | 8 ++------ src/main.zig | 2 +- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 6e97e9ca1a50f9038160e65e2531dbd908846a19..6599e8c34085ae331173e65edf15112e090c104d 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2828,11 +2828,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE }); }, }, - error.InvalidFormat => return comp.setMiscFailure( - .check_whole_cache, - "failed to check cache: invalid manifest file format", - .{}, - ), }; if (status == .hit and !ignore_hit) { // In this case the cache hit contains the full set of file system inputs. Nice! @@ -2845,7 +2840,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE comp.last_update_was_cache_hit = true; log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name}); - const bin_digest = man.finalBin(); + const bin_digest = man.hitDigest(); comp.digest = bin_digest; @@ -3054,7 +3049,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE try man.populateOtherManifest(pwc.manifest, pwc.prefix_map); } - const bin_digest = man.finalBin(); + const bin_digest = man.missDigest(); const hex_digest = Cache.binToHex(bin_digest); // Work around windows `AccessDenied` if any files within this @@ -5125,9 +5120,9 @@ pub fn translateC( var reader: std.Io.Reader = .fixed(stdout); const MessageHeader = std.zig.Server.Message.Header; const header = reader.takeStruct(MessageHeader, .little) catch |err| - fatal("unable to read translate-c MessageHeader: {s}", .{@errorName(err)}); + fatal("unable to read translate-c MessageHeader: {t}", .{err}); const body = reader.take(header.bytes_len) catch |err| - fatal("unable to read {}-byte translate-c message body: {s}", .{ header.bytes_len, @errorName(err) }); + fatal("unable to read {d}-byte translate-c message body: {t}", .{ header.bytes_len, err }); switch (header.tag) { .error_bundle => { const error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); @@ -5137,11 +5132,11 @@ pub fn translateC( .errors = error_bundle, }; }, - else => fatal("unexpected message type received from translate-c: {s}", .{@tagName(header.tag)}), + else => fatal("unexpected message type received from translate-c: {t}", .{header.tag}), } } - const bin_digest = man.finalBin(); + const bin_digest = man.missDigest(); const hex_digest = Cache.binToHex(bin_digest); const o_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; @@ -5878,10 +5873,10 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); - const digest = if (.hit == try man.check(child_progress_node)) man.final() else blk: { + const digest = if (.hit == try man.check(child_progress_node)) man.hitDigestHex() else blk: { // The digest only depends on the .manifest file, so we can // get the digest now and write the .res directly to the cache - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try fs.path.join(arena, &.{ "o", &digest }); var o_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, o_sub_path, .{}); @@ -5970,7 +5965,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename_noext = src_basename[0 .. src_basename.len - fs.path.extension(src_basename).len]; - const digest = if (.hit == try man.check(child_progress_node)) man.final() else blk: { + const digest = if (.hit == try man.check(child_progress_node)) man.hitDigestHex() else blk: { var zig_cache_tmp_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, "tmp", .{}); defer zig_cache_tmp_dir.close(io); @@ -6042,7 +6037,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 } // Rename into place. - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest }); var o_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, o_sub_path, .{}); defer o_dir.close(io); diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index f6593e64af7dc6f48ff2c450db77ce7c792c19eb..24a6ac4e12e4fe0749dde875bf88f8b8a574e4e0 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -266,13 +266,9 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P }, }, error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => { - comp.setMiscFailure(.windows_import_lib, "checking cache failed: invalid manifest file format", .{}); - return error.AlreadyReported; - }, }; if (status == .hit) { - const digest = man.final(); + const digest = man.hitDigestHex(); const sub_path = try std.fs.path.join(gpa, &.{ "o", &digest, final_lib_basename }); errdefer gpa.free(sub_path); @@ -291,7 +287,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P return crt_file_path; } - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); var o_dir = try comp.dirs.global_cache.handle.createDirPathOpen(io, o_sub_path, .{}); defer o_dir.close(io); diff --git a/src/main.zig b/src/main.zig index d7f6fe574c4c228166cf6fc90b00f46cd0cd6501..6bcb22f26e15180f6526022a4da12ea7393cbed8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4879,7 +4879,7 @@ fn cmdTranslateC( fatal("unable to process {q}: {t}", .{ c_source_file.src_path, err }); const result: Compilation.TranslateCResult = if (.hit == try man.check(prog_node)) .{ - .digest = man.finalBin(), + .digest = man.hitDigest(), .cache_hit = true, .errors = std.zig.ErrorBundle.empty, } else result: { -- 2.54.0 From a014c1a2876acf93a11cf4c1fdd255939ee0c6e0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2026 22:58:09 -0700 Subject: [PATCH 33/49] std.Build.Cache.DepTokenizer: slightly modernize --- lib/compiler/Maker/Step/Run.zig | 36 ++- lib/std/Build/Cache.zig | 159 +++++++--- lib/std/Build/Cache/DepTokenizer.zig | 438 ++++++++++++--------------- src/Compilation.zig | 51 +++- src/libs/freebsd.zig | 9 +- src/libs/glibc.zig | 11 +- src/libs/netbsd.zig | 11 +- src/libs/openbsd.zig | 9 +- 8 files changed, 380 insertions(+), 344 deletions(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 2e750ccfb9f963ed8bb544a5d99af61bb45fb77a..828b5eb170515c999e8c0e40da4d1b668adfb1a8 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -292,16 +292,32 @@ pub fn make( switch (arg.flags.tag) { .output_file => if (arg.flags.dep_file) { const generated_path = maker.generatedPath(arg.generated.value.?).*; - const result = if (has_side_effects) - man.addDepFile(generated_path.root_dir.handle, generated_path.sub_path) - else - man.addDepFilePost(generated_path.root_dir.handle, generated_path.sub_path); - result catch |err| switch (err) { - error.OutOfMemory, error.Canceled => |e| return e, - else => |e| return step.fail(maker, "failed adding to cache the file {f}: {t}", .{ - generated_path, e, - }), - }; + if (has_side_effects) { + var diagnostic: Cache.DepTokenizer.Token = undefined; + man.addInputDepFile(generated_path, &diagnostic) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidDepFile => return step.fail(maker, "failed adding dep file {f} to cache: {f}", .{ + generated_path, diagnostic, + }), + else => |e| return step.fail(maker, "failed adding dep file {f} to cache: {t}", .{ + generated_path, e, + }), + }; + } else { + var diagnostic: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined; + man.addDiscoveredDepFile(generated_path, &diagnostic) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidDepFile => return step.fail(maker, "failed adding dep file {f} to cache: {f}", .{ + generated_path, diagnostic.dep_tokenizer, + }), + error.FileSystemFailure => return step.fail(maker, "failed adding a path from dep file {f} to cache: {f}", .{ + generated_path, diagnostic.add_discovered_path, + }), + else => |e| return step.fail(maker, "failed adding dep file {f} to cache: {t}", .{ + generated_path, e, + }), + }; + } }, .output_directory => continue, else => unreachable, diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index a4864beb29c3926406868d315d146b8e32079af8..91c678ed2f94515d5104b42e162944f23920923b 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -759,6 +759,52 @@ pub const Manifest = struct { _ = try addInputPath(m, opt_path orelse return, options); } + pub const AddInputDepFileError = error{ + InvalidDepFile, + } || Allocator.Error || Io.File.OpenError || Io.File.Reader.Error; + + pub fn addInputDepFile(m: *Manifest, path: Path, diagnostic: ?*DepTokenizer.Token) AddInputDepFileError!void { + assert(m.manifest_file == null); + assert(m.state == .input); + + const cache = m.cache; + const gpa = cache.gpa; + const io = cache.io; + + // TODO: change DepTokenizer to be streaming rather than operating on slice of bytes. + const prev_len = m.all_input_content.items.len; + defer m.all_input_content.items.len = prev_len; + + const file = try path.root_dir.handle.openFile(io, path.sub_path, .{}); + defer file.close(io); + + var file_reader: Io.File.Reader = .init(file, io, &.{}); + file_reader.interface.appendRemainingUnlimited(gpa, &m.all_input_content) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return file_reader.err.?, + }; + + var aux: std.ArrayList(u8) = .empty; + defer aux.deinit(gpa); + + var it: DepTokenizer = .{ .bytes = m.all_input_content.items[prev_len..] }; + while (it.next()) |token| switch (token) { + .target, .target_must_resolve => {}, + .prereq => |file_path| { + _ = try m.addInputPath(.initCwd(file_path), .{}); + }, + .prereq_must_resolve => { + aux.clearRetainingCapacity(); + try token.resolve(gpa, &aux); + _ = try m.addInputPath(.initCwd(aux.items), .{}); + }, + else => |err| { + if (diagnostic) |d| d.* = err; + return error.InvalidDepFile; + }, + }; + } + pub const CheckResult = union(enum) { hit, incomplete_manifest, @@ -1614,6 +1660,13 @@ pub const Manifest = struct { stat_dir: Io.Dir.StatError, read_file: Io.File.ReadPositionalError, read_dir: Io.Dir.Reader.Error, + + pub fn format(this: @This(), w: *Io.Writer) Io.Writer.Error!void { + switch (this) { + .none => return w.writeAll("none"), + else => |err, tag| return w.print("{t}: {t}", .{ tag, err }), + } + } }; pub const AddDiscoveredPathError = error{ @@ -1726,6 +1779,59 @@ pub const Manifest = struct { } } + pub const AddDiscoveredDepFileError = error{ + InvalidDepFile, + } || AddDiscoveredPathError || Io.File.OpenError || Io.File.Reader.Error; + + pub const AddDiscoveredDepFileDiagnostic = union(enum) { + add_discovered_path: AddDiscoveredPathDiagnostic, + dep_tokenizer: DepTokenizer.Token, + }; + + pub fn addDiscoveredDepFile( + m: *Manifest, + path: Path, + diagnostic: ?*AddDiscoveredDepFileDiagnostic, + ) AddDiscoveredDepFileError!void { + assert(m.manifest_file != null); + transitionToMissDiscovered(m); + + const cache = m.cache; + const gpa = cache.gpa; + const io = cache.io; + + // TODO: change DepTokenizer to be streaming rather than operating on slice of bytes. + const prev_len = m.all_input_content.items.len; + defer m.all_input_content.items.len = prev_len; + + const file = try path.root_dir.handle.openFile(io, path.sub_path, .{}); + defer file.close(io); + + var file_reader: Io.File.Reader = .init(file, io, &.{}); + file_reader.interface.appendRemainingUnlimited(gpa, &m.all_input_content) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return file_reader.err.?, + }; + + var aux: std.ArrayList(u8) = .empty; + defer aux.deinit(gpa); + + var it: DepTokenizer = .{ .bytes = m.all_input_content.items[prev_len..] }; + while (it.next()) |token| switch (token) { + .target, .target_must_resolve => {}, + .prereq => |p| try m.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(p) } }), + .prereq_must_resolve => { + aux.clearRetainingCapacity(); + try token.resolve(gpa, &aux); + try m.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(aux.items) } }); + }, + else => |err| { + if (diagnostic) |d| d.* = .{ .dep_tokenizer = err }; + return error.InvalidDepFile; + }, + }; + } + fn populateFile( m: *Manifest, file: *File, @@ -1810,53 +1916,6 @@ pub const Manifest = struct { } } - pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { - assert(self.manifest_file == null); - return self.addDepFileMaybePost(dir, dep_file_sub_path); - } - - pub fn addDepFilePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { - assert(self.manifest_file != null); - return self.addDepFileMaybePost(dir, dep_file_sub_path); - } - - fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { - const gpa = self.cache.gpa; - const io = self.cache.io; - const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .unlimited); - defer gpa.free(dep_file_contents); - - var error_buf: std.ArrayList(u8) = .empty; - defer error_buf.deinit(gpa); - - var resolve_buf: std.ArrayList(u8) = .empty; - defer resolve_buf.deinit(gpa); - - var it: DepTokenizer = .{ .bytes = dep_file_contents }; - while (it.next()) |token| switch (token) { - // We don't care about targets, we only want the prereqs - // Clang is invoked in single-source mode but other programs may not - .target, .target_must_resolve => {}, - .prereq => |file_path| if (self.manifest_file == null) { - _ = try self.addInputPath(.initCwd(file_path), .{}); - } else try self.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(file_path) } }), - .prereq_must_resolve => { - resolve_buf.clearRetainingCapacity(); - try token.resolve(gpa, &resolve_buf); - if (self.manifest_file == null) { - _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); - } else try self.addDiscoveredPath(.{ - .discovered_path = .{ .unresolved = .initCwd(resolve_buf.items) }, - }); - }, - else => |err| { - try err.printError(gpa, &error_buf); - log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items }); - return error.InvalidDepFile; - }, - }; - } - /// See also `hitDigestHex`. pub fn hitDigest(m: *Manifest) BinDigest { assert(m.manifest_file != null); @@ -2533,3 +2592,9 @@ test "Manifest with files added after initial hash" { try testing.expect(!mem.eql(u8, &digest1, &digest3)); } } + +test { + _ = Path; + _ = Directory; + _ = DepTokenizer; +} diff --git a/lib/std/Build/Cache/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig index 4fe61603c8b768f6a60652f869adb8859a8cd53a..737ff54e55b513ca4792a3049e10a47c4884b828 100644 --- a/lib/std/Build/Cache/DepTokenizer.zig +++ b/lib/std/Build/Cache/DepTokenizer.zig @@ -1,14 +1,15 @@ const Tokenizer = @This(); -index: usize = 0, -bytes: []const u8, -state: State = .lhs, - const std = @import("std"); +const Io = std.Io; const testing = std.testing; const assert = std.debug.assert; const Allocator = std.mem.Allocator; +index: usize = 0, +bytes: []const u8, +state: State = .lhs, + pub fn next(self: *Tokenizer) ?Token { var start = self.index; var must_resolve = false; @@ -16,10 +17,7 @@ pub fn next(self: *Tokenizer) ?Token { const char = self.bytes[self.index]; switch (self.state) { .lhs => switch (char) { - '\t', '\n', '\r', ' ' => { - // silently ignore whitespace - self.index += 1; - }, + '\t', '\n', '\r', ' ' => self.index += 1, // Ignore whitespace. else => { start = self.index; self.state = .target; @@ -79,7 +77,7 @@ pub fn next(self: *Tokenizer) ?Token { self.index += 1; }, else => { - return errorIllegalChar(.expected_dollar_sign, self.index, char); + return errorIllegalChar(.expected_dollar, self.index, char); }, }, .target_colon => switch (char) { @@ -347,7 +345,7 @@ pub const Token = union(enum) { invalid_target: IndexAndChar, bad_target_escape: IndexAndChar, - expected_dollar_sign: IndexAndChar, + expected_dollar: IndexAndChar, continuation_eol: IndexAndChar, incomplete_escape: IndexAndChar, expected_colon: IndexAndChar, @@ -429,48 +427,23 @@ pub const Token = union(enum) { } } - pub fn printError(self: Token, gpa: Allocator, list: *std.ArrayList(u8)) error{OutOfMemory}!void { - switch (self) { - .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error - .incomplete_quoted_prerequisite, - .incomplete_target, - => |index_and_bytes| { - try list.print(gpa, "{s} '", .{self.errStr()}); - if (self == .incomplete_target) { - const tmp = Token{ .target_must_resolve = index_and_bytes.bytes }; - try tmp.resolve(gpa, list); - } else { - try printCharValues(gpa, list, index_and_bytes.bytes); - } - try list.print(gpa, "' at position {d}", .{index_and_bytes.index}); - }, + pub fn format(token: Token, w: *Io.Writer) Io.Writer.Error!void { + switch (token) { + .target, .target_must_resolve, .prereq, .prereq_must_resolve => return w.writeAll(@tagName(token)), + .incomplete_quoted_prerequisite, .incomplete_target => |info| return w.print("{t} {q} at position {d}", .{ + token, info.bytes, info.index, + }), .invalid_target, .bad_target_escape, - .expected_dollar_sign, + .expected_dollar, .continuation_eol, .incomplete_escape, .expected_colon, - => |index_and_char| { - try list.appendSlice(gpa, "illegal char "); - try printUnderstandableChar(gpa, list, index_and_char.char); - try list.print(gpa, " at position {d}: {s}", .{ index_and_char.index, self.errStr() }); - }, + => |info| return w.print("illegal char '{f}' at position {d}: {t}", .{ + std.zig.fmtChar(info.char), info.index, token, + }), } } - - fn errStr(self: Token) []const u8 { - return switch (self) { - .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error - .incomplete_quoted_prerequisite => "incomplete quoted prerequisite", - .incomplete_target => "incomplete target", - .invalid_target => "invalid target", - .bad_target_escape => "bad target escape", - .expected_dollar_sign => "expecting '$'", - .continuation_eol => "continuation expecting end-of-line", - .incomplete_escape => "incomplete escape", - .expected_colon => "expecting ':'", - }; - } }; test "empty file" { @@ -493,37 +466,37 @@ test "empty colon" { } test "empty target" { - try depTokenizer("foo.o:", "target = {foo.o}"); + try depTokenizer("foo.o:", "target = {\"foo.o\"}"); try depTokenizer( \\foo.o: \\bar.o: \\abcd.o: , - \\target = {foo.o} - \\target = {bar.o} - \\target = {abcd.o} + \\target = {"foo.o"} + \\target = {"bar.o"} + \\target = {"abcd.o"} ); } test "whitespace empty target" { - try depTokenizer("\nfoo.o:", "target = {foo.o}"); - try depTokenizer("\rfoo.o:", "target = {foo.o}"); - try depTokenizer("\r\nfoo.o:", "target = {foo.o}"); - try depTokenizer(" foo.o:", "target = {foo.o}"); + try depTokenizer("\nfoo.o:", "target = {\"foo.o\"}"); + try depTokenizer("\rfoo.o:", "target = {\"foo.o\"}"); + try depTokenizer("\r\nfoo.o:", "target = {\"foo.o\"}"); + try depTokenizer(" foo.o:", "target = {\"foo.o\"}"); } test "escape empty target" { - try depTokenizer("\\ foo.o:", "target = { foo.o}"); - try depTokenizer("\\#foo.o:", "target = {#foo.o}"); - try depTokenizer("\\\\foo.o:", "target = {\\foo.o}"); - try depTokenizer("$$foo.o:", "target = {$foo.o}"); + try depTokenizer("\\ foo.o:", "target = {\" foo.o\"}"); + try depTokenizer("\\#foo.o:", "target = {\"#foo.o\"}"); + try depTokenizer("\\\\foo.o:", "target = {\"\\\\foo.o\"}"); + try depTokenizer("$$foo.o:", "target = {\"$foo.o\"}"); } test "empty target linefeeds" { try depTokenizer("\n", ""); try depTokenizer("\r\n", ""); - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer( \\foo.o: , expect); @@ -541,7 +514,7 @@ test "empty target linefeeds" { } test "empty target linefeeds + continuations" { - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer( \\foo.o:\ , expect); @@ -559,7 +532,7 @@ test "empty target linefeeds + continuations" { } test "empty target linefeeds + hspace + continuations" { - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer( \\foo.o: \ , expect); @@ -577,7 +550,7 @@ test "empty target linefeeds + hspace + continuations" { } test "empty target + hspace + colon" { - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer("foo.o :", expect); try depTokenizer("foo.o\t\t\t:", expect); @@ -588,8 +561,8 @@ test "empty target + hspace + colon" { test "prereq" { const expect = - \\target = {foo.o} - \\prereq = {foo.c} + \\target = {"foo.o"} + \\prereq = {"foo.c"} ; try depTokenizer("foo.o: foo.c", expect); try depTokenizer( @@ -608,9 +581,9 @@ test "prereq" { test "prereq continuation" { const expect = - \\target = {foo.o} - \\prereq = {foo.h} - \\prereq = {bar.h} + \\target = {"foo.o"} + \\prereq = {"foo.h"} + \\prereq = {"bar.h"} ; try depTokenizer( \\foo.o: foo.h\ @@ -624,19 +597,19 @@ test "prereq continuation" { test "prereq continuation (CRLF)" { const expect = - \\target = {foo.o} - \\prereq = {foo.h} - \\prereq = {bar.h} + \\target = {"foo.o"} + \\prereq = {"foo.h"} + \\prereq = {"bar.h"} ; try depTokenizer("foo.o: foo.h\\\r\nbar.h", expect); } test "multiple prereqs" { const expect = - \\target = {foo.o} - \\prereq = {foo.c} - \\prereq = {foo.h} - \\prereq = {bar.h} + \\target = {"foo.o"} + \\prereq = {"foo.c"} + \\prereq = {"foo.h"} + \\prereq = {"bar.h"} ; try depTokenizer("foo.o: foo.c foo.h bar.h", expect); try depTokenizer( @@ -680,19 +653,19 @@ test "multiple targets and prereqs" { \\ one.h two.h \ \\ three.h four.h , - \\target = {foo.o} - \\prereq = {foo.c} - \\target = {bar.o} - \\prereq = {bar.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - \\target = {abc.o} - \\prereq = {abc.c} - \\prereq = {one.h} - \\prereq = {two.h} - \\prereq = {three.h} - \\prereq = {four.h} + \\target = {"foo.o"} + \\prereq = {"foo.c"} + \\target = {"bar.o"} + \\prereq = {"bar.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} + \\target = {"abc.o"} + \\prereq = {"abc.c"} + \\prereq = {"one.h"} + \\prereq = {"two.h"} + \\prereq = {"three.h"} + \\prereq = {"four.h"} ); try depTokenizer( \\ascii.o: ascii.c @@ -702,21 +675,21 @@ test "multiple targets and prereqs" { \\ macho.c\ \\ a.h b.h c.h , - \\target = {ascii.o} - \\prereq = {ascii.c} - \\target = {base64.o} - \\prereq = {base64.c} - \\prereq = {stdio.h} - \\target = {elf.o} - \\prereq = {elf.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - \\target = {macho.o} - \\prereq = {macho.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} + \\target = {"ascii.o"} + \\prereq = {"ascii.c"} + \\target = {"base64.o"} + \\prereq = {"base64.c"} + \\prereq = {"stdio.h"} + \\target = {"elf.o"} + \\prereq = {"elf.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} + \\target = {"macho.o"} + \\prereq = {"macho.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} ); try depTokenizer( \\a$$scii.o: ascii.c @@ -726,20 +699,20 @@ test "multiple targets and prereqs" { \\ "macho!.c" \ \\ a.h b.h c.h , - \\target = {a$scii.o} - \\prereq = {ascii.c} - \\target = {\base64.o} - \\prereq = {\base64.c} - \\prereq = {s t#dio.h} - \\target = {e\lf.o} - \\prereq = {e\lf.c} - \\prereq = {a.h$$} - \\prereq = {$$b.h c.h$$} - \\target = {macho.o} - \\prereq = {macho!.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} + \\target = {"a$scii.o"} + \\prereq = {"ascii.c"} + \\target = {"\\base64.o"} + \\prereq = {"\\base64.c"} + \\prereq = {"s t#dio.h"} + \\target = {"e\\lf.o"} + \\prereq = {"e\\lf.c"} + \\prereq = {"a.h$$"} + \\prereq = {"$$b.h c.h$$"} + \\target = {"macho.o"} + \\prereq = {"macho!.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} ); } @@ -750,12 +723,12 @@ test "windows quoted prereqs" { \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \ \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h" , - \\target = {c:\foo.o} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c} - \\target = {c:\foo2.o} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h} + \\target = {"c:\\foo.o"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo.c"} + \\target = {"c:\\foo2.o"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo2.c"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo1.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo2.h"} ); } @@ -779,23 +752,23 @@ test "windows mixed prereqs" { \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h" , - \\target = {cimport.o} - \\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h} - \\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h} + \\target = {"cimport.o"} + \\prereq = {"C:\\msys64\\home\\anon\\project\\zig\\master\\zig-cache\\o\\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\\cimport.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\stdio.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\vcruntime.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\sal.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\concurrencysal.h"} + \\prereq = {"C:\\msys64\\opt\\zig\\lib\\zig\\include\\vadefs.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\vadefs.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_wstdio.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_stdio_config.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\string.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_memory.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_memcpy_s.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\errno.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\vcruntime_string.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_wstring.h"} ); } @@ -810,14 +783,14 @@ test "windows funky targets" { \\C:\Users\anon\$$foo.o: \\C:\Users\anon\\\ \ \ \ \ foo.o: , - \\target = {C:\Users\anon\foo.o} - \\target = {C:\Users\anon\foo .o} - \\target = {C:\Users\anon\foo#.o} - \\target = {C:\Users\anon\foo$.o} - \\target = {C:\Users\anon\ foo.o} - \\target = {C:\Users\anon\#foo.o} - \\target = {C:\Users\anon\$foo.o} - \\target = {C:\Users\anon\ foo.o} + \\target = {"C:\\Users\\anon\\foo.o"} + \\target = {"C:\\Users\\anon\\foo .o"} + \\target = {"C:\\Users\\anon\\foo#.o"} + \\target = {"C:\\Users\\anon\\foo$.o"} + \\target = {"C:\\Users\\anon\\ foo.o"} + \\target = {"C:\\Users\\anon\\#foo.o"} + \\target = {"C:\\Users\\anon\\$foo.o"} + \\target = {"C:\\Users\\anon\\ foo.o"} ); } @@ -837,13 +810,13 @@ test "windows funky prereqs" { \\ somedir\\ \\ \c.c\ \\ , - \\target = {cimport.o} - \\prereq = {trailingbackslash\} - \\prereq = {C:\Users\John Smith\AppData\Local\zig\p\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\cimport.c} - \\prereq = {somedir\ a.c} - \\prereq = {somedir/ a.c} - \\prereq = {somedir\ b.c} - \\prereq = {somedir\ \ \c.c} + \\target = {"cimport.o"} + \\prereq = {"trailingbackslash\\"} + \\prereq = {"C:\\Users\\John Smith\\AppData\\Local\\zig\\p\\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\\cimport.c"} + \\prereq = {"somedir\\ a.c"} + \\prereq = {"somedir/ a.c"} + \\prereq = {"somedir\\ b.c"} + \\prereq = {"somedir\\ \\ \\c.c"} ); } @@ -852,171 +825,171 @@ test "windows drive and forward slashes" { \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \ \\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c , - \\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj} - \\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c} + \\target = {"C:/msys64/what/zig-cache\\tmp\\48ac4d78dd531abd-cxa_thread_atexit.obj"} + \\prereq = {"C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c"} ); } test "error incomplete escape - reverse_solidus" { try depTokenizer("\\", - \\ERROR: illegal char '\' at position 0: incomplete escape + \\ERROR: illegal char '\\' at position 0: incomplete_escape ); try depTokenizer("\t\\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); try depTokenizer("\n\\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); try depTokenizer("\r\\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); try depTokenizer("\r\n\\", - \\ERROR: illegal char '\' at position 2: incomplete escape + \\ERROR: illegal char '\\' at position 2: incomplete_escape ); try depTokenizer(" \\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); } test "error incomplete escape - dollar_sign" { try depTokenizer("$", - \\ERROR: illegal char '$' at position 0: incomplete escape + \\ERROR: illegal char '$' at position 0: incomplete_escape ); try depTokenizer("\t$", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); try depTokenizer("\n$", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); try depTokenizer("\r$", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); try depTokenizer("\r\n$", - \\ERROR: illegal char '$' at position 2: incomplete escape + \\ERROR: illegal char '$' at position 2: incomplete_escape ); try depTokenizer(" $", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); } test "error incomplete target" { try depTokenizer("foo.o", - \\ERROR: incomplete target 'foo.o' at position 0 + \\ERROR: incomplete_target "foo.o" at position 0 ); try depTokenizer("\tfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\nfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\rfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\r\nfoo.o", - \\ERROR: incomplete target 'foo.o' at position 2 + \\ERROR: incomplete_target "foo.o" at position 2 ); try depTokenizer(" foo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\\ foo.o", - \\ERROR: incomplete target ' foo.o' at position 0 + \\ERROR: incomplete_target "\\ foo.o" at position 0 ); try depTokenizer("\\#foo.o", - \\ERROR: incomplete target '#foo.o' at position 0 + \\ERROR: incomplete_target "\\#foo.o" at position 0 ); try depTokenizer("\\\\foo.o", - \\ERROR: incomplete target '\foo.o' at position 0 + \\ERROR: incomplete_target "\\\\foo.o" at position 0 ); try depTokenizer("$$foo.o", - \\ERROR: incomplete target '$foo.o' at position 0 + \\ERROR: incomplete_target "$$foo.o" at position 0 ); } test "error illegal char at position - bad target escape" { try depTokenizer("\\\t", - \\ERROR: illegal char \x09 at position 1: bad target escape + \\ERROR: illegal char '\t' at position 1: bad_target_escape ); try depTokenizer("\\\n", - \\ERROR: illegal char \x0A at position 1: bad target escape + \\ERROR: illegal char '\n' at position 1: bad_target_escape ); try depTokenizer("\\\r", - \\ERROR: illegal char \x0D at position 1: bad target escape + \\ERROR: illegal char '\r' at position 1: bad_target_escape ); try depTokenizer("\\\r\n", - \\ERROR: illegal char \x0D at position 1: bad target escape + \\ERROR: illegal char '\r' at position 1: bad_target_escape ); } test "error illegal char at position - expecting dollar_sign" { try depTokenizer("$\t", - \\ERROR: illegal char \x09 at position 1: expecting '$' + \\ERROR: illegal char '\t' at position 1: expected_dollar ); try depTokenizer("$\n", - \\ERROR: illegal char \x0A at position 1: expecting '$' + \\ERROR: illegal char '\n' at position 1: expected_dollar ); try depTokenizer("$\r", - \\ERROR: illegal char \x0D at position 1: expecting '$' + \\ERROR: illegal char '\r' at position 1: expected_dollar ); try depTokenizer("$\r\n", - \\ERROR: illegal char \x0D at position 1: expecting '$' + \\ERROR: illegal char '\r' at position 1: expected_dollar ); } test "error illegal char at position - invalid target" { try depTokenizer("foo\n.o", - \\ERROR: illegal char \x0A at position 3: invalid target + \\ERROR: illegal char '\n' at position 3: invalid_target ); try depTokenizer("foo\r.o", - \\ERROR: illegal char \x0D at position 3: invalid target + \\ERROR: illegal char '\r' at position 3: invalid_target ); try depTokenizer("foo\r\n.o", - \\ERROR: illegal char \x0D at position 3: invalid target + \\ERROR: illegal char '\r' at position 3: invalid_target ); } test "error target - continuation expecting end-of-line" { try depTokenizer("foo.o: \\\t", - \\target = {foo.o} - \\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char '\t' at position 8: continuation_eol ); try depTokenizer("foo.o: \\ ", - \\target = {foo.o} - \\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char ' ' at position 8: continuation_eol ); try depTokenizer("foo.o: \\x", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char 'x' at position 8: continuation_eol ); try depTokenizer("foo.o: \\\x0dx", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char 'x' at position 9: continuation_eol ); } test "error prereq - continuation expecting end-of-line" { try depTokenizer("foo.o: foo.h\\\x0dx", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char 'x' at position 14: continuation_eol ); } test "error illegal char at position - expecting colon" { try depTokenizer("foo\t.o:", - \\target = {foo} - \\ERROR: illegal char '.' at position 4: expecting ':' + \\target = {"foo"} + \\ERROR: illegal char '.' at position 4: expected_colon ); try depTokenizer("foo .o:", - \\target = {foo} - \\ERROR: illegal char '.' at position 4: expecting ':' + \\target = {"foo"} + \\ERROR: illegal char '.' at position 4: expected_colon ); try depTokenizer("foo \n.o:", - \\target = {foo} - \\ERROR: illegal char \x0A at position 4: expecting ':' + \\target = {"foo"} + \\ERROR: illegal char '\n' at position 4: expected_colon ); try depTokenizer("foo.o\t\n:", - \\target = {foo.o} - \\ERROR: illegal char \x0A at position 6: expecting ':' + \\target = {"foo.o"} + \\ERROR: illegal char '\n' at position 6: expected_colon ); } @@ -1028,70 +1001,35 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { var it: Tokenizer = .{ .bytes = input }; var buffer: std.ArrayList(u8) = .empty; - var resolve_buf: std.ArrayList(u8) = .empty; + var aux: std.ArrayList(u8) = .empty; var i: usize = 0; while (it.next()) |token| { if (i != 0) try buffer.appendSlice(arena, "\n"); switch (token) { .target, .prereq => |bytes| { - try buffer.appendSlice(arena, @tagName(token)); - try buffer.appendSlice(arena, " = {"); - for (bytes) |b| { - try buffer.append(arena, printable_char_tab[b]); - } - try buffer.appendSlice(arena, "}"); + try buffer.print(arena, "{t} = {{{q}}}", .{ token, bytes }); }, .target_must_resolve => { try buffer.appendSlice(arena, "target = {"); - try token.resolve(arena, &resolve_buf); - for (resolve_buf.items) |b| { - try buffer.append(arena, printable_char_tab[b]); - } - resolve_buf.items.len = 0; + aux.clearRetainingCapacity(); + try token.resolve(arena, &aux); + try buffer.print(arena, "{q}", .{aux.items}); try buffer.appendSlice(arena, "}"); }, .prereq_must_resolve => { try buffer.appendSlice(arena, "prereq = {"); - try token.resolve(arena, &resolve_buf); - for (resolve_buf.items) |b| { - try buffer.append(arena, printable_char_tab[b]); - } - resolve_buf.items.len = 0; + aux.clearRetainingCapacity(); + try token.resolve(arena, &aux); + try buffer.print(arena, "{q}", .{aux.items}); try buffer.appendSlice(arena, "}"); }, else => { - try buffer.appendSlice(arena, "ERROR: "); - try token.printError(arena, &buffer); + try buffer.print(arena, "ERROR: {f}", .{token}); break; }, } i += 1; } - if (std.mem.eql(u8, expect, buffer.items)) { - try testing.expect(true); - return; - } - try testing.expectEqualStrings(expect, buffer.items); } - -fn printCharValues(gpa: Allocator, list: *std.ArrayList(u8), bytes: []const u8) !void { - for (bytes) |b| try list.append(gpa, printable_char_tab[b]); -} - -fn printUnderstandableChar(gpa: Allocator, list: *std.ArrayList(u8), char: u8) !void { - if (std.ascii.isPrint(char)) { - try list.print(gpa, "'{c}'", .{char}); - } else { - try list.print(gpa, "\\x{X:0>2}", .{char}); - } -} - -// zig fmt: off -const printable_char_tab: [256]u8 = ( - "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++ - "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++ - "................................................................" ++ - "................................................................" -).*; diff --git a/src/Compilation.zig b/src/Compilation.zig index 6599e8c34085ae331173e65edf15112e090c104d..bc2bbbc18d94efc51a1726071482bc7a173ed9bc 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -5094,9 +5094,13 @@ pub fn translateC( try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, comp.thread_limit, &stdout); if (out_dep_path) |dep_file_path| add_deps: { + var diagnostic: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined; const dep_basename = fs.path.basename(dep_file_path); // Add the files depended on to the cache system, if a dep file was emitted - man.addDepFilePost(cache_tmp_dir, dep_basename) catch |err| switch (err) { + man.addDiscoveredDepFile(.{ + .root_dir = .{ .handle = cache_tmp_dir, .path = tmp_sub_path }, + .sub_path = dep_basename, + }, &diagnostic) catch |err| switch (err) { error.FileNotFound => break :add_deps, else => |e| return e, }; @@ -5105,14 +5109,17 @@ pub fn translateC( .whole => |whole| if (whole.cache_manifest) |whole_cache_manifest| { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try whole_cache_manifest.addDepFilePost(cache_tmp_dir, dep_basename); + try whole_cache_manifest.addDiscoveredDepFile(.{ + .root_dir = .{ .handle = cache_tmp_dir, .path = tmp_sub_path }, + .sub_path = dep_basename, + }, &diagnostic); }, .incremental, .none => {}, } // Just to save disk space, we delete the file because it is never needed again. cache_tmp_dir.deleteFile(io, dep_basename) catch |err| { - log.warn("failed to delete '{s}': {t}", .{ dep_file_path, err }); + log.warn("failed to delete dep file {q}: {t}", .{ dep_file_path, err }); }; } @@ -5527,7 +5534,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr const target = comp.getTarget(); assert(target.ofmt != .c); const o_ext = target.ofmt.fileExt(target.cpu.arch); - const digest = if (!comp.disable_c_depfile and try man.check(child_progress_node)) man.final() else blk: { + const digest = if (!comp.disable_c_depfile and .hit == try man.check(child_progress_node)) man.hitDigestHex() else miss: { var argv: std.array_list.Managed([]const u8) = .init(gpa); defer argv.deinit(); @@ -5654,11 +5661,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr // Just to save disk space, we delete the files that are never needed again. defer if (out_diag_path) |diag_file_path| zig_cache_tmp_dir.deleteFile(io, fs.path.basename(diag_file_path)) catch |err| switch (err) { error.FileNotFound => {}, // the file wasn't created due to an error we reported - else => log.warn("failed to delete '{s}': {s}", .{ diag_file_path, @errorName(err) }), + else => log.warn("failed to delete {q}: {t}", .{ diag_file_path, err }), }; defer if (out_dep_path) |dep_file_path| zig_cache_tmp_dir.deleteFile(io, fs.path.basename(dep_file_path)) catch |err| switch (err) { error.FileNotFound => {}, // the file wasn't created due to an error we reported - else => log.warn("failed to delete '{s}': {s}", .{ dep_file_path, @errorName(err) }), + else => log.warn("failed to delete {q}: {t}", .{ dep_file_path, err }), }; if (std.process.can_spawn) { if (comp.clang_passthrough_mode) { @@ -5747,8 +5754,10 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr const dep_basename = fs.path.basename(dep_file_path); if (comp.file_system_inputs != null) { + // TODO instead of this rely on passing the manifest contents directly + // Use the same file size limit as the cache code does for dependency files. - const dep_file_contents = try zig_cache_tmp_dir.readFileAlloc(io, dep_basename, gpa, .limited(Cache.manifest_file_size_max)); + const dep_file_contents = try zig_cache_tmp_dir.readFileAlloc(io, dep_basename, gpa, .unlimited); defer gpa.free(dep_file_contents); var str_buf: std.ArrayList(u8) = .empty; @@ -5763,9 +5772,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr try token.resolve(gpa, &str_buf); break :p try .fromUnresolved(arena, comp.dirs, &.{str_buf.items}); }, - else => |err| { - try err.printError(gpa, &str_buf); - log.err("failed parsing {s}: {s}", .{ dep_basename, str_buf.items }); + else => |err_token| { + log.err("failed parsing {s}: {f}", .{ dep_basename, err_token }); return error.InvalidDepFile; }, }; @@ -5774,13 +5782,26 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr } // Add the files depended on to the cache system. - try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); + var diagnostic: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined; + try man.addDiscoveredDepFile(.{ + .root_dir = .{ + .handle = zig_cache_tmp_dir, + .path = "tmp", + }, + .sub_path = dep_basename, + }, &diagnostic); switch (comp.cache_use) { .whole => |whole| { if (whole.cache_manifest) |whole_cache_manifest| { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename); + try whole_cache_manifest.addDiscoveredDepFile(.{ + .root_dir = .{ + .handle = zig_cache_tmp_dir, + .path = "tmp", + }, + .sub_path = dep_basename, + }, &diagnostic); } }, .incremental, .none => {}, @@ -5791,13 +5812,13 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr if (comp.disable_c_depfile) _ = try man.check(child_progress_node); // Rename into place. - const digest = man.final(); - const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest }); + const digest = man.missDigestHex(); + const o_sub_path = try fs.path.join(arena, &.{ "o", &digest }); var o_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, o_sub_path, .{}); defer o_dir.close(io); const tmp_basename = fs.path.basename(out_obj_path); try Io.Dir.rename(zig_cache_tmp_dir, tmp_basename, o_dir, o_basename, io); - break :blk digest; + break :miss digest; }; if (man.have_exclusive_lock) { diff --git a/src/libs/freebsd.zig b/src/libs/freebsd.zig index b02b3569af70d363eb73619f7bdd7cd514e0d1cc..36062854757ac0b1369c0712031f80e806f453f4 100644 --- a/src/libs/freebsd.zig +++ b/src/libs/freebsd.zig @@ -465,9 +465,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (try man.check(prog_node)) { - const digest = man.final(); - + if (.hit == try man.check(prog_node)) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -477,7 +476,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); var o_directory: Cache.Directory = .{ @@ -486,7 +485,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); diff --git a/src/libs/glibc.zig b/src/libs/glibc.zig index 08b9214d6db5d64a8fef5dcf4dde164e9f7f9f12..691da1bd0f047f527c37c4b850cdceea7dd036a8 100644 --- a/src/libs/glibc.zig +++ b/src/libs/glibc.zig @@ -705,9 +705,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (try man.check(prog_node)) { - const digest = man.final(); - + if (.hit == try man.check(prog_node)) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -717,8 +716,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); - const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); + const digest = man.missDigestHex(); + const o_sub_path = try path.join(arena, &.{ "o", &digest }); var o_directory: Cache.Directory = .{ .handle = try comp.dirs.global_cache.handle.createDirPathOpen(io, o_sub_path, .{}), @@ -726,7 +725,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); diff --git a/src/libs/netbsd.zig b/src/libs/netbsd.zig index 9d120d46ae2db878175c5ff5b84c8ebf7eaf7c9b..5445f5fcaab4906ada5cc8950ef403fb4fc825f6 100644 --- a/src/libs/netbsd.zig +++ b/src/libs/netbsd.zig @@ -413,9 +413,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (try man.check(prog_node)) { - const digest = man.final(); - + if (.hit == try man.check(prog_node)) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -425,8 +424,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); - const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); + const digest = man.missDigestHex(); + const o_sub_path = try path.join(arena, &.{ "o", &digest }); var o_directory: Cache.Directory = .{ .handle = try comp.dirs.global_cache.handle.createDirPathOpen(io, o_sub_path, .{}), @@ -434,7 +433,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); diff --git a/src/libs/openbsd.zig b/src/libs/openbsd.zig index e65fc524967c09300979897a0e5dd089e5fd8ad1..ab126f2a96aa9dce467620052d07a5af8f965e49 100644 --- a/src/libs/openbsd.zig +++ b/src/libs/openbsd.zig @@ -334,9 +334,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (try man.check(prog_node)) { - const digest = man.final(); - + if (.hit == try man.check(prog_node)) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -346,7 +345,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); var o_directory: Cache.Directory = .{ @@ -355,7 +354,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); -- 2.54.0 From 3bb9f1847b373256f495fdc602c5ab6a9f6e7724 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 00:32:16 -0700 Subject: [PATCH 34/49] std.Build.Cache: add transitionToMissDiscovered in addDiscoveredManifest --- lib/std/Build/Cache.zig | 61 +++++++++++++++++++++++++---------------- src/Compilation.zig | 4 +-- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 91c678ed2f94515d5104b42e162944f23920923b..fc651b8ecb034cb45d0bb2893aa9073a2357e537 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1675,11 +1675,12 @@ pub const Manifest = struct { FileSystemFailure, } || Allocator.Error || Io.Cancelable; - /// Add a file as a dependency of process being cached, after cache miss - /// occurs. + /// Add a file or directory as a dependency of process being cached, after cache miss occurs. /// /// See also: /// * `addInputPath` + /// * `addDiscoveredDepFile` + /// * `addDiscoveredManifest` pub fn addDiscoveredPath(m: *Manifest, options: AddDiscoveredPathOptions) AddDiscoveredPathError!void { assert(m.manifest_file != null); transitionToMissDiscovered(m); @@ -1788,6 +1789,11 @@ pub const Manifest = struct { dep_tokenizer: DepTokenizer.Token, }; + /// Add a GNU make style dep file as a dependency of process being cached, after cache miss occurs. + /// + /// See also: + /// * `addDiscoveredPath + /// * `addDiscoveredManifest` pub fn addDiscoveredDepFile( m: *Manifest, path: Path, @@ -2077,7 +2083,7 @@ pub const Manifest = struct { m.* = undefined; } - pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void { + pub fn populateFileSystemInputs(man: *const Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void { assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len); buf.clearRetainingCapacity(); const gpa = man.cache.gpa; @@ -2096,42 +2102,49 @@ pub const Manifest = struct { } } - pub fn populateOtherManifest(man: *Manifest, other: *Manifest, prefix_map: [5]u8) Allocator.Error!void { - const gpa = other.cache.gpa; - assert(other.manifest_file != null); - assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len); - assert(man.cache.prefixes_len == 5); + /// Add the full set of paths from another `Manifest` as a dependency of process being cached, after cache miss + /// occurs. + /// + /// See also: + /// * `addDiscoveredPath + /// * `addDiscoveredDepFile` + pub fn addDiscoveredManifest(m: *Manifest, discovered: *const Manifest, prefix_map: [5]u8) Allocator.Error!void { + const gpa = m.cache.gpa; + assert(m.manifest_file != null); + transitionToMissDiscovered(m); + assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == discovered.cache.prefixes_len); + assert(discovered.cache.prefixes_len == 5); - const orig_files_len = other.files.count(); - const orig_contents_len = other.contents.items.len; + const orig_files_len = m.files.count(); + const orig_contents_len = m.contents.items.len; errdefer { - other.files.shrinkRetainingCapacityContext(orig_files_len, .{ .contents = other.contents.items }); - other.contents.shrinkRetainingCapacity(orig_contents_len); + m.files.shrinkRetainingCapacityContext(orig_files_len, .{ .contents = m.contents.items }); + m.contents.shrinkRetainingCapacity(orig_contents_len); } - for (man.files.keys(), 0..) |off, file_index| { - try other.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = other.contents.items }); + for (discovered.files.keys(), 0..) |off, file_index| { + try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); - const next_off = if (file_index < man.files.count()) - @backingInt(man.files.keys()[file_index + 1]) + const next_off = if (file_index < discovered.files.count()) + @backingInt(discovered.files.keys()[file_index + 1]) else - man.contents.items.len; + discovered.contents.items.len; - const copy_bytes = man.contents.items[@backingInt(off)..next_off]; - const prev_contents_len: File.Offset = @fromBackingInt(@intCast(other.contents.items.len)); - try other.contents.appendSlice(gpa, copy_bytes); + const copy_bytes = discovered.contents.items[@backingInt(off)..next_off]; + const prev_contents_len: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); + try m.contents.appendSlice(gpa, copy_bytes); - const gop = other.files.getOrPutAssumeCapacityContext(prev_contents_len, .{ - .contents = other.contents.items, + const gop = m.files.getOrPutAssumeCapacityContext(prev_contents_len, .{ + .contents = m.contents.items, }); if (gop.found_existing) { - other.contents.shrinkRetainingCapacity(@backingInt(prev_contents_len)); + m.contents.shrinkRetainingCapacity(@backingInt(prev_contents_len)); continue; } // Flags are already copied but the prefix is supposed to be filtered by `prefix_map`. - const other_file = prev_contents_len.get(other.contents.items); + const other_file = prev_contents_len.get(m.contents.items); other_file.flags.prefix = @intCast(prefix_map[other_file.flags.prefix]); } } diff --git a/src/Compilation.zig b/src/Compilation.zig index bc2bbbc18d94efc51a1726071482bc7a173ed9bc..1cd82a32d6a8f0534fd7b57c7187f43e20a80ac1 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2835,7 +2835,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE if (comp.parent_whole_cache) |pwc| { try pwc.mutex.lock(io); defer pwc.mutex.unlock(io); - try man.populateOtherManifest(pwc.manifest, pwc.prefix_map); + try pwc.manifest.addDiscoveredManifest(&man, pwc.prefix_map); } comp.last_update_was_cache_hit = true; @@ -3046,7 +3046,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE if (comp.parent_whole_cache) |pwc| { try pwc.mutex.lock(io); defer pwc.mutex.unlock(io); - try man.populateOtherManifest(pwc.manifest, pwc.prefix_map); + try pwc.manifest.addDiscoveredManifest(&man, pwc.prefix_map); } const bin_digest = man.missDigest(); -- 2.54.0 From 3e3b0b835c178397bb3f8ac4fa0676382e481392 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 15:27:17 -0700 Subject: [PATCH 35/49] std.Build.Cache: fix addDiscoveredManifest --- lib/std/Build/Cache.zig | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index fc651b8ecb034cb45d0bb2893aa9073a2357e537..40bc3678a054c80da475f4ac1fcd930cb123ab70 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -2115,6 +2115,7 @@ pub const Manifest = struct { assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == discovered.cache.prefixes_len); assert(discovered.cache.prefixes_len == 5); + const discovered_files = discovered.files.keys(); const orig_files_len = m.files.count(); const orig_contents_len = m.contents.items.len; errdefer { @@ -2122,13 +2123,14 @@ pub const Manifest = struct { m.contents.shrinkRetainingCapacity(orig_contents_len); } - for (discovered.files.keys(), 0..) |off, file_index| { + for (discovered_files, 0..) |off, file_index| { try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); - const next_off = if (file_index < discovered.files.count()) - @backingInt(discovered.files.keys()[file_index + 1]) + const next_index = file_index + 1; + const next_off = if (discovered_files.len - next_index == 0) + discovered.contents.items.len else - discovered.contents.items.len; + @backingInt(discovered_files[next_index]); const copy_bytes = discovered.contents.items[@backingInt(off)..next_off]; const prev_contents_len: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); -- 2.54.0 From b9794c89ac4a6a82286382d04f22bb47374594ff Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 15:53:43 -0700 Subject: [PATCH 36/49] std.Build.Cache: remove stray hash mutation --- lib/std/Build/Cache.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 40bc3678a054c80da475f4ac1fcd930cb123ab70..9bb44df1a6aa0560e39382642f136de519f6e751 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1734,7 +1734,6 @@ pub const Manifest = struct { hasher.update(contents); hasher.final(&header.digest); } - m.hash.hasher.update(&header.digest); return; } -- 2.54.0 From 7911f377441a1717ac1ab82933af826926b29ff7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 15:53:51 -0700 Subject: [PATCH 37/49] Compilation: refactor a couple log messages --- src/Compilation.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 1cd82a32d6a8f0534fd7b57c7187f43e20a80ac1..fb4d0653f25f385a0d82e5a72ae73ac5f1805ab6 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -3007,7 +3007,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } if (build_options.enable_debug_extensions and comp.verbose_intern_pool) { - std.debug.print("intern pool stats for '{s}':\n", .{comp.root_name}); + std.debug.print("intern pool stats for {q}:\n", .{comp.root_name}); zcu.intern_pool.dump(); } @@ -3019,11 +3019,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE if (comp.link_depfile) |depfile_path| if (comp.bin_file) |lf| { assert(comp.file_system_inputs != null); - comp.createDepFile(depfile_path, lf.emit) catch |err| comp.setMiscFailure( - .link_depfile, - "unable to write linker dependency file: {t}", - .{err}, - ); + comp.createDepFile(depfile_path, lf.emit) catch |err| + comp.setMiscFailure(.link_depfile, "writing linker dependency file failed: {t}", .{err}); }; if (anyErrors(comp)) { -- 2.54.0 From 28feeaf77818c801bac4f7959ace39af5bdc40a5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 18:48:12 -0700 Subject: [PATCH 38/49] link: pass request_handle=true to addInputPath when passing already open handle, this is now required when the caller wants the cache system to keep the file handle open. --- lib/std/Build/Cache.zig | 2 ++ src/link.zig | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 9bb44df1a6aa0560e39382642f136de519f6e751..88fb82d74456fd011581463557d4eb66f5a03c66 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -650,8 +650,10 @@ pub const Manifest = struct { }; pub const AddInputPathOptions = struct { + /// If provided, will be closed when `check` is called, unless `request_handle` is also set. handle: PathHandle = .{ .file = null }, stat: ?Stat = null, + /// If set, file handle will remain open after `check` is called. request_handle: bool = false, /// Can request file or directory contents depending on `handle`. request_contents: bool = false, diff --git a/src/link.zig b/src/link.zig index df68e0216d3f20ab09837c55f849461c30fbd5a4..c97c752517928551c1753933a9c512417cb9e434 100644 --- a/src/link.zig +++ b/src/link.zig @@ -1500,11 +1500,11 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void { base.loadInput(input) catch |err| switch (err) { error.AlreadyReported => return, // error reported via diags else => |e| switch (input) { - .dso => |dso| diags.addParseError(dso.path, "failed to parse shared library: {s}", .{@errorName(e)}), - .object => |obj| diags.addParseError(obj.path, "failed to parse object: {s}", .{@errorName(e)}), - .archive => |obj| diags.addParseError(obj.path, "failed to parse archive: {s}", .{@errorName(e)}), - .res => |res| diags.addParseError(res.path, "failed to parse Windows resource: {s}", .{@errorName(e)}), - .dso_exact => diags.addError("failed to handle dso_exact: {s}", .{@errorName(e)}), + .dso => |dso| diags.addParseError(dso.path, "failed to parse shared library: {t}", .{e}), + .object => |obj| diags.addParseError(obj.path, "failed to parse object: {t}", .{e}), + .archive => |obj| diags.addParseError(obj.path, "failed to parse archive: {t}", .{e}), + .res => |res| diags.addParseError(res.path, "failed to parse Windows resource: {t}", .{e}), + .dso_exact => diags.addError("failed to handle dso_exact: {t}", .{e}), }, }; prog_node.completeOne(); @@ -1927,6 +1927,7 @@ pub fn hashInputs(man: *Cache.Manifest, link_inputs: []const Input) !void { .object, .archive => |obj| { _ = try man.addInputPath(obj.path, .{ .handle = .{ .file = obj.file }, + .request_handle = true, }); man.hash.add(obj.must_link); man.hash.add(obj.hidden); @@ -1934,11 +1935,13 @@ pub fn hashInputs(man: *Cache.Manifest, link_inputs: []const Input) !void { .res => |res| { _ = try man.addInputPath(res.path, .{ .handle = .{ .file = res.file }, + .request_handle = true, }); }, .dso => |dso| { _ = try man.addInputPath(dso.path, .{ .handle = .{ .file = dso.file }, + .request_handle = true, }); man.hash.add(dso.needed); man.hash.add(dso.weak); -- 2.54.0 From 5eb4a5d4b481a3003fe92f25f96f3d0b52a39e74 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 19:11:21 -0700 Subject: [PATCH 39/49] std.Build.Cache: use Io.File.INode type rather than u64. fixes build on Windows --- lib/std/Build/Cache.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 88fb82d74456fd011581463557d4eb66f5a03c66..8d00a3384607eb684fb3860754bfe821618dc1b5 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -439,7 +439,7 @@ pub const Manifest = struct { /// The data per tracked input file that is stored in the manifest file. pub const File = extern struct { size: u64 align(1), - inode: u64 align(1), + inode: Io.File.INode align(1), /// Nanoseconds. mtime: i64 align(1), /// To simplify the hashing logic, this value is computed from size, inode, and mtime -- 2.54.0 From 514989908c897e9ecaf47402ea483dd573ba3b30 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 19:14:19 -0700 Subject: [PATCH 40/49] test-cases: update to match new compile error formatting --- test/cases/compile_errors/bad_import.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cases/compile_errors/bad_import.zig b/test/cases/compile_errors/bad_import.zig index 13ce9d46ff8d45b72a40f5827fe1a111af3913c3..c532706dae893d64e943acd7153995f9badc413d 100644 --- a/test/cases/compile_errors/bad_import.zig +++ b/test/cases/compile_errors/bad_import.zig @@ -4,5 +4,5 @@ const bogus = @import( // error // -// bogus-does-not-exist.zig:1:1: error: unable to load 'bogus-does-not-exist.zig': FileNotFound +// bogus-does-not-exist.zig:1:1: error: unable to load "bogus-does-not-exist.zig": FileNotFound // :2:5: note: file imported here -- 2.54.0 From e8882ffcf0d97cabfc974bdbc1c592f911832623 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2026 20:25:47 -0700 Subject: [PATCH 41/49] std.Build.Cache: fix compilation on 32-bit --- lib/compiler/Maker/Step.zig | 5 +++-- lib/std/Build/Cache.zig | 5 ++++- src/Compilation.zig | 17 +++++++++++------ src/libs/mingw.zig | 4 ++++ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index c3ff8962b0d5cd940a766c7b4e6aef9ed7faeb9c..117e9c9df53e3626f0c4cfd259cd41bb41878779 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -753,12 +753,13 @@ fn failWithCacheError( switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, + .manifest_oversize => return s.fail(maker, "checking cache failed: {t}", .{man.diagnostic}), .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { - return s.fail(maker, "failed checking cache: {t} {t}", .{ man.diagnostic, e }); + return s.fail(maker, "checking cache failed: {t} {t}", .{ man.diagnostic, e }); }, .file_open, .file_stat, .file_read, .file_hash => |op| { const path = op.path(man); - return s.fail(maker, "failed checking cache: {f} {t} {t}", .{ path, man.diagnostic, op.err }); + return s.fail(maker, "checking cache failed: {f} {t} {t}", .{ path, man.diagnostic, op.err }); }, }, error.OutOfMemory, error.Canceled => |e| return e, diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 8d00a3384607eb684fb3860754bfe821618dc1b5..973488702c729c2e246b18db00db4daea831e8df 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -595,6 +595,7 @@ pub const Manifest = struct { none, manifest_create: Io.File.OpenError, manifest_stat: Io.File.StatError, + manifest_oversize, manifest_read: Io.File.ReadPositionalError, manifest_lock: Io.File.LockError, file_open: FileOp, @@ -1011,10 +1012,12 @@ pub const Manifest = struct { const io = m.cache.io; const manifest_file = m.manifest_file.?; - const manifest_size = if (manifest_file.stat(io)) |stat| stat.size else |err| switch (err) { + const manifest_stat = manifest_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| return fail(&m.diagnostic, .{ .manifest_stat = e }), }; + const manifest_size = std.math.cast(u32, manifest_stat.size) orelse + return fail(&m.diagnostic, .manifest_oversize); if (manifest_size == 0 or manifest_size < m.contents.items.len) { // Manifest file was never finalized. diff --git a/src/Compilation.zig b/src/Compilation.zig index fb4d0653f25f385a0d82e5a72ae73ac5f1805ab6..68c76dfd83bce307a45af5d0ed0c35f126316c64 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2817,13 +2817,18 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE error.Canceled, error.OutOfMemory => |e| return e, error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, - .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| return comp.setMiscFailure( - .check_whole_cache, - "failed to check cache: {t} {t}", - .{ man.diagnostic, e }, - ), + .manifest_oversize => { + return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {t}", .{ + man.diagnostic, + }); + }, + .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { + return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {t} {t}", .{ + man.diagnostic, e, + }); + }, .file_open, .file_stat, .file_read, .file_hash => |op| { - return comp.setMiscFailure(.check_whole_cache, "failed to check cache: {f} {t} {t}", .{ + return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {f} {t} {t}", .{ op.path(&man), man.diagnostic, op.err, }); }, diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index 24a6ac4e12e4fe0749dde875bf88f8b8a574e4e0..59d99a783af2af8053dcfb629ce1c48ee672f262 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -254,6 +254,10 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P const status = man.check(prog_node) catch |err| switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, + .manifest_oversize => { + comp.setMiscFailure(.windows_import_lib, "checking cache failed: {t}", .{man.diagnostic}); + return error.AlreadyReported; + }, .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { comp.setMiscFailure(.windows_import_lib, "checking cache failed: {t} {t}", .{ man.diagnostic, e }); return error.AlreadyReported; -- 2.54.0 From 597306838dcc4802c17e94ba156ccf3912e9934c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2026 17:00:34 -0700 Subject: [PATCH 42/49] frontend: better error reporting for cache check failures --- lib/compiler/Maker.zig | 8 +- lib/compiler/Maker/Step.zig | 23 ++- lib/std/Build/Cache.zig | 241 +++++++++++++++++------------ src/Compilation.zig | 80 ++++++---- src/Sema.zig | 12 +- src/libs/freebsd.zig | 15 +- src/libs/glibc.zig | 15 +- src/libs/mingw.zig | 25 +-- src/libs/netbsd.zig | 15 +- src/libs/openbsd.zig | 15 +- src/main.zig | 8 +- test/incremental/bad_import | 4 +- test/incremental/change_embed_file | 4 +- test/incremental/change_zon_file | 4 +- 14 files changed, 292 insertions(+), 177 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index d0bea06e27c69d07ac220e41c1add40e3b38c2ff..b372f8999270f53a0ba15e916f7ecb8200ec662a 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1399,7 +1399,13 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { defer compile_prog_node.end(); if (config_man) |man| { - if (.hit == try man.check(compile_prog_node)) { + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, compile_prog_node) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + error.CacheCheckFailed => fatal("checking cache failed: {f}", .{diagnostic.fmt(man)}), + }; + log.debug("configuration cache {f}", .{status.fmt(man)}); + if (status == .hit) { const digest = man.hitDigestHex(); const path: Path = .{ .root_dir = graph.local_cache_root, diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 117e9c9df53e3626f0c4cfd259cd41bb41878779..4e9eb614efcea395d8f763e1210de56070fb2743 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -726,7 +726,10 @@ pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.T /// Prefer `cacheHitWatched` unless you already added watch inputs /// separately from using the cache system. pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { - const hit = .hit == (man.check(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err)); + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, parent_progress_node) catch |err| + return failWithCacheError(s, maker, man, &diag, err); + const hit = .hit == status; // TODO cache miss reason in the build summary s.result_cached = hit; return hit; } @@ -736,7 +739,10 @@ pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_n /// /// Must be accompanied with `finalizeManifestAndWatch`. pub fn cacheHitWatched(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { - const hit = .hit == (man.check(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err)); + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, parent_progress_node) catch |err| + return failWithCacheError(s, maker, man, &diag, err); + const hit = .hit == status; // TODO cache miss reason in the build summary s.result_cached = hit; // The above call to hit() populates the manifest with files, so in case of // a hit, we need to populate watch inputs. @@ -748,20 +754,11 @@ fn failWithCacheError( s: *Step, maker: *Maker, man: *const Cache.Manifest, + diag: *const Cache.Manifest.CheckDiagnostic, err: Cache.Manifest.CheckError, ) error{ OutOfMemory, Canceled, MakeFailed } { switch (err) { - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_oversize => return s.fail(maker, "checking cache failed: {t}", .{man.diagnostic}), - .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { - return s.fail(maker, "checking cache failed: {t} {t}", .{ man.diagnostic, e }); - }, - .file_open, .file_stat, .file_read, .file_hash => |op| { - const path = op.path(man); - return s.fail(maker, "checking cache failed: {f} {t} {t}", .{ path, man.diagnostic, op.err }); - }, - }, + error.CacheCheckFailed => return s.fail(maker, "checking cache failed: {f}", .{diag.fmt(man)}), error.OutOfMemory, error.Canceled => |e| return e, } } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 973488702c729c2e246b18db00db4daea831e8df..3eaeef3ab92ab88f11c4da1418aa84fb51c960ef 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -289,7 +289,6 @@ pub const Manifest = struct { /// Indexes line up with `files`, but only up until `hit` is called. Uses /// `Cache.gpa`. input_paths: std.ArrayList(InputPath) = .empty, - diagnostic: Diagnostic = .none, /// Keeps track of the last time we performed a file system write to observe /// what time the file system thinks it is, according to its own granularity. recent_problematic_timestamp: Io.Timestamp = .zero, @@ -591,8 +590,7 @@ pub const Manifest = struct { } }; - pub const Diagnostic = union(enum) { - none, + pub const CheckDiagnostic = union(enum) { manifest_create: Io.File.OpenError, manifest_stat: Io.File.StatError, manifest_oversize, @@ -617,6 +615,31 @@ pub const Manifest = struct { }; } }; + + pub const Format = struct { + diagnostic: CheckDiagnostic, + manifest: *const Manifest, + + pub fn format(this: @This(), w: *Io.Writer) Io.Writer.Error!void { + switch (this.diagnostic) { + .manifest_oversize => return w.writeAll(@tagName(this.diagnostic)), + .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { + return w.print("{t} {t}", .{ this.diagnostic, e }); + }, + .file_open, .file_stat, .file_read, .file_hash => |op| { + const path = op.path(this.manifest); + return w.print("{t} {t} {f}", .{ this.diagnostic, op.err, path }); + }, + } + } + }; + + pub fn fmt(diagnostic: CheckDiagnostic, manifest: *const Manifest) Format { + return .{ + .diagnostic = diagnostic, + .manifest = manifest, + }; + } }; pub const Stat = struct { @@ -816,6 +839,28 @@ pub const Manifest = struct { directory_status_changed: File.Offset, metadata_changed: File.Offset, contents_changed: File.Offset, + + pub const Format = struct { + check_result: CheckResult, + manifest: *const Manifest, + + pub fn format(this: @This(), w: *Io.Writer) Io.Writer.Error!void { + switch (this.check_result) { + .hit, .incomplete_manifest, .invalid_manifest => return w.writeAll(@tagName(this.check_result)), + .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |off| { + const path = off.path(this.manifest.contents.items); + return w.print("{t} {s}", .{ this.check_result, path }); + }, + } + } + }; + + pub fn fmt(check_result: CheckResult, manifest: *const Manifest) Format { + return .{ + .check_result = check_result, + .manifest = manifest, + }; + } }; pub const CheckError = error{ @@ -836,13 +881,17 @@ pub const Manifest = struct { /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) CheckError!CheckResult { + pub fn check( + man: *Manifest, + diag: *CheckDiagnostic, + parent_progress_node: std.Progress.Node, + ) CheckError!CheckResult { const node = parent_progress_node.start("Reusing Cache Artifacts", 0); defer node.end(); - return checkProgressless(man); + return checkProgressless(man, diag); } - pub fn checkProgressless(man: *Manifest) CheckError!CheckResult { + pub fn checkProgressless(man: *Manifest, diag: *CheckDiagnostic) CheckError!CheckResult { assert(man.state == .input); assert(man.manifest_file == null); @@ -852,8 +901,6 @@ pub const Manifest = struct { man.hashFlagsAndPath(file_off, &man.hash.hasher); } - man.diagnostic = .none; - var input_digest: BinDigest = undefined; man.hash.hasher.final(&input_digest); const input_hex_digest = binToHex(input_digest); @@ -879,10 +926,7 @@ pub const Manifest = struct { man.manifest_file = man.cache.manifest_dir.openFile(io, manifest_file_path, .{ .mode = .read_write, .lock = .shared, - }) catch |e| { - man.diagnostic = .{ .manifest_create = e }; - return error.CacheCheckFailed; - }; + }) catch |e| return fail(diag, .{ .manifest_create = e }); break; }, error.FileNotFound => { @@ -895,10 +939,7 @@ pub const Manifest = struct { // disambiguates by returning EEXIST, indicating original // failure was a race, or ENOENT, indicating deletion of // the directory of our open handle. - if (!builtin.os.tag.isDarwin()) { - man.diagnostic = .{ .manifest_create = error.FileNotFound }; - return error.CacheCheckFailed; - } + if (!builtin.os.tag.isDarwin()) return fail(diag, .{ .manifest_create = error.FileNotFound }); if (man.cache.manifest_dir.createFile(io, manifest_file_path, .{ .read = true, @@ -912,22 +953,13 @@ pub const Manifest = struct { break; } else |excl_err| switch (excl_err) { error.WouldBlock, error.PathAlreadyExists => continue, - error.FileNotFound => { - man.diagnostic = .{ .manifest_create = error.FileNotFound }; - return error.CacheCheckFailed; - }, + error.FileNotFound => return fail(diag, .{ .manifest_create = error.FileNotFound }), error.Canceled => |e| return e, - else => |e| { - man.diagnostic = .{ .manifest_create = e }; - return error.CacheCheckFailed; - }, + else => |e| return fail(diag, .{ .manifest_create = e }), } }, error.Canceled => |e| return e, - else => |e| { - man.diagnostic = .{ .manifest_create = e }; - return error.CacheCheckFailed; - }, + else => |e| return fail(diag, .{ .manifest_create = e }), } } @@ -949,16 +981,16 @@ pub const Manifest = struct { hit: { const miss_result = miss: { - const result = try man.checkLocked(); + const result = try man.checkLocked(diag); if (result == .hit) { break :hit; - } else if (!try man.upgradeToExclusiveLock()) { + } else if (!try man.upgradeToExclusiveLock(diag)) { break :miss result; } // Missed with the shared lock, and upgraded to an exclusive lock. However, another process may have // modified the cache directory, so we need to check again before deciding to miss. man.shrinkFilesToInput(); - const refreshed_result = try man.checkLocked(); + const refreshed_result = try man.checkLocked(diag); if (refreshed_result == .hit) break :hit; break :miss refreshed_result; }; @@ -970,10 +1002,7 @@ pub const Manifest = struct { } if (man.want_shared_lock) { - man.downgradeToSharedLock() catch |err| { - man.diagnostic = .{ .manifest_lock = err }; - return error.CacheCheckFailed; - }; + man.downgradeToSharedLock() catch |err| return fail(diag, .{ .manifest_lock = err }); } man.state = .hit; @@ -1006,7 +1035,7 @@ pub const Manifest = struct { } /// Does not observe or modify `self.hash.hasher`. Asserts `self.files` contains only the original input files. - fn checkLocked(m: *Manifest) CheckError!CheckResult { + fn checkLocked(m: *Manifest, diag: *CheckDiagnostic) CheckError!CheckResult { assert(m.files.count() == m.input_paths.items.len); const gpa = m.cache.gpa; const io = m.cache.io; @@ -1014,15 +1043,15 @@ pub const Manifest = struct { const manifest_stat = manifest_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .manifest_stat = e }), + else => |e| return fail(diag, .{ .manifest_stat = e }), }; const manifest_size = std.math.cast(u32, manifest_stat.size) orelse - return fail(&m.diagnostic, .manifest_oversize); + return fail(diag, .manifest_oversize); if (manifest_size == 0 or manifest_size < m.contents.items.len) { // Manifest file was never finalized. try m.contents.ensureUnusedCapacity(gpa, 1); - return populateMissingInputFileHashes(m, 0, .incomplete_manifest); + return populateMissingInputFileHashes(m, diag, 0, .incomplete_manifest); } // We must not clobber existing `Manifest.contents` because it possibly contains prepopulated stat and digest @@ -1039,9 +1068,9 @@ pub const Manifest = struct { { const n = manifest_file.readPositionalAll(io, m.contents.items, 0) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .manifest_read = e }), + else => |e| return fail(diag, .{ .manifest_read = e }), }; - if (n != manifest_size) return missInput(m, 0, input_contents_len, manifest_size, .incomplete_manifest); + if (n != manifest_size) return missInput(m, diag, 0, input_contents_len, manifest_size, .incomplete_manifest); } const disk_contents = m.contents.items[0..manifest_size]; const input_contents = m.contents.items[manifest_size..][0..input_contents_len]; @@ -1050,12 +1079,12 @@ pub const Manifest = struct { var off: usize = 0; for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len], 0..) |*input_path, file_off, i| { if (@backingInt(file_off) + 1 >= disk_contents.len) - return missInput(m, i, input_contents_len, manifest_size, .incomplete_manifest); + return missInput(m, diag, i, input_contents_len, manifest_size, .incomplete_manifest); if (!mem.eql(u8, file_off.flagsAndPath(disk_contents), file_off.flagsAndPath(input_contents))) - return missInput(m, i, input_contents_len, manifest_size, .invalid_manifest); + return missInput(m, diag, i, input_contents_len, manifest_size, .invalid_manifest); - const result = try checkInputPath(m, file_off, input_path, disk_contents, input_contents); - if (result != .hit) return missInput(m, i + 1, input_contents_len, manifest_size, result); + const result = try checkInputPath(m, diag, file_off, input_path, disk_contents, input_contents); + if (result != .hit) return missInput(m, diag, i + 1, input_contents_len, manifest_size, result); off = @backingInt(file_off); } @@ -1074,7 +1103,7 @@ pub const Manifest = struct { if (path.len == 0) return .invalid_manifest; try m.files.putContext(gpa, file_off, {}, .{ .contents = disk_contents }); - const result = try checkDiscoveredPath(m, file_off, disk_contents); + const result = try checkDiscoveredPath(m, diag, file_off, disk_contents); if (result != .hit) return result; off += File.sizeOf(path.len); @@ -1098,6 +1127,7 @@ pub const Manifest = struct { /// missed one. fn missInput( m: *Manifest, + diag: *CheckDiagnostic, next_file_index: usize, input_contents_len: usize, manifest_size: usize, @@ -1114,18 +1144,19 @@ pub const Manifest = struct { m.contents.items[manifest_size + off ..][0..copy_len], ); m.contents.shrinkRetainingCapacity(input_contents_len); - return populateMissingInputFileHashes(m, next_file_index, result); + return populateMissingInputFileHashes(m, diag, next_file_index, result); } fn populateMissingInputFileHashes( m: *Manifest, + diag: *CheckDiagnostic, next_file_index: usize, result: CheckResult, ) CheckError!CheckResult { const file_offs = m.files.keys(); const contents = m.contents.items; for (file_offs[next_file_index..], m.input_paths.items[next_file_index..]) |input_file_off, *input_path| { - try populateInputPath(m, input_file_off, input_path, contents); + try populateInputPath(m, diag, input_file_off, input_path, contents); } return result; } @@ -1134,6 +1165,7 @@ pub const Manifest = struct { /// system. Only ensures that metadata and digest are available. fn populateInputPath( m: *Manifest, + diag: *CheckDiagnostic, file_off: File.Offset, input_path: *InputPath, contents: []u8, @@ -1156,21 +1188,21 @@ pub const Manifest = struct { const actual_stat = switch (input_path.getHandle(input_file)) { .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), }, .file => |opened_file| opened_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), }, .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1179,7 +1211,7 @@ pub const Manifest = struct { const actual_is_directory = actual_stat.kind == .directory; if (actual_is_directory != input_file.flags.is_directory) { // Since this is an input file, this is a failure, not a cache miss. - return fail(&m.diagnostic, .{ .file_stat = .{ + return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = if (actual_is_directory) error.IsDir else error.NotDir, } }); @@ -1210,7 +1242,7 @@ pub const Manifest = struct { .access_sub_paths = false, }) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + else => |e| return fail(diag, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1224,7 +1256,7 @@ pub const Manifest = struct { assert(!stat_path_ok); const actual_stat = opened_dir.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1235,7 +1267,7 @@ pub const Manifest = struct { const dir_contents_start = m.all_input_content.items.len; hashDir(gpa, io, opened_dir, &input_file.digest, &m.all_input_content) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1254,7 +1286,7 @@ pub const Manifest = struct { else parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + else => |e| return fail(diag, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1268,7 +1300,7 @@ pub const Manifest = struct { assert(!stat_path_ok); const actual_stat = opened_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1281,7 +1313,7 @@ pub const Manifest = struct { const start = m.all_input_content.items.len; hashFileAppend(io, opened_file, &input_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1293,7 +1325,7 @@ pub const Manifest = struct { }, .unrequested => hashFile(io, opened_file, &input_file.digest) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1306,6 +1338,7 @@ pub const Manifest = struct { /// Upon return, disk_contents digest and stat will be populated regardless of whether hit or miss occurs. fn checkInputPath( m: *Manifest, + diag: *CheckDiagnostic, file_off: File.Offset, input_path: *InputPath, disk_contents: []u8, @@ -1337,21 +1370,21 @@ pub const Manifest = struct { const actual_stat = switch (input_path.getHandle(input_file)) { .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), }, .file => |opened_file| opened_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), }, .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1362,7 +1395,7 @@ pub const Manifest = struct { const actual_is_directory = actual_stat.kind == .directory; if (actual_is_directory != disk_file.flags.is_directory) { // Since this is an input file, this is a failure, not a cache miss. - return fail(&m.diagnostic, .{ .file_stat = .{ + return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = if (actual_is_directory) error.IsDir else error.NotDir, } }); @@ -1393,7 +1426,7 @@ pub const Manifest = struct { .access_sub_paths = false, }) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + else => |e| return fail(diag, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1406,7 +1439,7 @@ pub const Manifest = struct { if (!input_path.have_stat) { const actual_stat = opened_dir.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1417,7 +1450,7 @@ pub const Manifest = struct { const dir_contents_start = m.all_input_content.items.len; hashDir(gpa, io, opened_dir, &disk_file.digest, &m.all_input_content) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1439,7 +1472,7 @@ pub const Manifest = struct { else parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + else => |e| return fail(diag, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1452,7 +1485,7 @@ pub const Manifest = struct { if (!input_path.have_stat) { const actual_stat = opened_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1465,7 +1498,7 @@ pub const Manifest = struct { const start = m.all_input_content.items.len; hashFileAppend(io, opened_file, &disk_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1477,7 +1510,7 @@ pub const Manifest = struct { }, .unrequested => hashFile(io, opened_file, &disk_file.digest) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1490,7 +1523,12 @@ pub const Manifest = struct { } } - fn checkDiscoveredPath(m: *Manifest, file_off: File.Offset, contents: []u8) CheckError!CheckResult { + fn checkDiscoveredPath( + m: *Manifest, + diag: *CheckDiagnostic, + file_off: File.Offset, + contents: []u8, + ) CheckError!CheckResult { const file = file_off.get(contents); const cache = m.cache; const gpa = cache.gpa; @@ -1502,7 +1540,7 @@ pub const Manifest = struct { const actual_stat = parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { error.FileNotFound => return .{ .path_deleted = file_off }, error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1525,7 +1563,7 @@ pub const Manifest = struct { error.FileNotFound => return .{ .path_deleted = file_off }, error.NotDir => return .{ .directory_status_changed = file_off }, error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + else => |e| return fail(diag, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1534,7 +1572,7 @@ pub const Manifest = struct { const actual_stat = opened_dir.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1545,7 +1583,7 @@ pub const Manifest = struct { defer m.all_input_content.shrinkRetainingCapacity(dir_contents_start); hashDir(gpa, io, opened_dir, &file.digest, &m.all_input_content) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1560,7 +1598,7 @@ pub const Manifest = struct { error.FileNotFound => return .{ .path_deleted = file_off }, error.IsDir => return .{ .directory_status_changed = file_off }, error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_open = .{ + else => |e| return fail(diag, .{ .file_open = .{ .file_offset = file_off, .err = e, } }), @@ -1569,7 +1607,7 @@ pub const Manifest = struct { const actual_stat = opened_file.stat(io) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_stat = .{ + else => |e| return fail(diag, .{ .file_stat = .{ .file_offset = file_off, .err = e, } }), @@ -1579,7 +1617,7 @@ pub const Manifest = struct { const prev_digest: BinDigest = file.digest; hashFile(io, opened_file, &file.digest) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| return fail(&m.diagnostic, .{ .file_read = .{ + else => |e| return fail(diag, .{ .file_read = .{ .file_offset = file_off, .err = e, } }), @@ -1591,8 +1629,7 @@ pub const Manifest = struct { return .hit; } - fn fail(ptr: *Diagnostic, d: Diagnostic) error{CacheCheckFailed} { - assert(ptr.* == .none); + fn fail(ptr: *CheckDiagnostic, d: CheckDiagnostic) error{CacheCheckFailed} { ptr.* = d; return error.CacheCheckFailed; } @@ -2002,22 +2039,19 @@ pub const Manifest = struct { self.have_exclusive_lock = false; } - fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool { - if (self.have_exclusive_lock) return false; - assert(self.manifest_file != null); - const io = self.cache.io; + fn upgradeToExclusiveLock(m: *Manifest, diag: *CheckDiagnostic) error{CacheCheckFailed}!bool { + if (m.have_exclusive_lock) return false; + assert(m.manifest_file != null); + const io = m.cache.io; if (std.process.can_spawn or !builtin.single_threaded) { - const manifest_file = self.manifest_file.?; + const manifest_file = m.manifest_file.?; // Here we intentionally have a period where the lock is released, in case there are // other processes holding a shared lock. manifest_file.unlock(io); - manifest_file.lock(io, .exclusive) catch |err| { - self.diagnostic = .{ .manifest_lock = err }; - return error.CacheCheckFailed; - }; + manifest_file.lock(io, .exclusive) catch |err| return fail(diag, .{ .manifest_lock = err }); } - self.have_exclusive_lock = true; + m.have_exclusive_lock = true; return true; } @@ -2327,7 +2361,8 @@ test "cache file and then recall it" { .sub_path = temp_file, }, .{}); - try testing.expectEqual(.incomplete_manifest, try man.check(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); digest1 = man.missDigestHex(); try man.finalize(); @@ -2345,7 +2380,8 @@ test "cache file and then recall it" { }, .{}); // Cache hit! We just "built" the same file - try testing.expectEqual(.hit, try man.check(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.hit, try man.check(&diag, .none)); digest2 = man.hitDigestHex(); try testing.expectEqual(false, man.have_exclusive_lock); @@ -2406,7 +2442,8 @@ test "check that changing a file causes cache miss" { .sub_path = temp_file, }, .{ .request_contents = true }); - try testing.expectEqual(.incomplete_manifest, try man.check(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); @@ -2428,9 +2465,10 @@ test "check that changing a file causes cache miss" { }, .{ .request_contents = true }); // The one input file changed. + var diag: Manifest.CheckDiagnostic = undefined; try testing.expectEqual( @as(Manifest.CheckResult, .{ .contents_changed = temp_file_idx.offset(&man) }), - try man.check(.none), + try man.check(&diag, .none), ); try testing.expectEqualStrings(updated_temp_file_contents, temp_file_idx.contents(&man)); @@ -2473,7 +2511,8 @@ test "no file inputs" { man.hash.addBytes("1234"); - try testing.expectEqual(.incomplete_manifest, try man.check(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); digest1 = man.missDigestHex(); @@ -2485,7 +2524,8 @@ test "no file inputs" { man.hash.addBytes("1234"); - try testing.expectEqual(.hit, try man.check(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.hit, try man.check(&diag, .none)); digest2 = man.hitDigestHex(); try testing.expectEqual(false, man.have_exclusive_lock); } @@ -2545,7 +2585,8 @@ test "Manifest with files added after initial hash" { .sub_path = temp_file1, }, .{}); - try testing.expectEqual(.incomplete_manifest, try man.check(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .{ .root_dir = tmp_directory, @@ -2565,7 +2606,8 @@ test "Manifest with files added after initial hash" { .sub_path = temp_file1, }, .{}); - try testing.expect(.hit == try man.check(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expect(.hit == try man.check(&diag, .none)); digest2 = man.hitDigestHex(); try testing.expectEqual(false, man.have_exclusive_lock); @@ -2591,7 +2633,8 @@ test "Manifest with files added after initial hash" { .sub_path = temp_file1, }, .{}); - switch (try man.check(.none)) { + var diag: Manifest.CheckDiagnostic = undefined; + switch (try man.check(&diag, .none)) { .contents_changed => |off| { try testing.expectEqualStrings(temp_file2, off.path(man.contents.items)); }, diff --git a/src/Compilation.zig b/src/Compilation.zig index 68c76dfd83bce307a45af5d0ed0c35f126316c64..5b3131bf8734ffec53c0bfed5fc04d119716b313 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2813,27 +2813,14 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE man.want_shared_lock = false; } - const status = man.check(main_progress_node) catch |err| switch (err) { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, main_progress_node) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_oversize => { - return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {t}", .{ - man.diagnostic, - }); - }, - .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { - return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {t} {t}", .{ - man.diagnostic, e, - }); - }, - .file_open, .file_stat, .file_read, .file_hash => |op| { - return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {f} {t} {t}", .{ - op.path(&man), man.diagnostic, op.err, - }); - }, + error.CacheCheckFailed => { + return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {f}", .{diag.fmt(&man)}); }, }; + log.debug("{s} whole cache {f} ignore_hit={}", .{ comp.root_name, status.fmt(&man), ignore_hit }); if (status == .hit and !ignore_hit) { // In this case the cache hit contains the full set of file system inputs. Nice! if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); @@ -2844,7 +2831,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } comp.last_update_was_cache_hit = true; - log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name}); const bin_digest = man.hitDigest(); comp.digest = bin_digest; @@ -2853,7 +2839,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE whole.lock = man.toOwnedLock(); return; } - log.debug("CacheMode.whole cache miss for {s}", .{comp.root_name}); if (ignore_hit) { // Okay, now set this back so that `Manifest.finalize` will downgrade our lock later. @@ -5536,7 +5521,18 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr const target = comp.getTarget(); assert(target.ofmt != .c); const o_ext = target.ofmt.fileExt(target.cpu.arch); - const digest = if (!comp.disable_c_depfile and .hit == try man.check(child_progress_node)) man.hitDigestHex() else miss: { + const digest = d: { + if (!comp.disable_c_depfile) { + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, child_progress_node) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + error.CacheCheckFailed => { + return comp.failCObj(c_object, "checking cache failed: {f}", .{diagnostic.fmt(&man)}); + }, + }; + log.debug("C object cache {f}", .{status.fmt(&man)}); + if (status == .hit) break :d man.hitDigestHex(); + } var argv: std.array_list.Managed([]const u8) = .init(gpa); defer argv.deinit(); @@ -5811,16 +5807,28 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr } // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. - if (comp.disable_c_depfile) _ = try man.check(child_progress_node); + const digest = inner: { + if (comp.disable_c_depfile) { + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, child_progress_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + return comp.failCObj(c_object, "checking cache failed: {f}", .{diagnostic.fmt(&man)}); + }, + }; + log.debug("ignored C object cache {f}", .{status.fmt(&man)}); + if (status == .hit) break :inner man.hitDigestHex(); + } + break :inner man.missDigestHex(); + }; // Rename into place. - const digest = man.missDigestHex(); const o_sub_path = try fs.path.join(arena, &.{ "o", &digest }); var o_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, o_sub_path, .{}); defer o_dir.close(io); const tmp_basename = fs.path.basename(out_obj_path); try Io.Dir.rename(zig_cache_tmp_dir, tmp_basename, o_dir, o_basename, io); - break :miss digest; + break :d digest; }; if (man.have_exclusive_lock) { @@ -5896,7 +5904,15 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); - const digest = if (.hit == try man.check(child_progress_node)) man.hitDigestHex() else blk: { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, child_progress_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => return comp.failWin32Resource(win32_resource, "checking cache failed: {f}", .{ + diag.fmt(&man), + }), + }; + log.debug("win32 resource cache {f}", .{status.fmt(&man)}); + const digest = if (status == .hit) man.hitDigestHex() else miss: { // The digest only depends on the .manifest file, so we can // get the digest now and write the .res directly to the cache const digest = man.missDigestHex(); @@ -5961,7 +5977,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 try spawnZigRc(comp, win32_resource, arena, argv.items, child_progress_node); - break :blk digest; + break :miss digest; }; if (man.have_exclusive_lock) { @@ -5988,7 +6004,15 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename_noext = src_basename[0 .. src_basename.len - fs.path.extension(src_basename).len]; - const digest = if (.hit == try man.check(child_progress_node)) man.hitDigestHex() else blk: { + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, child_progress_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => return comp.failWin32Resource(win32_resource, "checking cache failed: {f}", .{ + diagnostic.fmt(&man), + }), + }; + log.debug("win32 resource cache {f}", .{status.fmt(&man)}); + const digest = if (status == .hit) man.hitDigestHex() else miss: { var zig_cache_tmp_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, "tmp", .{}); defer zig_cache_tmp_dir.close(io); @@ -6066,7 +6090,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 defer o_dir.close(io); const tmp_basename = fs.path.basename(out_res_path); try Io.Dir.rename(zig_cache_tmp_dir, tmp_basename, o_dir, res_filename, io); - break :blk digest; + break :miss digest; }; if (man.have_exclusive_lock) { diff --git a/src/Sema.zig b/src/Sema.zig index 1090ecd135c01218c2020d740b8eada0bc5fef26..a3533e99233d4e8f8e0d61184d65733906d1ad86 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -12974,7 +12974,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A const result = ef_idx.get(zcu); if (result.val == .none) { - return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(result.err.?) }); + return sema.fail(block, operand_src, "failed opening {q}: {t}", .{ name, result.err.? }); } return Air.internedToRef(result.val); @@ -15911,9 +15911,7 @@ fn zirClosureGet(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstDat const file, const src_base_node = Zcu.LazySrcLoc.resolveBaseNode(block.src_base_inst, zcu).?; const tree = file.getTree(zcu) catch |err| { // In this case we emit a warning + a less precise source location. - log.warn("unable to load {f}: {s}", .{ - file.path.fmt(zcu.comp), @errorName(err), - }); + log.warn("failed loading {qf}: {t}", .{ file.path.fmt(zcu.comp), err }); break :name null; }; const node = src_node.toAbsolute(src_base_node); @@ -15939,9 +15937,7 @@ fn zirClosureGet(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstDat const file, const src_base_node = Zcu.LazySrcLoc.resolveBaseNode(block.src_base_inst, zcu).?; const tree = file.getTree(zcu) catch |err| { // In this case we emit a warning + a less precise source location. - log.warn("unable to load {f}: {s}", .{ - file.path.fmt(zcu.comp), @errorName(err), - }); + log.warn("failed loading {qf}: {t}", .{ file.path.fmt(zcu.comp), err }); break :name null; }; const node = src_node.toAbsolute(src_base_node); @@ -15950,7 +15946,7 @@ fn zirClosureGet(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstDat }; const msg = if (name) |some| - try sema.errMsg(src, "'{s}' not accessible from inner function", .{some}) + try sema.errMsg(src, "{q} not accessible from inner function", .{some}) else try sema.errMsg(src, "variable not accessible from inner function", .{}); errdefer msg.destroy(sema.gpa); diff --git a/src/libs/freebsd.zig b/src/libs/freebsd.zig index 36062854757ac0b1369c0712031f80e806f453f4..584a6b63b69adae9e3427334567bd41357056543 100644 --- a/src/libs/freebsd.zig +++ b/src/libs/freebsd.zig @@ -465,7 +465,20 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (.hit == try man.check(prog_node)) { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .freebsd_shared_objects, + "compiling FreeBSD libc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + }; + log.debug("freebsd_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/glibc.zig b/src/libs/glibc.zig index 691da1bd0f047f527c37c4b850cdceea7dd036a8..86c5360cc17cce248afeac8ddcfb5bb4b2f9ceca 100644 --- a/src/libs/glibc.zig +++ b/src/libs/glibc.zig @@ -705,7 +705,20 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (.hit == try man.check(prog_node)) { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .glibc_shared_objects, + "compiling glibc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + }; + log.debug("glibc_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index 59d99a783af2af8053dcfb629ce1c48ee672f262..61dbd5dfb064aa52631efd77837b23013a8214dc 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -251,26 +251,17 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P const final_lib_basename = try std.fmt.allocPrint(gpa, "{s}.lib", .{lib_name}); errdefer gpa.free(final_lib_basename); - const status = man.check(prog_node) catch |err| switch (err) { - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_oversize => { - comp.setMiscFailure(.windows_import_lib, "checking cache failed: {t}", .{man.diagnostic}); - return error.AlreadyReported; - }, - .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { - comp.setMiscFailure(.windows_import_lib, "checking cache failed: {t} {t}", .{ man.diagnostic, e }); - return error.AlreadyReported; - }, - .file_open, .file_stat, .file_read, .file_hash => |op| { - comp.setMiscFailure(.windows_import_lib, "checking cache failed: {f} {t} {t}", .{ - op.path(&man), man.diagnostic, op.err, - }); - return error.AlreadyReported; - }, + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.CacheCheckFailed => { + comp.setMiscFailure(.windows_import_lib, "{s} cache check failed: {f}", .{ + final_lib_basename, diag.fmt(&man), + }); + return error.AlreadyReported; }, error.OutOfMemory, error.Canceled => |e| return e, }; + log.debug("{s} cache {f}", .{ final_lib_basename, status.fmt(&man) }); if (status == .hit) { const digest = man.hitDigestHex(); const sub_path = try std.fs.path.join(gpa, &.{ "o", &digest, final_lib_basename }); diff --git a/src/libs/netbsd.zig b/src/libs/netbsd.zig index 5445f5fcaab4906ada5cc8950ef403fb4fc825f6..46a34b96a55943f84a92914521a730c27d0d19a7 100644 --- a/src/libs/netbsd.zig +++ b/src/libs/netbsd.zig @@ -413,7 +413,20 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (.hit == try man.check(prog_node)) { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .netbsd_shared_objects, + "compiling NetBSD libc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + error.OutOfMemory, error.Canceled => |e| return e, + }; + log.debug("netbsd_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/openbsd.zig b/src/libs/openbsd.zig index ab126f2a96aa9dce467620052d07a5af8f965e49..4bea660a65eed944eb93d48f27c5cc70a0369a8d 100644 --- a/src/libs/openbsd.zig +++ b/src/libs/openbsd.zig @@ -334,7 +334,20 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye .request_contents = true, }); - if (.hit == try man.check(prog_node)) { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .openbsd_shared_objects, + "compiling OpenBSD libc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + }; + log.debug("openbsd_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/main.zig b/src/main.zig index 6bcb22f26e15180f6526022a4da12ea7393cbed8..a4299f4f896ba5f1118a1e8d81dfa5a9522cff1c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4878,7 +4878,13 @@ fn cmdTranslateC( Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| fatal("unable to process {q}: {t}", .{ c_source_file.src_path, err }); - const result: Compilation.TranslateCResult = if (.hit == try man.check(prog_node)) .{ + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => fatal("translate-c checking cache failed: {f}", .{diag.fmt(&man)}), + }; + std.log.debug("translate-c cache {f}", .{status.fmt(&man)}); + const result: Compilation.TranslateCResult = if (status == .hit) .{ .digest = man.hitDigest(), .cache_hit = true, .errors = std.zig.ErrorBundle.empty, diff --git a/test/incremental/bad_import b/test/incremental/bad_import index ab6369bee38a0d8c0c3e01b42458cc52051e4f5f..fd91fa66050e2ae3cb1407382d5f173bdb2fe884 100644 --- a/test/incremental/bad_import +++ b/test/incremental/bad_import @@ -10,7 +10,7 @@ const io = std.Io.Threaded.global_single_threaded.io(); comptime { _ = @import("bad.zig"); } -#expect_error=bad.zig:1:1: error: unable to load 'bad.zig': FileNotFound +#expect_error=bad.zig:1:1: error: unable to load "bad.zig": FileNotFound #expect_error=foo.zig:2:17: note: file imported here #update=change bad import @@ -18,7 +18,7 @@ comptime { comptime { _ = @import("this_is/not_real.zig"); } -#expect_error=this_is/not_real.zig:1:1: error: unable to load 'not_real.zig': FileNotFound +#expect_error=this_is/not_real.zig:1:1: error: unable to load "not_real.zig": FileNotFound #expect_error=foo.zig:2:17: note: file imported here #update=remove import of 'foo.zig' diff --git a/test/incremental/change_embed_file b/test/incremental/change_embed_file index 34efcdcc5a4fb112e35c862173e85afd96a15ddd..8b515d2d980c9ee68da186c8fa149b5b19c2ed20 100644 --- a/test/incremental/change_embed_file +++ b/test/incremental/change_embed_file @@ -17,7 +17,7 @@ Hello again, World! #update=delete file #rm_file=string.txt -#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound +#expect_error=main.zig:2:27: error: failed opening "string.txt": FileNotFound #update=remove reference to file #file=main.zig @@ -37,7 +37,7 @@ pub fn main() !void { try std.Io.File.stdout().writeStreamingAll(io, string); } const io = std.Io.Threaded.global_single_threaded.io(); -#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound +#expect_error=main.zig:2:27: error: failed opening "string.txt": FileNotFound #update=recreate file #file=string.txt diff --git a/test/incremental/change_zon_file b/test/incremental/change_zon_file index 080dc235199d21b5dded5136bd6baf04b72142a0..7616c53ea80094d746102df13d51e387a3d1bc40 100644 --- a/test/incremental/change_zon_file +++ b/test/incremental/change_zon_file @@ -17,7 +17,7 @@ const io = std.Io.Threaded.global_single_threaded.io(); #update=delete file #rm_file=message.zon -#expect_error=message.zon:1:1: error: unable to load 'message.zon': FileNotFound +#expect_error=message.zon:1:1: error: unable to load "message.zon": FileNotFound #expect_error=main.zig:2:37: note: file imported here #update=remove reference to ZON file @@ -28,7 +28,7 @@ pub fn main() !void { try std.Io.File.stdout().writeStreamingAll(io, "a hardcoded string\n"); } const io = std.Io.Threaded.global_single_threaded.io(); -#expect_error=message.zon:1:1: error: unable to load 'message.zon': FileNotFound +#expect_error=message.zon:1:1: error: unable to load "message.zon": FileNotFound #expect_error=main.zig:2:37: note: file imported here #update=recreate ZON file -- 2.54.0 From 4cfd2563309eca3e79c2f2aac104c963925d3166 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2026 22:23:39 -0700 Subject: [PATCH 43/49] build.zig: metadata dependency on .git/logs/HEAD --- build.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.zig b/build.zig index 8c18ea4841bcd00e19239273014f3e11699781f8..6b94dd1d195da678e7ef1335d68464e8fd9dcadc 100644 --- a/build.zig +++ b/build.zig @@ -272,12 +272,13 @@ pub fn build(b: *std.Build) !void { const io = b.graph.io; const git_file = b.root.openFile(io, ".git", .{ .allow_directory = false }) catch |err| switch (err) { error.IsDir => { - b.dependOnFileContents(b.path(".git/logs/HEAD")); + b.dependOnFileMetadata(b.path(".git/logs/HEAD")); break :git; }, else => |e| return e, }; defer git_file.close(io); + b.dependOnFileContents(b.path(".git")); var line_buffer: ["gitdir: ".len + std.Io.Dir.max_path_bytes + 1]u8 = undefined; var git_file_reader = git_file.reader(io, &line_buffer); if (std.mem.cutPrefix(u8, std.mem.trimEnd(u8, try git_file_reader.interface.allocRemaining( @@ -285,7 +286,7 @@ pub fn build(b: *std.Build) !void { .limited("gitdir: ".len + std.Io.Dir.max_path_bytes + "\r\n".len), ), "\r\n"), "gitdir: ")) |git_dir| { const head_file = b.pathJoin(&.{ git_dir, "logs", "HEAD" }); - b.dependOnFileContents(if (std.Io.Dir.path.isAbsolute(head_file)) + b.dependOnFileMetadata(if (std.Io.Dir.path.isAbsolute(head_file)) b.graph.cwdRelativePath(head_file) else b.path(head_file)); -- 2.54.0 From f04384393e1a0fd4056b258d779625b37cb57cd8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2026 22:24:20 -0700 Subject: [PATCH 44/49] CLI: add a debug log for cwd path --- src/main.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.zig b/src/main.zig index a4299f4f896ba5f1118a1e8d81dfa5a9522cff1c..41b4e841d36f4bfe598cb23fd300d6ffe248b0f8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3270,6 +3270,7 @@ fn buildOutputType( }; const cwd_path = try std.zig.getResolvedCwd(io, arena); + std.log.debug("cwd_path={s}", .{cwd_path}); // This `init` calls `fatal` on error. var dirs: std.zig.Directories = .init(arena, io, .{ -- 2.54.0 From 117fc710eed02e82682f80e1b77fe822e6037e27 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2026 22:24:47 -0700 Subject: [PATCH 45/49] incr-check: log the subprocess argv and cwd --- tools/incr-check.zig | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 21c24e9048203c80a5562f6c3b47fce31cb2e80d..638dfb853e63fd48fe7e033a3eb2711c4f4dafb7 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -163,7 +163,6 @@ pub fn main(init: std.process.Init) !void { "--global-cache-dir", ".global-cache", }); - try child_args.append(arena, "--listen=-"); if (opt_resolved_lib_dir) |resolved_lib_dir| { try child_args.appendSlice(arena, &.{ "--zig-lib-dir", resolved_lib_dir }); @@ -187,6 +186,7 @@ pub fn main(init: std.process.Init) !void { for (case.modules) |mod| { try child_args.append(arena, try std.fmt.allocPrint(arena, "-M{s}={s}", .{ mod.name, mod.file })); } + try child_args.append(arena, "--listen=-"); const zig_prog_node = prog_node.start("zig", 0); defer zig_prog_node.end(); @@ -210,6 +210,16 @@ pub fn main(init: std.process.Init) !void { try cc_child_args.append(arena, "-o"); } + const allow_compiler_stderr = debug_log_args.items.len != 0; + + if (allow_compiler_stderr) { + const cmd: std.zig.SubprocessCommand = .{ + .argv = child_args.items, + .cwd = tmp_dir_path, + }; + std.log.scoped(.spawn).info("{f}", .{cmd}); + } + var child = try std.process.spawn(io, .{ .argv = child_args.items, .stdin = .pipe, @@ -233,7 +243,7 @@ pub fn main(init: std.process.Init) !void { .tmp_dir = tmp_dir, .tmp_dir_path = tmp_dir_path, .child = &child, - .allow_compiler_stderr = debug_log_args.items.len != 0, + .allow_compiler_stderr = allow_compiler_stderr, .quiet = quiet, .preserve_tmp_on_fatal = preserve_tmp, .cc_child_args = &cc_child_args, -- 2.54.0 From 1cc2f45fce41ea20e5c14b0d45ee256b4fe01b52 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2026 02:30:21 -0700 Subject: [PATCH 46/49] compiler: lock when setting libs/mingw error --- src/libs/mingw.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index 61dbd5dfb064aa52631efd77837b23013a8214dc..bfab89ca4ab86ec81d4366bca85fd72bf5eef7a6 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -254,7 +254,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P var diag: Cache.Manifest.CheckDiagnostic = undefined; const status = man.check(&diag, prog_node) catch |err| switch (err) { error.CacheCheckFailed => { - comp.setMiscFailure(.windows_import_lib, "{s} cache check failed: {f}", .{ + comp.lockAndSetMiscFailure(.windows_import_lib, "{s} cache check failed: {f}", .{ final_lib_basename, diag.fmt(&man), }); return error.AlreadyReported; -- 2.54.0 From 31ffe4be432ba9db5a7afc58e453fe96a8b9225d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2026 02:30:33 -0700 Subject: [PATCH 47/49] test-cases: update some expected error messages --- .../accessing_runtime_parameter_from_outer_function.zig | 2 +- test/cases/compile_errors/embedFile_with_bogus_file.zig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig b/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig index 37bc8f5388edb4c18bc72e3cd9084797742f7b2f..b86775c86925e40959aa4bbe3969e6c2d2e06dc5 100644 --- a/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig +++ b/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig @@ -14,5 +14,5 @@ export fn entry() void { // error // -// :4:24: error: 'y' not accessible from inner function +// :4:24: error: "y" not accessible from inner function // :3:9: note: crossed function definition here diff --git a/test/cases/compile_errors/embedFile_with_bogus_file.zig b/test/cases/compile_errors/embedFile_with_bogus_file.zig index c769b4b2c6f48b9248ad3eee33022e8c8b6546cb..d27847189ae141a894c81cd52f29a92161e57e75 100644 --- a/test/cases/compile_errors/embedFile_with_bogus_file.zig +++ b/test/cases/compile_errors/embedFile_with_bogus_file.zig @@ -6,4 +6,4 @@ export fn entry() usize { // error // -// :1:29: error: unable to open 'bogus.txt': FileNotFound +// :1:29: error: failed opening "bogus.txt": FileNotFound -- 2.54.0 From 5549d9bbe3d091751394b78c205e357891013cdb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2026 14:52:32 -0700 Subject: [PATCH 48/49] incr-check: pass local and global cache overrides also to C compilation and simplify by using `zig build-exe` rather than `zig cc` --- tools/incr-check.zig | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 638dfb853e63fd48fe7e033a3eb2711c4f4dafb7..c8034260dbc2af7a256151c7c43f4429531e390b 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -152,16 +152,11 @@ pub fn main(init: std.process.Init) !void { var child_args: std.ArrayList([]const u8) = .empty; try child_args.appendSlice(arena, &.{ - resolved_zig_exe, - "build-exe", - "-fincremental", - "-fno-ubsan-rt", - "-target", - try target_query.zigTriple(arena), - "--cache-dir", - ".local-cache", - "--global-cache-dir", - ".global-cache", + resolved_zig_exe, "build-exe", + "-fincremental", "-fno-ubsan-rt", + "-target", try target_query.zigTriple(arena), + "--cache-dir", ".local-cache", + "--global-cache-dir", ".global-cache", }); if (opt_resolved_lib_dir) |resolved_lib_dir| { @@ -199,15 +194,13 @@ pub fn main(init: std.process.Init) !void { resolved_zig_exe; try cc_child_args.appendSlice(arena, &.{ - resolved_cc_zig_exe, - "cc", - "-target", - try target_query.zigTriple(arena), - "-I", - opt_resolved_lib_dir.?, // verified earlier + resolved_cc_zig_exe, "build-exe", + "-target", try target_query.zigTriple(arena), + "--cache-dir", ".local-cache", + "--global-cache-dir", ".global-cache", + "-I", opt_resolved_lib_dir.?, // verified earlier + "-lc", }); - - try cc_child_args.append(arena, "-o"); } const allow_compiler_stderr = debug_log_args.items.len != 0; @@ -669,9 +662,18 @@ const Eval = struct { const child_prog_node = prog_node.start("build cbe output", 0); defer child_prog_node.end(); - try eval.cc_child_args.appendSlice(eval.arena, &.{ out_path, c_path }); + try eval.cc_child_args.append(eval.arena, try std.fmt.allocPrint(eval.arena, "-femit-bin={s}", .{out_path})); + try eval.cc_child_args.append(eval.arena, c_path); defer eval.cc_child_args.items.len -= 2; + if (eval.allow_compiler_stderr) { + const cmd: std.zig.SubprocessCommand = .{ + .argv = eval.cc_child_args.items, + .cwd = eval.tmp_dir_path, + }; + std.log.scoped(.spawn).info("{f}", .{cmd}); + } + const result = std.process.run(eval.arena, eval.io, .{ .argv = eval.cc_child_args.items, .cwd = .{ .path = eval.tmp_dir_path }, -- 2.54.0 From 56571a70d99a896777146e9588d9f6a04124c9a5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2026 16:05:26 -0700 Subject: [PATCH 49/49] std.Build.Cache: fix flagsAndPath implementation it was scanning for sentinel starting from flags rather than flags + 1, which could lead to the scenario where the path is empty because flags is zero --- lib/std/Build/Cache.zig | 14 ++++++-------- src/Compilation.zig | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 3eaeef3ab92ab88f11c4da1418aa84fb51c960ef..d0a7d93e16b1207429222ee77a96361a345290e4 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -499,9 +499,10 @@ pub const Manifest = struct { pub fn flagsAndPath(off: File.Offset, contents: []const u8) [:0]const u8 { const flags_off = @offsetOf(File, "flags"); comptime assert(@offsetOf(File, "path_start") - flags_off == 1); - const hash_start = @backingInt(off) + flags_off; - const hash_end = mem.findScalarPos(u8, contents, hash_start, 0).?; - return contents[hash_start..hash_end :0]; + const start = @backingInt(off) + flags_off; + // Scan for the sentinel starting from the path_start offset because flags might be zero. + const end = mem.findScalarPos(u8, contents, start + 1, 0).?; + return contents[start..end :0]; } pub fn pathFallible(off: File.Offset, contents: []const u8) error{InvalidFormat}![:0]const u8 { @@ -898,7 +899,8 @@ pub const Manifest = struct { // This is *not* hashing the contents of the input files. It is the // flags (including prefix) and path only. for (man.files.keys()[0..man.input_paths.items.len]) |file_off| { - man.hashFlagsAndPath(file_off, &man.hash.hasher); + const flags_and_path = file_off.flagsAndPath(man.contents.items); + man.hash.hasher.update(mem.absorbSentinel(flags_and_path)); } var input_digest: BinDigest = undefined; @@ -2288,10 +2290,6 @@ pub const Manifest = struct { hasher.update(contents.items[contents_start..][0..contents_len]); hasher.final(bin_digest); } - - fn hashFlagsAndPath(m: *const Manifest, off: File.Offset, hasher: *Hasher) void { - hasher.update(off.flagsAndPath(m.contents.items)); - } }; /// Create/Write a file, close it, then grab its stat.mtime timestamp. diff --git a/src/Compilation.zig b/src/Compilation.zig index 5b3131bf8734ffec53c0bfed5fc04d119716b313..d94b7146b59afed2774f83d89b63604794a1ddbd 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -5530,7 +5530,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr return comp.failCObj(c_object, "checking cache failed: {f}", .{diagnostic.fmt(&man)}); }, }; - log.debug("C object cache {f}", .{status.fmt(&man)}); + log.debug("C object {q} cache {f}", .{ c_object.src.src_path, status.fmt(&man) }); if (status == .hit) break :d man.hitDigestHex(); } var argv: std.array_list.Managed([]const u8) = .init(gpa); -- 2.54.0