From 22c141605b532eb7d62bfecb020868e580476b93 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 19 Oct 2026 14:58:10 -0700 Subject: [PATCH] WIP: combine file_system_inputs and Cache.Manifest --- lib/compiler/Maker.zig | 1 + lib/compiler/Maker/Step.zig | 89 +++++--------- lib/compiler/Maker/WebServer.zig | 24 +++- lib/std/Build/Cache.zig | 126 +++++++++----------- lib/std/zig.zig | 64 +++++----- lib/std/zig/Server.zig | 66 ++++++----- src/Compilation.zig | 196 +++++++------------------------ src/Zcu/PerThread.zig | 4 - src/main.zig | 75 +++--------- 9 files changed, 238 insertions(+), 407 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 66ba521c040d736e679af5bf58773bfcecfc55d2..63c995d934bc730e4fc57ee480b9a82360025df4 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1573,6 +1573,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{ .argv = build_configurer_argv.items, .cache_root = graph.local_cache_root, + .cache = &graph.cache, .root_name = configurer_exe_name, .environ_map = &graph.environ_map, .cache_manifest = config_man, diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 791d5b402cb21a526a46bca54eeadcc8b79c7e9d..75e0931cc06ca2cf5934f86283791193870f55d7 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -659,68 +659,33 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi // This message indicates the end of the update. if (watch) break; }, - .emit_digest => { - const emit_digest = body_r.takeStruct(std.zig.Server.Message.EmitDigest, .little) catch unreachable; - const digest = body_r.takeArray(Cache.bin_digest_len) catch unreachable; - s.result_cache_status = if (emit_digest.flags.cache_hit) .hit else .incomplete_manifest; - result = .{ .bin = digest.* }; - }, - .file_system_inputs => { - clearWatchInputs(s, maker); - const conf = &maker.scanned_config.configuration; - const conf_step = step_index.ptr(conf); - var it = std.mem.splitScalar(u8, body, 0); - while (it.next()) |prefixed_path| { - const prefix_index: std.zig.Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1)); - const sub_path = try arena.dupe(u8, prefixed_path[1..]); - const sub_path_dirname = Dir.path.dirname(sub_path) orelse ""; - switch (prefix_index) { - .cwd => { - const path: Path = .{ - .root_dir = .cwd(), - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); - }, - .zig_lib => zl: { - switch (conf_step.extended.get(conf.extra)) { - .compile => |compile| if (compile.zig_lib_dir.value) |zig_lib_dir| { - const resolved = try maker.resolveLazyPathIndex(arena, zig_lib_dir, step_index); - const appended = try resolved.join(arena, sub_path); - try addWatchInputPath(s, maker, appended); - break :zl; - }, - else => {}, - } - const path: Path = .{ - .root_dir = graph.zig_lib_directory, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); - }, - .local_cache => { - const path: Path = .{ - .root_dir = graph.local_cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); - }, - .global_cache => { - const path: Path = .{ - .root_dir = graph.global_cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); - }, - .build_root => { - const path: Path = .{ - .root_dir = graph.build_root_directory, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); - }, - } - } + .cache_manifest => { + const cache_manifest = body_r.takeStruct(std.zig.Server.Message.CacheManifest, .little) catch unreachable; + var diag: Cache.Manifest.CheckDiagnostic = undefined; + var man = graph.cache.obtain(); + defer man.deinit(); + man.openFile(&cache_manifest.input_hash, &diag) catch |err| switch (err) { + else => |e| return e, + error.CacheCheckFailed => return s.fail(maker, "opening cache manifest failed: {f}", .{ + diag.fmt(&man), + }), + }; + + s.result_cache_status = switch (cache_manifest.cache_status.result) { + .hit => .hit, + .incomplete_manifest => .incomplete_manifest, + .invalid_manifest => .invalid_manifest, + inline .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |tag| b: { + const off: Cache.Manifest.File.Offset = @fromBackingInt(cache_manifest.cache_status.file_off); + break :b @unionInit( + ResultCacheStatus, + @tagName(tag), + try gpa.dupeSentinel(u8, off.path(man.contents.items), 0), + ); + }, + }; + result = .{ .bin = cache_manifest.discovered_hash }; + try setWatchInputsFromManifest(s, maker, &man); }, .time_report => if (maker.web_server) |ws| { const tr = body_r.takeStruct(std.zig.Server.Message.TimeReport, .little) catch unreachable; diff --git a/lib/compiler/Maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig index 55f1e545f180eaf28eb041403140e86caf9d2d4b..17be4a5a627a68653ccdf4a66a9734a7266caba1 100644 --- a/lib/compiler/Maker/WebServer.zig +++ b/lib/compiler/Maker/WebServer.zig @@ -650,15 +650,37 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim const compile_prog_node = ws.root_prog_node.start("Compile WebAssembly Component", 0); defer compile_prog_node.end(); + var cache: Cache = .{ + .gpa = gpa, + .io = io, + .manifest_dir = try graph.global_cache_root.handle.createDirPathOpen(io, "h", .{}), + .cwd = graph.cache.cwd, + }; + defer cache.manifest_dir.close(io); + cache.addPrefix(.cwd()); + cache.addPrefix(graph.zig_lib_directory); + cache.addPrefix(graph.global_cache_root); + const result = try std.zig.buildExeSubprocess(gpa, io, .{ .argv = argv.items, .cache_root = graph.global_cache_root, + .cache = &cache, + .prefix_map = .{ + 0, // cwd is the same + 1, // zig lib dir is the same + 2, // local cache is mapped to global cache + 2, // global cache is the same + 0, // build root is not provided + }, .root_name = root_name, .arch_os_abi = arch_os_abi, .cpu_features = cpu_features, .progress_node = compile_prog_node, }); - if (!result.cache_hit) log.info("source changes detected; rebuilt wasm component", .{}); + switch (result.cache_result) { + .hit => {}, + else => log.info("source changes detected: {t}; rebuilt wasm component", .{result.cache_result}), + } return result.path; } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index c4d1c106022d436b7b32295c627c0898797b8906..e716bbf68b575d3fc2e77ded88565181adc606f8 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -286,7 +286,7 @@ pub const Manifest = struct { want_refresh_timestamp: bool = true, /// Uses `Cache.gpa`. files: Files = .empty, - /// Indexes line up with `files`, but only up until `hit` is called. Uses + /// Indexes line up with `files`, but only up until `check` is called. Uses /// `Cache.gpa`. input_paths: std.ArrayList(InputPath) = .empty, /// Keeps track of the last time we performed a file system write to observe @@ -693,12 +693,12 @@ pub const Manifest = struct { }; /// Add a file or directory path as a dependency of process being cached. - /// When `hit` is called, the contents will be checked to ensure + /// When `check` is called, the contents will be checked to ensure /// that it matches the contents from previous times. /// /// The contents of the input file may be requested and subsequently /// obtained via methods of the returned `InputPath.Index` after calling - /// `hit`. + /// `check`. /// /// Contents of a directory are considered to be the sorted list of file /// names of direct entries, separated by null byte. Each file name is @@ -910,7 +910,57 @@ pub const Manifest = struct { var input_digest: BinDigest = undefined; man.hash.hasher.final(&input_digest); - const input_hex_digest = binToHex(input_digest); + + try openFile(man, &input_digest, diag); + + man.want_refresh_timestamp = true; + + // We're going to construct a second hash. Its input will begin with the digest we've already computed + // (`input_digest`), and then it'll have the digests of each input file, including discovered files (see + // `addDiscoveredPath`). If this is a hit, we learn the set of discovered files from the manifest on disk. If + // this is a miss, we'll learn those from future calls to `addDiscoveredPath` etc. As such, the state of + // `man.hash.hasher` after this function depends on whether this is a hit or a miss. + // + // If we return `CheckResult.hit`, then `man.hash.hasher` must already include the digests of the discovered + // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests + // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered + // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user + // can use `final`, and will at some point `finalize` the file list to disk. + man.hash.hasher = hasher_init; + man.hash.hasher.update(&input_digest); + + hit: { + const miss_result = miss: { + const result = try man.checkLocked(diag); + if (result == .hit) { + break :hit; + } else if (!try man.upgradeToExclusiveLock(diag)) { + break :miss result; + } + // Missed with the shared lock, and upgraded to an exclusive lock. However, another process may have + // modified the cache directory, so we need to check again before deciding to miss. + man.shrinkFilesToInput(); + const refreshed_result = try man.checkLocked(diag); + if (refreshed_result == .hit) break :hit; + break :miss refreshed_result; + }; + + // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a + // cache miss, which is needed because they will be used in the manifest digest. + man.state = .miss; + return miss_result; + } + + if (man.want_shared_lock) { + man.downgradeToSharedLock() catch |err| return fail(diag, .{ .manifest_lock = err }); + } + + man.state = .hit; + return .hit; + } + + pub fn openFile(man: *Manifest, input_digest: *const BinDigest, diag: *CheckDiagnostic) CheckError!void { + const input_hex_digest = binToHex(input_digest.*); const manifest_file_path = &input_hex_digest; const io = man.cache.io; @@ -969,51 +1019,6 @@ pub const Manifest = struct { else => |e| return fail(diag, .{ .manifest_create = e }), } } - - man.want_refresh_timestamp = true; - - // We're going to construct a second hash. Its input will begin with the digest we've already computed - // (`input_digest`), and then it'll have the digests of each input file, including discovered files (see - // `addDiscoveredPath`). If this is a hit, we learn the set of discovered files from the manifest on disk. If - // this is a miss, we'll learn those from future calls to `addDiscoveredPath` etc. As such, the state of - // `man.hash.hasher` after this function depends on whether this is a hit or a miss. - // - // If we return `CheckResult.hit`, then `man.hash.hasher` must already include the digests of the discovered - // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests - // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered - // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user - // can use `final`, and will at some point `finalize` the file list to disk. - man.hash.hasher = hasher_init; - man.hash.hasher.update(&input_digest); - - hit: { - const miss_result = miss: { - const result = try man.checkLocked(diag); - if (result == .hit) { - break :hit; - } else if (!try man.upgradeToExclusiveLock(diag)) { - break :miss result; - } - // Missed with the shared lock, and upgraded to an exclusive lock. However, another process may have - // modified the cache directory, so we need to check again before deciding to miss. - man.shrinkFilesToInput(); - const refreshed_result = try man.checkLocked(diag); - if (refreshed_result == .hit) break :hit; - break :miss refreshed_result; - }; - - // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a - // cache miss, which is needed because they will be used in the manifest digest. - man.state = .miss; - return miss_result; - } - - if (man.want_shared_lock) { - man.downgradeToSharedLock() catch |err| return fail(diag, .{ .manifest_lock = err }); - } - - man.state = .hit; - return .hit; } fn transitionToMissDiscovered(m: *Manifest) void { @@ -1673,7 +1678,7 @@ pub const Manifest = struct { if (timestamp.nanoseconds < man.recent_problematic_timestamp.nanoseconds) return false; - // This flag prevents multiple filesystem writes for the same hit() call. + // This flag prevents multiple filesystem writes for the same `check` call. if (man.want_refresh_timestamp) { man.want_refresh_timestamp = false; @@ -2120,7 +2125,7 @@ pub const Manifest = struct { } /// Releases the manifest file and frees any memory the Manifest was using. - /// `Manifest.hit` must be called first. + /// `Manifest.check` must be called first. /// /// Don't forget to call `finalize` before this! pub fn deinit(m: *Manifest) void { @@ -2142,25 +2147,6 @@ pub const Manifest = struct { m.* = undefined; } - pub fn populateFileSystemInputs(man: *const Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void { - assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len); - buf.clearRetainingCapacity(); - const gpa = man.cache.gpa; - const files = man.files.keys(); - if (files.len > 0) { - const contents = man.contents.items; - for (files) |file| { - const file_path = file.path(contents); - try buf.ensureUnusedCapacity(gpa, file_path.len + 2); - buf.appendAssumeCapacity(@as(u8, file.get(contents).flags.prefix) + 1); - buf.appendSliceAssumeCapacity(file_path); - buf.appendAssumeCapacity(0); - } - // The null byte is a separator, not a terminator. - buf.items.len -= 1; - } - } - /// Add the full set of paths from another `Manifest` as a dependency of process being cached, after cache miss /// occurs. /// diff --git a/lib/std/zig.zig b/lib/std/zig.zig index b1537b6c166e6cd73c57447d993f219c4274f22a..b023f4e5af695fc25e72f0e3a0297d33098baa95 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1826,10 +1826,12 @@ pub fn isUpDir(p: []const u8) bool { pub const BuildExeSubprocessOptions = struct { argv: []const []const u8, cache_root: Cache.Directory, + cache: *Cache, root_name: []const u8, environ_map: ?*std.process.Environ.Map = null, cache_manifest: ?*Cache.Manifest = null, + prefix_map: [5]u8 = .{ 0, 1, 2, 3, 4 }, arch_os_abi: ?[]const u8 = null, cpu_features: ?[]const u8 = null, progress_node: std.Progress.Node = .none, @@ -1848,7 +1850,7 @@ pub const BuildExeSubprocessError = error{ pub const BuildExeSubprocessResult = struct { received_fs_inputs: bool, - cache_hit: bool, + cache_result: Cache.Manifest.CheckResult, path: Cache.Path, }; @@ -1911,7 +1913,7 @@ pub fn buildExeSubprocess( const error_bundle = options.error_bundle orelse &default_error_bundle; var received_fs_inputs = false; - var cache_hit = false; + var cache_result: Cache.Manifest.CheckResult = .incomplete_manifest; var eos_err: error{EndOfStream}!void = {}; @@ -1948,37 +1950,41 @@ pub fn buildExeSubprocess( else => |e| return e, }; }, - .emit_digest => { - const EmitDigest = Server.Message.EmitDigest; - const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); - cache_hit = ebp_hdr.flags.cache_hit; - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; + .cache_manifest => { + var body_r: std.Io.Reader = .fixed(body); + const cache_manifest = body_r.takeStruct(std.zig.Server.Message.CacheManifest, .little) catch unreachable; + var diag: Cache.Manifest.CheckDiagnostic = undefined; + var man = options.cache.obtain(); + defer man.deinit(); + man.openFile(&cache_manifest.input_hash, &diag) catch |err| switch (err) { + else => |e| return e, + error.CacheCheckFailed => { + log.err("opening cache manifest failed: {f}", .{diag.fmt(&man)}); + return error.AlreadyReported; + }, + }; + + cache_result = switch (cache_manifest.cache_status.result) { + .hit => .hit, + .incomplete_manifest => .incomplete_manifest, + .invalid_manifest => .invalid_manifest, + .path_deleted => .{ .path_deleted = @fromBackingInt(cache_manifest.cache_status.file_off) }, + .directory_status_changed => .{ .directory_status_changed = @fromBackingInt(cache_manifest.cache_status.file_off) }, + .metadata_changed => .{ .metadata_changed = @fromBackingInt(cache_manifest.cache_status.file_off) }, + .contents_changed => .{ .contents_changed = @fromBackingInt(cache_manifest.cache_status.file_off) }, + }; + + if (options.cache_manifest) |parent_man| { + try parent_man.addDiscoveredManifest(&man, options.prefix_map); + received_fs_inputs = true; + } + if (result) |r| gpa.free(r.sub_path); result = .{ .root_dir = options.cache_root, - .sub_path = try Dir.path.join(gpa, &.{ "o", &Cache.binToHex(digest.*) }), + .sub_path = try Dir.path.join(gpa, &.{ "o", &Cache.binToHex(cache_manifest.discovered_hash) }), }; }, - .file_system_inputs => if (options.cache_manifest) |man| { - received_fs_inputs = true; - var it = mem.splitScalar(u8, body, 0); - while (it.next()) |prefixed_path| { - const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1)); - const sub_path = prefixed_path[1..]; - man.addDiscoveredPath(.{ - .discovered_path = .{ .prefixed = .{ - .prefix = @intCast(@backingInt(prefix)), - .sub_path = sub_path, - } }, - }) catch |err| switch (err) { - error.Canceled, error.OutOfMemory => |e| return e, - else => |e| { - log.err("adding {t} {s} to cache failed: {t}", .{ prefix, sub_path, e }); - return error.AlreadyReported; - }, - }; - } - }, else => {}, // ignore other messages } } @@ -2049,7 +2055,7 @@ pub fn buildExeSubprocess( defer gpa.free(bin_name); return .{ .received_fs_inputs = received_fs_inputs, - .cache_hit = cache_hit, + .cache_result = cache_result, .path = try base_path.join(gpa, bin_name), }; } diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 7fb59ac5a45a762d5bfc38f979e883f765a01c9c..5392143fd365aa32f34e48e438c564e7419ad1c6 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -33,8 +33,6 @@ pub const Message = struct { zig_version, /// Body is an ErrorBundle. error_bundle, - /// Body is a EmitDigest. - emit_digest, /// Body is a TestMetadata test_metadata, /// Body is a TestResults @@ -45,14 +43,9 @@ pub const Message = struct { /// runner to enforce unit test time limits is relatively accurate under extreme system load /// (where there may be a non-trivial delay before the test process is scheduled). test_started, - /// Body is a series of strings, delimited by null bytes. - /// Each string is a prefixed file path. - /// The first byte indicates the file prefix path (see prefixes fields - /// of Cache). This byte is sent over the wire incremented so that null - /// bytes are not confused with string terminators. - /// The remaining bytes is the file path relative to that prefix. - /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir) - file_system_inputs, + /// All the information needed to place a file watch on file system inputs. Same memory layout as + /// `std.Build.Cache.Manifest.contents`. Body is a `CacheManifest`. + cache_manifest, /// Body is: /// - a u64le that indicates the file path within the cache used /// to store coverage information. The integer is a hash of the PCs @@ -208,14 +201,33 @@ pub const Message = struct { }; }; - /// Trailing: - /// * the hex digest of the cache directory within the /o/ subdirectory. - pub const EmitDigest = extern struct { - flags: Flags, + pub const CacheManifest = extern struct { + /// Hit or miss, and if miss, what kind of miss, and which file or directory caused it. + cache_status: CacheStatus, + /// Binary digest of the cache inputs. Tells where to locate the manifest file. + input_hash: std.Build.Cache.BinDigest, + /// Binary digest of hashed input_hash plus each file's hash. Tells where to locate the cached + /// artifacts. This is technically denormalized data assuming the cache manifest file is intact. + discovered_hash: std.Build.Cache.BinDigest, - pub const Flags = packed struct(u8) { - cache_hit: bool, - reserved: u7 = 0, + pub const CacheStatus = packed struct(u32) { + result: Result, + /// Presence determined by cache_status. + file_off: u29, + + pub const Result = enum(u3) { + hit, + incomplete_manifest, + invalid_manifest, + /// Has file_off + path_deleted, + /// Has file_off + directory_status_changed, + /// Has file_off + metadata_changed, + /// Has file_off + contents_changed, + }; }; }; @@ -317,30 +329,24 @@ pub fn serveBroadcastFuzzInputMessage(s: *const Server, test_i: u32, bytes: []co try s.out.flush(); } -pub fn serveEmitDigest( - s: *Server, - digest: *const [Cache.bin_digest_len]u8, - header: OutMessage.EmitDigest, -) !void { +pub fn serveCacheManifestUnflushed(s: *Server, header: OutMessage.CacheManifest) !void { try s.serveMessageHeader(.{ - .tag = .emit_digest, - .bytes_len = @intCast(digest.len + @sizeOf(OutMessage.EmitDigest)), + .tag = .cache_manifest, + .bytes_len = @sizeOf(OutMessage.CacheManifest), }); try s.out.writeStruct(header, .little); - try s.out.writeAll(digest); - try s.out.flush(); } pub fn serveTestResults(s: *Server, msg: OutMessage.TestResults) !void { try s.serveMessageHeader(.{ .tag = .test_results, - .bytes_len = @intCast(@sizeOf(OutMessage.TestResults)), + .bytes_len = @sizeOf(OutMessage.TestResults), }); try s.out.writeStruct(msg, .little); try s.out.flush(); } -pub fn serveErrorBundle(s: *Server, tag: Message.Tag, error_bundle: std.zig.ErrorBundle) !void { +pub fn serveErrorBundleUnflushed(s: *Server, tag: Message.Tag, error_bundle: std.zig.ErrorBundle) !void { const eb_hdr: OutMessage.ErrorBundle = .{ .extra_len = @intCast(error_bundle.extra.len), .string_bytes_len = @intCast(error_bundle.string_bytes.len), @@ -354,6 +360,10 @@ pub fn serveErrorBundle(s: *Server, tag: Message.Tag, error_bundle: std.zig.Erro try s.out.writeStruct(eb_hdr, .little); try s.out.writeSliceEndian(u32, error_bundle.extra, .little); try s.out.writeAll(error_bundle.string_bytes); +} + +pub fn serveErrorBundle(s: *Server, tag: Message.Tag, error_bundle: std.zig.ErrorBundle) !void { + try serveErrorBundleUnflushed(s, tag, error_bundle); try s.out.flush(); } diff --git a/src/Compilation.zig b/src/Compilation.zig index 5a98fa435ce9f77181cb491f02a656b446cca4e2..e1ffd9945961e89631e451f37286327abf04b5a6 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -176,7 +176,14 @@ debug_compile_errors: bool, /// Do not check this field directly. Instead, use the `debugIncremental` wrapper function. debug_incremental: bool, alloc_failure_occurred: bool = false, -last_update_was_cache_hit: bool = false, + +last_update_cache_status: Cache.Manifest.CheckResult, +/// Corresponds to .zig-cache/h/* filenames. +/// Remains stable across updates. +input_digest: [Cache.bin_digest_len]u8, +/// Corresponds to .zig-cache/o/* filenames. +/// May change across updates. +discovered_digest: [Cache.bin_digest_len]u8, c_source_files: []const CSourceFile, rc_source_files: []const RcSourceFile, @@ -251,12 +258,6 @@ llvm_opt_bisect_limit: c_int, time_report: ?TimeReport, -file_system_inputs: ?*std.ArrayList(u8), - -/// This is the digest of the cache for the current compilation. -/// This digest will be known after update() is called. -digest: ?[Cache.bin_digest_len]u8 = null, - /// Non-`null` iff we are emitting a binary. /// Does not change for the lifetime of this `Compilation`. /// Cwd-relative if `cache_use == .none`. Otherwise, relative to our subdirectory in the cache. @@ -1600,9 +1601,6 @@ pub const CreateOptions = struct { error_limit: ?Zcu.ErrorInt = null, global_cc_argv: []const []const u8 = &.{}, - /// Tracks all files that can cause the Compilation to be invalidated and need a rebuild. - file_system_inputs: ?*std.ArrayList(u8) = null, - parent_whole_cache: ?ParentWholeCache = null, environ_map: *const std.process.Environ.Map, @@ -2147,7 +2145,6 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, .force_undefined_symbols = options.force_undefined_symbols, .link_eh_frame_hdr = link_eh_frame_hdr, .global_cc_argv = options.global_cc_argv, - .file_system_inputs = options.file_system_inputs, .parent_whole_cache = options.parent_whole_cache, .link_diags = .init(gpa, io), .oneshot_prelink_tasks = .empty, @@ -2158,6 +2155,11 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, .emit_llvm_bc = try options.emit_llvm_bc.resolve(arena, &options, .llvm_bc), .emit_docs = try options.emit_docs.resolve(arena, &options, .docs), .environ_map = options.environ_map, + + // These are populated during `update`. + .last_update_cache_status = undefined, + .input_digest = undefined, + .discovered_digest = undefined, }; errdefer { @@ -2300,7 +2302,8 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, // In the case of incremental cache mode, this `artifact_directory` // is computed based on a hash of non-linker inputs, and it is where all // build artifacts are stored (even while in-progress). - comp.digest = hash.peekBin(); + comp.input_digest = hash.peekBin(); + comp.computed_digest = comp.input_digest; const digest = hash.final(); const artifact_sub_dir = "o" ++ fs.path.sep_str ++ digest; @@ -2817,7 +2820,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE log.debug("{s} whole cache {f} ignore_hit={}", .{ comp.root_name, status.fmt(&man), ignore_hit }); if (status == .hit and !ignore_hit) { // In this case the cache hit contains the full set of file system inputs. Nice! - if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); if (comp.parent_whole_cache) |pwc| { try pwc.mutex.lock(io); defer pwc.mutex.unlock(io); @@ -2881,29 +2883,14 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE try comp.c_object_work_queue.ensureUnusedCapacity(gpa, comp.c_objects.items.len); for (comp.c_objects.items) |c_object| { comp.c_object_work_queue.pushBackAssumeCapacity(c_object); - try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{c_object.src.src_path})); } } - for (comp.link_inputs) |input| { - const path = input.path(); - try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{ - path.root_dir.path orelse ".", - path.sub_path, - })); - } - // For compiling Win32 resources, we rely on the cache hash system to avoid duplicating work. // Add a Job for each Win32 resource file. try comp.win32_resource_work_queue.ensureUnusedCapacity(gpa, comp.win32_resources.items.len); for (comp.win32_resources.items) |win32_resource| { comp.win32_resource_work_queue.pushBackAssumeCapacity(win32_resource); - switch (win32_resource.src) { - .rc => |f| { - try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{f.src_path})); - }, - .manifest => {}, - } } if (comp.zcu) |zcu| { @@ -2911,11 +2898,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE zcu.skip_analysis_this_update = false; - // TODO: doing this in `resolveReferences` later could avoid adding inputs for dead embedfiles. Investigate! - for (zcu.embed_table.keys()) |embed_file| { - try comp.appendFileSystemInput(embed_file.path); - } - zcu.analysis_roots_len = 0; zcu.analysis_roots_buffer[zcu.analysis_roots_len] = zcu.std_mod; @@ -3003,7 +2985,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } if (comp.link_depfile) |depfile_path| if (comp.bin_file) |lf| { - assert(comp.file_system_inputs != null); comp.createDepFile(depfile_path, lf.emit) catch |err| comp.setMiscFailure(.link_depfile, "writing linker dependency file failed: {t}", .{err}); }; @@ -3024,7 +3005,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE switch (comp.cache_use) { .none, .incremental => try flush(comp, arena), .whole => |whole| { - if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); if (comp.parent_whole_cache) |pwc| { try pwc.mutex.lock(io); defer pwc.mutex.unlock(io); @@ -3129,39 +3109,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } } -/// Thread-safe. Assumes that `comp.mutex` is *not* already held by the caller. -pub fn appendFileSystemInput(comp: *Compilation, path: Compilation.Path) Allocator.Error!void { - const gpa = comp.gpa; - const io = comp.io; - const fsi = comp.file_system_inputs orelse return; - const prefixes = comp.cache_parent.prefixes(); - - const want_prefix_dir: Cache.Directory = switch (path.root) { - .zig_lib => comp.dirs.zig_lib, - .global_cache => comp.dirs.global_cache, - .local_cache => comp.dirs.local_cache, - .build_root => comp.dirs.build_root, - .none => .cwd(), - }; - const prefix: u8 = for (prefixes, 1..) |prefix_dir, i| { - if (prefix_dir.eql(want_prefix_dir)) { - break @intCast(i); - } - } else std.debug.panic( - "missing prefix directory {t} ('{f}') for {q}", - .{ path.root, want_prefix_dir, path.sub_path }, - ); - - // There may be concurrent calls to this function from C object workers and/or the main thread. - comp.mutex.lockUncancelable(io); - defer comp.mutex.unlock(io); - - try fsi.ensureUnusedCapacity(gpa, path.sub_path.len + 3); - if (fsi.items.len > 0) fsi.appendAssumeCapacity(0); - fsi.appendAssumeCapacity(prefix); - fsi.appendSliceAssumeCapacity(path.sub_path); -} - fn resolveEmitPath(comp: *Compilation, path: []const u8) Cache.Path { return .{ .root_dir = switch (comp.cache_use) { @@ -4649,6 +4596,7 @@ fn writeDepFile( w: *std.Io.Writer, ) std.Io.Writer.Error!void { const prefixes = comp.cache_parent.prefixes(); + if (true) @panic("TODO reimplement this based on Cache.Manifest data"); const fsi = comp.file_system_inputs.?.items; try w.print("{f}:", .{bin_file}); @@ -4667,11 +4615,8 @@ fn writeDepFile( } fn workerDocsCopy(comp: *Compilation) void { - docsCopyFallible(comp) catch |err| return comp.lockAndSetMiscFailure( - .docs_copy, - "unable to copy autodocs artifacts: {s}", - .{@errorName(err)}, - ); + docsCopyFallible(comp) catch |err| + return comp.lockAndSetMiscFailure(.docs_copy, "unable to copy autodocs artifacts: {t}", .{err}); } fn docsCopyFallible(comp: *Compilation) anyerror!void { @@ -4680,11 +4625,9 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void { const docs_path = comp.resolveEmitPath(comp.emit_docs.?); var out_dir = docs_path.root_dir.handle.createDirPathOpen(io, docs_path.sub_path, .{}) catch |err| { - return comp.lockAndSetMiscFailure( - .docs_copy, - "unable to create output directory '{f}': {s}", - .{ docs_path, @errorName(err) }, - ); + return comp.lockAndSetMiscFailure(.docs_copy, "unable to create output directory {qf}: {t}", .{ + docs_path, err, + }); }; defer out_dir.close(io); @@ -4695,11 +4638,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void { } var tar_file = out_dir.createFile(io, "sources.tar", .{}) catch |err| { - return comp.lockAndSetMiscFailure( - .docs_copy, - "unable to create '{f}/sources.tar': {s}", - .{ docs_path, @errorName(err) }, - ); + return comp.lockAndSetMiscFailure(.docs_copy, "unable to create '{f}/sources.tar': {t}", .{ docs_path, err }); }; defer tar_file.close(io); @@ -5719,64 +5658,17 @@ fn updateCObject( .sub_path = fs.path.basename(raw_dep_file_path), }; - if (comp.file_system_inputs != null) { - // TODO instead of this rely on passing the manifest contents directly - - // Use the same file size limit as the cache code does for dependency files. - const dep_file_contents = dep_file_path.root_dir.handle.readFileAlloc(io, dep_file_path.sub_path, gpa, .unlimited) catch |err| switch (err) { - error.OutOfMemory, error.Canceled => |e| return e, - else => |e| return comp.failCObjRetryable( - c_object, - "failed to read '{f}': {t}", - .{ dep_file_path, e }, - ), - }; - defer gpa.free(dep_file_contents); - - var str_buf: std.ArrayList(u8) = .empty; - defer str_buf.deinit(gpa); - - var it: std.Build.Cache.DepTokenizer = .{ .bytes = dep_file_contents }; - while (it.next()) |token| { - const input_path: Compilation.Path = switch (token) { - .target, .target_must_resolve => continue, - .prereq => |file_path| try .fromUnresolved(arena, comp.dirs, &.{file_path}), - .prereq_must_resolve => p: { - try token.resolve(gpa, &str_buf); - break :p try .fromUnresolved(arena, comp.dirs, &.{str_buf.items}); - }, - else => |err_token| return comp.failCObjRetryable( - c_object, - "failed parsing '{f}': {f}", - .{ dep_file_path, err_token }, - ), - }; - try comp.appendFileSystemInput(input_path); - } - } - // Add the files depended on to the cache system. var diags: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined; man.addDiscoveredDepFile(dep_file_path, &diags) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - - error.InvalidDepFile => return comp.failCObjRetryable( - c_object, - "failed to parse '{f}': {f}", - .{ dep_file_path, diags.dep_tokenizer }, - ), - error.FileSystemFailure => return comp.failCObjRetryable( - c_object, - "failed to parse '{f}': {f}", - .{ dep_file_path, diags.add_discovered_path }, - ), - else => |e| return comp.failCObjRetryable( - c_object, - "failed to read '{f}': {t}", - .{ dep_file_path, e }, - ), + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidDepFile => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{ + dep_file_path, diags.dep_tokenizer, + }), + error.FileSystemFailure => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{ + dep_file_path, diags.add_discovered_path, + }), + else => |e| return comp.failCObjRetryable(c_object, "failed to read {qf}: {t}", .{ dep_file_path, e }), }; switch (comp.cache_use) { .whole => |whole| { @@ -5784,25 +5676,17 @@ fn updateCObject( try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); whole_cache_manifest.addDiscoveredDepFile(dep_file_path, &diags) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, + error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidDepFile => return comp.failCObjRetryable( - c_object, - "failed to parse '{f}': {f}", - .{ dep_file_path, diags.dep_tokenizer }, - ), - error.FileSystemFailure => return comp.failCObjRetryable( - c_object, - "failed to parse '{f}': {f}", - .{ dep_file_path, diags.add_discovered_path }, - ), - else => |e| return comp.failCObjRetryable( - c_object, - "failed to read '{f}': {t}", - .{ dep_file_path, e }, - ), + error.InvalidDepFile => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{ + dep_file_path, diags.dep_tokenizer, + }), + error.FileSystemFailure => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{ + dep_file_path, diags.add_discovered_path, + }), + else => |e| return comp.failCObjRetryable(c_object, "failed to read {qf}: {t}", .{ + dep_file_path, e, + }), }; } }, diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index ec764ddfdefc972b4ff9bfbc3d8a920cdabe9762..4b558f2c20a75ebce4edd4988c844374376bb5e0 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -2643,8 +2643,6 @@ fn computeAliveFiles(pt: Zcu.PerThread) Allocator.Error!bool { .success => {}, } - try comp.appendFileSystemInput(file.path); - switch (file.getMode()) { .zig => {}, // continue to logic below .zon => continue, // ZON can't import anything @@ -3020,8 +3018,6 @@ fn newEmbedFile(pt: Zcu.PerThread, path: Compilation.Path) EmbedFileError!*Zcu.E var opt_ip_str: ?InternPool.String = null; try pt.updateEmbedFile(new_file, &opt_ip_str); - try comp.appendFileSystemInput(path); - // Add the file contents to the `whole` cache manifest if necessary. cache: { const whole = switch (zcu.comp.cache_use) { diff --git a/src/main.zig b/src/main.zig index 607cd7b643a5a94d06d710dc25feea16c55568c7..7bb9ebc30428031f995c17b7d260e48da577a473 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3627,9 +3627,6 @@ fn buildOutputType( break :b .whole; }; - var file_system_inputs: std.ArrayList(u8) = .empty; - defer file_system_inputs.deinit(gpa); - // Deduplicate rpath entries var rpath_dedup = std.array_hash_map.String(void){}; for (create_module.rpath_list.items) |rpath| { @@ -3762,7 +3759,6 @@ fn buildOutputType( // than to any particular module. This feature can greatly reduce CLI // noise when --search-prefix and -M are combined. .global_cc_argv = try cc_argv.toOwnedSlice(arena), - .file_system_inputs = &file_system_inputs, .debug_compiler_runtime_libs = debug_compiler_runtime_libs, .environ_map = environ_map, }) catch |err| switch (err) { @@ -3818,7 +3814,6 @@ fn buildOutputType( arg_mode, all_args, runtime_args_start, - environ_map, ); return cleanExit(io); }, @@ -3845,7 +3840,6 @@ fn buildOutputType( arg_mode, all_args, runtime_args_start, - environ_map, ); return cleanExit(io); }, @@ -3858,7 +3852,7 @@ fn buildOutputType( defer root_prog_node.end(); if (arg_mode == .translate_c) { - return cmdTranslateC(comp, arena, null, null, root_prog_node, environ_map); + return cmdTranslateC(comp, arena, null, root_prog_node, environ_map); } updateModule(comp, color, root_prog_node) catch |err| switch (err) { @@ -4400,7 +4394,6 @@ fn serve( arg_mode: ArgMode, all_args: []const []const u8, runtime_args_start: ?usize, - environ_map: *process.Environ.Map, ) !void { const gpa = comp.gpa; const io = comp.io; @@ -4422,8 +4415,6 @@ fn serve( const main_progress_node = std.Progress.start(io, .{}); defer main_progress_node.end(); - const file_system_inputs = comp.file_system_inputs.?; - const IncrementalDebugServer = if (build_options.enable_debug_extensions and !builtin.single_threaded) @import("IncrementalDebugServer.zig") else @@ -4446,29 +4437,8 @@ fn serve( switch (hdr.tag) { .exit => return cleanExit(io), .update => { - file_system_inputs.clearRetainingCapacity(); - if (arg_mode == .translate_c) { - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - var output: Compilation.TranslateCResult = undefined; - try cmdTranslateC(comp, arena, &output, file_system_inputs, main_progress_node, environ_map); - defer output.deinit(gpa); - - if (file_system_inputs.items.len != 0) { - try server.serveStringMessage(.file_system_inputs, file_system_inputs.items); - } - - if (output.errors.errorMessageCount() != 0) { - try server.serveErrorBundle(.error_bundle, output.errors); - } else { - try server.serveEmitDigest(&output.digest, .{ - .flags = .{ .cache_hit = output.cache_hit }, - }); - } - - continue; + @panic("TODO delete translate-c"); } try comp.makeBinFileWritable(); @@ -4496,7 +4466,6 @@ fn serve( //); }, .hot_update => { - file_system_inputs.clearRetainingCapacity(); if (child_pid) |pid| { try comp.hotCodeSwap(main_progress_node, pid); try serveUpdateResults(&server, comp); @@ -4530,14 +4499,6 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void { var error_bundle = try comp.getAllErrorsAlloc(); defer error_bundle.deinit(gpa); - if (comp.file_system_inputs) |file_system_inputs| { - if (file_system_inputs.items.len == 0) { - assert(error_bundle.errorMessageCount() > 0); - } else { - try s.serveStringMessage(.file_system_inputs, file_system_inputs.items); - } - } - if (comp.time_report) |*tr| { var decls_len: u32 = 0; @@ -4613,22 +4574,25 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void { }, }); try s.out.writeVecAll(&slices); - try s.out.flush(); } - if (error_bundle.errorMessageCount() > 0) { - try s.serveErrorBundle(.error_bundle, error_bundle); - return; - } - - if (comp.digest) |digest| { - try s.serveEmitDigest(&digest, .{ - .flags = .{ .cache_hit = comp.last_update_was_cache_hit }, - }); - } + try s.serveCacheManifestUnflushed(.{ + .cache_status = switch (comp.last_update_cache_status) { + .hit => .{ .result = .hit, .file_off = 0 }, + .incomplete_manifest => .{ .result = .incomplete_manifest, .file_off = 0 }, + .invalid_manifest => .{ .result = .invalid_manifest, .file_off = 0 }, + .path_deleted => |off| .{ .result = .path_deleted, .file_off = off }, + .directory_status_changed => |off| .{ .result = .directory_status_changed, .file_off = off }, + .metadata_changed => |off| .{ .result = .metadata_changed, .file_off = off }, + .contents_changed => |off| .{ .result = .contents_changed, .file_off = off }, + }, + .input_hash = comp.input_hash, + .discovered_hash = comp.discovered_hash, + }); - // Serve empty error bundle to indicate the update is done. - try s.serveErrorBundle(.error_bundle, std.zig.ErrorBundle.empty); + // Even if error bundle is empty, serve it to indicate the update is done. + try s.serveErrorBundleUnflushed(.error_bundle, error_bundle); + try s.out.flush(); } fn runOrTest( @@ -4889,7 +4853,6 @@ fn cmdTranslateC( comp: *Compilation, arena: Allocator, fancy_output: ?*Compilation.TranslateCResult, - file_system_inputs: ?*std.ArrayList(u8), prog_node: std.Progress.Node, environ_map: *process.Environ.Map, ) !void { @@ -4934,7 +4897,6 @@ fn cmdTranslateC( if (result.errors.errorMessageCount() != 0) { if (fancy_output) |p| { - if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf); p.* = result; return; } else { @@ -4948,7 +4910,6 @@ fn cmdTranslateC( break :result result; }; - if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf); if (fancy_output) |p| { p.* = result; } else { -- 2.54.0