diff --git a/build.zig b/build.zig index 94283dffd11afcf9f126f2418d21081710be4361..6b94dd1d195da678e7ef1335d68464e8fd9dcadc 100644 --- a/build.zig +++ b/build.zig @@ -263,8 +263,8 @@ pub fn build(b: *std.Build) !void { const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git."); const version_slice = if (opt_version_string) |version| version else v: { if (!std.process.can_spawn) { - std.debug.print("error: version info cannot be retrieved from git. Zig version must be provided using -Dversion-string\n", .{}); - std.process.exit(1); + std.log.info("version info can be provided explicitly via \"-Dversion-string\"", .{}); + std.process.fatal("version info cannot be retrieved from git", .{}); } // Ensure git version changes get picked up. @@ -272,12 +272,13 @@ pub fn build(b: *std.Build) !void { const io = b.graph.io; const git_file = b.root.openFile(io, ".git", .{ .allow_directory = false }) catch |err| switch (err) { error.IsDir => { - b.dependOnFileContents(b.path(".git/logs/HEAD")); + b.dependOnFileMetadata(b.path(".git/logs/HEAD")); break :git; }, else => |e| return e, }; defer git_file.close(io); + b.dependOnFileContents(b.path(".git")); var line_buffer: ["gitdir: ".len + std.Io.Dir.max_path_bytes + 1]u8 = undefined; var git_file_reader = git_file.reader(io, &line_buffer); if (std.mem.cutPrefix(u8, std.mem.trimEnd(u8, try git_file_reader.interface.allocRemaining( @@ -285,7 +286,7 @@ pub fn build(b: *std.Build) !void { .limited("gitdir: ".len + std.Io.Dir.max_path_bytes + "\r\n".len), ), "\r\n"), "gitdir: ")) |git_dir| { const head_file = b.pathJoin(&.{ git_dir, "logs", "HEAD" }); - b.dependOnFileContents(if (std.Io.Dir.path.isAbsolute(head_file)) + b.dependOnFileMetadata(if (std.Io.Dir.path.isAbsolute(head_file)) b.graph.cwdRelativePath(head_file) else b.path(head_file)); @@ -310,8 +311,9 @@ pub fn build(b: *std.Build) !void { 0 => { // Tagged release version (e.g. 0.10.0). if (!mem.eql(u8, git_describe, version_string)) { - std.debug.print("Zig version '{s}' does not match Git tag '{s}'\n", .{ version_string, git_describe }); - std.process.exit(1); + std.process.fatal("zig version {q} does not match Git tag {q}", .{ + version_string, git_describe, + }); } break :v version_string; }, @@ -324,13 +326,14 @@ pub fn build(b: *std.Build) !void { const ancestor_ver = try std.SemanticVersion.parse(tagged_ancestor); if (zig_version.order(ancestor_ver) != .gt) { - std.debug.print("Zig version '{f}' must be greater than tagged ancestor '{f}'\n", .{ zig_version, ancestor_ver }); - std.process.exit(1); + std.process.fatal("zig version {f} must be greater than tagged ancestor {qf}", .{ + zig_version, ancestor_ver, + }); } // Check that the commit hash is prefixed with a 'g' (a Git convention). if (commit_id.len < 1 or commit_id[0] != 'g') { - std.debug.print("Unexpected `git describe` output: {s}\n", .{git_describe}); + std.log.warn("unexpected \"git describe\" output: {s}", .{git_describe}); break :v version_string; } @@ -338,7 +341,7 @@ pub fn build(b: *std.Build) !void { break :v b.fmt("{s}-dev.{s}+{s}", .{ version_string, commit_height, commit_id[1..] }); }, else => { - std.debug.print("Unexpected `git describe` output: {s}\n", .{git_describe}); + std.log.warn("unexpected \"git describe\" output: {s}", .{git_describe}); break :v version_string; }, } @@ -354,7 +357,8 @@ pub fn build(b: *std.Build) !void { const file_contents = cwd.readFileAlloc(io, config_h_path, arena, .limited(max_config_h_bytes)) catch unreachable; break :blk parseConfigH(b, file_contents); } else { - std.log.warn("config.h could not be located automatically. Consider providing it explicitly via \"-Dconfig_h\"", .{}); + std.log.warn("config.h could not be located automatically", .{}); + std.log.info("config.h can be provided explicitly via \"-Dconfig_h\"", .{}); break :blk null; } }; diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 6038cad96d84e824b062749abcc63d44625a0451..b372f8999270f53a0ba15e916f7ecb8200ec662a 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -199,12 +199,13 @@ pub fn main(init: process.Init.Minimal) !void { .random_seed = parseRandomSeed(seed_arg), }; - const cmd = stringToEnum(enum { libc, init, fetch, build }, cmd_name) orelse + const cmd = stringToEnum(enum { libc, init, fetch, build, @"cache-cat" }, cmd_name) orelse fatal("bad command name: {q}", .{cmd_name}); switch (cmd) { .libc => return cmdLibC(gpa, &graph, args[arg_i..]), .init => return cmdInit(gpa, &graph, args[arg_i..]), .fetch => return cmdFetch(gpa, &graph, args[arg_i..]), + .@"cache-cat" => return cmdCacheCat(gpa, &graph, args[arg_i..]), .build => {}, } @@ -702,8 +703,8 @@ pub fn main(init: process.Init.Minimal) !void { configure: while (true) { // Set of files that, if modified, imply that recompiling and rerunning // configurer is needed. - var configure_source_files: Cache.Manifest.Files = .empty; - defer Cache.Manifest.freeFiles(gpa, &configure_source_files); + var configure_source_files: Cache.Manifest.SelfContainedFiles = .empty; + defer configure_source_files.deinit(gpa); // If this fails, we can still start the server and wait for user // to request a rebuild. If it returns error.FailedButCacheIntact @@ -1043,7 +1044,7 @@ const ConfigureOptions = struct { fetch_only: bool, print_configuration: PrintConfiguration, forks: []Fork, - src_files: *Cache.Manifest.Files, + src_files: *Cache.Manifest.SelfContainedFiles, }; fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { @@ -1398,8 +1399,14 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { defer compile_prog_node.end(); if (config_man) |man| { - if (try man.hit(compile_prog_node)) { - const digest = man.final(); + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, compile_prog_node) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + error.CacheCheckFailed => fatal("checking cache failed: {f}", .{diagnostic.fmt(man)}), + }; + log.debug("configuration cache {f}", .{status.fmt(man)}); + if (status == .hit) { + const digest = man.hitDigestHex(); const path: Path = .{ .root_dir = graph.local_cache_root, .sub_path = try arena.print("c/{s}", .{&digest}), @@ -1497,11 +1504,12 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { } if (config_man) |man| for (configuration.path_deps) |path_dep| { - switch (path_dep.flags.mode) { - .directory => {}, // TODO - .contents => try man.addPathPost(try confPathDepToCachePath(arena, graph, &configuration, path_dep)), - .metadata => {}, // TODO - } + const path = try confPathDepToCachePath(arena, graph, &configuration, path_dep); + try man.addDiscoveredPath(.{ + .discovered_path = .{ .unresolved = path }, + .handle = if (path_dep.flags.is_directory) .{ .dir = null } else .{ .file = null }, + .metadata_only = path_dep.flags.metadata_only, + }); }; // If it is poisoned, there is no point in moving it to cached @@ -1510,7 +1518,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { break :cp .{ config_tmp_path, null }; } else { const man = config_man.?; - const digest = man.final(); + const digest = man.missDigestHex(); const final_path: Path = .{ .root_dir = graph.local_cache_root, .sub_path = try arena.print("c/{s}", .{&digest}), @@ -1541,7 +1549,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { config_tmp_path, final_path, e, }); }; - man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); + man.finalize() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); options.src_files.* = man.takeFiles(); break :cp .{ final_path, man.toOwnedLock() }; } @@ -1563,10 +1571,10 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { const configuration = c: { var file = configuration_path.root_dir.handle.openFile(io, configuration_path.sub_path, .{}) catch |err| - fatal("failed to open configuration file {f}: {t}", .{ configuration_path, err }); + fatal("failed to open configuration file {qf}: {t}", .{ configuration_path, err }); defer file.close(io); break :c Configuration.loadFile(arena, io, file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ configuration_path, err }); + fatal("failed to load configuration file {qf}: {t}", .{ configuration_path, err }); }; // Technically if the configuration is marked as poisoned, we could // already delete the file now, but we leave it around in case the @@ -1879,6 +1887,101 @@ fn cmdFetch(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { return process.cleanExit(io); } +fn cmdCacheCat(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { + const io = graph.io; + + var arg_i: usize = 0; + var contents: std.ArrayList(u8) = .empty; + defer contents.deinit(gpa); + + while (nextArg(args, &arg_i)) |arg| { + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try Io.File.stdout().writeStreamingAll(io, usage_cache_cat); + return process.cleanExit(io); + } else { + fatal("unrecognized parameter: {q}", .{arg}); + } + } else { + var file = Dir.cwd().openFile(io, arg, .{}) catch |err| fatal("opening {q} failed: {t}", .{ arg, err }); + defer file.close(io); + + var manifest_reader = file.reader(io, &.{}); // Reads positionally from zero. + contents.clearRetainingCapacity(); + manifest_reader.interface.appendRemainingUnlimited(gpa, &contents) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => switch (manifest_reader.err.?) { + error.Canceled => |e| return e, + else => |e| fatal("reading from {q} failed: {t}", .{ arg, e }), + }, + }; + const hex_digest = Dir.path.basename(arg); + cacheCatOne(hex_digest, contents.items, initStdoutWriter(io)) catch |err| switch (err) { + error.WriteFailed => fatal("writing to stdout failed: {t}", .{stdout_writer_allocation.err.?}), + else => |e| fatal("parsing {q} failed: {t}", .{ arg, e }), + }; + try stdout_writer_allocation.flush(); + } + } +} + +fn cacheCatOne(input_hex_digest: []const u8, contents: []const u8, writer: *Io.Writer) !void { + var bin_digest: Cache.BinDigest = undefined; + _ = try fmt.hexToBytes(&bin_digest, input_hex_digest); + + var hh: Cache.HashHelper = .{}; + hh.hasher.update(&bin_digest); + + var serializer: std.zon.Serializer = .{ .writer = writer }; + var top_level = try serializer.beginStruct(.{}); + try top_level.field("input_hash", input_hex_digest, .{}); + var files_tuple = try top_level.beginTupleField("files", .{}); + var off: usize = 0; + while (off + 1 < contents.len) { + const file_off: Cache.Manifest.File.Offset = @fromBackingInt(@intCast(off)); + const file = try file_off.getFallibleConst(contents); + const path = try file_off.pathFallible(contents); + if (path.len == 0) return error.InvalidFormat; + + var file_obj = try files_tuple.beginStructField(.{ .whitespace_style = .{ .wrap = false } }); + try file_obj.field("size", file.size, .{}); + try file_obj.field("inode", file.inode, .{}); + try file_obj.field("mtime", file.mtime, .{}); + const hex_digest = Cache.binToHex(file.digest); + try file_obj.field("digest", @as([]const u8, &hex_digest), .{}); + if (file.flags.is_directory) try file_obj.field("directory", true, .{}); + if (file.flags.metadata_only) try file_obj.field("metadata", true, .{}); + try file_obj.field("prefix", file.flags.prefix, .{}); + try file_obj.field("path", path, .{}); + try file_obj.end(); + + hh.hasher.update(&file.digest); + + off += Cache.Manifest.File.sizeOf(path.len); + } + + try files_tuple.end(); + + var discovered_bin_digest: Cache.BinDigest = undefined; + hh.hasher.final(&discovered_bin_digest); + const discovered_hex_digest = Cache.binToHex(discovered_bin_digest); + try top_level.field("discovered_hash", @as([]const u8, &discovered_hex_digest), .{}); + + try top_level.end(); + try writer.writeByte('\n'); +} + +const usage_cache_cat = + \\Usage: zig cache-cat + \\ + \\ Prints .zig-cache/h/* manifest files in text form. + \\ + \\Options: + \\ -h, --help Print this help and exit + \\ + \\ +; + const usage_fetch = \\Usage: zig fetch [options] \\Usage: zig fetch [options] @@ -2220,7 +2323,7 @@ fn resolveTopLevelSteps(maker: *Maker, step_names: []const []const u8) ![]const fn prepare( maker: *Maker, step_indices: []const Configuration.Step.Index, - configure_source_files: *const Cache.Manifest.Files, + configure_source_files: *const Cache.Manifest.SelfContainedFiles, ) !void { const gpa = maker.gpa; const graph = maker.graph; diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 0fd82226cce06989c0acea46f4f22e4dcd6d6e58..4e9eb614efcea395d8f763e1210de56070fb2743 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -726,53 +726,48 @@ pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.T /// Prefer `cacheHitWatched` unless you already added watch inputs /// separately from using the cache system. pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { - s.result_cached = man.hit(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err); - return s.result_cached; + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, parent_progress_node) catch |err| + return failWithCacheError(s, maker, man, &diag, err); + const hit = .hit == status; // TODO cache miss reason in the build summary + s.result_cached = hit; + return hit; } /// Clears previous watch inputs, if any, and then populates watch inputs from /// the full set of files picked up by the cache manifest. /// -/// Must be accompanied with `writeManifestAndWatch`. +/// Must be accompanied with `finalizeManifestAndWatch`. pub fn cacheHitWatched(s: *Step, maker: *Maker, man: *Cache.Manifest, parent_progress_node: std.Progress.Node) !bool { - const is_hit = man.hit(parent_progress_node) catch |err| return failWithCacheError(s, maker, man, err); - s.result_cached = is_hit; + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, parent_progress_node) catch |err| + return failWithCacheError(s, maker, man, &diag, err); + const hit = .hit == status; // TODO cache miss reason in the build summary + s.result_cached = hit; // The above call to hit() populates the manifest with files, so in case of // a hit, we need to populate watch inputs. - if (is_hit) try setWatchInputsFromManifest(s, maker, man); - return is_hit; + if (hit) try setWatchInputsFromManifest(s, maker, man); + return hit; } fn failWithCacheError( s: *Step, maker: *Maker, man: *const Cache.Manifest, - err: Cache.Manifest.HitError, + diag: *const Cache.Manifest.CheckDiagnostic, + err: Cache.Manifest.CheckError, ) error{ OutOfMemory, Canceled, MakeFailed } { switch (err) { - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail(maker, "failed checking cache: {t} {t}", .{ - man.diagnostic, e, - }), - .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - return s.fail(maker, "failed checking cache: {s}{c}{s} {t} {t}", .{ - prefix, Dir.path.sep, pp.sub_path, man.diagnostic, op.err, - }); - }, - }, + error.CacheCheckFailed => return s.fail(maker, "checking cache failed: {f}", .{diag.fmt(man)}), error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => return s.fail(maker, "failed checking cache: invalid manifest file format", .{}), } } -/// Prefer `writeManifestAndWatch` unless you already added watch inputs +/// Prefer `finalizeManifestAndWatch` unless you already added watch inputs /// separately from using the cache system. -pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { +pub fn finalizeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { if (s.test_results.isSuccess()) { - man.writeManifest() catch |err| switch (err) { + man.finalize() catch |err| switch (err) { error.Canceled => |e| return e, else => |e| try s.addError(maker, "failed writing cache manifest: {t}", .{e}), }; @@ -783,29 +778,29 @@ pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { /// the full set of files picked up by the cache manifest. /// /// Must be accompanied with `cacheHitWatched`. -pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { - try writeManifest(s, maker, man); +pub fn finalizeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { + try finalizeManifest(s, maker, man); try setWatchInputsFromManifest(s, maker, man); } pub fn setWatchInputsFromManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { - return setWatchInputsFromManifestFiles(s, maker, &man.files, man.cache.prefixes()); + return setWatchInputsFromManifestFiles(s, maker, &man.borrowFiles(), man.cache.prefixes()); } pub fn setWatchInputsFromManifestFiles( s: *Step, maker: *Maker, - files: *const Cache.Manifest.Files, + scf: *const Cache.Manifest.SelfContainedFiles, prefixes: []const Cache.Directory, ) !void { const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena clearWatchInputs(s, maker); - for (files.keys()) |file| { + for (scf.files.keys()) |file_offset| { // The file path data is freed when the cache manifest is cleaned up at the end of `make`. - const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); + const sub_path = try arena.dupe(u8, scf.path(file_offset)); try addWatchInputFromPath(s, maker, .{ - .root_dir = prefixes[file.prefixed_path.prefix], + .root_dir = prefixes[file_offset.get(scf.contents.items).flags.prefix], .sub_path = Dir.path.dirname(sub_path) orelse "", }, Dir.path.basename(sub_path)); } diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index 20152955fe469b751648032e54152a7aa2b53f67..a539e2463128445541037d2feab024faa673681f 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -137,7 +137,7 @@ pub fn make( man.hash.addBytes(output); if (try step.cacheHit(maker, &man, progress_node)) { - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_ch.generated_dir).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), @@ -145,7 +145,7 @@ pub fn make( return; } - const digest = man.final(); + const digest = man.missDigestHex(); // If output_path has directory parts, deal with them. Example: // output_dir is zig-cache/o/HASH @@ -169,7 +169,7 @@ pub fn make( .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), }; - try step.writeManifest(maker, &man); + try step.finalizeManifest(maker, &man); } fn ensureAllValuesUsed( diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 49424d4b35daea41bb6a485e1fb2a37acfe5e468..52e3f9c7d4f20e0dc9dcd6623b57e647b8a26ca2 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -34,7 +34,7 @@ pub fn make( defer man.deinit(); const input_path = try maker.resolveLazyPath(arena, input_lazy_path, step_index); - _ = try man.addFilePath(input_path, null); + _ = try man.addInputPath(input_path, .{}); man.hash.addOptionalBytes(only_section); man.hash.addOptionalBytes(opt_basename); man.hash.addOptionalBytes(opt_debug_basename); @@ -48,7 +48,7 @@ pub fn make( if (try step.cacheHit(maker, &man, progress_node)) { // Cache hit, skip subprocess execution. - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_oc.output_file).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), @@ -67,7 +67,7 @@ pub fn make( // We don't find out more input files while executing objcopy so we can // already obtain the digest and use it directly as the output path. - const digest = man.final(); + const digest = man.missDigestHex(); const dest_path: Path = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), @@ -165,7 +165,7 @@ pub fn make( maker.generatedPath(conf_oc.output_file).* = dest_path; - step.writeManifest(maker, &man) catch |err| switch (err) { + step.finalizeManifest(maker, &man) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| try step.addError(maker, "failed writing cache manifest: {t}", .{e}), }; diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index 7c31f144d59a20f70f1e6550320b8f8c05237276..b9e676ef6542343afe8a018e3f3bdff240a18936 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -42,7 +42,7 @@ pub fn make( const lazy_path = arg.path.get(conf); try step.addWatchInput(maker, arena, lazy_path); const arg_path = try maker.resolveLazyPath(arena, lazy_path, step_index); - _ = try man.addFilePath(arg_path, null); + _ = try man.addInputPath(arg_path, .{}); try args_bytes.print(arena, "pub const {f}: []const u8 = \"{f}\";\n", .{ std.zig.fmtId(name), arg_path.fmtEscapeString(), }); @@ -54,7 +54,7 @@ pub fn make( const basename = "options.zig"; if (try step.cacheHitWatched(maker, &man, progress_node)) { - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_options.generated_file).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), @@ -63,7 +63,7 @@ pub fn make( return; } - const digest = man.final(); + const digest = man.missDigestHex(); const out_path: Cache.Path = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), @@ -95,7 +95,7 @@ pub fn make( }, }; - try step.writeManifestAndWatch(maker, &man); + try step.finalizeManifestAndWatch(maker, &man); maker.generatedPath(conf_options.generated_file).* = out_path; } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index c8219543c2891c24ee5003f76a81efdf91f0d5a1..828b5eb170515c999e8c0e40da4d1b668adfb1a8 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -97,7 +97,7 @@ pub fn make( man.hash.add(arg.flags.make_absolute); man.hash.addBytesZ(prefix); man.hash.addBytesZ(suffix); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .path_directory => { const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; @@ -135,7 +135,7 @@ pub fn make( argv_list.appendAssumeCapacity(result.written()); man.hash.addBytesZ(prefix); man.hash.addBytesZ(suffix); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .artifact => { const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; @@ -155,7 +155,7 @@ pub fn make( man.hash.add(arg.flags.make_absolute); man.hash.addBytesZ(prefix); man.hash.addBytesZ(suffix); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .output_file, .output_directory => { const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; @@ -211,7 +211,7 @@ pub fn make( }, .lazy_path => |lazy_path| { const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); }, .none => {}, } @@ -240,7 +240,7 @@ pub fn make( for (conf_run.file_inputs.slice) |lazy_path| { const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); - _ = try man.addFilePath(file_path, null); + _ = try man.addInputPath(file_path, .{}); } if (conf_run.cwd.value) |lazy_path| { @@ -261,7 +261,7 @@ pub fn make( if (!has_side_effects and try step.cacheHitWatched(maker, &man, progress_node)) { // Cache hit; skip running command. - const digest = man.final(); + const digest = man.hitDigestHex(); try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); step.result_cached = true; @@ -270,12 +270,12 @@ pub fn make( if (!any_dep_files) { // We already know the final output paths; use them directly. - const digest = if (has_side_effects) man.hash.final() else man.final(); + const digest = if (has_side_effects) man.hash.final() else man.missDigestHex(); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); try populateGeneratedPathsCreateDirs(arena, run_index, maker, output_dir_path, output_placeholders.items, argv_list.items); try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); - if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); + if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man); return; } @@ -292,23 +292,39 @@ pub fn make( switch (arg.flags.tag) { .output_file => if (arg.flags.dep_file) { const generated_path = maker.generatedPath(arg.generated.value.?).*; - const result = if (has_side_effects) - man.addDepFile(generated_path.root_dir.handle, generated_path.sub_path) - else - man.addDepFilePost(generated_path.root_dir.handle, generated_path.sub_path); - result catch |err| switch (err) { - error.OutOfMemory, error.Canceled => |e| return e, - else => |e| return step.fail(maker, "failed adding to cache the file {f}: {t}", .{ - generated_path, e, - }), - }; + if (has_side_effects) { + var diagnostic: Cache.DepTokenizer.Token = undefined; + man.addInputDepFile(generated_path, &diagnostic) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidDepFile => return step.fail(maker, "failed adding dep file {f} to cache: {f}", .{ + generated_path, diagnostic, + }), + else => |e| return step.fail(maker, "failed adding dep file {f} to cache: {t}", .{ + generated_path, e, + }), + }; + } else { + var diagnostic: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined; + man.addDiscoveredDepFile(generated_path, &diagnostic) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidDepFile => return step.fail(maker, "failed adding dep file {f} to cache: {f}", .{ + generated_path, diagnostic.dep_tokenizer, + }), + error.FileSystemFailure => return step.fail(maker, "failed adding a path from dep file {f} to cache: {f}", .{ + generated_path, diagnostic.add_discovered_path, + }), + else => |e| return step.fail(maker, "failed adding dep file {f} to cache: {t}", .{ + generated_path, e, + }), + }; + } }, .output_directory => continue, else => unreachable, } } - const digest = if (has_side_effects) man.hash.final() else man.final(); + const digest = if (has_side_effects) man.hash.final() else man.missDigestHex(); const any_output = output_placeholders.items.len > 0 or conf_run.captured_stdout.value != null or conf_run.captured_stderr.value != null; @@ -344,7 +360,7 @@ pub fn make( }; } - if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); + if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man); try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index 65b19bdbae8137e94b407fded33c31af71805bc2..6b142a02338315221f403397db090178b096ab1a 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -55,7 +55,7 @@ pub fn make( man.hash.addBytes(copy.sub_path.slice(conf)); const src_lazy_path = copy.src_file.get(conf); const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); - _ = try man.addFilePath(source_path, null); + _ = try man.addInputPath(source_path, .{}); try step.addWatchInput(maker, arena, src_lazy_path); } @@ -96,7 +96,7 @@ pub fn make( }, .file => { const entry_path = try src_dir_path.join(arena, entry.path); - _ = try man.addFilePath(entry_path, null); + _ = try man.addInputPath(entry_path, .{}); total_items += 1; }, else => continue, @@ -105,7 +105,7 @@ pub fn make( } if (try step.cacheHit(maker, &man, progress_node)) { - const digest = man.final(); + const digest = man.hitDigestHex(); maker.generatedPath(conf_wf.generated_directory).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), @@ -114,7 +114,7 @@ pub fn make( return; } - const digest = man.final(); + const digest = man.missDigestHex(); const out_path: Path = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), @@ -122,7 +122,7 @@ pub fn make( progress_node.setEstimatedTotalItems(total_items); try operate(maker, step_index, open_dir_cache, out_path, progress_node); - try step.writeManifest(maker, &man); + try step.finalizeManifest(maker, &man); maker.generatedPath(conf_wf.generated_directory).* = out_path; }, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 55271b4a6a4e2977093675e15b82cf40fe54e8cd..47993f66d4edb8905c6c497288825c103e80ba9a 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -58,7 +58,8 @@ available_deps: AvailableDeps, pub const ConfigureDependency = struct { lazy_path: LazyPath, - mode: std.Build.Configuration.PathDep.Mode, + is_directory: bool, + metadata_only: bool, }; pub const ReleaseMode = enum { @@ -102,7 +103,8 @@ pub const Graph = struct { /// Populated by calling one of: /// * `dependOnFileContents` /// * `dependOnFileMetadata` - /// * `dependOnDirectory` + /// * `dependOnDirectoryContents` + /// * `dependOnDirectoryMetadata` configure_dependencies: ArrayList(ConfigureDependency) = .empty, /// If the cache is poisoned means that the **configure logic** had side @@ -2602,7 +2604,8 @@ pub fn dependOnFileContents(b: *Build, lazy_path: LazyPath) void { const graph = b.graph; graph.configure_dependencies.append(graph.arena, .{ .lazy_path = lazy_path.dupe(graph), - .mode = .contents, + .is_directory = false, + .metadata_only = false, }) catch @panic("OOM"); } @@ -2627,7 +2630,8 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void { const graph = b.graph; graph.configure_dependencies.append(graph.arena, .{ .lazy_path = lazy_path.dupe(graph), - .mode = .metadata, + .is_directory = false, + .metadata_only = true, }) catch @panic("OOM"); } @@ -2636,6 +2640,35 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void { /// This is an alternative to `Graph.poisonCache` that avoids making every invocation /// of `zig build` into a cache miss. /// +/// If any file is created, deleted, or renamed in this directory, leaving the +/// directory in a different state than last configuration with respect to +/// existence and naming of entries, the configure phase will be repeated. +/// +/// Only a subset of `LazyPath` are supported: +/// - Relative to cwd +/// - Relative to any package root +/// - Relative to zig cache or zig installation +/// +/// If the directory would be inside one of the search prefixes, then the dependency +/// cannot be tracked; `Graph.poisonCache` must be used instead. +/// +/// Not recursive. +pub fn dependOnDirectoryContents(b: *Build, lazy_path: LazyPath) void { + validateConfigureDependency(lazy_path); + const graph = b.graph; + graph.configure_dependencies.append(graph.arena, .{ + .lazy_path = lazy_path.dupe(graph), + .is_directory = true, + .metadata_only = false, + }) catch @panic("OOM"); +} + +/// Indicates that the build.zig logic depends on a particular directory's last +/// modification date. +/// +/// This is an alternative to `Graph.poisonCache` that avoids making every invocation +/// of `zig build` into a cache miss. +/// /// If any file is created, deleted, or renamed in this directory, the /// configure phase will be repeated. /// @@ -2646,12 +2679,15 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void { /// /// If the directory would be inside one of the search prefixes, then the dependency /// cannot be tracked; `Graph.poisonCache` must be used instead. -pub fn dependOnDirectory(b: *Build, lazy_path: LazyPath) void { +/// +/// Not recursive. +pub fn dependOnDirectoryMetadata(b: *Build, lazy_path: LazyPath) void { validateConfigureDependency(lazy_path); const graph = b.graph; graph.configure_dependencies.append(graph.arena, .{ .lazy_path = lazy_path.dupe(graph), - .mode = .directory, + .is_directory = true, + .metadata_only = true, }) catch @panic("OOM"); } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 94051c51658796081d413cc18133a97c93814d55..d0a7d93e16b1207429222ee77a96361a345290e4 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -48,8 +48,7 @@ pub fn obtain(cache: *Cache) Manifest { .cache = cache, .hash = cache.hash, .manifest_file = null, - .manifest_dirty = false, - .hex_digest = undefined, + .state = .input, }; } @@ -57,71 +56,66 @@ pub fn prefixes(cache: *const Cache) []const Directory { return cache.prefixes_buffer[0..cache.prefixes_len]; } -pub const PrefixedPath = struct { - prefix: u8, +const PrefixIndex = u6; + +const PrefixedPath = struct { + prefix: PrefixIndex, sub_path: []const u8, - - fn eql(a: PrefixedPath, b: PrefixedPath) bool { - return a.prefix == b.prefix and std.mem.eql(u8, a.sub_path, b.sub_path); - } - - fn hash(pp: PrefixedPath) u32 { - return @truncate(std.hash.Wyhash.hash(pp.prefix, pp.sub_path)); - } }; -fn findPrefixPath(cache: *const Cache, path: Path) !PrefixedPath { - const gpa = cache.gpa; - const resolved_path = try std.fs.path.resolve(gpa, &.{ - cache.cwd, path.root_dir.path orelse ".", path.subPathOrDot(), - }); - errdefer gpa.free(resolved_path); - return findPrefixResolved(cache, resolved_path); +fn appendPrefixedPath(cache: *const Cache, contents: *std.ArrayList(u8), prefixed_path: PrefixedPath) !PrefixIndex { + const end = contents.items.len + prefixed_path.sub_path.len; + const needed_alignment = @alignOf(Manifest.File) - (end % @alignOf(Manifest.File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + // Extra +1 here is due to the requirement to keep at least 1 unused capacity in `Manifest.contents` at all times. + try contents.ensureTotalCapacity(cache.gpa, end + needed_alignment + 1); + contents.appendSliceAssumeCapacity(prefixed_path.sub_path); + contents.appendNTimesAssumeCapacity(0, needed_alignment); + return prefixed_path.prefix; } -fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath { - const gpa = cache.gpa; - const resolved_path = try std.fs.path.resolve(gpa, &.{file_path}); - errdefer gpa.free(resolved_path); - return findPrefixResolved(cache, resolved_path); -} - -/// Takes ownership of `resolved_path` on success. -fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath { +fn resolveAppendPath(cache: *const Cache, contents: *std.ArrayList(u8), path: Path) !PrefixIndex { const gpa = cache.gpa; const cwd = cache.cwd; + const path_start = contents.items.len; + + const resolved_path = try std.fs.path.resolveAlloc(gpa, &.{ + path.root_dir.path orelse cwd, + path.subPathOrDot(), + }); + defer gpa.free(resolved_path); + for (cache.prefixes(), 0..) |prefix, i| { - const p = prefix.path orelse continue; - const sub_path = getPrefixSubpath(gpa, cwd, p, resolved_path) catch |err| switch (err) { - error.NotASubPath => continue, - else => |e| return e, - }; - // Free the resolved path since we're not going to return it - gpa.free(resolved_path); - return .{ - .prefix = @intCast(i), - .sub_path = sub_path, - }; - } + const pp = prefix.path orelse continue; + contents.shrinkRetainingCapacity(path_start); + try std.fs.path.relativeAppend(gpa, contents, cwd, null, pp, resolved_path); + const relative = contents.items[path_start..]; - return .{ - .prefix = 0, - .sub_path = resolved_path, - }; -} + var component_iterator: std.fs.path.NativeComponentIterator = .init(relative); + if (component_iterator.root() != null) continue; + const first_component = component_iterator.first(); + if (first_component != null and mem.eql(u8, first_component.?.name, "..")) continue; -fn getPrefixSubpath(gpa: Allocator, cwd: []const u8, prefix: []const u8, path: []u8) ![]u8 { - const relative = try std.fs.path.relative(gpa, cwd, null, prefix, path); - errdefer gpa.free(relative); - var component_iterator: std.fs.path.NativeComponentIterator = .init(relative); - if (component_iterator.root() != null) { - return error.NotASubPath; - } - const first_component = component_iterator.first(); - if (first_component != null and std.mem.eql(u8, first_component.?.name, "..")) { - return error.NotASubPath; + const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + // Extra +1 here is due to the requirement to keep at least 1 unused capacity in `Manifest.contents` at all + // times. + try contents.ensureUnusedCapacity(gpa, needed_alignment + 1); + contents.appendNTimesAssumeCapacity(0, needed_alignment); + + return @intCast(i); } - return relative; + + contents.shrinkRetainingCapacity(path_start); + try contents.appendSlice(gpa, resolved_path); + + const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + // Extra +1 here is due to the requirement to keep at least 1 unused capacity in `Manifest.contents` at all times. + try contents.ensureUnusedCapacity(gpa, needed_alignment + 1); + contents.appendNTimesAssumeCapacity(0, needed_alignment); + + return 0; } /// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 @@ -130,10 +124,6 @@ pub const hex_digest_len = bin_digest_len * 2; pub const BinDigest = [bin_digest_len]u8; pub const HexDigest = [hex_digest_len]u8; -/// This is currently just an arbitrary non-empty string that can't match another manifest line. -const manifest_header = "0"; -pub const manifest_file_size_max = 100 * 1024 * 1024; - /// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it /// provides enough collision resistance for the Manifest use cases, while being one of our /// fastest options right now. @@ -143,56 +133,12 @@ pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); /// Refresh this with new random bytes when the manifest /// format is modified in a non-backwards-compatible way. pub const hasher_init: Hasher = Hasher.init(&.{ - 0x33, 0x52, 0xa2, 0x84, - 0xcf, 0x17, 0x56, 0x57, - 0x01, 0xbb, 0xcd, 0xe4, - 0x77, 0xd6, 0xf0, 0x60, + 0x02, 0xe9, 0xfa, 0xfe, + 0xe0, 0x95, 0x81, 0x55, + 0xf5, 0x5a, 0x15, 0xcb, + 0x4a, 0xf4, 0x00, 0x09, }); -pub const File = struct { - prefixed_path: PrefixedPath, - max_file_size: ?usize, - /// Populated if the user calls `addOpenedFile`. - /// The handle is not owned here. - handle: ?Io.File, - stat: Stat, - bin_digest: BinDigest, - contents: ?[]const u8, - - pub const Stat = struct { - inode: Io.File.INode, - size: u64, - mtime: Io.Timestamp, - - pub fn fromFs(fs_stat: Io.File.Stat) Stat { - return .{ - .inode = fs_stat.inode, - .size = fs_stat.size, - .mtime = fs_stat.mtime, - }; - } - }; - - pub fn deinit(self: *File, gpa: Allocator) void { - gpa.free(self.prefixed_path.sub_path); - if (self.contents) |contents| { - gpa.free(contents); - self.contents = null; - } - self.* = undefined; - } - - pub fn updateMaxSize(file: *File, new_max_size: ?usize) void { - const new = new_max_size orelse return; - file.max_file_size = if (file.max_file_size) |old| @max(old, new) else new; - } - - pub fn updateHandle(file: *File, new_handle: ?Io.File) void { - const handle = new_handle orelse return; - file.handle = handle; - } -}; - pub const HashHelper = struct { hasher: Hasher = hasher_init, @@ -299,7 +245,7 @@ pub const HashHelper = struct { pub fn binToHex(bin_digest: BinDigest) HexDigest { var out_digest: HexDigest = undefined; - var w: std.Io.Writer = .fixed(&out_digest); + var w: Io.Writer = .fixed(&out_digest); w.printHex(&bin_digest, .lower) catch unreachable; return out_digest; } @@ -319,33 +265,337 @@ pub const Lock = struct { } }; +/// Format: a series of consecutive `Manifest.File`, followed by a final +/// terminating zero byte to distinguish empty manifest file from manifest with +/// zero files. pub const Manifest = struct { cache: *Cache, /// Current state for incremental hashing. hash: HashHelper, + /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. manifest_file: ?Io.File, - manifest_dirty: bool, - /// Set this flag to true before calling hit() in order to indicate that - /// upon a cache hit, the code using the cache will not modify the files - /// within the cache directory. This allows multiple processes to utilize - /// the same cache directory at the same time. + state: State, + /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code + /// using the cache will not modify the files within the cache directory. This allows multiple processes + /// to utilize the same cache directory at the same time. want_shared_lock: bool = true, have_exclusive_lock: bool = false, // Indicate that we want isProblematicTimestamp to perform a filesystem write in // order to obtain a problematic timestamp for the next call. Calls after that // will then use the same timestamp, to avoid unnecessary filesystem writes. want_refresh_timestamp: bool = true, - files: Files = .{}, - hex_digest: HexDigest, - diagnostic: Diagnostic = .none, + /// Uses `Cache.gpa`. + files: Files = .empty, + /// Indexes line up with `files`, but only up until `hit` is called. Uses + /// `Cache.gpa`. + input_paths: std.ArrayList(InputPath) = .empty, /// Keeps track of the last time we performed a file system write to observe /// what time the file system thinks it is, according to its own granularity. recent_problematic_timestamp: Io.Timestamp = .zero, + /// The entire manifest file contents, except for the final terminating + /// zero byte. However maintains always at least 1 unused capacity so the + /// final terminating byte can be added without allocation. Uses + /// `Cache.gpa`. + contents: std.ArrayList(u8) = .empty, + /// All contents from all `input_paths` whose contents were requested, + /// concatenated. Total byte size will be less than `max_input_content_len` + /// otherwise an error is returned. + /// + /// Data is invalidated when `addDiscoveredPath` is called. + all_input_content: std.ArrayList(u8) = .empty, + max_input_content_len: usize = std.math.maxInt(u32), - pub const Diagnostic = union(enum) { - none, + pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); + + pub const State = enum { + /// Call `addInputPath`, `addInputPathOptional`, and methods of `Manifest.hash`. Once all input files and state + /// have been added, call `check`. + input, + /// `check` has already been called and a hit occurred. Call `hitDigest` to find cache artifact directory. + hit, + /// `hitDigest` has already been called. No further operations are allowed. + hit_digested, + /// `check` has already been called and a miss occurred. `missDigest` has not been called yet. Miss diagnostic + /// data may still be accessed. Call `addDiscoveredPath` to add discovered files and directories. + miss, + /// A miss occurred, `missDigest` has not been called yet. Call `addDiscoveredPath` to add more discovered files + /// and directories. Miss diagnostic data may no longer be accessed. + miss_discovered, + /// `missDigest` has been called, but `finalize` has not been called yet. + miss_digested, + /// The cache manifest has been written to disk such that the next check for the same inputs will yield a hit. + /// No further operations are allowed. + miss_finalized, + }; + + /// Source files and directories whose prefix and relative path are + /// included when computing the cache manifest digest. It's the information + /// needed to lazily hash the input files only when a cache miss occurs. + /// + /// `File.prefix`, `File.path`, and `File.mode` will be always populated, + /// but the other fields of `File` will be populated depending on the + /// fields of `InputPath`. + pub const InputPath = struct { + /// Determines whether `check` will keep the handle open or close it. + request_handle: bool, + /// Determines whether `handle` is populated. + have_handle: bool, + /// Determines whether `File.size`, `File.inode`, and `File.mtime` are populated. + have_stat: bool, + /// Determines whether `File.digest` is populated. + have_digest: bool, + contents: struct { + pop_off: enum(u32) { unpopulated = std.math.maxInt(u32), _ }, + req_len: enum(u32) { unrequested = std.math.maxInt(u32) - 1, requested = std.math.maxInt(u32), _ }, + + pub const requested: @This() = .{ + .pop_off = .unpopulated, + .req_len = .requested, + }; + + pub const unrequested: @This() = .{ + .pop_off = .unpopulated, + .req_len = .unrequested, + }; + + pub fn populated(p: Populated) @This() { + return .{ + .pop_off = @fromBackingInt(p.off), + .req_len = @fromBackingInt(p.len), + }; + } + + pub const Unwrapped = union(enum) { + unrequested, + requested, + populated: Populated, + }; + + pub const Populated = struct { + off: u32, + len: u32, + }; + + pub fn unwrap(this: @This()) Unwrapped { + return switch (this.pop_off) { + .unpopulated => switch (this.req_len) { + .unrequested => .unrequested, + .requested => .requested, + _ => unreachable, + }, + _ => .{ .populated = .{ + .off = @backingInt(this.pop_off), + .len = @backingInt(this.req_len), + } }, + }; + } + + pub fn get(this: @This(), all_contents: []const u8) ?[]const u8 { + return switch (this.unwrap()) { + .unrequested, .requested => null, + .populated => |p| all_contents[p.off..][0..p.len], + }; + } + }, + /// `have_handle` determines whether this is populated. + handle: union { + file: Io.File, + dir: Io.Dir, + }, + + pub const Handle = union(enum) { + none, + file: Io.File, + dir: Io.Dir, + }; + + /// Index into `Manifest.input_paths`. + pub const Index = enum(u32) { + _, + + pub fn get(i: @This(), manifest: *const Manifest) InputPath { + return manifest.input_paths.items[@backingInt(i)]; + } + + pub fn offset(i: @This(), manifest: *const Manifest) File.Offset { + return manifest.files.keys()[@backingInt(i)]; + } + + pub fn contents(i: @This(), manifest: *const Manifest) []const u8 { + const input_path = get(i, manifest); + return input_path.contents.get(manifest.all_input_content.items).?; + } + }; + + pub fn getHandle(this: @This(), file: *const File) Handle { + return if (!this.have_handle) .none else switch (file.flags.is_directory) { + false => .{ .file = this.handle.file }, + true => .{ .dir = this.handle.dir }, + }; + } + }; + + /// The data per tracked input file that is stored in the manifest file. + pub const File = extern struct { + size: u64 align(1), + inode: Io.File.INode align(1), + /// Nanoseconds. + mtime: i64 align(1), + /// To simplify the hashing logic, this value is computed from size, inode, and mtime + /// in `hashFromMetadata` in the case that `Flags.metadata_only` is `true`. + digest: BinDigest align(1), + /// Starting with this field and continuing into the path, excluding the null byte, + /// is the string that is hashed for the manifest digest. + flags: Flags align(1), + /// Terminated by zero byte, then followed by padding until 8-byte aligned. + path_start: [0]u8 align(1), + + pub const Flags = packed struct(u8) { + is_directory: bool, + metadata_only: bool, + prefix: PrefixIndex, + }; + + /// Prefixes path names in encoded directory contents. Starts numbering + /// at `1` so that null byte can be used unambiguously as entry + /// separator. + pub const Kind = enum(u8) { + file = 1, + directory = 2, + other = 3, + + pub fn fromStat(kind: Io.File.Kind) @This() { + return switch (kind) { + .file => .file, + .directory => .directory, + else => .other, + }; + } + }; + + /// Byte index within `Manifest.contents` where the entry starts. + pub const Offset = enum(u32) { + _, + + pub fn get(offset: Offset, contents: []u8) *File { + return @constCast(getConst(offset, contents)); + } + + pub fn getConst(offset: Offset, contents: []const u8) *const File { + return @ptrCast(@alignCast(contents[@backingInt(offset)..][0..@sizeOf(File)])); + } + + pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File { + // TODO make @constCast support in-memory coercion across error unions and optionals + return @constCast(try getFallibleConst(offset, contents)); + } + + pub fn getFallibleConst(offset: Offset, contents: []const u8) error{InvalidFormat}!*const File { + if (@backingInt(offset) + @sizeOf(File) >= contents.len) return error.InvalidFormat; + if (!mem.isAligned(@backingInt(offset), @alignOf(File))) return error.InvalidFormat; + return getConst(offset, contents); + } + + pub fn flagsAndPath(off: File.Offset, contents: []const u8) [:0]const u8 { + const flags_off = @offsetOf(File, "flags"); + comptime assert(@offsetOf(File, "path_start") - flags_off == 1); + const start = @backingInt(off) + flags_off; + // Scan for the sentinel starting from the path_start offset because flags might be zero. + const end = mem.findScalarPos(u8, contents, start + 1, 0).?; + return contents[start..end :0]; + } + + pub fn pathFallible(off: File.Offset, contents: []const u8) error{InvalidFormat}![:0]const u8 { + const path_start = @backingInt(off) + @offsetOf(File, "path_start"); + const path_end = mem.findScalarPos(u8, contents, path_start, 0) orelse return error.InvalidFormat; + return contents[path_start..path_end :0]; + } + + pub fn path(off: File.Offset, contents: []const u8) [:0]const u8 { + return pathFallible(off, contents) catch unreachable; + } + }; + + /// Intentionally matches if the files are different only by flags other than prefix. + pub const HashContext = struct { + contents: []const u8, + + pub fn hash(this: @This(), off: Offset) u32 { + const file_prefix = off.getConst(this.contents).flags.prefix; + const file_path = off.path(this.contents); + return @truncate(std.hash.Wyhash.hash(file_prefix, file_path)); + } + + pub fn eql(this: @This(), a_off: Offset, b_off: Offset, b_index: usize) bool { + _ = b_index; + const a_prefix = a_off.getConst(this.contents).flags.prefix; + const b_prefix = b_off.getConst(this.contents).flags.prefix; + if (a_prefix != b_prefix) return false; + const a_path = a_off.path(this.contents); + const b_path = b_off.path(this.contents); + return mem.eql(u8, a_path, b_path); + } + }; + + fn hashFromMetadata(file: *File) void { + var hasher = hasher_init; + hasher.update(mem.asBytes(&file.size)); + hasher.update(mem.asBytes(&file.inode)); + hasher.update(mem.asBytes(&file.mtime)); + hasher.final(&file.digest); + } + + fn stat(file: *const File) Stat { + return .{ + .size = file.size, + .inode = file.inode, + .mtime = .fromNanoseconds(file.mtime), + }; + } + + fn setStatUnchecked(file: *File, s: Stat) void { + file.size = s.size; + file.inode = s.inode; + file.mtime = @intCast(s.mtime.toNanoseconds()); + } + + fn setStat(file: *File, m: *Manifest, s: Stat) Io.Cancelable!void { + setStatUnchecked(file, s); + + if (try m.isProblematicTimestamp(s.mtime)) { + // The actual file has an unreliable timestamp; force it to be hashed. + file.mtime = 0; + file.inode = 0; + } + } + + /// Returns true if the stat was changed. Updates the `file` with the new stat value. + fn setStatChanged(file: *File, m: *Manifest, s: Stat) Io.Cancelable!bool { + if (s.size == file.size and + s.mtime.nanoseconds == file.mtime and + s.inode == file.inode) + { + return false; + } else { + try setStat(file, m, s); + return true; + } + } + + /// `path_len` does not include the null byte. + pub fn sizeOf(path_len: usize) usize { + const end = @offsetOf(File, "path_start") + path_len; + const needed_alignment = @alignOf(File) - (end % @alignOf(File)); + assert(needed_alignment >= 1); // Always need at least a null byte. + return end + needed_alignment; + } + }; + + pub const CheckDiagnostic = union(enum) { manifest_create: Io.File.OpenError, - manifest_read: Io.File.Reader.Error, + manifest_stat: Io.File.StatError, + manifest_oversize, + manifest_read: Io.File.ReadPositionalError, manifest_lock: Io.File.LockError, file_open: FileOp, file_stat: FileOp, @@ -353,115 +603,274 @@ pub const Manifest = struct { file_hash: FileOp, pub const FileOp = struct { - file_index: usize, + file_offset: File.Offset, err: anyerror, + + /// Returned `Path` references `Manifest.contents`. + pub fn path(fo: FileOp, manifest: *const Manifest) Path { + const contents = manifest.contents.items; + const prefix = fo.file_offset.get(contents).flags.prefix; + return .{ + .root_dir = manifest.cache.prefixes()[prefix], + .sub_path = fo.file_offset.path(contents), + }; + } }; - }; - pub const Files = std.array_hash_map.Custom(File, void, FilesContext, false); + pub const Format = struct { + diagnostic: CheckDiagnostic, + manifest: *const Manifest, + + pub fn format(this: @This(), w: *Io.Writer) Io.Writer.Error!void { + switch (this.diagnostic) { + .manifest_oversize => return w.writeAll(@tagName(this.diagnostic)), + .manifest_create, .manifest_stat, .manifest_read, .manifest_lock => |e| { + return w.print("{t} {t}", .{ this.diagnostic, e }); + }, + .file_open, .file_stat, .file_read, .file_hash => |op| { + const path = op.path(this.manifest); + return w.print("{t} {t} {f}", .{ this.diagnostic, op.err, path }); + }, + } + } + }; - pub const FilesContext = struct { - pub fn hash(fc: FilesContext, file: File) u32 { - _ = fc; - return file.prefixed_path.hash(); + pub fn fmt(diagnostic: CheckDiagnostic, manifest: *const Manifest) Format { + return .{ + .diagnostic = diagnostic, + .manifest = manifest, + }; } + }; + + pub const Stat = struct { + size: u64, + inode: Io.File.INode, + mtime: Io.Timestamp, - pub fn eql(fc: FilesContext, a: File, b: File, b_index: usize) bool { - _ = fc; - _ = b_index; - return a.prefixed_path.eql(b.prefixed_path); + pub fn init(other: Io.File.Stat) Stat { + return .{ + .size = other.size, + .inode = other.inode, + .mtime = other.mtime, + }; } }; - const FilesAdapter = struct { - pub fn eql(context: @This(), a: PrefixedPath, b: File, b_index: usize) bool { - _ = context; - _ = b_index; - return a.eql(b.prefixed_path); + pub const PathHandle = union(enum) { + file: ?Io.File, + /// If provided, this handle must be opened with iteration capability. + dir: ?Io.Dir, + + pub fn isDirectory(this: @This()) bool { + return this == .dir; } - pub fn hash(context: @This(), key: PrefixedPath) u32 { - _ = context; - return key.hash(); + pub fn have(this: @This()) bool { + return switch (this) { + .file => |opt_file| opt_file != null, + .dir => |opt_dir| opt_dir != null, + }; } }; - /// Add a file as a dependency of process being cached. When `hit` is - /// called, the file's contents will be checked to ensure that it matches - /// the contents from previous times. + pub const AddInputPathOptions = struct { + /// If provided, will be closed when `check` is called, unless `request_handle` is also set. + handle: PathHandle = .{ .file = null }, + stat: ?Stat = null, + /// If set, file handle will remain open after `check` is called. + request_handle: bool = false, + /// Can request file or directory contents depending on `handle`. + request_contents: bool = false, + /// Content hashing skipped; any difference in metadata implies cache + /// miss. + metadata_only: bool = false, + }; + + /// Add a file or directory path as a dependency of process being cached. + /// When `hit` is called, the contents will be checked to ensure + /// that it matches the contents from previous times. /// - /// Max file size will be used to determine the amount of space the file contents - /// are allowed to take up in memory. If max_file_size is null, then the contents - /// will not be loaded into memory. + /// The contents of the input file may be requested and subsequently + /// obtained via methods of the returned `InputPath.Index` after calling + /// `hit`. /// - /// Returns the index of the entry in the `files` array list. You can use it - /// to access the contents of the file after calling `hit()` like so: + /// Contents of a directory are considered to be the sorted list of file + /// names of direct entries, separated by null byte. Each file name is + /// prefixed by `Io.File.Kind` byte, +1 so that the zero tag is not aliased + /// by the entry separator. /// - /// ``` - /// var file_contents = cache_hash.files.keys()[file_index].contents.?; - /// ``` - pub fn addFilePath(m: *Manifest, file_path: Path, max_file_size: ?usize) !usize { - return addOpenedFile(m, file_path, null, max_file_size); - } - - /// Same as `addFilePath` except the file has already been opened. - pub fn addOpenedFile(m: *Manifest, path: Path, handle: ?Io.File, max_file_size: ?usize) !usize { - const gpa = m.cache.gpa; - try m.files.ensureUnusedCapacity(gpa, 1); - const resolved_path = try std.fs.path.resolve(gpa, &.{ - path.root_dir.path orelse ".", - path.subPathOrDot(), - }); - errdefer gpa.free(resolved_path); - const prefixed_path = try m.cache.findPrefixResolved(resolved_path); - return addFileInner(m, prefixed_path, handle, max_file_size); - } - - fn addFileInner(self: *Manifest, prefixed_path: PrefixedPath, handle: ?Io.File, max_file_size: ?usize) usize { - const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{}); + /// See also: + /// * `addDiscoveredPath` + pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { + assert(m.state == .input); + const cache = m.cache; + const gpa = cache.gpa; + const is_directory = options.handle.isDirectory(); + + try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); + try m.input_paths.ensureUnusedCapacity(gpa, 1); + + const new_file_offset: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); + try m.contents.appendNTimes(gpa, 0, @offsetOf(File, "path_start")); + errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); + + const new_prefix = try cache.resolveAppendPath(&m.contents, path); + assert(mem.isAligned(m.contents.items.len, @alignOf(File))); + new_file_offset.get(m.contents.items).flags = .{ + .prefix = new_prefix, + .is_directory = is_directory, + .metadata_only = options.metadata_only, + }; + + const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ .contents = m.contents.items }); + m.files.lockPointers(); + defer m.files.unlockPointers(); + if (gop.found_existing) { - self.cache.gpa.free(prefixed_path.sub_path); - gop.key_ptr.updateMaxSize(max_file_size); - gop.key_ptr.updateHandle(handle); - return gop.index; + m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); + const existing_input_file = &m.input_paths.items[gop.index]; + switch (options.handle) { + .file => |opt_file| if (opt_file) |file| { + existing_input_file.handle = .{ .file = file }; + existing_input_file.have_handle = true; + }, + .dir => |opt_dir| if (opt_dir) |dir| { + existing_input_file.handle = .{ .dir = dir }; + existing_input_file.have_handle = true; + }, + } + if (options.request_contents) switch (existing_input_file.contents.unwrap()) { + .requested, .unrequested => existing_input_file.contents = .requested, + .populated => {}, + }; + const existing_header = m.files.keys()[gop.index].get(m.contents.items); + if (options.stat) |stat| { + existing_input_file.have_stat = true; + existing_header.size = stat.size; + existing_header.inode = stat.inode; + existing_header.mtime = @intCast(stat.mtime.toNanoseconds()); + } + // If it trips, the same file path has been added to the cache + // manifest both as a directory and as a normal file, making the + // intended caching behavior ambiguous. + assert(existing_header.flags.is_directory == is_directory); + if (!options.metadata_only) + existing_header.flags.metadata_only = false; + } else { + m.input_paths.appendAssumeCapacity(.{ + .request_handle = options.request_handle, + .have_handle = options.handle.have(), + .handle = switch (options.handle) { + .file => |opt_file| if (opt_file) |file| .{ .file = file } else undefined, + .dir => |opt_dir| if (opt_dir) |dir| .{ .dir = dir } else undefined, + }, + .contents = if (options.request_contents) .requested else .unrequested, + .have_digest = false, + .have_stat = options.stat != null, + }); + assert(m.input_paths.items.len - 1 == gop.index); + if (options.stat) |stat| { + const header = new_file_offset.get(m.contents.items); + header.size = stat.size; + header.inode = stat.inode; + header.mtime = @intCast(stat.mtime.toNanoseconds()); + } } - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .contents = null, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - .handle = handle, + return @fromBackingInt(@intCast(gop.index)); + } + + pub fn addInputPathOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { + assert(m.state == .input); + m.hash.add(opt_path != null); + _ = try addInputPath(m, opt_path orelse return, options); + } + + pub const AddInputDepFileError = error{ + InvalidDepFile, + } || Allocator.Error || Io.File.OpenError || Io.File.Reader.Error; + + pub fn addInputDepFile(m: *Manifest, path: Path, diagnostic: ?*DepTokenizer.Token) AddInputDepFileError!void { + assert(m.manifest_file == null); + assert(m.state == .input); + + const cache = m.cache; + const gpa = cache.gpa; + const io = cache.io; + + // TODO: change DepTokenizer to be streaming rather than operating on slice of bytes. + const prev_len = m.all_input_content.items.len; + defer m.all_input_content.items.len = prev_len; + + const file = try path.root_dir.handle.openFile(io, path.sub_path, .{}); + defer file.close(io); + + var file_reader: Io.File.Reader = .init(file, io, &.{}); + file_reader.interface.appendRemainingUnlimited(gpa, &m.all_input_content) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return file_reader.err.?, }; - self.hash.add(prefixed_path.prefix); - self.hash.addBytes(prefixed_path.sub_path); + var aux: std.ArrayList(u8) = .empty; + defer aux.deinit(gpa); - return gop.index; + var it: DepTokenizer = .{ .bytes = m.all_input_content.items[prev_len..] }; + while (it.next()) |token| switch (token) { + .target, .target_must_resolve => {}, + .prereq => |file_path| { + _ = try m.addInputPath(.initCwd(file_path), .{}); + }, + .prereq_must_resolve => { + aux.clearRetainingCapacity(); + try token.resolve(gpa, &aux); + _ = try m.addInputPath(.initCwd(aux.items), .{}); + }, + else => |err| { + if (diagnostic) |d| d.* = err; + return error.InvalidDepFile; + }, + }; } - pub fn addOptionalFilePath(self: *Manifest, optional_file_path: ?Path) !void { - self.hash.add(optional_file_path != null); - const file_path = optional_file_path orelse return; - _ = try self.addFilePath(file_path, null); - } + pub const CheckResult = union(enum) { + hit, + incomplete_manifest, + invalid_manifest, + path_deleted: File.Offset, + directory_status_changed: File.Offset, + metadata_changed: File.Offset, + contents_changed: File.Offset, - pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { - assert(self.manifest_file == null); - return self.addDepFileMaybePost(dir, dep_file_sub_path); - } + pub const Format = struct { + check_result: CheckResult, + manifest: *const Manifest, + + pub fn format(this: @This(), w: *Io.Writer) Io.Writer.Error!void { + switch (this.check_result) { + .hit, .incomplete_manifest, .invalid_manifest => return w.writeAll(@tagName(this.check_result)), + .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |off| { + const path = off.path(this.manifest.contents.items); + return w.print("{t} {s}", .{ this.check_result, path }); + }, + } + } + }; + + pub fn fmt(check_result: CheckResult, manifest: *const Manifest) Format { + return .{ + .check_result = check_result, + .manifest = manifest, + }; + } + }; - pub const HitError = error{ + pub const CheckError = error{ /// Unable to check the cache for a reason that has been recorded into /// the `diagnostic` field. CacheCheckFailed, - /// A cache manifest file exists however it could not be parsed. - InvalidFormat, - OutOfMemory, - Canceled, - }; + } || Allocator.Error || Io.Cancelable; - /// Check the cache to see if the input exists in it. If it exists, returns `true`. + /// Check the cache to see if the input exists in it. /// A hex encoding of its hash is available by calling `final`. /// /// This function will also acquire an exclusive lock to the manifest file. This means @@ -473,52 +882,53 @@ pub const Manifest = struct { /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn hit(man: *Manifest, parent_progress_node: std.Progress.Node) HitError!bool { + pub fn check( + man: *Manifest, + diag: *CheckDiagnostic, + parent_progress_node: std.Progress.Node, + ) CheckError!CheckResult { const node = parent_progress_node.start("Reusing Cache Artifacts", 0); defer node.end(); - return hitInner(man); + return checkProgressless(man, diag); } - pub fn hitInner(self: *Manifest) HitError!bool { - assert(self.manifest_file == null); + pub fn checkProgressless(man: *Manifest, diag: *CheckDiagnostic) CheckError!CheckResult { + assert(man.state == .input); + assert(man.manifest_file == null); - self.diagnostic = .none; + // This is *not* hashing the contents of the input files. It is the + // flags (including prefix) and path only. + for (man.files.keys()[0..man.input_paths.items.len]) |file_off| { + const flags_and_path = file_off.flagsAndPath(man.contents.items); + man.hash.hasher.update(mem.absorbSentinel(flags_and_path)); + } - const ext = ".txt"; - var manifest_file_path: [hex_digest_len + ext.len]u8 = undefined; + var input_digest: BinDigest = undefined; + man.hash.hasher.final(&input_digest); + const input_hex_digest = binToHex(input_digest); - var bin_digest: BinDigest = undefined; - self.hash.hasher.final(&bin_digest); - - self.hex_digest = binToHex(bin_digest); - - @memcpy(manifest_file_path[0..self.hex_digest.len], &self.hex_digest); - manifest_file_path[hex_digest_len..][0..ext.len].* = ext.*; - - const io = self.cache.io; + const manifest_file_path = &input_hex_digest; + const io = man.cache.io; // We'll try to open the cache with an exclusive lock, but if that would block // and `want_shared_lock` is set, a shared lock might be sufficient, so we'll // open with a shared lock instead. while (true) { - if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{ + if (man.cache.manifest_dir.createFile(io, manifest_file_path, .{ .read = true, .truncate = false, .lock = .exclusive, - .lock_nonblocking = self.want_shared_lock, + .lock_nonblocking = man.want_shared_lock, })) |manifest_file| { - self.manifest_file = manifest_file; - self.have_exclusive_lock = true; + man.manifest_file = manifest_file; + man.have_exclusive_lock = true; break; } else |err| switch (err) { error.WouldBlock => { - self.manifest_file = self.cache.manifest_dir.openFile(io, &manifest_file_path, .{ + man.manifest_file = man.cache.manifest_dir.openFile(io, manifest_file_path, .{ .mode = .read_write, .lock = .shared, - }) catch |e| { - self.diagnostic = .{ .manifest_create = e }; - return error.CacheCheckFailed; - }; + }) catch |e| return fail(diag, .{ .manifest_create = e }); break; }, error.FileNotFound => { @@ -531,317 +941,699 @@ pub const Manifest = struct { // disambiguates by returning EEXIST, indicating original // failure was a race, or ENOENT, indicating deletion of // the directory of our open handle. - if (!builtin.os.tag.isDarwin()) { - self.diagnostic = .{ .manifest_create = error.FileNotFound }; - return error.CacheCheckFailed; - } + if (!builtin.os.tag.isDarwin()) return fail(diag, .{ .manifest_create = error.FileNotFound }); - if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{ + if (man.cache.manifest_dir.createFile(io, manifest_file_path, .{ .read = true, .truncate = false, .lock = .exclusive, - .lock_nonblocking = self.want_shared_lock, + .lock_nonblocking = man.want_shared_lock, .exclusive = true, })) |manifest_file| { - self.manifest_file = manifest_file; - self.have_exclusive_lock = true; + man.manifest_file = manifest_file; + man.have_exclusive_lock = true; break; } else |excl_err| switch (excl_err) { error.WouldBlock, error.PathAlreadyExists => continue, - error.FileNotFound => { - self.diagnostic = .{ .manifest_create = error.FileNotFound }; - return error.CacheCheckFailed; - }, + error.FileNotFound => return fail(diag, .{ .manifest_create = error.FileNotFound }), error.Canceled => |e| return e, - else => |e| { - self.diagnostic = .{ .manifest_create = e }; - return error.CacheCheckFailed; - }, + else => |e| return fail(diag, .{ .manifest_create = e }), } }, error.Canceled => |e| return e, - else => |e| { - self.diagnostic = .{ .manifest_create = e }; - return error.CacheCheckFailed; - }, + else => |e| return fail(diag, .{ .manifest_create = e }), } } - self.want_refresh_timestamp = true; + man.want_refresh_timestamp = true; - const input_file_count = self.files.entries.len; - - // We're going to construct a second hash. Its input will begin with the digest we've - // already computed (`bin_digest`), and then it'll have the digests of each input file, - // including "post" files (see `addFilePost`). If this is a hit, we learn the set of "post" - // files from the manifest on disk. If this is a miss, we'll learn those from future calls - // to `addFilePost` etc. As such, the state of `self.hash.hasher` after this function - // depends on whether this is a hit or a miss. + // We're going to construct a second hash. Its input will begin with the digest we've already computed + // (`input_digest`), and then it'll have the digests of each input file, including discovered files (see + // `addDiscoveredPath`). If this is a hit, we learn the set of discovered files from the manifest on disk. If + // this is a miss, we'll learn those from future calls to `addDiscoveredPath` etc. As such, the state of + // `man.hash.hasher` after this function depends on whether this is a hit or a miss. // - // If we return `true` indicating a cache hit, then `self.hash.hasher` must already include - // the digests of the "post" files, so the caller can call `final`. Otherwise, on a cache - // miss, `self.hash.hasher` will include the digests of all non-"post" files -- that is, - // the ones we've already been told about. The rest will be discovered through calls to - // `addFilePost` etc, which will update the hasher. After all files are added, the user can - // use `final`, and will at some point `writeManifest` the file list to disk. - - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); + // If we return `CacheStatus.hit`, then `man.hash.hasher` must already include the digests of the discovered + // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests + // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered + // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user + // can use `final`, and will at some point `finalize` the file list to disk. + man.hash.hasher = hasher_init; + man.hash.hasher.update(&input_digest); hit: { - const file_digests_populated: usize = digests: { - switch (try self.hitWithCurrentLock()) { - .hit => break :hit, - .miss => |m| if (!try self.upgradeToExclusiveLock()) { - break :digests m.file_digests_populated; - }, - } - // We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone - // else might have modified the digest, so we need to check again before deciding to miss. - // Before trying again, we must reset `self.hash.hasher` and `self.files`. - // This is basically just the first half of `unhit`. - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - while (self.files.count() != input_file_count) { - var file = self.files.pop().?; - file.key.deinit(self.cache.gpa); - } - switch (try self.hitWithCurrentLock()) { - .hit => break :hit, - .miss => |m| break :digests m.file_digests_populated, + const miss_result = miss: { + const result = try man.checkLocked(diag); + if (result == .hit) { + break :hit; + } else if (!try man.upgradeToExclusiveLock(diag)) { + break :miss result; } + // Missed with the shared lock, and upgraded to an exclusive lock. However, another process may have + // modified the cache directory, so we need to check again before deciding to miss. + man.shrinkFilesToInput(); + const refreshed_result = try man.checkLocked(diag); + if (refreshed_result == .hit) break :hit; + break :miss refreshed_result; }; - // This is a guaranteed cache miss. We're almost ready to return `false`, but there's a - // little bookkeeping to do first. The first `file_digests_populated` entries in `files` - // have their `bin_digest` populated; there may be some left in `input_file_count` which - // we'll need to populate ourselves. Other than that, this is basically `unhit`. - self.manifest_dirty = true; - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - while (self.files.count() != input_file_count) { - var file = self.files.pop().?; - file.key.deinit(self.cache.gpa); - } - for (self.files.keys(), 0..) |*file, idx| { - if (idx < file_digests_populated) { - // `bin_digest` is already populated by `hitWithCurrentLock`, so we can use it directly. - self.hash.hasher.update(&file.bin_digest); - } else { - self.populateFileHash(file) catch |err| { - self.diagnostic = .{ .file_hash = .{ - .file_index = idx, - .err = err, - } }; - return error.CacheCheckFailed; - }; - } - } - return false; + // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a + // cache miss, which is needed because they will be used in the manifest digest. + man.state = .miss; + return miss_result; } - if (self.want_shared_lock) { - self.downgradeToSharedLock() catch |err| { - self.diagnostic = .{ .manifest_lock = err }; - return error.CacheCheckFailed; - }; + if (man.want_shared_lock) { + man.downgradeToSharedLock() catch |err| return fail(diag, .{ .manifest_lock = err }); } - return true; + man.state = .hit; + return .hit; } - /// Assumes that `self.hash.hasher` has been updated only with the original digest and that - /// `self.files` contains only the original input files. - fn hitWithCurrentLock(self: *Manifest) HitError!union(enum) { - hit, - miss: struct { - file_digests_populated: usize, - }, - } { - const gpa = self.cache.gpa; - const io = self.cache.io; - const input_file_count = self.files.entries.len; - var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded - var manifest_reader = self.manifest_file.?.reader(io, &tiny_buffer); // Reads positionally from zero. - const limit: std.Io.Limit = .limited(manifest_file_size_max); - const file_contents = manifest_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.StreamTooLong => return error.OutOfMemory, - error.ReadFailed => { - self.diagnostic = .{ .manifest_read = manifest_reader.err.? }; - return error.CacheCheckFailed; + fn transitionToMissDiscovered(m: *Manifest) void { + switch (m.state) { + .input => unreachable, + .hit => unreachable, + .hit_digested => unreachable, + .miss => { + m.shrinkFilesToInput(); + m.state = .miss_discovered; }, + .miss_discovered => {}, + .miss_digested => unreachable, + .miss_finalized => unreachable, + } + } + + fn shrinkFilesToInput(m: *Manifest) void { + if (m.files.count() <= m.input_paths.items.len) return; + // Reads from files hash map whose data is destroyed on the next line. + const off = m.files.keys()[m.input_paths.items.len]; + // Reads from the unshrunken contents whose data is destroyed on the next line. + m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items }); + m.contents.shrinkRetainingCapacity(@backingInt(off)); + assert(mem.isAligned(m.contents.items.len, @alignOf(File))); + } + + /// Does not observe or modify `self.hash.hasher`. Asserts `self.files` contains only the original input files. + fn checkLocked(m: *Manifest, diag: *CheckDiagnostic) CheckError!CheckResult { + assert(m.files.count() == m.input_paths.items.len); + const gpa = m.cache.gpa; + const io = m.cache.io; + const manifest_file = m.manifest_file.?; + + const manifest_stat = manifest_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .manifest_stat = e }), }; - defer gpa.free(file_contents); - - var any_file_changed = false; - var line_iter = mem.tokenizeScalar(u8, file_contents, '\n'); - var idx: usize = 0; - const header_valid = valid: { - const line = line_iter.next() orelse break :valid false; - break :valid std.mem.eql(u8, line, manifest_header); + const manifest_size = std.math.cast(u32, manifest_stat.size) orelse + return fail(diag, .manifest_oversize); + + if (manifest_size == 0 or manifest_size < m.contents.items.len) { + // Manifest file was never finalized. + try m.contents.ensureUnusedCapacity(gpa, 1); + return populateMissingInputFileHashes(m, diag, 0, .incomplete_manifest); + } + + // We must not clobber existing `Manifest.contents` because it possibly contains prepopulated stat and digest + // information which we must compare with the file system. Since non input files are typically greater in number + // than input files, the strategy here is to move the contents to make room for entirety of the manifest + // contents on disk, and then later truncate the contents array. + // + // The file on disk already supposedly includes the extra null byte that we have to maintain. + const input_contents_len = m.contents.items.len; + // After this resize, every return statement needs to modify m.contents length. + try m.contents.resize(gpa, input_contents_len + manifest_size); + errdefer m.contents.shrinkRetainingCapacity(input_contents_len); + @memcpy(m.contents.items[manifest_size..][0..input_contents_len], m.contents.items[0..input_contents_len]); + { + const n = manifest_file.readPositionalAll(io, m.contents.items, 0) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .manifest_read = e }), + }; + if (n != manifest_size) return missInput(m, diag, 0, input_contents_len, manifest_size, .incomplete_manifest); + } + const disk_contents = m.contents.items[0..manifest_size]; + const input_contents = m.contents.items[manifest_size..][0..input_contents_len]; + + // First the input files section, which must match our input files, otherwise it's invalid or incomplete. + var off: usize = 0; + for (m.input_paths.items, m.files.keys()[0..m.input_paths.items.len], 0..) |*input_path, file_off, i| { + if (@backingInt(file_off) + 1 >= disk_contents.len) + return missInput(m, diag, i, input_contents_len, manifest_size, .incomplete_manifest); + if (!mem.eql(u8, file_off.flagsAndPath(disk_contents), file_off.flagsAndPath(input_contents))) + return missInput(m, diag, i, input_contents_len, manifest_size, .invalid_manifest); + + const result = try checkInputPath(m, diag, file_off, input_path, disk_contents, input_contents); + if (result != .hit) return missInput(m, diag, i + 1, input_contents_len, manifest_size, result); + off = @backingInt(file_off); + } + + // Guess number of files based on manifest contents len to reduce allocations. + // This is not an upper bound; subsequent insertions may potentially allocate. + try m.files.ensureUnusedCapacityContext(gpa, disk_contents.len / (@sizeOf(File) + 32), .{ + .contents = disk_contents, + }); + + // Validate and check discovered files. + while (off + 1 < disk_contents.len) { + const file_off: File.Offset = @fromBackingInt(@intCast(off)); + const file = file_off.getFallible(disk_contents) catch return .invalid_manifest; + if (file.flags.prefix >= m.cache.prefixes_len) return .invalid_manifest; + const path = file_off.pathFallible(disk_contents) catch return .invalid_manifest; + if (path.len == 0) return .invalid_manifest; + + try m.files.putContext(gpa, file_off, {}, .{ .contents = disk_contents }); + const result = try checkDiscoveredPath(m, diag, file_off, disk_contents); + if (result != .hit) return result; + + off += File.sizeOf(path.len); + } + + // Final terminating zero byte to distinguish empty manifest file from + // manifest with zero files. + const file_valid = off + 1 == disk_contents.len and disk_contents[off] == 0; + if (!file_valid) return .incomplete_manifest; + + // Since it's a cache hit, we accept the input file contents from disk and discard the other copy. + // Furthermore, don't track the trailing zero byte in contents. + m.contents.shrinkRetainingCapacity(manifest_size - 1); + return .hit; + } + + /// Restore input file contents to prepare for cache miss workflow. We want to keep everything prior to `off` to + /// avoid redundant computation of digest and stat. + /// + /// Furthermore, this function is responsible for populating the remaining input file digests and metadata after the + /// missed one. + fn missInput( + m: *Manifest, + diag: *CheckDiagnostic, + next_file_index: usize, + input_contents_len: usize, + manifest_size: usize, + result: CheckResult, + ) CheckError!CheckResult { + const file_offs = m.files.keys(); + const off = if (file_offs.len - next_file_index == 0) + input_contents_len + else + @backingInt(file_offs[next_file_index]); + const copy_len = input_contents_len - off; + if (manifest_size > 0) @memcpy( + m.contents.items[off..][0..copy_len], + m.contents.items[manifest_size + off ..][0..copy_len], + ); + m.contents.shrinkRetainingCapacity(input_contents_len); + return populateMissingInputFileHashes(m, diag, next_file_index, result); + } + + fn populateMissingInputFileHashes( + m: *Manifest, + diag: *CheckDiagnostic, + next_file_index: usize, + result: CheckResult, + ) CheckError!CheckResult { + const file_offs = m.files.keys(); + const contents = m.contents.items; + for (file_offs[next_file_index..], m.input_paths.items[next_file_index..]) |input_file_off, *input_path| { + try populateInputPath(m, diag, input_file_off, input_path, contents); + } + return result; + } + + /// Similar to `checkInputPath` and `checkDiscoveredPath` but does not compare metadata or digest against file + /// system. Only ensures that metadata and digest are available. + fn populateInputPath( + m: *Manifest, + diag: *CheckDiagnostic, + file_off: File.Offset, + input_path: *InputPath, + contents: []u8, + ) CheckError!void { + const cache = m.cache; + const io = cache.io; + const input_file = file_off.get(contents); + const parent_dir = cache.prefixes()[input_file.flags.prefix].handle; + const file_path = file_off.path(contents); + const gpa = cache.gpa; + + if (input_path.have_digest) return; + + const stat_path_ok = switch (input_path.contents.unwrap()) { + .populated => true, + .unrequested, .requested => input_file.flags.metadata_only, }; - if (!header_valid) { - return .{ .miss = .{ .file_digests_populated = 0 } }; - } - while (line_iter.next()) |line| { - defer idx += 1; - - var iter = mem.tokenizeScalar(u8, line, ' '); - const size = iter.next() orelse return error.InvalidFormat; - const inode = iter.next() orelse return error.InvalidFormat; - const mtime_nsec_str = iter.next() orelse return error.InvalidFormat; - const digest_str = iter.next() orelse return error.InvalidFormat; - const prefix_str = iter.next() orelse return error.InvalidFormat; - const file_path = iter.rest(); - - const stat_size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; - const stat_inode = fmt.parseInt(Io.File.INode, inode, 10) catch return error.InvalidFormat; - const stat_mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; - const file_bin_digest = b: { - if (digest_str.len != hex_digest_len) return error.InvalidFormat; - var bd: BinDigest = undefined; - _ = fmt.hexToBytes(&bd, digest_str) catch return error.InvalidFormat; - break :b bd; + if (!input_path.have_stat and stat_path_ok) { + // Since this is an input file, FileNotFound counts as a failure, not a cache miss. + const actual_stat = switch (input_path.getHandle(input_file)) { + .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .file => |opened_file| opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, }; + const actual_is_directory = actual_stat.kind == .directory; + if (actual_is_directory != input_file.flags.is_directory) { + // Since this is an input file, this is a failure, not a cache miss. + return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = if (actual_is_directory) error.IsDir else error.NotDir, + } }); + } + try input_file.setStat(m, .init(actual_stat)); + } - const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat; - if (prefix >= self.cache.prefixes_len) return error.InvalidFormat; + if (input_file.flags.metadata_only) { + assert(stat_path_ok); + input_file.hashFromMetadata(); + return; + } - if (file_path.len == 0) return error.InvalidFormat; + if (input_path.contents.get(m.all_input_content.items)) |file_or_dir_contents| { + assert(stat_path_ok); + var hasher = hasher_init; + hasher.update(file_or_dir_contents); + hasher.final(&input_file.digest); + return; + } - const cache_hash_file = f: { - const prefixed_path: PrefixedPath = .{ - .prefix = prefix, - .sub_path = file_path, // expires with file_contents + if (input_file.flags.is_directory) { + const opened_dir = if (input_path.have_handle) + input_path.handle.dir + else + parent_dir.openDir(io, file_path, .{ + .iterate = true, + .access_sub_paths = false, + }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), }; - if (idx < input_file_count) { - const file = &self.files.keys()[idx]; - if (!file.prefixed_path.eql(prefixed_path)) - return error.InvalidFormat; - - file.stat = .{ - .size = stat_size, - .inode = stat_inode, - .mtime = .{ .nanoseconds = stat_mtime }, - }; - file.bin_digest = file_bin_digest; - break :f file; - } - const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = self.files.pop(); - if (!gop.found_existing) { - gop.key_ptr.* = .{ - .prefixed_path = .{ - .prefix = prefix, - .sub_path = try gpa.dupe(u8, file_path), - }, - .contents = null, - .max_file_size = null, - .handle = null, - .stat = .{ - .size = stat_size, - .inode = stat_inode, - .mtime = .{ .nanoseconds = stat_mtime }, - }, - .bin_digest = file_bin_digest, - }; - } - break :f gop.key_ptr; + defer if (!input_path.request_handle) opened_dir.close(io) else { + input_path.handle = .{ .dir = opened_dir }; + input_path.have_handle = true; }; - const pp = cache_hash_file.prefixed_path; - const dir = self.cache.prefixes()[pp.prefix].handle; - const this_file = dir.openFile(io, pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => { - // Every digest before this one has been populated successfully. - return .{ .miss = .{ .file_digests_populated = idx } }; - }, - error.Canceled => |e| return e, - else => |e| { - self.diagnostic = .{ .file_open = .{ - .file_index = idx, + if (!input_path.have_stat) { + assert(!stat_path_ok); + const actual_stat = opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, .err = e, - } }; - return error.CacheCheckFailed; - }, + } }), + }; + try input_file.setStat(m, .init(actual_stat)); + } + + const dir_contents_start = m.all_input_content.items.len; + hashDir(gpa, io, opened_dir, &input_file.digest, &m.all_input_content) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), }; - defer this_file.close(io); - - const actual_stat = this_file.stat(io) catch |err| { - self.diagnostic = .{ .file_stat = .{ - .file_index = idx, - .err = err, - } }; - return error.CacheCheckFailed; + defer switch (input_path.contents.unwrap()) { + .requested => input_path.contents = .populated(.{ + .off = @intCast(dir_contents_start), + .len = @intCast(m.all_input_content.items.len - dir_contents_start), + }), + .unrequested => m.all_input_content.shrinkRetainingCapacity(dir_contents_start), + .populated => unreachable, }; - const size_match = actual_stat.size == cache_hash_file.stat.size; - const mtime_match = actual_stat.mtime.nanoseconds == cache_hash_file.stat.mtime.nanoseconds; - const inode_match = actual_stat.inode == cache_hash_file.stat.inode; - - if (!size_match or !mtime_match or !inode_match) { - cache_hash_file.stat = .{ - .size = actual_stat.size, - .mtime = actual_stat.mtime, - .inode = actual_stat.inode, + } else { + const opened_file = if (input_path.have_handle) + input_path.handle.file + else + parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), }; + defer if (!input_path.request_handle) opened_file.close(io) else { + input_path.handle = .{ .file = opened_file }; + input_path.have_handle = true; + }; + + if (!input_path.have_stat) { + assert(!stat_path_ok); + const actual_stat = opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + try input_file.setStat(m, .init(actual_stat)); + } + + switch (input_path.contents.unwrap()) { + .requested => { + const start = m.all_input_content.items.len; + hashFileAppend(io, opened_file, &input_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + input_path.contents = .populated(.{ + .off = @intCast(start), + .len = @intCast(m.all_input_content.items.len - start), + }); + }, + .unrequested => hashFile(io, opened_file, &input_file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .populated => unreachable, + } + } + } + + /// Upon return, disk_contents digest and stat will be populated regardless of whether hit or miss occurs. + fn checkInputPath( + m: *Manifest, + diag: *CheckDiagnostic, + file_off: File.Offset, + input_path: *InputPath, + disk_contents: []u8, + input_contents: []u8, + ) CheckError!CheckResult { + const cache = m.cache; + const io = cache.io; + const gpa = cache.gpa; + const disk_file = file_off.get(disk_contents); + const input_file = file_off.get(input_contents); + const parent_dir = cache.prefixes()[disk_file.flags.prefix].handle; + const file_path = file_off.path(disk_contents); + + assert(disk_file.flags == input_file.flags); + + if (input_path.have_digest) { + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) + return .hit; + } - if (try self.isProblematicTimestamp(cache_hash_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - cache_hash_file.stat.mtime = .zero; - cache_hash_file.stat.inode = 0; + if (input_path.have_stat) { + const changed = try disk_file.setStatChanged(m, input_file.stat()); + if (!changed) return .hit; + } + + if (disk_file.flags.metadata_only) { + if (!input_path.have_stat) { + // Since this is an input file, FileNotFound counts as a failure, not a cache miss. + const actual_stat = switch (input_path.getHandle(input_file)) { + .none => parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .file => |opened_file| opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .dir => |opened_dir| opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }, + }; + // In the other case (have_stat=true), this check is redundant since we already asserted the flags are + // identical, which contains the prefix. + const actual_is_directory = actual_stat.kind == .directory; + if (actual_is_directory != disk_file.flags.is_directory) { + // Since this is an input file, this is a failure, not a cache miss. + return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = if (actual_is_directory) error.IsDir else error.NotDir, + } }); } + const stat: Stat = .init(actual_stat); + const changed = try disk_file.setStatChanged(m, stat); + disk_file.hashFromMetadata(); + if (!changed) return .hit; + } + return .{ .metadata_changed = file_off }; + } + + if (input_path.contents.get(m.all_input_content.items)) |file_or_dir_contents| { + var hasher = hasher_init; + hasher.update(file_or_dir_contents); + hasher.final(&disk_file.digest); + + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; + return .{ .contents_changed = file_off }; + } - var actual_digest: BinDigest = undefined; - hashFile(io, this_file, &actual_digest) catch |err| { - self.diagnostic = .{ .file_read = .{ - .file_index = idx, - .err = err, - } }; - return error.CacheCheckFailed; + if (disk_file.flags.is_directory) { + const opened_dir = if (input_path.have_handle) + input_path.handle.dir + else + parent_dir.openDir(io, file_path, .{ + .iterate = true, + .access_sub_paths = false, + }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), }; + defer if (!input_path.request_handle) opened_dir.close(io) else { + input_path.handle = .{ .dir = opened_dir }; + input_path.have_handle = true; + }; - if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { - cache_hash_file.bin_digest = actual_digest; - // keep going until we have the input file digests - any_file_changed = true; - } + if (!input_path.have_stat) { + const actual_stat = opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + if (!try disk_file.setStatChanged(m, .init(actual_stat))) return .hit; } - if (!any_file_changed) { - self.hash.hasher.update(&cache_hash_file.bin_digest); + const dir_contents_start = m.all_input_content.items.len; + hashDir(gpa, io, opened_dir, &disk_file.digest, &m.all_input_content) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer switch (input_path.contents.unwrap()) { + .requested => input_path.contents = .populated(.{ + .off = @intCast(dir_contents_start), + .len = @intCast(m.all_input_content.items.len - dir_contents_start), + }), + .unrequested => m.all_input_content.shrinkRetainingCapacity(dir_contents_start), + .populated => unreachable, + }; + + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; + return .{ .contents_changed = file_off }; + } else { + const opened_file = if (input_path.have_handle) + input_path.handle.file + else + parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer if (!input_path.request_handle) opened_file.close(io) else { + input_path.handle = .{ .file = opened_file }; + input_path.have_handle = true; + }; + + if (!input_path.have_stat) { + const actual_stat = opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + if (!try disk_file.setStatChanged(m, .init(actual_stat))) return .hit; } + + switch (input_path.contents.unwrap()) { + .requested => { + const start = m.all_input_content.items.len; + hashFileAppend(io, opened_file, &disk_file.digest, &m.all_input_content, gpa) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + input_path.contents = .populated(.{ + .off = @intCast(start), + .len = @intCast(m.all_input_content.items.len - start), + }); + }, + .unrequested => hashFile(io, opened_file, &disk_file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }, + .populated => unreachable, + } + + if (mem.eql(u8, &disk_file.digest, &input_file.digest)) return .hit; + return .{ .contents_changed = file_off }; } + } + + fn checkDiscoveredPath( + m: *Manifest, + diag: *CheckDiagnostic, + file_off: File.Offset, + contents: []u8, + ) CheckError!CheckResult { + const file = file_off.get(contents); + const cache = m.cache; + const gpa = cache.gpa; + const io = cache.io; + const parent_dir = cache.prefixes()[file.flags.prefix].handle; + const file_path = file_off.path(contents); + + if (file.flags.metadata_only) { + const actual_stat = parent_dir.statFile(io, file_path, .{}) catch |err| switch (err) { + error.FileNotFound => return .{ .path_deleted = file_off }, + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; - // If the manifest was somehow missing one of our input files, or if any file hash has changed, - // then this is a cache miss. However, we have successfully populated some or all of the file - // digests. - if (any_file_changed or idx < input_file_count) { - return .{ .miss = .{ .file_digests_populated = idx } }; + const actual_is_directory = actual_stat.kind == .directory; + if (actual_is_directory != file.flags.is_directory) return .{ .directory_status_changed = file_off }; + + const changed = try file.setStatChanged(m, .init(actual_stat)); + file.hashFromMetadata(); + if (changed) return .{ .metadata_changed = file_off }; + return .hit; } - return .hit; - } + if (file.flags.is_directory) { + const opened_dir = parent_dir.openDir(io, file_path, .{ + .iterate = true, + .access_sub_paths = false, + }) catch |err| switch (err) { + error.FileNotFound => return .{ .path_deleted = file_off }, + error.NotDir => return .{ .directory_status_changed = file_off }, + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer opened_dir.close(io); - /// Reset `self.hash.hasher` to the state it should be in after `hit` returns `false`. - /// The hasher contains the original input digest, and all original input file digests (i.e. - /// not including post files). - /// Assumes that `bin_digest` is populated for all files up to `input_file_count`. As such, - /// this is not necessarily safe to call within `hit`. - pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void { - // Reset the hash. - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); + const actual_stat = opened_dir.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + if (try file.setStatChanged(m, .init(actual_stat))) { + const prev_digest: BinDigest = file.digest; + const dir_contents_start = m.all_input_content.items.len; + defer m.all_input_content.shrinkRetainingCapacity(dir_contents_start); + hashDir(gpa, io, opened_dir, &file.digest, &m.all_input_content) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; - // Remove files not in the initial hash. - while (self.files.count() != input_file_count) { - var file = self.files.pop().?; - file.key.deinit(self.cache.gpa); + if (!mem.eql(u8, &file.digest, &prev_digest)) return .{ .contents_changed = file_off }; + } + return .hit; } - for (self.files.keys()) |file| { - self.hash.hasher.update(&file.bin_digest); + const opened_file = parent_dir.openFile(io, file_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => return .{ .path_deleted = file_off }, + error.IsDir => return .{ .directory_status_changed = file_off }, + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_open = .{ + .file_offset = file_off, + .err = e, + } }), + }; + defer opened_file.close(io); + + const actual_stat = opened_file.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_stat = .{ + .file_offset = file_off, + .err = e, + } }), + }; + + if (try file.setStatChanged(m, .init(actual_stat))) { + const prev_digest: BinDigest = file.digest; + hashFile(io, opened_file, &file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| return fail(diag, .{ .file_read = .{ + .file_offset = file_off, + .err = e, + } }), + }; + + if (!mem.eql(u8, &file.digest, &prev_digest)) return .{ .contents_changed = file_off }; } + + return .hit; + } + + fn fail(ptr: *CheckDiagnostic, d: CheckDiagnostic) error{CacheCheckFailed} { + ptr.* = d; + return error.CacheCheckFailed; } fn isProblematicTimestamp(man: *Manifest, timestamp: Io.Timestamp) error{Canceled}!bool { @@ -887,303 +1679,354 @@ pub const Manifest = struct { return timestamp.nanoseconds >= man.recent_problematic_timestamp.nanoseconds; } - fn populateFileHash(self: *Manifest, ch_file: *File) !void { - const io = self.cache.io; - - if (ch_file.handle) |handle| { - return populateFileHashHandle(self, ch_file, handle); - } else { - const pp = ch_file.prefixed_path; - const dir = self.cache.prefixes()[pp.prefix].handle; - const handle = try dir.openFile(io, pp.sub_path, .{}); - defer handle.close(io); - return populateFileHashHandle(self, ch_file, handle); + pub const AddDiscoveredPathOptions = struct { + discovered_path: DiscoveredPath, + handle: PathHandle = .{ .file = null }, + stat: ?Stat = null, + /// If it is a directory, there is a special encoding required for contents, which + /// is null-separated sorted entries, each one prefixed with `File.Kind`. + contents: ?[]const u8 = null, + metadata_only: bool = false, + /// Populated if and only if `error.FileSystemFailure` is returned from `addDiscoveredPath`. + diagnostic: ?*AddDiscoveredPathDiagnostic = null, + }; + + pub const DiscoveredPath = union(enum) { + unresolved: Path, + prefixed: PrefixedPath, + }; + + pub const AddDiscoveredPathDiagnostic = union(enum) { + none, + open_file: Io.File.OpenError, + open_dir: Io.Dir.OpenError, + stat_file: Io.File.StatError, + stat_dir: Io.Dir.StatError, + read_file: Io.File.ReadPositionalError, + read_dir: Io.Dir.Reader.Error, + + pub fn format(this: @This(), w: *Io.Writer) Io.Writer.Error!void { + switch (this) { + .none => return w.writeAll("none"), + else => |err, tag| return w.print("{t}: {t}", .{ tag, err }), + } } - } + }; - fn populateFileHashHandle(self: *Manifest, ch_file: *File, io_file: Io.File) !void { - const io = self.cache.io; - const gpa = self.cache.gpa; + pub const AddDiscoveredPathError = error{ + /// If this is returned, diagnostic will be populated. + /// If contents and stat are both provided, this is unreachable. + FileSystemFailure, + } || Allocator.Error || Io.Cancelable; - const actual_stat = try io_file.stat(io); - ch_file.stat = .{ - .size = actual_stat.size, - .mtime = actual_stat.mtime, - .inode = actual_stat.inode, + /// Add a file or directory as a dependency of process being cached, after cache miss occurs. + /// + /// See also: + /// * `addInputPath` + /// * `addDiscoveredDepFile` + /// * `addDiscoveredManifest` + pub fn addDiscoveredPath(m: *Manifest, options: AddDiscoveredPathOptions) AddDiscoveredPathError!void { + assert(m.manifest_file != null); + transitionToMissDiscovered(m); + const cache = m.cache; + const gpa = cache.gpa; + const io = cache.io; + const is_directory = options.handle == .dir; + + try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); + + const new_file_offset: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); + try m.contents.appendNTimes(gpa, 0, @offsetOf(File, "path_start")); + errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); + + const new_prefix = switch (options.discovered_path) { + .unresolved => |unresolved| try cache.resolveAppendPath(&m.contents, unresolved), + .prefixed => |prefixed| try cache.appendPrefixedPath(&m.contents, prefixed), + }; + assert(mem.isAligned(m.contents.items.len, @alignOf(File))); + new_file_offset.get(m.contents.items).flags = .{ + .prefix = new_prefix, + .is_directory = is_directory, + .metadata_only = options.metadata_only, }; - if (try self.isProblematicTimestamp(ch_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - ch_file.stat.mtime = .zero; - ch_file.stat.inode = 0; - } - - if (ch_file.max_file_size) |max_file_size| { - if (ch_file.stat.size > max_file_size) return error.FileTooBig; - - // Hash while reading from disk, to keep the contents in the cpu - // cache while doing hashing. - const contents = try gpa.alloc(u8, @intCast(ch_file.stat.size)); - errdefer gpa.free(contents); - - var hasher = hasher_init; - var off: usize = 0; - while (true) { - const bytes_read = try io_file.readPositional(io, &.{contents[off..]}, off); - if (bytes_read == 0) break; - hasher.update(contents[off..][0..bytes_read]); - off += bytes_read; + const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ .contents = m.contents.items }); + m.files.lockPointers(); + defer m.files.unlockPointers(); + + const file_offset = if (gop.found_existing) h: { + m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); + const existing_off = gop.key_ptr.*; + const header = existing_off.get(m.contents.items); + // If it trips, the same file path has been added to the cache + // manifest both as a directory and as a normal file, making the + // intended caching behavior ambiguous. + assert(header.flags.is_directory == is_directory); + if (!options.metadata_only) + header.flags.metadata_only = false; + break :h existing_off; + } else new_file_offset; + + const header = file_offset.get(m.contents.items); + + if (options.stat) |stat| { + try header.setStat(m, stat); + if (header.flags.metadata_only) { + header.hashFromMetadata(); + } else if (options.contents) |contents| { + var hasher = hasher_init; + hasher.update(contents); + hasher.final(&header.digest); } - hasher.final(&ch_file.bin_digest); - - ch_file.contents = contents; - } else { - try hashFile(io, io_file, &ch_file.bin_digest); + return; } - self.hash.hasher.update(&ch_file.bin_digest); - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. This is useful for processes that don't know all the files that - /// are depended on ahead of time. For example, a source file that can import other files - /// will need to be recompiled if the imported file is changed. - pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 { - assert(self.manifest_file != null); - - const gpa = self.cache.gpa; - const prefixed_path = try self.cache.findPrefix(file_path); - errdefer gpa.free(prefixed_path.sub_path); - - const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = self.files.pop(); - - if (gop.found_existing) { - gpa.free(prefixed_path.sub_path); - return gop.key_ptr.contents.?; + const need_stat = options.stat == null; + const metadata_only = header.flags.metadata_only; + const prefix = header.flags.prefix; + + switch (options.handle) { + .dir => |opt_handle| if (opt_handle) |handle| { + try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); + } else { + const dir = cache.prefixes()[prefix].handle; + const sub_path = file_offset.path(m.contents.items); + const handle = dir.openDir(io, sub_path, .{ + .access_sub_paths = false, + .iterate = true, + }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (options.diagnostic) |d| d.* = .{ .open_dir = e }; + return error.FileSystemFailure; + }, + }; + defer handle.close(io); + try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); + }, + + .file => |opt_handle| if (opt_handle) |handle| { + try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); + } else { + const dir = cache.prefixes()[prefix].handle; + const sub_path = file_offset.path(m.contents.items); + const handle = dir.openFile(io, sub_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (options.diagnostic) |d| d.* = .{ .open_file = e }; + return error.FileSystemFailure; + }, + }; + defer handle.close(io); + try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); + }, } - - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - .handle = null, - }; - - self.files.lockPointers(); - defer self.files.unlockPointers(); - - try self.populateFileHash(gop.key_ptr); - return gop.key_ptr.contents.?; - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. - /// - /// This is useful for processes that don't know the all the files that are - /// depended on ahead of time. For example, a source file that can import - /// other files will need to be recompiled if the imported file is changed. - pub fn addFilePost(man: *Manifest, file_path: []const u8) !void { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; - const prefixed_path = try man.cache.findPrefix(file_path); - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try addPrefixedPathPost(man, prefixed_path); - } - - pub fn addPathPost(man: *Manifest, path: Path) !void { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; - const prefixed_path: PrefixedPath = try man.cache.findPrefixPath(path); - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try addPrefixedPathPost(man, prefixed_path); } - /// Low level function. `prefixed_path` references cloned memory. Returns - /// whether or not `prefixed_path.sub_path` should be kept. - pub fn addPrefixedPathPost(man: *Manifest, prefixed_path: PrefixedPath) !bool { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; + pub const AddDiscoveredDepFileError = error{ + InvalidDepFile, + } || AddDiscoveredPathError || Io.File.OpenError || Io.File.Reader.Error; - const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = man.files.pop(); + pub const AddDiscoveredDepFileDiagnostic = union(enum) { + add_discovered_path: AddDiscoveredPathDiagnostic, + dep_tokenizer: DepTokenizer.Token, + }; - if (gop.found_existing) return false; - - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = null, - .handle = null, - .stat = undefined, - .bin_digest = undefined, - .contents = null, + /// Add a GNU make style dep file as a dependency of process being cached, after cache miss occurs. + /// + /// See also: + /// * `addDiscoveredPath + /// * `addDiscoveredManifest` + pub fn addDiscoveredDepFile( + m: *Manifest, + path: Path, + diagnostic: ?*AddDiscoveredDepFileDiagnostic, + ) AddDiscoveredDepFileError!void { + assert(m.manifest_file != null); + transitionToMissDiscovered(m); + + const cache = m.cache; + const gpa = cache.gpa; + const io = cache.io; + + // TODO: change DepTokenizer to be streaming rather than operating on slice of bytes. + const prev_len = m.all_input_content.items.len; + defer m.all_input_content.items.len = prev_len; + + const file = try path.root_dir.handle.openFile(io, path.sub_path, .{}); + defer file.close(io); + + var file_reader: Io.File.Reader = .init(file, io, &.{}); + file_reader.interface.appendRemainingUnlimited(gpa, &m.all_input_content) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return file_reader.err.?, }; - man.files.lockPointers(); - defer man.files.unlockPointers(); - - try man.populateFileHash(gop.key_ptr); - return true; + var aux: std.ArrayList(u8) = .empty; + defer aux.deinit(gpa); + + var it: DepTokenizer = .{ .bytes = m.all_input_content.items[prev_len..] }; + while (it.next()) |token| switch (token) { + .target, .target_must_resolve => {}, + .prereq => |p| try m.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(p) } }), + .prereq_must_resolve => { + aux.clearRetainingCapacity(); + try token.resolve(gpa, &aux); + try m.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(aux.items) } }); + }, + else => |err| { + if (diagnostic) |d| d.* = .{ .dep_tokenizer = err }; + return error.InvalidDepFile; + }, + }; } - /// Like `addFilePost` but when the file contents have already been loaded from disk. - pub fn addFilePostContents( - man: *Manifest, - file_path: []const u8, - bytes: []const u8, - stat: File.Stat, - ) !void { - assert(man.manifest_file != null); - const gpa = man.cache.gpa; - const prefixed_path = try man.cache.findPrefix(file_path); - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try addPrefixedPathPostContents(man, prefixed_path, bytes, stat); + fn populateFile( + m: *Manifest, + file: *File, + need_stat: bool, + handle: Io.File, + contents: ?[]const u8, + metadata_only: bool, + diagnostic: ?*AddDiscoveredPathDiagnostic, + ) AddDiscoveredPathError!void { + const io = m.cache.io; + + if (need_stat) { + const stat = handle.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .stat_file = e }; + return error.FileSystemFailure; + }, + }; + try file.setStat(m, .init(stat)); + } + if (metadata_only) { + file.hashFromMetadata(); + return; + } + if (contents) |bytes| { + var hasher = hasher_init; + hasher.update(bytes); + hasher.final(&file.digest); + } else { + hashFile(io, handle, &file.digest) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .read_file = e }; + return error.FileSystemFailure; + }, + }; + } } - /// Low level function. `prefixed_path` references cloned memory. Returns - /// whether or not `prefixed_path.sub_path` should be kept. - pub fn addPrefixedPathPostContents( - man: *Manifest, - prefixed_path: PrefixedPath, - bytes: []const u8, - stat: File.Stat, - ) !bool { - const gpa = man.cache.gpa; - const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = man.files.pop(); - - if (gop.found_existing) return false; - - const new_file = gop.key_ptr; - - new_file.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = null, - .handle = null, - .stat = stat, - .bin_digest = undefined, - .contents = null, - }; - - if (try man.isProblematicTimestamp(new_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - new_file.stat.mtime = .zero; - new_file.stat.inode = 0; + fn populateDirectory( + m: *Manifest, + file: *File, + need_stat: bool, + handle: Io.Dir, + contents: ?[]const u8, + metadata_only: bool, + diagnostic: ?*AddDiscoveredPathDiagnostic, + ) AddDiscoveredPathError!void { + const cache = m.cache; + const io = cache.io; + const gpa = cache.gpa; + + if (need_stat) { + const stat = handle.stat(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .stat_dir = e }; + return error.FileSystemFailure; + }, + }; + try file.setStat(m, .init(stat)); } - - { + if (metadata_only) { + file.hashFromMetadata(); + return; + } + if (contents) |bytes| { var hasher = hasher_init; hasher.update(bytes); - hasher.final(&new_file.bin_digest); + hasher.final(&file.digest); + } else { + const prev_contents_len = m.all_input_content.items.len; + defer m.all_input_content.shrinkRetainingCapacity(prev_contents_len); + hashDir(gpa, io, handle, &file.digest, &m.all_input_content) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + else => |e| { + if (diagnostic) |d| d.* = .{ .read_dir = e }; + return error.FileSystemFailure; + }, + }; } + } - man.hash.hasher.update(&new_file.bin_digest); - return true; + /// See also `hitDigestHex`. + pub fn hitDigest(m: *Manifest) BinDigest { + assert(m.manifest_file != null); + assert(m.state == .hit); + m.state = .hit_digested; + return finalDigest(m); } - pub fn addDepFilePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { - assert(self.manifest_file != null); - return self.addDepFileMaybePost(dir, dep_file_sub_path); + pub fn hitDigestHex(m: *Manifest) HexDigest { + return binToHex(m.hitDigest()); } - fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void { - const gpa = self.cache.gpa; - const io = self.cache.io; - const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(manifest_file_size_max)); - defer gpa.free(dep_file_contents); - - var error_buf: std.ArrayList(u8) = .empty; - defer error_buf.deinit(gpa); - - var resolve_buf: std.ArrayList(u8) = .empty; - defer resolve_buf.deinit(gpa); - - var it: DepTokenizer = .{ .bytes = dep_file_contents }; - while (it.next()) |token| { - switch (token) { - // We don't care about targets, we only want the prereqs - // Clang is invoked in single-source mode but other programs may not - .target, .target_must_resolve => {}, - .prereq => |file_path| if (self.manifest_file == null) { - _ = try self.addFilePath(.initCwd(file_path), null); - } else try self.addFilePost(file_path), - .prereq_must_resolve => { - resolve_buf.clearRetainingCapacity(); - try token.resolve(gpa, &resolve_buf); - if (self.manifest_file == null) { - _ = try self.addFilePath(.initCwd(resolve_buf.items), null); - } else try self.addFilePost(resolve_buf.items); - }, - else => |err| { - try err.printError(gpa, &error_buf); - log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items }); - return error.InvalidDepFile; - }, - } - } + /// See also `missDigestHex`. + pub fn missDigest(m: *Manifest) BinDigest { + assert(m.manifest_file != null); + transitionToMissDiscovered(m); + m.state = .miss_digested; + return finalDigest(m); } - /// Returns a binary hash of the inputs. - pub fn finalBin(self: *Manifest) BinDigest { - assert(self.manifest_file != null); + pub fn missDigestHex(m: *Manifest) HexDigest { + return binToHex(m.missDigest()); + } - // We don't close the manifest file yet, because we want to - // keep it locked until the API user is done using it. - // We also don't write out the manifest yet, because until - // cache_release is called we still might be working on creating - // the artifacts to cache. + fn finalDigest(m: *Manifest) BinDigest { + const contents = m.contents.items; + const hasher = &m.hash.hasher; + for (m.files.keys()) |off| { + const file = off.get(contents); + hasher.update(&file.digest); + } + // We don't close the manifest file yet, because we want to keep it locked until the API user is done using it. + // We also don't write out the manifest yet, because until `finalize` is called we still might be working on + // creating the artifacts to cache. var bin_digest: BinDigest = undefined; - self.hash.hasher.final(&bin_digest); + hasher.final(&bin_digest); return bin_digest; } - /// Returns a hex encoded hash of the inputs. - pub fn final(self: *Manifest) HexDigest { - const bin_digest = self.finalBin(); - return binToHex(bin_digest); - } - /// If `want_shared_lock` is true, this function automatically downgrades the /// lock from exclusive to shared. - pub fn writeManifest(self: *Manifest) !void { - assert(self.have_exclusive_lock); - const io = self.cache.io; - const manifest_file = self.manifest_file.?; - if (self.manifest_dirty) { - self.manifest_dirty = false; + pub fn finalize(m: *Manifest) !void { + const io = m.cache.io; + const manifest_file = m.manifest_file.?; - var buffer: [4000]u8 = undefined; - var fw = manifest_file.writer(io, &buffer); - writeDirtyManifestToStream(self, &fw) catch |err| switch (err) { - error.WriteFailed => return fw.err.?, - else => |e| return e, - }; - } + assert(m.state == .miss_digested); + assert(m.have_exclusive_lock); + + { + m.contents.appendAssumeCapacity(0); + defer _ = m.contents.pop().?; + + try manifest_file.setLength(io, m.contents.items.len); + try manifest_file.writePositionalAll(io, m.contents.items, 0); - if (self.want_shared_lock) { - try self.downgradeToSharedLock(); + m.state = .miss_finalized; } - } - fn writeDirtyManifestToStream(self: *Manifest, fw: *Io.File.Writer) !void { - try fw.interface.writeAll(manifest_header ++ "\n"); - for (self.files.keys()) |file| { - try fw.interface.print("{d} {d} {d} {x} {d} {s}\n", .{ - file.stat.size, - file.stat.inode, - file.stat.mtime, - &file.bin_digest, - file.prefixed_path.prefix, - file.prefixed_path.sub_path, - }); + if (m.want_shared_lock) { + try m.downgradeToSharedLock(); } - try fw.end(); } fn downgradeToSharedLock(self: *Manifest) !void { @@ -1198,53 +2041,74 @@ pub const Manifest = struct { self.have_exclusive_lock = false; } - fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool { - if (self.have_exclusive_lock) return false; - assert(self.manifest_file != null); - const io = self.cache.io; + fn upgradeToExclusiveLock(m: *Manifest, diag: *CheckDiagnostic) error{CacheCheckFailed}!bool { + if (m.have_exclusive_lock) return false; + assert(m.manifest_file != null); + const io = m.cache.io; if (std.process.can_spawn or !builtin.single_threaded) { - const manifest_file = self.manifest_file.?; + const manifest_file = m.manifest_file.?; // Here we intentionally have a period where the lock is released, in case there are // other processes holding a shared lock. manifest_file.unlock(io); - manifest_file.lock(io, .exclusive) catch |err| { - self.diagnostic = .{ .manifest_lock = err }; - return error.CacheCheckFailed; - }; + manifest_file.lock(io, .exclusive) catch |err| return fail(diag, .{ .manifest_lock = err }); } - self.have_exclusive_lock = true; + m.have_exclusive_lock = true; return true; } /// Obtain only the data needed to maintain a lock on the manifest file. /// The `Manifest` remains safe to deinit. /// - /// Don't forget to call `writeManifest` before this! + /// Don't forget to call `finalize` before this! pub fn toOwnedLock(self: *Manifest) Lock { defer self.manifest_file = null; return .{ .manifest_file = self.manifest_file.? }; } - pub fn takeFiles(man: *Manifest) Files { - defer man.files = .empty; - return man.files; + pub const SelfContainedFiles = struct { + /// References memory inside `contents`. + files: Files, + contents: std.ArrayList(u8), + + pub const empty: @This() = .{ + .files = .empty, + .contents = .empty, + }; + + pub fn deinit(scf: *SelfContainedFiles, gpa: Allocator) void { + scf.files.deinit(gpa); + scf.contents.deinit(gpa); + scf.* = undefined; + } + + pub fn path(scf: *const SelfContainedFiles, file_offset: File.Offset) [:0]const u8 { + return file_offset.path(scf.contents.items); + } + }; + + pub fn takeFiles(m: *Manifest) SelfContainedFiles { + defer m.files = .empty; + defer m.contents = .empty; + return borrowFiles(m); } - pub fn freeFiles(gpa: Allocator, files: *Files) void { - for (files.keys()) |*file| file.deinit(gpa); - files.deinit(gpa); + pub fn borrowFiles(m: *const Manifest) SelfContainedFiles { + return .{ + .files = m.files, + .contents = m.contents, + }; } /// Releases the manifest file and frees any memory the Manifest was using. /// `Manifest.hit` must be called first. /// - /// Don't forget to call `writeManifest` before this! - pub fn deinit(man: *Manifest) void { - const io = man.cache.io; - const gpa = man.cache.gpa; + /// Don't forget to call `finalize` before this! + pub fn deinit(m: *Manifest) void { + const io = m.cache.io; + const gpa = m.cache.gpa; - if (man.manifest_file) |file| { + if (m.manifest_file) |file| { if (builtin.os.tag == .windows) { // See Lock.release for why this is required on Windows file.unlock(io); @@ -1252,20 +2116,25 @@ pub const Manifest = struct { file.close(io); } - freeFiles(gpa, &man.files); - man.* = undefined; + m.files.deinit(gpa); + m.contents.deinit(gpa); + m.input_paths.deinit(gpa); + m.all_input_content.deinit(gpa); + m.* = undefined; } - pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void { + pub fn populateFileSystemInputs(man: *const Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void { assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len); buf.clearRetainingCapacity(); const gpa = man.cache.gpa; const files = man.files.keys(); if (files.len > 0) { + const contents = man.contents.items; for (files) |file| { - try buf.ensureUnusedCapacity(gpa, file.prefixed_path.sub_path.len + 2); - buf.appendAssumeCapacity(file.prefixed_path.prefix + 1); - buf.appendSliceAssumeCapacity(file.prefixed_path.sub_path); + const file_path = file.path(contents); + try buf.ensureUnusedCapacity(gpa, file_path.len + 2); + buf.appendAssumeCapacity(@as(u8, file.get(contents).flags.prefix) + 1); + buf.appendSliceAssumeCapacity(file_path); buf.appendAssumeCapacity(0); } // The null byte is a separator, not a terminator. @@ -1273,53 +2142,157 @@ pub const Manifest = struct { } } - pub fn populateOtherManifest(man: *Manifest, other: *Manifest, prefix_map: [5]u8) Allocator.Error!void { - const gpa = other.cache.gpa; - assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len); - assert(man.cache.prefixes_len == 5); - for (man.files.keys()) |file| { - const prefixed_path: PrefixedPath = .{ - .prefix = prefix_map[file.prefixed_path.prefix], - .sub_path = try gpa.dupe(u8, file.prefixed_path.sub_path), - }; - errdefer gpa.free(prefixed_path.sub_path); - - const gop = try other.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = other.files.pop(); + /// Add the full set of paths from another `Manifest` as a dependency of process being cached, after cache miss + /// occurs. + /// + /// See also: + /// * `addDiscoveredPath + /// * `addDiscoveredDepFile` + pub fn addDiscoveredManifest(m: *Manifest, discovered: *const Manifest, prefix_map: [5]u8) Allocator.Error!void { + const gpa = m.cache.gpa; + assert(m.manifest_file != null); + transitionToMissDiscovered(m); + assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == discovered.cache.prefixes_len); + assert(discovered.cache.prefixes_len == 5); + + const discovered_files = discovered.files.keys(); + const orig_files_len = m.files.count(); + const orig_contents_len = m.contents.items.len; + errdefer { + m.files.shrinkRetainingCapacityContext(orig_files_len, .{ .contents = m.contents.items }); + m.contents.shrinkRetainingCapacity(orig_contents_len); + } + + for (discovered_files, 0..) |off, file_index| { + try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); + + const next_index = file_index + 1; + const next_off = if (discovered_files.len - next_index == 0) + discovered.contents.items.len + else + @backingInt(discovered_files[next_index]); + + const copy_bytes = discovered.contents.items[@backingInt(off)..next_off]; + const prev_contents_len: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); + try m.contents.appendSlice(gpa, copy_bytes); + + const gop = m.files.getOrPutAssumeCapacityContext(prev_contents_len, .{ + .contents = m.contents.items, + }); if (gop.found_existing) { - gpa.free(prefixed_path.sub_path); + m.contents.shrinkRetainingCapacity(@backingInt(prev_contents_len)); continue; } - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = file.max_file_size, - .handle = file.handle, - .stat = file.stat, - .bin_digest = file.bin_digest, - .contents = null, - }; + // Flags are already copied but the prefix is supposed to be filtered by `prefix_map`. + const other_file = prev_contents_len.get(m.contents.items); + other_file.flags.prefix = @intCast(prefix_map[other_file.flags.prefix]); + } + } + + fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void { + var buffer: [2048]u8 = undefined; + var hasher = hasher_init; + var offset: u64 = 0; + while (true) { + const n = try file.readPositional(io, &.{&buffer}, offset); + if (n == 0) break; + hasher.update(buffer[0..n]); + offset += n; + } + hasher.final(bin_digest); + } + + fn hashFileAppend( + io: Io, + file: Io.File, + bin_digest: *[Hasher.mac_length]u8, + al: *std.ArrayList(u8), + gpa: Allocator, + ) (Io.File.ReadPositionalError || Allocator.Error)!void { + var hasher = hasher_init; + var offset: u64 = 0; + while (true) { + try al.ensureUnusedCapacity(gpa, 2048); + const buffer = al.unusedCapacitySlice(); + const n = try file.readPositional(io, &.{buffer}, offset); + if (n == 0) break; + hasher.update(buffer[0..n]); + offset += n; + al.items.len += n; + } + hasher.final(bin_digest); + } + + const HashDirError = Io.Dir.Reader.Error || Allocator.Error; + + /// Appends the sorted, encoded directory entries to `contents`. + fn hashDir( + gpa: Allocator, + io: Io, + dir: Io.Dir, + bin_digest: *[Hasher.mac_length]u8, + contents: *std.ArrayList(u8), + ) HashDirError!void { + var buffer: [@max(2048, Io.Dir.Reader.min_buffer_len)]u8 align(@alignOf(usize)) = undefined; + var reader: Io.Dir.Reader = .init(dir, &buffer); + var entry_buffer: [16]Io.Dir.Entry = undefined; - other.hash.hasher.update(&gop.key_ptr.bin_digest); + const contents_start = contents.items.len; + errdefer contents.shrinkRetainingCapacity(contents_start); + + // Each index points into `contents`. + var entries_list: std.ArrayList(u32) = .empty; + defer entries_list.deinit(gpa); + + while (reader.state != .finished) { + const entries = entry_buffer[0..try reader.read(io, &entry_buffer)]; + for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| { + off.* = @intCast(contents.items.len); + // As an optimization, make the reservation also count the duplication + // of the contents buffer that will be required after sorting. + try contents.ensureUnusedCapacity(gpa, (contents.items.len + entry.name.len + 2 - contents_start) * 2); + contents.appendAssumeCapacity(@backingInt(Manifest.File.Kind.fromStat(entry.kind))); + contents.appendSliceAssumeCapacity(entry.name); + contents.appendAssumeCapacity(0); + } } + + const Sort = struct { + contents: [*:0]const u8, + pub fn lessThan(this: @This(), lhs: u32, rhs: u32) bool { + // This comparison includes the kind byte. + return mem.lessThanZ(u8, this.contents + lhs, this.contents + rhs); + } + }; + mem.sortUnstable(u32, entries_list.items, @as(Sort, .{ + .contents = @ptrCast(contents.items.ptr), + }), Sort.lessThan); + + // Duplicate the contents such that we may refer to it while creating a + // sorted copy in the original position (at contents_start). We will then + // offset all the entries_list offsets by contents len when reading from the unsorted copy. + const contents_len = contents.items.len - contents_start; + @memcpy(contents.addManyAsSliceAssumeCapacity(contents_len), contents.items[contents_start..][0..contents_len]); + + var new_offset: usize = contents_start; + for (entries_list.items) |wrong_offset| { + const offset = wrong_offset + contents_len; + // Includes the kind prefix which we also want to copy. + const entry: [*:0]const u8 = @ptrCast(contents.items[offset..]); + new_offset += mem.copySentinelInclusive(u8, 0, contents.items[new_offset..], entry); + } + assert(new_offset == contents_start + contents_len); + contents.shrinkRetainingCapacity(contents_start + contents_len); + + var hasher = hasher_init; + hasher.update(contents.items[contents_start..][0..contents_len]); + hasher.final(bin_digest); } }; -fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void { - var buffer: [2048]u8 = undefined; - var hasher = hasher_init; - var offset: u64 = 0; - while (true) { - const n = try file.readPositional(io, &.{&buffer}, offset); - if (n == 0) break; - hasher.update(buffer[0..n]); - offset += n; - } - hasher.final(bin_digest); -} - -// Create/Write a file, close it, then grab its stat.mtime timestamp. +/// Create/Write a file, close it, then grab its stat.mtime timestamp. fn testGetCurrentFileTimestamp(io: Io, dir: Io.Dir) !Io.Timestamp { const test_out_file = "test-filetimestamp.tmp"; @@ -1348,11 +2321,16 @@ test "cache file and then recall it" { const temp_manifest_dir = "temp_manifest_dir"; try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = "Hello, world!\n" }); + const tmp_directory: Directory = .{ + .path = try std.fs.path.join(testing.allocator, &.{std.testing.TmpDir.parent_dir_path}), + .handle = tmp.dir, + }; + defer testing.allocator.free(tmp_directory.path.?); // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } var digest1: HexDigest = undefined; @@ -1365,45 +2343,53 @@ test "cache file and then recall it" { .manifest_dir = try tmp.dir.createDirPathOpen(io, temp_manifest_dir, .{}), .cwd = cwd, }; - cache.addPrefix(.{ .path = null, .handle = tmp.dir }); + cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + cache.addPrefix(tmp_directory); defer cache.manifest_dir.close(io); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file), null); + man.hash.add(true); + man.hash.add(@as(u16, 1234)); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{}); - // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); - digest1 = ch.final(); - try ch.writeManifest(); + digest1 = man.missDigestHex(); + try man.finalize(); } { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file), null); + man.hash.add(true); + man.hash.add(@as(u16, 1234)); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{}); // Cache hit! We just "built" the same file - try testing.expect(try ch.hit(.none)); - digest2 = ch.final(); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.hit, try man.check(&diag, .none)); + digest2 = man.hitDigestHex(); - try testing.expectEqual(false, ch.have_exclusive_lock); + try testing.expectEqual(false, man.have_exclusive_lock); } try testing.expectEqual(digest1, digest2); } } -test "check that changing a file makes cache fail" { +test "check that changing a file causes cache miss" { const io = testing.io; var tmp = testing.tmpDir(.{}); @@ -1418,11 +2404,16 @@ test "check that changing a file makes cache fail" { const updated_temp_file_contents = "Hello, world; but updated!\n"; try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = original_temp_file_contents }); + const tmp_directory: Directory = .{ + .path = try std.fs.path.join(testing.allocator, &.{std.testing.TmpDir.parent_dir_path}), + .handle = tmp.dir, + }; + defer testing.allocator.free(tmp_directory.path.?); // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } var digest1: HexDigest = undefined; @@ -1435,47 +2426,57 @@ test "check that changing a file makes cache fail" { .manifest_dir = try tmp.dir.createDirPathOpen(io, temp_manifest_dir, .{}), .cwd = cwd, }; - cache.addPrefix(.{ .path = null, .handle = tmp.dir }); + cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + cache.addPrefix(tmp_directory); defer cache.manifest_dir.close(io); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFilePath(.initCwd(temp_file), 100); + man.hash.addBytes("1234"); + const temp_file_idx = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{ .request_contents = true }); - // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); - try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.keys()[temp_file_idx].contents.?)); + try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); - digest1 = ch.final(); + digest1 = man.missDigestHex(); - try ch.writeManifest(); + try man.finalize(); } try tmp.dir.writeFile(io, .{ .sub_path = temp_file, .data = updated_temp_file_contents }); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFilePath(.initCwd(temp_file), 100); + man.hash.addBytes("1234"); + const temp_file_idx = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file, + }, .{ .request_contents = true }); - // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit(.none)); + // The one input file changed. + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual( + @as(Manifest.CheckResult, .{ .contents_changed = temp_file_idx.offset(&man) }), + try man.check(&diag, .none), + ); - // The cache system does not keep the contents of re-hashed input files. - try testing.expect(ch.files.keys()[temp_file_idx].contents == null); + try testing.expectEqualStrings(updated_temp_file_contents, temp_file_idx.contents(&man)); - digest2 = ch.final(); + digest2 = man.missDigestHex(); - try ch.writeManifest(); + try man.finalize(); } - try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); + try testing.expect(!mem.eql(u8, &digest1, &digest2)); } } @@ -1508,12 +2509,12 @@ test "no file inputs" { man.hash.addBytes("1234"); - // There should be nothing in the cache - try testing.expectEqual(false, try man.hit(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); - digest1 = man.final(); + digest1 = man.missDigestHex(); - try man.writeManifest(); + try man.finalize(); } { var man = cache.obtain(); @@ -1521,15 +2522,16 @@ test "no file inputs" { man.hash.addBytes("1234"); - try testing.expect(try man.hit(.none)); - digest2 = man.final(); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.hit, try man.check(&diag, .none)); + digest2 = man.hitDigestHex(); try testing.expectEqual(false, man.have_exclusive_lock); } try testing.expectEqual(digest1, digest2); } -test "Manifest with files added after initial hash work" { +test "Manifest with files added after initial hash" { const io = testing.io; var tmp = testing.tmpDir(.{}); @@ -1544,11 +2546,16 @@ test "Manifest with files added after initial hash work" { try tmp.dir.writeFile(io, .{ .sub_path = temp_file1, .data = "Hello, world!\n" }); try tmp.dir.writeFile(io, .{ .sub_path = temp_file2, .data = "Hello world the second!\n" }); + const tmp_directory: Directory = .{ + .path = try std.fs.path.join(testing.allocator, &.{std.testing.TmpDir.parent_dir_path}), + .handle = tmp.dir, + }; + defer testing.allocator.free(tmp_directory.path.?); // Wait for file timestamps to tick const initial_time = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } var digest1: HexDigest = undefined; @@ -1562,35 +2569,46 @@ test "Manifest with files added after initial hash work" { .manifest_dir = try tmp.dir.createDirPathOpen(io, temp_manifest_dir, .{}), .cwd = cwd, }; - cache.addPrefix(.{ .path = null, .handle = tmp.dir }); + cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + cache.addPrefix(tmp_directory); defer cache.manifest_dir.close(io); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file1), null); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file1, + }, .{}); - // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expectEqual(.incomplete_manifest, try man.check(&diag, .none)); - _ = try ch.addFilePost(temp_file2); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .{ + .root_dir = tmp_directory, + .sub_path = temp_file2, + } } }); - digest1 = ch.final(); - try ch.writeManifest(); + digest1 = man.missDigestHex(); + try man.finalize(); } { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file1), null); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file1, + }, .{}); - try testing.expect(try ch.hit(.none)); - digest2 = ch.final(); + var diag: Manifest.CheckDiagnostic = undefined; + try testing.expect(.hit == try man.check(&diag, .none)); + digest2 = man.hitDigestHex(); - try testing.expectEqual(false, ch.have_exclusive_lock); + try testing.expectEqual(false, man.have_exclusive_lock); } try testing.expect(mem.eql(u8, &digest1, &digest2)); @@ -1600,26 +2618,43 @@ test "Manifest with files added after initial hash work" { // Wait for file timestamps to tick const initial_time2 = try testGetCurrentFileTimestamp(io, tmp.dir); while ((try testGetCurrentFileTimestamp(io, tmp.dir)).nanoseconds == initial_time2.nanoseconds) { - try std.Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); + try Io.Clock.Duration.sleep(.{ .clock = .boot, .raw = .fromNanoseconds(1) }, io); } { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); - _ = try ch.addFilePath(.initCwd(temp_file1), null); + man.hash.addBytes("1234"); + _ = try man.addInputPath(.{ + .root_dir = tmp_directory, + .sub_path = temp_file1, + }, .{}); - // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit(.none)); + var diag: Manifest.CheckDiagnostic = undefined; + switch (try man.check(&diag, .none)) { + .contents_changed => |off| { + try testing.expectEqualStrings(temp_file2, off.path(man.contents.items)); + }, + else => return error.TestFailed, + } - _ = try ch.addFilePost(temp_file2); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .{ + .root_dir = tmp_directory, + .sub_path = temp_file2, + } } }); - digest3 = ch.final(); + digest3 = man.missDigestHex(); - try ch.writeManifest(); + try man.finalize(); } try testing.expect(!mem.eql(u8, &digest1, &digest3)); } } + +test { + _ = Path; + _ = Directory; + _ = DepTokenizer; +} diff --git a/lib/std/Build/Cache/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig index 4fe61603c8b768f6a60652f869adb8859a8cd53a..737ff54e55b513ca4792a3049e10a47c4884b828 100644 --- a/lib/std/Build/Cache/DepTokenizer.zig +++ b/lib/std/Build/Cache/DepTokenizer.zig @@ -1,14 +1,15 @@ const Tokenizer = @This(); -index: usize = 0, -bytes: []const u8, -state: State = .lhs, - const std = @import("std"); +const Io = std.Io; const testing = std.testing; const assert = std.debug.assert; const Allocator = std.mem.Allocator; +index: usize = 0, +bytes: []const u8, +state: State = .lhs, + pub fn next(self: *Tokenizer) ?Token { var start = self.index; var must_resolve = false; @@ -16,10 +17,7 @@ pub fn next(self: *Tokenizer) ?Token { const char = self.bytes[self.index]; switch (self.state) { .lhs => switch (char) { - '\t', '\n', '\r', ' ' => { - // silently ignore whitespace - self.index += 1; - }, + '\t', '\n', '\r', ' ' => self.index += 1, // Ignore whitespace. else => { start = self.index; self.state = .target; @@ -79,7 +77,7 @@ pub fn next(self: *Tokenizer) ?Token { self.index += 1; }, else => { - return errorIllegalChar(.expected_dollar_sign, self.index, char); + return errorIllegalChar(.expected_dollar, self.index, char); }, }, .target_colon => switch (char) { @@ -347,7 +345,7 @@ pub const Token = union(enum) { invalid_target: IndexAndChar, bad_target_escape: IndexAndChar, - expected_dollar_sign: IndexAndChar, + expected_dollar: IndexAndChar, continuation_eol: IndexAndChar, incomplete_escape: IndexAndChar, expected_colon: IndexAndChar, @@ -429,48 +427,23 @@ pub const Token = union(enum) { } } - pub fn printError(self: Token, gpa: Allocator, list: *std.ArrayList(u8)) error{OutOfMemory}!void { - switch (self) { - .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error - .incomplete_quoted_prerequisite, - .incomplete_target, - => |index_and_bytes| { - try list.print(gpa, "{s} '", .{self.errStr()}); - if (self == .incomplete_target) { - const tmp = Token{ .target_must_resolve = index_and_bytes.bytes }; - try tmp.resolve(gpa, list); - } else { - try printCharValues(gpa, list, index_and_bytes.bytes); - } - try list.print(gpa, "' at position {d}", .{index_and_bytes.index}); - }, + pub fn format(token: Token, w: *Io.Writer) Io.Writer.Error!void { + switch (token) { + .target, .target_must_resolve, .prereq, .prereq_must_resolve => return w.writeAll(@tagName(token)), + .incomplete_quoted_prerequisite, .incomplete_target => |info| return w.print("{t} {q} at position {d}", .{ + token, info.bytes, info.index, + }), .invalid_target, .bad_target_escape, - .expected_dollar_sign, + .expected_dollar, .continuation_eol, .incomplete_escape, .expected_colon, - => |index_and_char| { - try list.appendSlice(gpa, "illegal char "); - try printUnderstandableChar(gpa, list, index_and_char.char); - try list.print(gpa, " at position {d}: {s}", .{ index_and_char.index, self.errStr() }); - }, + => |info| return w.print("illegal char '{f}' at position {d}: {t}", .{ + std.zig.fmtChar(info.char), info.index, token, + }), } } - - fn errStr(self: Token) []const u8 { - return switch (self) { - .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error - .incomplete_quoted_prerequisite => "incomplete quoted prerequisite", - .incomplete_target => "incomplete target", - .invalid_target => "invalid target", - .bad_target_escape => "bad target escape", - .expected_dollar_sign => "expecting '$'", - .continuation_eol => "continuation expecting end-of-line", - .incomplete_escape => "incomplete escape", - .expected_colon => "expecting ':'", - }; - } }; test "empty file" { @@ -493,37 +466,37 @@ test "empty colon" { } test "empty target" { - try depTokenizer("foo.o:", "target = {foo.o}"); + try depTokenizer("foo.o:", "target = {\"foo.o\"}"); try depTokenizer( \\foo.o: \\bar.o: \\abcd.o: , - \\target = {foo.o} - \\target = {bar.o} - \\target = {abcd.o} + \\target = {"foo.o"} + \\target = {"bar.o"} + \\target = {"abcd.o"} ); } test "whitespace empty target" { - try depTokenizer("\nfoo.o:", "target = {foo.o}"); - try depTokenizer("\rfoo.o:", "target = {foo.o}"); - try depTokenizer("\r\nfoo.o:", "target = {foo.o}"); - try depTokenizer(" foo.o:", "target = {foo.o}"); + try depTokenizer("\nfoo.o:", "target = {\"foo.o\"}"); + try depTokenizer("\rfoo.o:", "target = {\"foo.o\"}"); + try depTokenizer("\r\nfoo.o:", "target = {\"foo.o\"}"); + try depTokenizer(" foo.o:", "target = {\"foo.o\"}"); } test "escape empty target" { - try depTokenizer("\\ foo.o:", "target = { foo.o}"); - try depTokenizer("\\#foo.o:", "target = {#foo.o}"); - try depTokenizer("\\\\foo.o:", "target = {\\foo.o}"); - try depTokenizer("$$foo.o:", "target = {$foo.o}"); + try depTokenizer("\\ foo.o:", "target = {\" foo.o\"}"); + try depTokenizer("\\#foo.o:", "target = {\"#foo.o\"}"); + try depTokenizer("\\\\foo.o:", "target = {\"\\\\foo.o\"}"); + try depTokenizer("$$foo.o:", "target = {\"$foo.o\"}"); } test "empty target linefeeds" { try depTokenizer("\n", ""); try depTokenizer("\r\n", ""); - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer( \\foo.o: , expect); @@ -541,7 +514,7 @@ test "empty target linefeeds" { } test "empty target linefeeds + continuations" { - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer( \\foo.o:\ , expect); @@ -559,7 +532,7 @@ test "empty target linefeeds + continuations" { } test "empty target linefeeds + hspace + continuations" { - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer( \\foo.o: \ , expect); @@ -577,7 +550,7 @@ test "empty target linefeeds + hspace + continuations" { } test "empty target + hspace + colon" { - const expect = "target = {foo.o}"; + const expect = "target = {\"foo.o\"}"; try depTokenizer("foo.o :", expect); try depTokenizer("foo.o\t\t\t:", expect); @@ -588,8 +561,8 @@ test "empty target + hspace + colon" { test "prereq" { const expect = - \\target = {foo.o} - \\prereq = {foo.c} + \\target = {"foo.o"} + \\prereq = {"foo.c"} ; try depTokenizer("foo.o: foo.c", expect); try depTokenizer( @@ -608,9 +581,9 @@ test "prereq" { test "prereq continuation" { const expect = - \\target = {foo.o} - \\prereq = {foo.h} - \\prereq = {bar.h} + \\target = {"foo.o"} + \\prereq = {"foo.h"} + \\prereq = {"bar.h"} ; try depTokenizer( \\foo.o: foo.h\ @@ -624,19 +597,19 @@ test "prereq continuation" { test "prereq continuation (CRLF)" { const expect = - \\target = {foo.o} - \\prereq = {foo.h} - \\prereq = {bar.h} + \\target = {"foo.o"} + \\prereq = {"foo.h"} + \\prereq = {"bar.h"} ; try depTokenizer("foo.o: foo.h\\\r\nbar.h", expect); } test "multiple prereqs" { const expect = - \\target = {foo.o} - \\prereq = {foo.c} - \\prereq = {foo.h} - \\prereq = {bar.h} + \\target = {"foo.o"} + \\prereq = {"foo.c"} + \\prereq = {"foo.h"} + \\prereq = {"bar.h"} ; try depTokenizer("foo.o: foo.c foo.h bar.h", expect); try depTokenizer( @@ -680,19 +653,19 @@ test "multiple targets and prereqs" { \\ one.h two.h \ \\ three.h four.h , - \\target = {foo.o} - \\prereq = {foo.c} - \\target = {bar.o} - \\prereq = {bar.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - \\target = {abc.o} - \\prereq = {abc.c} - \\prereq = {one.h} - \\prereq = {two.h} - \\prereq = {three.h} - \\prereq = {four.h} + \\target = {"foo.o"} + \\prereq = {"foo.c"} + \\target = {"bar.o"} + \\prereq = {"bar.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} + \\target = {"abc.o"} + \\prereq = {"abc.c"} + \\prereq = {"one.h"} + \\prereq = {"two.h"} + \\prereq = {"three.h"} + \\prereq = {"four.h"} ); try depTokenizer( \\ascii.o: ascii.c @@ -702,21 +675,21 @@ test "multiple targets and prereqs" { \\ macho.c\ \\ a.h b.h c.h , - \\target = {ascii.o} - \\prereq = {ascii.c} - \\target = {base64.o} - \\prereq = {base64.c} - \\prereq = {stdio.h} - \\target = {elf.o} - \\prereq = {elf.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - \\target = {macho.o} - \\prereq = {macho.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} + \\target = {"ascii.o"} + \\prereq = {"ascii.c"} + \\target = {"base64.o"} + \\prereq = {"base64.c"} + \\prereq = {"stdio.h"} + \\target = {"elf.o"} + \\prereq = {"elf.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} + \\target = {"macho.o"} + \\prereq = {"macho.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} ); try depTokenizer( \\a$$scii.o: ascii.c @@ -726,20 +699,20 @@ test "multiple targets and prereqs" { \\ "macho!.c" \ \\ a.h b.h c.h , - \\target = {a$scii.o} - \\prereq = {ascii.c} - \\target = {\base64.o} - \\prereq = {\base64.c} - \\prereq = {s t#dio.h} - \\target = {e\lf.o} - \\prereq = {e\lf.c} - \\prereq = {a.h$$} - \\prereq = {$$b.h c.h$$} - \\target = {macho.o} - \\prereq = {macho!.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} + \\target = {"a$scii.o"} + \\prereq = {"ascii.c"} + \\target = {"\\base64.o"} + \\prereq = {"\\base64.c"} + \\prereq = {"s t#dio.h"} + \\target = {"e\\lf.o"} + \\prereq = {"e\\lf.c"} + \\prereq = {"a.h$$"} + \\prereq = {"$$b.h c.h$$"} + \\target = {"macho.o"} + \\prereq = {"macho!.c"} + \\prereq = {"a.h"} + \\prereq = {"b.h"} + \\prereq = {"c.h"} ); } @@ -750,12 +723,12 @@ test "windows quoted prereqs" { \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \ \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h" , - \\target = {c:\foo.o} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c} - \\target = {c:\foo2.o} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h} + \\target = {"c:\\foo.o"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo.c"} + \\target = {"c:\\foo2.o"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo2.c"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo1.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\foo2.h"} ); } @@ -779,23 +752,23 @@ test "windows mixed prereqs" { \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h" , - \\target = {cimport.o} - \\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h} - \\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h} + \\target = {"cimport.o"} + \\prereq = {"C:\\msys64\\home\\anon\\project\\zig\\master\\zig-cache\\o\\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\\cimport.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\stdio.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\vcruntime.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\sal.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\concurrencysal.h"} + \\prereq = {"C:\\msys64\\opt\\zig\\lib\\zig\\include\\vadefs.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\vadefs.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_wstdio.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_stdio_config.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\string.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_memory.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_memcpy_s.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\errno.h"} + \\prereq = {"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.21.27702\\lib\\x64\\\\..\\..\\include\\vcruntime_string.h"} + \\prereq = {"C:\\Program Files (x86)\\Windows Kits\\10\\\\Include\\10.0.17763.0\\ucrt\\corecrt_wstring.h"} ); } @@ -810,14 +783,14 @@ test "windows funky targets" { \\C:\Users\anon\$$foo.o: \\C:\Users\anon\\\ \ \ \ \ foo.o: , - \\target = {C:\Users\anon\foo.o} - \\target = {C:\Users\anon\foo .o} - \\target = {C:\Users\anon\foo#.o} - \\target = {C:\Users\anon\foo$.o} - \\target = {C:\Users\anon\ foo.o} - \\target = {C:\Users\anon\#foo.o} - \\target = {C:\Users\anon\$foo.o} - \\target = {C:\Users\anon\ foo.o} + \\target = {"C:\\Users\\anon\\foo.o"} + \\target = {"C:\\Users\\anon\\foo .o"} + \\target = {"C:\\Users\\anon\\foo#.o"} + \\target = {"C:\\Users\\anon\\foo$.o"} + \\target = {"C:\\Users\\anon\\ foo.o"} + \\target = {"C:\\Users\\anon\\#foo.o"} + \\target = {"C:\\Users\\anon\\$foo.o"} + \\target = {"C:\\Users\\anon\\ foo.o"} ); } @@ -837,13 +810,13 @@ test "windows funky prereqs" { \\ somedir\\ \\ \c.c\ \\ , - \\target = {cimport.o} - \\prereq = {trailingbackslash\} - \\prereq = {C:\Users\John Smith\AppData\Local\zig\p\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\cimport.c} - \\prereq = {somedir\ a.c} - \\prereq = {somedir/ a.c} - \\prereq = {somedir\ b.c} - \\prereq = {somedir\ \ \c.c} + \\target = {"cimport.o"} + \\prereq = {"trailingbackslash\\"} + \\prereq = {"C:\\Users\\John Smith\\AppData\\Local\\zig\\p\\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\\cimport.c"} + \\prereq = {"somedir\\ a.c"} + \\prereq = {"somedir/ a.c"} + \\prereq = {"somedir\\ b.c"} + \\prereq = {"somedir\\ \\ \\c.c"} ); } @@ -852,171 +825,171 @@ test "windows drive and forward slashes" { \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \ \\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c , - \\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj} - \\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c} + \\target = {"C:/msys64/what/zig-cache\\tmp\\48ac4d78dd531abd-cxa_thread_atexit.obj"} + \\prereq = {"C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c"} ); } test "error incomplete escape - reverse_solidus" { try depTokenizer("\\", - \\ERROR: illegal char '\' at position 0: incomplete escape + \\ERROR: illegal char '\\' at position 0: incomplete_escape ); try depTokenizer("\t\\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); try depTokenizer("\n\\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); try depTokenizer("\r\\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); try depTokenizer("\r\n\\", - \\ERROR: illegal char '\' at position 2: incomplete escape + \\ERROR: illegal char '\\' at position 2: incomplete_escape ); try depTokenizer(" \\", - \\ERROR: illegal char '\' at position 1: incomplete escape + \\ERROR: illegal char '\\' at position 1: incomplete_escape ); } test "error incomplete escape - dollar_sign" { try depTokenizer("$", - \\ERROR: illegal char '$' at position 0: incomplete escape + \\ERROR: illegal char '$' at position 0: incomplete_escape ); try depTokenizer("\t$", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); try depTokenizer("\n$", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); try depTokenizer("\r$", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); try depTokenizer("\r\n$", - \\ERROR: illegal char '$' at position 2: incomplete escape + \\ERROR: illegal char '$' at position 2: incomplete_escape ); try depTokenizer(" $", - \\ERROR: illegal char '$' at position 1: incomplete escape + \\ERROR: illegal char '$' at position 1: incomplete_escape ); } test "error incomplete target" { try depTokenizer("foo.o", - \\ERROR: incomplete target 'foo.o' at position 0 + \\ERROR: incomplete_target "foo.o" at position 0 ); try depTokenizer("\tfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\nfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\rfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\r\nfoo.o", - \\ERROR: incomplete target 'foo.o' at position 2 + \\ERROR: incomplete_target "foo.o" at position 2 ); try depTokenizer(" foo.o", - \\ERROR: incomplete target 'foo.o' at position 1 + \\ERROR: incomplete_target "foo.o" at position 1 ); try depTokenizer("\\ foo.o", - \\ERROR: incomplete target ' foo.o' at position 0 + \\ERROR: incomplete_target "\\ foo.o" at position 0 ); try depTokenizer("\\#foo.o", - \\ERROR: incomplete target '#foo.o' at position 0 + \\ERROR: incomplete_target "\\#foo.o" at position 0 ); try depTokenizer("\\\\foo.o", - \\ERROR: incomplete target '\foo.o' at position 0 + \\ERROR: incomplete_target "\\\\foo.o" at position 0 ); try depTokenizer("$$foo.o", - \\ERROR: incomplete target '$foo.o' at position 0 + \\ERROR: incomplete_target "$$foo.o" at position 0 ); } test "error illegal char at position - bad target escape" { try depTokenizer("\\\t", - \\ERROR: illegal char \x09 at position 1: bad target escape + \\ERROR: illegal char '\t' at position 1: bad_target_escape ); try depTokenizer("\\\n", - \\ERROR: illegal char \x0A at position 1: bad target escape + \\ERROR: illegal char '\n' at position 1: bad_target_escape ); try depTokenizer("\\\r", - \\ERROR: illegal char \x0D at position 1: bad target escape + \\ERROR: illegal char '\r' at position 1: bad_target_escape ); try depTokenizer("\\\r\n", - \\ERROR: illegal char \x0D at position 1: bad target escape + \\ERROR: illegal char '\r' at position 1: bad_target_escape ); } test "error illegal char at position - expecting dollar_sign" { try depTokenizer("$\t", - \\ERROR: illegal char \x09 at position 1: expecting '$' + \\ERROR: illegal char '\t' at position 1: expected_dollar ); try depTokenizer("$\n", - \\ERROR: illegal char \x0A at position 1: expecting '$' + \\ERROR: illegal char '\n' at position 1: expected_dollar ); try depTokenizer("$\r", - \\ERROR: illegal char \x0D at position 1: expecting '$' + \\ERROR: illegal char '\r' at position 1: expected_dollar ); try depTokenizer("$\r\n", - \\ERROR: illegal char \x0D at position 1: expecting '$' + \\ERROR: illegal char '\r' at position 1: expected_dollar ); } test "error illegal char at position - invalid target" { try depTokenizer("foo\n.o", - \\ERROR: illegal char \x0A at position 3: invalid target + \\ERROR: illegal char '\n' at position 3: invalid_target ); try depTokenizer("foo\r.o", - \\ERROR: illegal char \x0D at position 3: invalid target + \\ERROR: illegal char '\r' at position 3: invalid_target ); try depTokenizer("foo\r\n.o", - \\ERROR: illegal char \x0D at position 3: invalid target + \\ERROR: illegal char '\r' at position 3: invalid_target ); } test "error target - continuation expecting end-of-line" { try depTokenizer("foo.o: \\\t", - \\target = {foo.o} - \\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char '\t' at position 8: continuation_eol ); try depTokenizer("foo.o: \\ ", - \\target = {foo.o} - \\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char ' ' at position 8: continuation_eol ); try depTokenizer("foo.o: \\x", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char 'x' at position 8: continuation_eol ); try depTokenizer("foo.o: \\\x0dx", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char 'x' at position 9: continuation_eol ); } test "error prereq - continuation expecting end-of-line" { try depTokenizer("foo.o: foo.h\\\x0dx", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line + \\target = {"foo.o"} + \\ERROR: illegal char 'x' at position 14: continuation_eol ); } test "error illegal char at position - expecting colon" { try depTokenizer("foo\t.o:", - \\target = {foo} - \\ERROR: illegal char '.' at position 4: expecting ':' + \\target = {"foo"} + \\ERROR: illegal char '.' at position 4: expected_colon ); try depTokenizer("foo .o:", - \\target = {foo} - \\ERROR: illegal char '.' at position 4: expecting ':' + \\target = {"foo"} + \\ERROR: illegal char '.' at position 4: expected_colon ); try depTokenizer("foo \n.o:", - \\target = {foo} - \\ERROR: illegal char \x0A at position 4: expecting ':' + \\target = {"foo"} + \\ERROR: illegal char '\n' at position 4: expected_colon ); try depTokenizer("foo.o\t\n:", - \\target = {foo.o} - \\ERROR: illegal char \x0A at position 6: expecting ':' + \\target = {"foo.o"} + \\ERROR: illegal char '\n' at position 6: expected_colon ); } @@ -1028,70 +1001,35 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { var it: Tokenizer = .{ .bytes = input }; var buffer: std.ArrayList(u8) = .empty; - var resolve_buf: std.ArrayList(u8) = .empty; + var aux: std.ArrayList(u8) = .empty; var i: usize = 0; while (it.next()) |token| { if (i != 0) try buffer.appendSlice(arena, "\n"); switch (token) { .target, .prereq => |bytes| { - try buffer.appendSlice(arena, @tagName(token)); - try buffer.appendSlice(arena, " = {"); - for (bytes) |b| { - try buffer.append(arena, printable_char_tab[b]); - } - try buffer.appendSlice(arena, "}"); + try buffer.print(arena, "{t} = {{{q}}}", .{ token, bytes }); }, .target_must_resolve => { try buffer.appendSlice(arena, "target = {"); - try token.resolve(arena, &resolve_buf); - for (resolve_buf.items) |b| { - try buffer.append(arena, printable_char_tab[b]); - } - resolve_buf.items.len = 0; + aux.clearRetainingCapacity(); + try token.resolve(arena, &aux); + try buffer.print(arena, "{q}", .{aux.items}); try buffer.appendSlice(arena, "}"); }, .prereq_must_resolve => { try buffer.appendSlice(arena, "prereq = {"); - try token.resolve(arena, &resolve_buf); - for (resolve_buf.items) |b| { - try buffer.append(arena, printable_char_tab[b]); - } - resolve_buf.items.len = 0; + aux.clearRetainingCapacity(); + try token.resolve(arena, &aux); + try buffer.print(arena, "{q}", .{aux.items}); try buffer.appendSlice(arena, "}"); }, else => { - try buffer.appendSlice(arena, "ERROR: "); - try token.printError(arena, &buffer); + try buffer.print(arena, "ERROR: {f}", .{token}); break; }, } i += 1; } - if (std.mem.eql(u8, expect, buffer.items)) { - try testing.expect(true); - return; - } - try testing.expectEqualStrings(expect, buffer.items); } - -fn printCharValues(gpa: Allocator, list: *std.ArrayList(u8), bytes: []const u8) !void { - for (bytes) |b| try list.append(gpa, printable_char_tab[b]); -} - -fn printUnderstandableChar(gpa: Allocator, list: *std.ArrayList(u8), char: u8) !void { - if (std.ascii.isPrint(char)) { - try list.print(gpa, "'{c}'", .{char}); - } else { - try list.print(gpa, "\\x{X:0>2}", .{char}); - } -} - -// zig fmt: off -const printable_char_tab: [256]u8 = ( - "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++ - "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++ - "................................................................" ++ - "................................................................" -).*; diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index e216a272e95bd69e462d494123bfcf3c7083669c..f32fb26ec5ba4a2b6f1cb4bf52638cbb55c034a7 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1873,12 +1873,11 @@ pub const PathDep = extern struct { pkg: Package.OptionalIndex, pub const Flags = packed struct(u32) { - mode: Mode, + is_directory: bool, + metadata_only: bool, base: LazyPath.Relative.Base, - _: u16 = 0, + _: u22 = 0, }; - - pub const Mode = enum(u8) { directory, contents, metadata }; }; pub const InstallDestDir = enum(u32) { diff --git a/lib/std/Build/Serialize.zig b/lib/std/Build/Serialize.zig index 7582df0a43f1ef3264da6e7332c4fab8bf87d108..eafc8e0ab7063ac42f80183e125f1c1b6c11358a 100644 --- a/lib/std/Build/Serialize.zig +++ b/lib/std/Build/Serialize.zig @@ -34,7 +34,8 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi .cwd_relative => .cwd, .relative => |r| r.base, }, - .mode = src.mode, + .is_directory = src.is_directory, + .metadata_only = src.metadata_only, }, .sub = switch (src.lazy_path) { .src_path => |sp| try wc.addString(sp.sub_path), diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 372a7890b734638cf9fd7d4a8f99ca5a29c13c01..947f01694bcf6a1dea849bb7bbadef92c3f57096 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -397,6 +397,7 @@ pub fn appendRemainingAligned( pub const UnlimitedAllocError = Allocator.Error || ShortError; pub fn appendRemainingUnlimited(r: *Reader, gpa: Allocator, list: *ArrayList(u8)) UnlimitedAllocError!void { + list.pointer_stability.assertUnlocked(); var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.allocatedSlice()); a.writer.end = list.items.len; list.* = .empty; diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index 7b6a004603a2d557dd2af751f322536dfa23b1a6..7c34f58dba4d4151e1df3426eeccbbc1a7064b16 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -776,6 +776,9 @@ pub fn Aligned(comptime T: type, comptime alignment: ?mem.Alignment) type { /// /// Asserts what the capacity is equal to the length. /// Never invalidates element pointers. + /// + /// See also: + /// * `shrinkToLen` pub fn toOwnedSliceAssert(self: *Self) Slice { assert(self.items.len == self.capacity); const items = self.items; diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 7c010cf4c5dd4c08e89604000f2a3d262f6f78ad..d7ee376afb152587681de2ea8875c2c653a72f2e 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -592,6 +592,9 @@ pub const TmpDir = struct { const random_bytes_count = 12; const sub_path_len = std.base64.url_safe.Encoder.calcSize(random_bytes_count); + /// Deprecated. + pub const parent_dir_path: []const u8 = ".zig-cache" ++ std.fs.path.sep_str ++ "tmp"; + pub fn cleanup(self: *TmpDir) void { self.dir.close(io); self.parent_dir.deleteTree(io, &self.sub_path) catch {}; diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 46556a225263153d2b404f283d2ae80d1a64eee1..57bda0bbfd85ad057dd45dae7b482f226012b2ae 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1838,12 +1838,12 @@ pub fn buildExeSubprocess( var it = mem.splitScalar(u8, body, 0); while (it.next()) |prefixed_path| { const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1)); - const sub_path = try gpa.dupe(u8, prefixed_path[1..]); - var keep = false; - defer if (!keep) gpa.free(sub_path); - keep = man.addPrefixedPathPost(.{ - .prefix = @backingInt(prefix), - .sub_path = sub_path, + const sub_path = prefixed_path[1..]; + man.addDiscoveredPath(.{ + .discovered_path = .{ .prefixed = .{ + .prefix = @intCast(@backingInt(prefix)), + .sub_path = sub_path, + } }, }) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, else => |e| { diff --git a/src/Compilation.zig b/src/Compilation.zig index 1663f740c6e14d370c4e8c4da8865cdf125223a1..d94b7146b59afed2774f83d89b63604794a1ddbd 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -701,39 +701,12 @@ pub const Path = struct { }; } - pub fn addToCacheManifestPostHit(p: Path, man: *Cache.Manifest, dirs: *const std.zig.Directories) !void { - comptime assert(0 == @backingInt(std.zig.Server.Message.PathPrefix.cwd)); - comptime assert(1 == @backingInt(std.zig.Server.Message.PathPrefix.zig_lib)); - comptime assert(2 == @backingInt(std.zig.Server.Message.PathPrefix.local_cache)); - comptime assert(3 == @backingInt(std.zig.Server.Message.PathPrefix.global_cache)); - comptime assert(4 == @backingInt(std.zig.Server.Message.PathPrefix.build_root)); - comptime assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == 5); - const gpa = man.cache.gpa; - const prefixed_path: Cache.PrefixedPath = .{ - .prefix = switch (p.root) { - .none => { - const path = try p.toAbsolute(dirs, gpa); - defer gpa.free(path); - return man.addFilePost(path); - }, - .zig_lib => 1, - .local_cache => 2, - .global_cache => 3, - .build_root => 4, - }, - .sub_path = try gpa.dupe(u8, p.sub_path), - }; - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try man.addPrefixedPathPost(prefixed_path); - } - - pub fn addToCacheManifestPostHitContents( + pub fn addToCacheManifestAsDiscovered( p: Path, man: *Cache.Manifest, dirs: *const std.zig.Directories, - bytes: []const u8, - stat: Cache.File.Stat, + contents: ?[]const u8, + stat: ?Cache.Manifest.Stat, ) !void { comptime assert(0 == @backingInt(std.zig.Server.Message.PathPrefix.cwd)); comptime assert(1 == @backingInt(std.zig.Server.Message.PathPrefix.zig_lib)); @@ -741,24 +714,29 @@ pub const Path = struct { comptime assert(3 == @backingInt(std.zig.Server.Message.PathPrefix.global_cache)); comptime assert(4 == @backingInt(std.zig.Server.Message.PathPrefix.build_root)); comptime assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == 5); - const gpa = man.cache.gpa; - const prefixed_path: Cache.PrefixedPath = .{ - .prefix = switch (p.root) { - .none => { - const path = try p.toAbsolute(dirs, gpa); - defer gpa.free(path); - return man.addFilePostContents(path, bytes, stat); + try man.addDiscoveredPath(.{ + .discovered_path = .{ .prefixed = .{ + .prefix = switch (p.root) { + .none => { + const gpa = man.cache.gpa; + const path = try p.toAbsolute(dirs, gpa); + defer gpa.free(path); + return man.addDiscoveredPath(.{ + .discovered_path = .{ .unresolved = .initCwd(path) }, + .contents = contents, + .stat = stat, + }); + }, + .zig_lib => 1, + .local_cache => 2, + .global_cache => 3, + .build_root => 4, }, - .zig_lib => 1, - .local_cache => 2, - .global_cache => 3, - .build_root => 4, - }, - .sub_path = try gpa.dupe(u8, p.sub_path), - }; - var keep = false; - defer if (!keep) gpa.free(prefixed_path.sub_path); - keep = try man.addPrefixedPathPostContents(prefixed_path, bytes, stat); + .sub_path = p.sub_path, + } }, + .contents = contents, + .stat = stat, + }); } }; @@ -1341,19 +1319,19 @@ pub const cache_helpers = struct { } } - pub fn hashCSource(self: *Cache.Manifest, c_source: CSourceFile) !void { - _ = try self.addFilePath(.initCwd(c_source.src_path), null); + pub fn hashCSource(man: *Cache.Manifest, c_source: CSourceFile) !void { + _ = try man.addInputPath(.initCwd(c_source.src_path), .{}); // Hash the extra flags, with special care to call addFile for file parameters. // TODO this logic can likely be improved by utilizing clang_options_data.zig. const file_args = [_][]const u8{"-include"}; var arg_i: usize = 0; while (arg_i < c_source.extra_flags.len) : (arg_i += 1) { const arg = c_source.extra_flags[arg_i]; - self.hash.addBytes(arg); + man.hash.addBytes(arg); for (file_args) |file_arg| { if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) { arg_i += 1; - _ = try self.addFilePath(.initCwd(c_source.extra_flags[arg_i]), null); + _ = try man.addInputPath(.initCwd(c_source.extra_flags[arg_i]), .{}); } } } @@ -2835,41 +2813,25 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE man.want_shared_lock = false; } - const is_hit = man.hit(main_progress_node) catch |err| switch (err) { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, main_progress_node) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return comp.setMiscFailure( - .check_whole_cache, - "failed to check cache: {t} {t}", - .{ man.diagnostic, e }, - ), - .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix]; - return comp.setMiscFailure(.check_whole_cache, "failed to check cache: {f}{s} {t} {t}", .{ - prefix, pp.sub_path, man.diagnostic, op.err, - }); - }, + error.CacheCheckFailed => { + return comp.setMiscFailure(.check_whole_cache, "checking cache failed: {f}", .{diag.fmt(&man)}); }, - error.InvalidFormat => return comp.setMiscFailure( - .check_whole_cache, - "failed to check cache: invalid manifest file format", - .{}, - ), }; - if (is_hit and !ignore_hit) { + log.debug("{s} whole cache {f} ignore_hit={}", .{ comp.root_name, status.fmt(&man), ignore_hit }); + if (status == .hit and !ignore_hit) { // In this case the cache hit contains the full set of file system inputs. Nice! if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); if (comp.parent_whole_cache) |pwc| { try pwc.mutex.lock(io); defer pwc.mutex.unlock(io); - try man.populateOtherManifest(pwc.manifest, pwc.prefix_map); + try pwc.manifest.addDiscoveredManifest(&man, pwc.prefix_map); } comp.last_update_was_cache_hit = true; - log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name}); - const bin_digest = man.finalBin(); + const bin_digest = man.hitDigest(); comp.digest = bin_digest; @@ -2877,10 +2839,9 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE whole.lock = man.toOwnedLock(); return; } - log.debug("CacheMode.whole cache miss for {s}", .{comp.root_name}); if (ignore_hit) { - // Okay, now set this back so that `writeManifest` will downgrade our lock later. + // Okay, now set this back so that `Manifest.finalize` will downgrade our lock later. man.want_shared_lock = true; } @@ -3036,7 +2997,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } if (build_options.enable_debug_extensions and comp.verbose_intern_pool) { - std.debug.print("intern pool stats for '{s}':\n", .{comp.root_name}); + std.debug.print("intern pool stats for {q}:\n", .{comp.root_name}); zcu.intern_pool.dump(); } @@ -3048,11 +3009,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE if (comp.link_depfile) |depfile_path| if (comp.bin_file) |lf| { assert(comp.file_system_inputs != null); - comp.createDepFile(depfile_path, lf.emit) catch |err| comp.setMiscFailure( - .link_depfile, - "unable to write linker dependency file: {t}", - .{err}, - ); + comp.createDepFile(depfile_path, lf.emit) catch |err| + comp.setMiscFailure(.link_depfile, "writing linker dependency file failed: {t}", .{err}); }; if (anyErrors(comp)) { @@ -3075,10 +3033,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE if (comp.parent_whole_cache) |pwc| { try pwc.mutex.lock(io); defer pwc.mutex.unlock(io); - try man.populateOtherManifest(pwc.manifest, pwc.prefix_map); + try pwc.manifest.addDiscoveredManifest(&man, pwc.prefix_map); } - const bin_digest = man.finalBin(); + const bin_digest = man.missDigest(); const hex_digest = Cache.binToHex(bin_digest); // Work around windows `AccessDenied` if any files within this @@ -3168,7 +3126,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } // Failure here only means an unnecessary cache miss. - man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); + man.finalize() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); assert(whole.lock == null); whole.lock = man.toOwnedLock(); @@ -3410,7 +3368,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes try link.hashInputs(man, comp.link_inputs); for (comp.c_objects.items) |c_object| { - _ = try man.addFilePath(.initCwd(c_object.src.src_path), null); + _ = try man.addInputPath(.initCwd(c_object.src.src_path), .{}); man.hash.addOptional(c_object.src.ext); man.hash.addListOfBytes(c_object.src.extra_flags); } @@ -3418,11 +3376,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes for (comp.win32_resources.items) |win32_resource| { switch (win32_resource.src) { .rc => |rc_src| { - _ = try man.addFilePath(.initCwd(rc_src.src_path), null); + _ = try man.addInputPath(.initCwd(rc_src.src_path), .{}); man.hash.addListOfBytes(rc_src.extra_flags); }, .manifest => |manifest_path| { - _ = try man.addFilePath(.initCwd(manifest_path), null); + _ = try man.addInputPath(.initCwd(manifest_path), .{}); }, } } @@ -3454,8 +3412,8 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes const opts = comp.cache_use.whole.lf_open_opts; - try man.addOptionalFilePath(opts.linker_script); - try man.addOptionalFilePath(opts.version_script); + try man.addInputPathOptional(opts.linker_script, .{}); + try man.addInputPathOptional(opts.version_script, .{}); man.hash.add(opts.allow_undefined_version); man.hash.addOptional(opts.enable_new_dtags); @@ -3514,7 +3472,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes // Mach-O specific stuff try link.File.MachO.hashAddFrameworks(man, opts.frameworks); - try man.addOptionalFilePath(opts.entitlements); + try man.addInputPathOptional(opts.entitlements, .{}); man.hash.addOptional(opts.pagezero_size); man.hash.addOptional(opts.headerpad_size); man.hash.add(opts.headerpad_max_install_names); @@ -5123,9 +5081,13 @@ pub fn translateC( try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, comp.thread_limit, &stdout); if (out_dep_path) |dep_file_path| add_deps: { + var diagnostic: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined; const dep_basename = fs.path.basename(dep_file_path); // Add the files depended on to the cache system, if a dep file was emitted - man.addDepFilePost(cache_tmp_dir, dep_basename) catch |err| switch (err) { + man.addDiscoveredDepFile(.{ + .root_dir = .{ .handle = cache_tmp_dir, .path = tmp_sub_path }, + .sub_path = dep_basename, + }, &diagnostic) catch |err| switch (err) { error.FileNotFound => break :add_deps, else => |e| return e, }; @@ -5134,14 +5096,17 @@ pub fn translateC( .whole => |whole| if (whole.cache_manifest) |whole_cache_manifest| { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try whole_cache_manifest.addDepFilePost(cache_tmp_dir, dep_basename); + try whole_cache_manifest.addDiscoveredDepFile(.{ + .root_dir = .{ .handle = cache_tmp_dir, .path = tmp_sub_path }, + .sub_path = dep_basename, + }, &diagnostic); }, .incremental, .none => {}, } // Just to save disk space, we delete the file because it is never needed again. cache_tmp_dir.deleteFile(io, dep_basename) catch |err| { - log.warn("failed to delete '{s}': {t}", .{ dep_file_path, err }); + log.warn("failed to delete dep file {q}: {t}", .{ dep_file_path, err }); }; } @@ -5149,9 +5114,9 @@ pub fn translateC( var reader: std.Io.Reader = .fixed(stdout); const MessageHeader = std.zig.Server.Message.Header; const header = reader.takeStruct(MessageHeader, .little) catch |err| - fatal("unable to read translate-c MessageHeader: {s}", .{@errorName(err)}); + fatal("unable to read translate-c MessageHeader: {t}", .{err}); const body = reader.take(header.bytes_len) catch |err| - fatal("unable to read {}-byte translate-c message body: {s}", .{ header.bytes_len, @errorName(err) }); + fatal("unable to read {d}-byte translate-c message body: {t}", .{ header.bytes_len, err }); switch (header.tag) { .error_bundle => { const error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); @@ -5161,11 +5126,11 @@ pub fn translateC( .errors = error_bundle, }; }, - else => fatal("unexpected message type received from translate-c: {s}", .{@tagName(header.tag)}), + else => fatal("unexpected message type received from translate-c: {t}", .{header.tag}), } } - const bin_digest = man.finalBin(); + const bin_digest = man.missDigest(); const hex_digest = Cache.binToHex(bin_digest); const o_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; @@ -5556,7 +5521,18 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr const target = comp.getTarget(); assert(target.ofmt != .c); const o_ext = target.ofmt.fileExt(target.cpu.arch); - const digest = if (!comp.disable_c_depfile and try man.hit(child_progress_node)) man.final() else blk: { + const digest = d: { + if (!comp.disable_c_depfile) { + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, child_progress_node) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + error.CacheCheckFailed => { + return comp.failCObj(c_object, "checking cache failed: {f}", .{diagnostic.fmt(&man)}); + }, + }; + log.debug("C object {q} cache {f}", .{ c_object.src.src_path, status.fmt(&man) }); + if (status == .hit) break :d man.hitDigestHex(); + } var argv: std.array_list.Managed([]const u8) = .init(gpa); defer argv.deinit(); @@ -5683,11 +5659,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr // Just to save disk space, we delete the files that are never needed again. defer if (out_diag_path) |diag_file_path| zig_cache_tmp_dir.deleteFile(io, fs.path.basename(diag_file_path)) catch |err| switch (err) { error.FileNotFound => {}, // the file wasn't created due to an error we reported - else => log.warn("failed to delete '{s}': {s}", .{ diag_file_path, @errorName(err) }), + else => log.warn("failed to delete {q}: {t}", .{ diag_file_path, err }), }; defer if (out_dep_path) |dep_file_path| zig_cache_tmp_dir.deleteFile(io, fs.path.basename(dep_file_path)) catch |err| switch (err) { error.FileNotFound => {}, // the file wasn't created due to an error we reported - else => log.warn("failed to delete '{s}': {s}", .{ dep_file_path, @errorName(err) }), + else => log.warn("failed to delete {q}: {t}", .{ dep_file_path, err }), }; if (std.process.can_spawn) { if (comp.clang_passthrough_mode) { @@ -5776,8 +5752,10 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr const dep_basename = fs.path.basename(dep_file_path); if (comp.file_system_inputs != null) { + // TODO instead of this rely on passing the manifest contents directly + // Use the same file size limit as the cache code does for dependency files. - const dep_file_contents = try zig_cache_tmp_dir.readFileAlloc(io, dep_basename, gpa, .limited(Cache.manifest_file_size_max)); + const dep_file_contents = try zig_cache_tmp_dir.readFileAlloc(io, dep_basename, gpa, .unlimited); defer gpa.free(dep_file_contents); var str_buf: std.ArrayList(u8) = .empty; @@ -5792,9 +5770,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr try token.resolve(gpa, &str_buf); break :p try .fromUnresolved(arena, comp.dirs, &.{str_buf.items}); }, - else => |err| { - try err.printError(gpa, &str_buf); - log.err("failed parsing {s}: {s}", .{ dep_basename, str_buf.items }); + else => |err_token| { + log.err("failed parsing {s}: {f}", .{ dep_basename, err_token }); return error.InvalidDepFile; }, }; @@ -5803,13 +5780,26 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr } // Add the files depended on to the cache system. - try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); + var diagnostic: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined; + try man.addDiscoveredDepFile(.{ + .root_dir = .{ + .handle = zig_cache_tmp_dir, + .path = "tmp", + }, + .sub_path = dep_basename, + }, &diagnostic); switch (comp.cache_use) { .whole => |whole| { if (whole.cache_manifest) |whole_cache_manifest| { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename); + try whole_cache_manifest.addDiscoveredDepFile(.{ + .root_dir = .{ + .handle = zig_cache_tmp_dir, + .path = "tmp", + }, + .sub_path = dep_basename, + }, &diagnostic); } }, .incremental, .none => {}, @@ -5817,16 +5807,28 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr } // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. - if (comp.disable_c_depfile) _ = try man.hit(child_progress_node); + const digest = inner: { + if (comp.disable_c_depfile) { + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, child_progress_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + return comp.failCObj(c_object, "checking cache failed: {f}", .{diagnostic.fmt(&man)}); + }, + }; + log.debug("ignored C object cache {f}", .{status.fmt(&man)}); + if (status == .hit) break :inner man.hitDigestHex(); + } + break :inner man.missDigestHex(); + }; // Rename into place. - const digest = man.final(); - const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest }); + const o_sub_path = try fs.path.join(arena, &.{ "o", &digest }); var o_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, o_sub_path, .{}); defer o_dir.close(io); const tmp_basename = fs.path.basename(out_obj_path); try Io.Dir.rename(zig_cache_tmp_dir, tmp_basename, o_dir, o_basename, io); - break :blk digest; + break :d digest; }; if (man.have_exclusive_lock) { @@ -5834,11 +5836,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr // possible we had a hit and the manifest is dirty, for example if the file mtime changed but // the contents were the same, we hit the cache but the manifest is dirty and we need to update // it to prevent doing a full file content comparison the next time around. - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ - c_object.src.src_path, @errorName(err), - }); - }; + man.finalize() catch |err| + log.warn("failed to write cache manifest when compiling {q}: {t}", .{ c_object.src.src_path, err }); } const o_basename = try std.fmt.allocPrint(arena, "{s}{s}", .{ o_basename_noext, o_ext }); @@ -5900,15 +5899,23 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // the XML data as a RT_MANIFEST resource. This means we can skip preprocessing, // include paths, CLI options, etc. if (win32_resource.src == .manifest) { - _ = try man.addFilePath(.initCwd(src_path), null); + _ = try man.addInputPath(.initCwd(src_path), .{}); const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); - const digest = if (try man.hit(child_progress_node)) man.final() else blk: { + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, child_progress_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => return comp.failWin32Resource(win32_resource, "checking cache failed: {f}", .{ + diag.fmt(&man), + }), + }; + log.debug("win32 resource cache {f}", .{status.fmt(&man)}); + const digest = if (status == .hit) man.hitDigestHex() else miss: { // The digest only depends on the .manifest file, so we can // get the digest now and write the .res directly to the cache - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try fs.path.join(arena, &.{ "o", &digest }); var o_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, o_sub_path, .{}); @@ -5970,13 +5977,12 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 try spawnZigRc(comp, win32_resource, arena, argv.items, child_progress_node); - break :blk digest; + break :miss digest; }; if (man.have_exclusive_lock) { - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ src_path, @errorName(err) }); - }; + man.finalize() catch |err| + log.warn("failed to write cache manifest when compiling {q}: {t}", .{ src_path, err }); } win32_resource.status = .{ @@ -5993,12 +5999,20 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // We now know that we're compiling an .rc file const rc_src = win32_resource.src.rc; - _ = try man.addFilePath(.initCwd(rc_src.src_path), null); + _ = try man.addInputPath(.initCwd(rc_src.src_path), .{}); man.hash.addListOfBytes(rc_src.extra_flags); const rc_basename_noext = src_basename[0 .. src_basename.len - fs.path.extension(src_basename).len]; - const digest = if (try man.hit(child_progress_node)) man.final() else blk: { + var diagnostic: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diagnostic, child_progress_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => return comp.failWin32Resource(win32_resource, "checking cache failed: {f}", .{ + diagnostic.fmt(&man), + }), + }; + log.debug("win32 resource cache {f}", .{status.fmt(&man)}); + const digest = if (status == .hit) man.hitDigestHex() else miss: { var zig_cache_tmp_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, "tmp", .{}); defer zig_cache_tmp_dir.close(io); @@ -6055,12 +6069,14 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 return comp.failWin32Resource(win32_resource, "depfile from zig rc has unexpected format", .{}); } const dep_file_path = element.string; - try man.addFilePost(dep_file_path); + try man.addDiscoveredPath(.{ .discovered_path = .{ .unresolved = .initCwd(dep_file_path) } }); switch (comp.cache_use) { .whole => |whole| if (whole.cache_manifest) |whole_cache_manifest| { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try whole_cache_manifest.addFilePost(dep_file_path); + try whole_cache_manifest.addDiscoveredPath(.{ + .discovered_path = .{ .unresolved = .initCwd(dep_file_path) }, + }); }, .incremental, .none => {}, } @@ -6068,13 +6084,13 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 } // Rename into place. - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest }); var o_dir = try comp.dirs.local_cache.handle.createDirPathOpen(io, o_sub_path, .{}); defer o_dir.close(io); const tmp_basename = fs.path.basename(out_res_path); try Io.Dir.rename(zig_cache_tmp_dir, tmp_basename, o_dir, res_filename, io); - break :blk digest; + break :miss digest; }; if (man.have_exclusive_lock) { @@ -6082,8 +6098,8 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // possible we had a hit and the manifest is dirty, for example if the file mtime changed but // the contents were the same, we hit the cache but the manifest is dirty and we need to update // it to prevent doing a full file content comparison the next time around. - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ rc_src.src_path, @errorName(err) }); + man.finalize() catch |err| { + log.warn("failed to write cache manifest when compiling {q}: {t}", .{ rc_src.src_path, err }); }; } diff --git a/src/Sema.zig b/src/Sema.zig index 1090ecd135c01218c2020d740b8eada0bc5fef26..a3533e99233d4e8f8e0d61184d65733906d1ad86 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -12974,7 +12974,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A const result = ef_idx.get(zcu); if (result.val == .none) { - return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(result.err.?) }); + return sema.fail(block, operand_src, "failed opening {q}: {t}", .{ name, result.err.? }); } return Air.internedToRef(result.val); @@ -15911,9 +15911,7 @@ fn zirClosureGet(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstDat const file, const src_base_node = Zcu.LazySrcLoc.resolveBaseNode(block.src_base_inst, zcu).?; const tree = file.getTree(zcu) catch |err| { // In this case we emit a warning + a less precise source location. - log.warn("unable to load {f}: {s}", .{ - file.path.fmt(zcu.comp), @errorName(err), - }); + log.warn("failed loading {qf}: {t}", .{ file.path.fmt(zcu.comp), err }); break :name null; }; const node = src_node.toAbsolute(src_base_node); @@ -15939,9 +15937,7 @@ fn zirClosureGet(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstDat const file, const src_base_node = Zcu.LazySrcLoc.resolveBaseNode(block.src_base_inst, zcu).?; const tree = file.getTree(zcu) catch |err| { // In this case we emit a warning + a less precise source location. - log.warn("unable to load {f}: {s}", .{ - file.path.fmt(zcu.comp), @errorName(err), - }); + log.warn("failed loading {qf}: {t}", .{ file.path.fmt(zcu.comp), err }); break :name null; }; const node = src_node.toAbsolute(src_base_node); @@ -15950,7 +15946,7 @@ fn zirClosureGet(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstDat }; const msg = if (name) |some| - try sema.errMsg(src, "'{s}' not accessible from inner function", .{some}) + try sema.errMsg(src, "{q} not accessible from inner function", .{some}) else try sema.errMsg(src, "variable not accessible from inner function", .{}); errdefer msg.destroy(sema.gpa); diff --git a/src/Zcu.zig b/src/Zcu.zig index d6f9da974052860224583dd47cd3b6e44c90d056..56118073660df5a55f77517a459fe5f666c1d442 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -974,7 +974,7 @@ pub const File = struct { success, }, /// Whether this is populated depends on `status`. - stat: Cache.File.Stat, + stat: Cache.Manifest.Stat, /// Whether this file is the generated file of a "builtin" module. This matters because those /// files are generated and stored in-nemory rather than being read off-disk. The rest of the @@ -1246,7 +1246,7 @@ pub const EmbedFile = struct { val: InternPool.Index, /// If this is `null` and `val` is `.none`, the file has never been loaded. err: ?(Io.File.OpenError || Io.File.StatError || Io.File.Reader.Error || error{UnexpectedEof}), - stat: Cache.File.Stat, + stat: Cache.Manifest.Stat, pub const Index = enum(u32) { _, diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index d63b4778c1d5c075237f3fc9d62112fc9916974a..f0b1764f463dbe9d085e0fca931d20286e99255f 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -224,11 +224,12 @@ pub fn update( const result = res: { try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - if (file.source) |source| { - break :res file.path.addToCacheManifestPostHitContents(man, &comp.dirs, source, file.stat); - } else { - break :res file.path.addToCacheManifestPostHit(man, &comp.dirs); - } + break :res file.path.addToCacheManifestAsDiscovered( + man, + &comp.dirs, + file.source, + if (file.source != null) file.stat else null, + ); }; result catch |err| switch (err) { error.OutOfMemory => |e| return e, @@ -348,11 +349,8 @@ pub fn update( } } fn workerUpdateBuiltinFile(comp: *Compilation, file: *Zcu.File) void { - Builtin.updateFileOnDisk(file, comp) catch |err| comp.lockAndSetMiscFailure( - .write_builtin_zig, - "unable to write '{f}': {s}", - .{ file.path.fmt(comp), @errorName(err) }, - ); + Builtin.updateFileOnDisk(file, comp) catch |err| + comp.lockAndSetMiscFailure(.write_builtin_zig, "unable to write {qf}: {t}", .{ file.path.fmt(comp), err }); } fn workerUpdateFile( comp: *Compilation, @@ -371,7 +369,9 @@ fn workerUpdateFile( const active = comp.zcu.?.activate(tid); defer active.deactivate(); active.pt.updateFile(file_index, file) catch |err| { - active.pt.reportRetryableFileError(file_index, "unable to load '{s}': {s}", .{ std.fs.path.basename(file.path.sub_path), @errorName(err) }) catch |oom| switch (oom) { + active.pt.reportRetryableFileError(file_index, "unable to load {q}: {t}", .{ + std.fs.path.basename(file.path.sub_path), err, + }) catch |oom| switch (oom) { error.OutOfMemory => { comp.mutex.lockUncancelable(io); defer comp.mutex.unlock(io); @@ -2836,26 +2836,21 @@ fn updateBuiltinModule(pt: Zcu.PerThread, opts: Builtin) Allocator.Error!struct assert(mem.eql(u8, import_path, "std")); // the single import is of 'std' } - Builtin.updateFileOnDisk(file, comp) catch |err| comp.setMiscFailure( - .write_builtin_zig, - "unable to write '{f}': {s}", - .{ file.path.fmt(comp), @errorName(err) }, - ); + Builtin.updateFileOnDisk(file, comp) catch |err| + comp.setMiscFailure(.write_builtin_zig, "unable to write {qf}: {t}", .{ file.path.fmt(comp), err }); return .{ .file = file_index, .module_root = mod, }; } +pub const EmbedFileError = error{ImportOutsideModulePath} || Allocator.Error || Io.Cancelable; + pub fn embedFile( pt: Zcu.PerThread, cur_file: *Zcu.File, import_string: []const u8, -) error{ - OutOfMemory, - Canceled, - ImportOutsideModulePath, -}!Zcu.EmbedFile.Index { +) EmbedFileError!Zcu.EmbedFile.Index { const zcu = pt.zcu; const gpa = zcu.gpa; @@ -2938,7 +2933,7 @@ fn updateEmbedFileInner( }; defer file.close(io); - const stat: Cache.File.Stat = .fromFs(try file.stat(io)); + const stat: Cache.Manifest.Stat = .init(try file.stat(io)); if (ef.val != .none) { const old_stat = ef.stat; @@ -2995,10 +2990,7 @@ fn updateEmbedFileInner( } /// Assumes that `path` is allocated into `gpa`. Takes ownership of `path` on success. -fn newEmbedFile( - pt: Zcu.PerThread, - path: Compilation.Path, -) !*Zcu.EmbedFile { +fn newEmbedFile(pt: Zcu.PerThread, path: Compilation.Path) EmbedFileError!*Zcu.EmbedFile { const zcu = pt.zcu; const comp = zcu.comp; const io = comp.io; @@ -3035,7 +3027,10 @@ fn newEmbedFile( try whole.cache_manifest_mutex.lock(io); defer whole.cache_manifest_mutex.unlock(io); - try path.addToCacheManifestPostHitContents(man, &comp.dirs, contents, new_file.stat); + path.addToCacheManifestAsDiscovered(man, &comp.dirs, contents, new_file.stat) catch |err| switch (err) { + error.FileSystemFailure => unreachable, // contents and stat are both provided + else => |e| return e, + }; } return new_file; diff --git a/src/libs/freebsd.zig b/src/libs/freebsd.zig index d7b53387347e27ab502c13282dc99f8c75763df4..584a6b63b69adae9e3427334567bd41357056543 100644 --- a/src/libs/freebsd.zig +++ b/src/libs/freebsd.zig @@ -458,14 +458,28 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_os_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); - - if (try man.hit(prog_node)) { - const digest = man.final(); + }, .{ + .request_contents = true, + }); + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .freebsd_shared_objects, + "compiling FreeBSD libc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + }; + log.debug("freebsd_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -475,7 +489,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); var o_directory: Cache.Directory = .{ @@ -484,7 +498,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); @@ -980,9 +994,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for FreeBSD libc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for FreeBSD libc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/glibc.zig b/src/libs/glibc.zig index ebc36af06304aa28b760477167ce0b04570abac3..86c5360cc17cce248afeac8ddcfb5bb4b2f9ceca 100644 --- a/src/libs/glibc.zig +++ b/src/libs/glibc.zig @@ -698,14 +698,28 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); - - if (try man.hit(prog_node)) { - const digest = man.final(); + }, .{ + .request_contents = true, + }); + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .glibc_shared_objects, + "compiling glibc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + }; + log.debug("glibc_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -715,8 +729,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); - const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); + const digest = man.missDigestHex(); + const o_sub_path = try path.join(arena, &.{ "o", &digest }); var o_directory: Cache.Directory = .{ .handle = try comp.dirs.global_cache.handle.createDirPathOpen(io, o_sub_path, .{}), @@ -724,7 +738,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); @@ -1129,9 +1143,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for glibc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for glibc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/mingw.zig b/src/libs/mingw.zig index a91c24291d0a395b189712c4d8f199bf13cd57d0..bfab89ca4ab86ec81d4366bca85fd72bf5eef7a6 100644 --- a/src/libs/mingw.zig +++ b/src/libs/mingw.zig @@ -246,35 +246,24 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P var man = cache.obtain(); defer man.deinit(); - _ = try man.addFilePath(def_file_path, null); + _ = try man.addInputPath(def_file_path, .{}); const final_lib_basename = try std.fmt.allocPrint(gpa, "{s}.lib", .{lib_name}); errdefer gpa.free(final_lib_basename); - const is_hit = man.hit(prog_node) catch |err| switch (err) { - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| { - comp.setMiscFailure(.windows_import_lib, "checking cache failed: {t} {t}", .{ man.diagnostic, e }); - return error.AlreadyReported; - }, - .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix]; - comp.setMiscFailure(.windows_import_lib, "checking cache failed: {f}{s} {t} {t}", .{ - prefix, pp.sub_path, man.diagnostic, op.err, - }); - return error.AlreadyReported; - }, - }, - error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => { - comp.setMiscFailure(.windows_import_lib, "checking cache failed: invalid manifest file format", .{}); + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure(.windows_import_lib, "{s} cache check failed: {f}", .{ + final_lib_basename, diag.fmt(&man), + }); return error.AlreadyReported; }, + error.OutOfMemory, error.Canceled => |e| return e, }; - if (is_hit) { - const digest = man.final(); + log.debug("{s} cache {f}", .{ final_lib_basename, status.fmt(&man) }); + if (status == .hit) { + const digest = man.hitDigestHex(); const sub_path = try std.fs.path.join(gpa, &.{ "o", &digest, final_lib_basename }); errdefer gpa.free(sub_path); @@ -293,7 +282,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P return crt_file_path; } - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); var o_dir = try comp.dirs.global_cache.handle.createDirPathOpen(io, o_sub_path, .{}); defer o_dir.close(io); @@ -365,9 +354,8 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8, prog_node: std.P try file_writer.interface.flush(); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for DLL import {s}.lib: {s}", .{ lib_name, @errorName(err) }); - }; + man.finalize() catch |err| + log.warn("failed to write cache manifest for DLL import {s}.lib: {t}", .{ lib_name, err }); comp.mutex.lockUncancelable(io); defer comp.mutex.unlock(io); diff --git a/src/libs/netbsd.zig b/src/libs/netbsd.zig index b0fd269b22279b860426781e41a21e5297989c75..46a34b96a55943f84a92914521a730c27d0d19a7 100644 --- a/src/libs/netbsd.zig +++ b/src/libs/netbsd.zig @@ -406,14 +406,28 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); - - if (try man.hit(prog_node)) { - const digest = man.final(); + }, .{ + .request_contents = true, + }); + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .netbsd_shared_objects, + "compiling NetBSD libc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + error.OutOfMemory, error.Canceled => |e| return e, + }; + log.debug("netbsd_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -423,8 +437,8 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); - const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); + const digest = man.missDigestHex(); + const o_sub_path = try path.join(arena, &.{ "o", &digest }); var o_directory: Cache.Directory = .{ .handle = try comp.dirs.global_cache.handle.createDirPathOpen(io, o_sub_path, .{}), @@ -432,7 +446,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); @@ -641,9 +655,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for NetBSD libc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for NetBSD libc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/libs/openbsd.zig b/src/libs/openbsd.zig index ba65dd46943a10fc8e61f2216e6762debf549e4a..4bea660a65eed944eb93d48f27c5cc70a0369a8d 100644 --- a/src/libs/openbsd.zig +++ b/src/libs/openbsd.zig @@ -327,14 +327,28 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye man.hash.add(target.abi); man.hash.add(target_version); - const abilists_index = try man.addFilePath(.{ + const abilists_index = try man.addInputPath(.{ .root_dir = comp.dirs.zig_lib, .sub_path = abilists_path, - }, abilists_max_size); - - if (try man.hit(prog_node)) { - const digest = man.final(); + }, .{ + .request_contents = true, + }); + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => { + comp.lockAndSetMiscFailure( + .openbsd_shared_objects, + "compiling OpenBSD libc shared objects: checking cache failed: {f}", + .{diag.fmt(&man)}, + ); + return error.AlreadyReported; + }, + }; + log.debug("openbsd_shared_objects cache {f}", .{status.fmt(&man)}); + if (status == .hit) { + const digest = man.hitDigestHex(); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), .dir_path = .{ @@ -344,7 +358,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }); } - const digest = man.final(); + const digest = man.missDigestHex(); const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); var o_directory: Cache.Directory = .{ @@ -353,7 +367,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye }; defer o_directory.handle.close(io); - const abilists_contents = man.files.keys()[abilists_index].contents.?; + const abilists_contents = abilists_index.contents(&man); const metadata = try loadMetaData(gpa, abilists_contents); defer metadata.destroy(gpa); @@ -562,9 +576,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) anye try buildSharedLib(comp, arena, o_directory, asm_file_basename, lib, prog_node); } - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for OpenBSD libc stubs: {s}", .{@errorName(err)}); - }; + man.finalize() catch |err| log.warn("failed to write cache manifest for OpenBSD libc stubs: {t}", .{err}); return queueSharedObjects(comp, .{ .lock = man.toOwnedLock(), diff --git a/src/link.zig b/src/link.zig index babee6e563e072fd863c3622be6107fdac6a35fd..c97c752517928551c1753933a9c512417cb9e434 100644 --- a/src/link.zig +++ b/src/link.zig @@ -1500,11 +1500,11 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void { base.loadInput(input) catch |err| switch (err) { error.AlreadyReported => return, // error reported via diags else => |e| switch (input) { - .dso => |dso| diags.addParseError(dso.path, "failed to parse shared library: {s}", .{@errorName(e)}), - .object => |obj| diags.addParseError(obj.path, "failed to parse object: {s}", .{@errorName(e)}), - .archive => |obj| diags.addParseError(obj.path, "failed to parse archive: {s}", .{@errorName(e)}), - .res => |res| diags.addParseError(res.path, "failed to parse Windows resource: {s}", .{@errorName(e)}), - .dso_exact => diags.addError("failed to handle dso_exact: {s}", .{@errorName(e)}), + .dso => |dso| diags.addParseError(dso.path, "failed to parse shared library: {t}", .{e}), + .object => |obj| diags.addParseError(obj.path, "failed to parse object: {t}", .{e}), + .archive => |obj| diags.addParseError(obj.path, "failed to parse archive: {t}", .{e}), + .res => |res| diags.addParseError(res.path, "failed to parse Windows resource: {t}", .{e}), + .dso_exact => diags.addError("failed to handle dso_exact: {t}", .{e}), }, }; prog_node.completeOne(); @@ -1925,15 +1925,24 @@ pub fn hashInputs(man: *Cache.Manifest, link_inputs: []const Input) !void { man.hash.add(@as(@typeInfo(Input).@"union".tag_type.?, link_input)); switch (link_input) { .object, .archive => |obj| { - _ = try man.addOpenedFile(obj.path, obj.file, null); + _ = try man.addInputPath(obj.path, .{ + .handle = .{ .file = obj.file }, + .request_handle = true, + }); man.hash.add(obj.must_link); man.hash.add(obj.hidden); }, .res => |res| { - _ = try man.addOpenedFile(res.path, res.file, null); + _ = try man.addInputPath(res.path, .{ + .handle = .{ .file = res.file }, + .request_handle = true, + }); }, .dso => |dso| { - _ = try man.addOpenedFile(dso.path, dso.file, null); + _ = try man.addInputPath(dso.path, .{ + .handle = .{ .file = dso.file }, + .request_handle = true, + }); man.hash.add(dso.needed); man.hash.add(dso.weak); man.hash.add(dso.reexport); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 416eab37e05651c42cfedb4e025bca62b9cac016..f04d7582bd6dac07669861f0d920c86efa6d6d1c 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1120,7 +1120,7 @@ fn parseDso( const handle = dso.file; - const stat = Stat.fromFs(try handle.stat(io)); + const stat: Stat = .init(try handle.stat(io)); var header = try SharedObject.parseHeader(gpa, io, diags, dso.path, handle, stat, target); defer header.deinit(gpa); @@ -4407,7 +4407,7 @@ const mem = std.mem; const Allocator = std.mem.Allocator; const Hash = std.hash.Wyhash; const Path = std.Build.Cache.Path; -const Stat = std.Build.Cache.File.Stat; +const Stat = std.Build.Cache.Manifest.Stat; const codegen = @import("../codegen.zig"); const eh_frame = @import("Elf/eh_frame.zig"); diff --git a/src/link/Elf/SharedObject.zig b/src/link/Elf/SharedObject.zig index 052093dce3acdf01c893ec6af73368f55523d234..3030a45bff3806d93b1af418d6eaa49a7a649558 100644 --- a/src/link/Elf/SharedObject.zig +++ b/src/link/Elf/SharedObject.zig @@ -7,7 +7,7 @@ const elf = std.elf; const log = std.log.scoped(.elf); const mem = std.mem; const Path = std.Build.Cache.Path; -const Stat = std.Build.Cache.File.Stat; +const Stat = std.Build.Cache.Manifest.Stat; const Allocator = mem.Allocator; const Elf = @import("../Elf.zig"); diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 72e34b943e1d565e6c3360cfd017b4d9c7ba3f5e..c32301b826482026993bfb7e8763357cd57dbf06 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -155,7 +155,7 @@ pub fn hashAddFrameworks(man: *Cache.Manifest, hm: []const Framework) !void { for (hm) |value| { man.hash.add(value.needed); man.hash.add(value.weak); - _ = try man.addFilePath(value.path, null); + _ = try man.addInputPath(value.path, .{}); } } diff --git a/src/main.zig b/src/main.zig index a2218a480f8f38b30c4963402e8bfe413df2b8bd..41b4e841d36f4bfe598cb23fd300d6ffe248b0f8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -107,6 +107,7 @@ const normal_usage = \\ \\ env Print lib path, std path, cache directory, and version \\ help Print this help and exit + \\ cache-cat Print a zig-cache manifest file as zon \\ std View standard library documentation in a browser \\ libc Display native libc paths file or validate one \\ targets List available compilation targets @@ -242,6 +243,10 @@ const Cmd = enum { ar, build, + @"cache-cat", + fetch, + init, + libc, clang, @"-cc1", @@ -258,10 +263,7 @@ const Cmd = enum { fmt, objcopy, objdump, - fetch, - libc, std, - init, targets, version, env, @@ -351,7 +353,7 @@ fn mainArgs( dev.check(.ar_command); return process.exit(try llvmArMain(arena, args)); }, - .build, .fetch, .init, .libc => { + .build, .fetch, .init, .libc, .@"cache-cat" => { return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "maker", .root_src_path = "Maker.zig", @@ -3268,6 +3270,7 @@ fn buildOutputType( }; const cwd_path = try std.zig.getResolvedCwd(io, arena); + std.log.debug("cwd_path={s}", .{cwd_path}); // This `init` calls `fatal` on error. var dirs: std.zig.Directories = .init(arena, io, .{ @@ -4876,8 +4879,14 @@ fn cmdTranslateC( Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| fatal("unable to process {q}: {t}", .{ c_source_file.src_path, err }); - const result: Compilation.TranslateCResult = if (try man.hit(prog_node)) .{ - .digest = man.finalBin(), + var diag: Cache.Manifest.CheckDiagnostic = undefined; + const status = man.check(&diag, prog_node) catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.CacheCheckFailed => fatal("translate-c checking cache failed: {f}", .{diag.fmt(&man)}), + }; + std.log.debug("translate-c cache {f}", .{status.fmt(&man)}); + const result: Compilation.TranslateCResult = if (status == .hit) .{ + .digest = man.hitDigest(), .cache_hit = true, .errors = std.zig.ErrorBundle.empty, } else result: { @@ -4904,7 +4913,7 @@ fn cmdTranslateC( } } - man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + man.finalize() catch |err| warn("failed to write cache manifest: {t}", .{err}); break :result result; }; diff --git a/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig b/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig index 37bc8f5388edb4c18bc72e3cd9084797742f7b2f..b86775c86925e40959aa4bbe3969e6c2d2e06dc5 100644 --- a/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig +++ b/test/cases/compile_errors/accessing_runtime_parameter_from_outer_function.zig @@ -14,5 +14,5 @@ export fn entry() void { // error // -// :4:24: error: 'y' not accessible from inner function +// :4:24: error: "y" not accessible from inner function // :3:9: note: crossed function definition here diff --git a/test/cases/compile_errors/bad_import.zig b/test/cases/compile_errors/bad_import.zig index 13ce9d46ff8d45b72a40f5827fe1a111af3913c3..c532706dae893d64e943acd7153995f9badc413d 100644 --- a/test/cases/compile_errors/bad_import.zig +++ b/test/cases/compile_errors/bad_import.zig @@ -4,5 +4,5 @@ const bogus = @import( // error // -// bogus-does-not-exist.zig:1:1: error: unable to load 'bogus-does-not-exist.zig': FileNotFound +// bogus-does-not-exist.zig:1:1: error: unable to load "bogus-does-not-exist.zig": FileNotFound // :2:5: note: file imported here diff --git a/test/cases/compile_errors/embedFile_with_bogus_file.zig b/test/cases/compile_errors/embedFile_with_bogus_file.zig index c769b4b2c6f48b9248ad3eee33022e8c8b6546cb..d27847189ae141a894c81cd52f29a92161e57e75 100644 --- a/test/cases/compile_errors/embedFile_with_bogus_file.zig +++ b/test/cases/compile_errors/embedFile_with_bogus_file.zig @@ -6,4 +6,4 @@ export fn entry() usize { // error // -// :1:29: error: unable to open 'bogus.txt': FileNotFound +// :1:29: error: failed opening "bogus.txt": FileNotFound diff --git a/test/incremental/bad_import b/test/incremental/bad_import index ab6369bee38a0d8c0c3e01b42458cc52051e4f5f..fd91fa66050e2ae3cb1407382d5f173bdb2fe884 100644 --- a/test/incremental/bad_import +++ b/test/incremental/bad_import @@ -10,7 +10,7 @@ const io = std.Io.Threaded.global_single_threaded.io(); comptime { _ = @import("bad.zig"); } -#expect_error=bad.zig:1:1: error: unable to load 'bad.zig': FileNotFound +#expect_error=bad.zig:1:1: error: unable to load "bad.zig": FileNotFound #expect_error=foo.zig:2:17: note: file imported here #update=change bad import @@ -18,7 +18,7 @@ comptime { comptime { _ = @import("this_is/not_real.zig"); } -#expect_error=this_is/not_real.zig:1:1: error: unable to load 'not_real.zig': FileNotFound +#expect_error=this_is/not_real.zig:1:1: error: unable to load "not_real.zig": FileNotFound #expect_error=foo.zig:2:17: note: file imported here #update=remove import of 'foo.zig' diff --git a/test/incremental/change_embed_file b/test/incremental/change_embed_file index 34efcdcc5a4fb112e35c862173e85afd96a15ddd..8b515d2d980c9ee68da186c8fa149b5b19c2ed20 100644 --- a/test/incremental/change_embed_file +++ b/test/incremental/change_embed_file @@ -17,7 +17,7 @@ Hello again, World! #update=delete file #rm_file=string.txt -#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound +#expect_error=main.zig:2:27: error: failed opening "string.txt": FileNotFound #update=remove reference to file #file=main.zig @@ -37,7 +37,7 @@ pub fn main() !void { try std.Io.File.stdout().writeStreamingAll(io, string); } const io = std.Io.Threaded.global_single_threaded.io(); -#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound +#expect_error=main.zig:2:27: error: failed opening "string.txt": FileNotFound #update=recreate file #file=string.txt diff --git a/test/incremental/change_zon_file b/test/incremental/change_zon_file index 080dc235199d21b5dded5136bd6baf04b72142a0..7616c53ea80094d746102df13d51e387a3d1bc40 100644 --- a/test/incremental/change_zon_file +++ b/test/incremental/change_zon_file @@ -17,7 +17,7 @@ const io = std.Io.Threaded.global_single_threaded.io(); #update=delete file #rm_file=message.zon -#expect_error=message.zon:1:1: error: unable to load 'message.zon': FileNotFound +#expect_error=message.zon:1:1: error: unable to load "message.zon": FileNotFound #expect_error=main.zig:2:37: note: file imported here #update=remove reference to ZON file @@ -28,7 +28,7 @@ pub fn main() !void { try std.Io.File.stdout().writeStreamingAll(io, "a hardcoded string\n"); } const io = std.Io.Threaded.global_single_threaded.io(); -#expect_error=message.zon:1:1: error: unable to load 'message.zon': FileNotFound +#expect_error=message.zon:1:1: error: unable to load "message.zon": FileNotFound #expect_error=main.zig:2:37: note: file imported here #update=recreate ZON file diff --git a/test/src/Cases.zig b/test/src/Cases.zig index d04bc01972bbb8b0fe09885471971fc5b5f1e8a0..5dbff266e78f9b3f585bdbd87b99ac8e2f27f09f 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -319,7 +319,7 @@ pub fn addCompile( pub fn addFromDir(ctx: *Cases, dir: Io.Dir, path_from_root: []const u8, b: *std.Build) void { var current_file: []const u8 = "none"; ctx.addFromDirInner(dir, path_from_root, ¤t_file, b) catch |err| { - std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}\n", .{ + std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}", .{ current_file, err, }); }; @@ -348,7 +348,7 @@ fn addFromDirInner( try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path)); }, .directory => { - b.dependOnDirectory(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); + b.dependOnDirectoryContents(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); }, else => continue, } diff --git a/test/tests.zig b/test/tests.zig index 18c9fe113a8d607caa374aaca7cad68386707e40..645152b8457384367c4fa7a7933c5ca2034ad56d 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -3339,7 +3339,7 @@ pub fn addCases( var cases = @import("src/Cases.zig").init(gpa, arena, io); - b.dependOnDirectory(b.path("test/cases")); + b.dependOnDirectoryContents(b.path("test/cases")); var dir = try b.root.openDir(io, "test/cases", .{ .iterate = true }); defer dir.close(io); @@ -3401,7 +3401,7 @@ pub fn addIncrementalTests( }), }); - b.dependOnDirectory(b.path("test/incremental")); + b.dependOnDirectoryContents(b.path("test/incremental")); var dir = try b.root.openDir(io, "test/incremental", .{ .iterate = true }); defer dir.close(io); @@ -3417,7 +3417,7 @@ pub fn addIncrementalTests( switch (entry.kind) { .file => {}, .directory => { - b.dependOnDirectory(b.path(b.pathJoin(&.{ "test", "incremental", entry.path }))); + b.dependOnDirectoryContents(b.path(b.pathJoin(&.{ "test", "incremental", entry.path }))); }, else => continue, } diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 21c24e9048203c80a5562f6c3b47fce31cb2e80d..c8034260dbc2af7a256151c7c43f4429531e390b 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -152,18 +152,12 @@ pub fn main(init: std.process.Init) !void { var child_args: std.ArrayList([]const u8) = .empty; try child_args.appendSlice(arena, &.{ - resolved_zig_exe, - "build-exe", - "-fincremental", - "-fno-ubsan-rt", - "-target", - try target_query.zigTriple(arena), - "--cache-dir", - ".local-cache", - "--global-cache-dir", - ".global-cache", + resolved_zig_exe, "build-exe", + "-fincremental", "-fno-ubsan-rt", + "-target", try target_query.zigTriple(arena), + "--cache-dir", ".local-cache", + "--global-cache-dir", ".global-cache", }); - try child_args.append(arena, "--listen=-"); if (opt_resolved_lib_dir) |resolved_lib_dir| { try child_args.appendSlice(arena, &.{ "--zig-lib-dir", resolved_lib_dir }); @@ -187,6 +181,7 @@ pub fn main(init: std.process.Init) !void { for (case.modules) |mod| { try child_args.append(arena, try std.fmt.allocPrint(arena, "-M{s}={s}", .{ mod.name, mod.file })); } + try child_args.append(arena, "--listen=-"); const zig_prog_node = prog_node.start("zig", 0); defer zig_prog_node.end(); @@ -199,15 +194,23 @@ pub fn main(init: std.process.Init) !void { resolved_zig_exe; try cc_child_args.appendSlice(arena, &.{ - resolved_cc_zig_exe, - "cc", - "-target", - try target_query.zigTriple(arena), - "-I", - opt_resolved_lib_dir.?, // verified earlier + resolved_cc_zig_exe, "build-exe", + "-target", try target_query.zigTriple(arena), + "--cache-dir", ".local-cache", + "--global-cache-dir", ".global-cache", + "-I", opt_resolved_lib_dir.?, // verified earlier + "-lc", }); + } - try cc_child_args.append(arena, "-o"); + const allow_compiler_stderr = debug_log_args.items.len != 0; + + if (allow_compiler_stderr) { + const cmd: std.zig.SubprocessCommand = .{ + .argv = child_args.items, + .cwd = tmp_dir_path, + }; + std.log.scoped(.spawn).info("{f}", .{cmd}); } var child = try std.process.spawn(io, .{ @@ -233,7 +236,7 @@ pub fn main(init: std.process.Init) !void { .tmp_dir = tmp_dir, .tmp_dir_path = tmp_dir_path, .child = &child, - .allow_compiler_stderr = debug_log_args.items.len != 0, + .allow_compiler_stderr = allow_compiler_stderr, .quiet = quiet, .preserve_tmp_on_fatal = preserve_tmp, .cc_child_args = &cc_child_args, @@ -659,9 +662,18 @@ const Eval = struct { const child_prog_node = prog_node.start("build cbe output", 0); defer child_prog_node.end(); - try eval.cc_child_args.appendSlice(eval.arena, &.{ out_path, c_path }); + try eval.cc_child_args.append(eval.arena, try std.fmt.allocPrint(eval.arena, "-femit-bin={s}", .{out_path})); + try eval.cc_child_args.append(eval.arena, c_path); defer eval.cc_child_args.items.len -= 2; + if (eval.allow_compiler_stderr) { + const cmd: std.zig.SubprocessCommand = .{ + .argv = eval.cc_child_args.items, + .cwd = eval.tmp_dir_path, + }; + std.log.scoped(.spawn).info("{f}", .{cmd}); + } + const result = std.process.run(eval.arena, eval.io, .{ .argv = eval.cc_child_args.items, .cwd = .{ .path = eval.tmp_dir_path },