authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-19 14:58:10-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-19 14:58:10-07:00
log22c141605b532eb7d62bfecb020868e580476b93
treeb9d0f06006f4c36f05aa9bfe6e6f94add6c9fcd3
parent64caa0e324d522915769dd5cc8a11374f919f01f

WIP: combine file_system_inputs and Cache.Manifest

9 files changed, 240 insertions(+), 409 deletions(-)

lib/compiler/Maker.zig+1
...@@ -1573,6 +1573,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1573,6 +1573,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1573 const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{1573 const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{
1574 .argv = build_configurer_argv.items,1574 .argv = build_configurer_argv.items,
1575 .cache_root = graph.local_cache_root,1575 .cache_root = graph.local_cache_root,
1576 .cache = &graph.cache,
1576 .root_name = configurer_exe_name,1577 .root_name = configurer_exe_name,
1577 .environ_map = &graph.environ_map,1578 .environ_map = &graph.environ_map,
1578 .cache_manifest = config_man,1579 .cache_manifest = config_man,
lib/compiler/Maker/Step.zig+27-62
...@@ -659,68 +659,33 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi...@@ -659,68 +659,33 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi
659 // This message indicates the end of the update.659 // This message indicates the end of the update.
660 if (watch) break;660 if (watch) break;
661 },661 },
662 .emit_digest => {662 .cache_manifest => {
663 const emit_digest = body_r.takeStruct(std.zig.Server.Message.EmitDigest, .little) catch unreachable;663 const cache_manifest = body_r.takeStruct(std.zig.Server.Message.CacheManifest, .little) catch unreachable;
664 const digest = body_r.takeArray(Cache.bin_digest_len) catch unreachable;664 var diag: Cache.Manifest.CheckDiagnostic = undefined;
665 s.result_cache_status = if (emit_digest.flags.cache_hit) .hit else .incomplete_manifest;665 var man = graph.cache.obtain();
666 result = .{ .bin = digest.* };666 defer man.deinit();
667 },667 man.openFile(&cache_manifest.input_hash, &diag) catch |err| switch (err) {
668 .file_system_inputs => {668 else => |e| return e,
669 clearWatchInputs(s, maker);669 error.CacheCheckFailed => return s.fail(maker, "opening cache manifest failed: {f}", .{
670 const conf = &maker.scanned_config.configuration;670 diag.fmt(&man),
671 const conf_step = step_index.ptr(conf);671 }),
672 var it = std.mem.splitScalar(u8, body, 0);672 };
673 while (it.next()) |prefixed_path| {673
674 const prefix_index: std.zig.Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1));674 s.result_cache_status = switch (cache_manifest.cache_status.result) {
675 const sub_path = try arena.dupe(u8, prefixed_path[1..]);675 .hit => .hit,
676 const sub_path_dirname = Dir.path.dirname(sub_path) orelse "";676 .incomplete_manifest => .incomplete_manifest,
677 switch (prefix_index) {677 .invalid_manifest => .invalid_manifest,
678 .cwd => {678 inline .path_deleted, .directory_status_changed, .metadata_changed, .contents_changed => |tag| b: {
679 const path: Path = .{679 const off: Cache.Manifest.File.Offset = @fromBackingInt(cache_manifest.cache_status.file_off);
680 .root_dir = .cwd(),680 break :b @unionInit(
681 .sub_path = sub_path_dirname,681 ResultCacheStatus,
682 };682 @tagName(tag),
683 try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path));683 try gpa.dupeSentinel(u8, off.path(man.contents.items), 0),
684 },684 );
685 .zig_lib => zl: {685 },
686 switch (conf_step.extended.get(conf.extra)) {686 };
687 .compile => |compile| if (compile.zig_lib_dir.value) |zig_lib_dir| {687 result = .{ .bin = cache_manifest.discovered_hash };
688 const resolved = try maker.resolveLazyPathIndex(arena, zig_lib_dir, step_index);688 try setWatchInputsFromManifest(s, maker, &man);
689 const appended = try resolved.join(arena, sub_path);
690 try addWatchInputPath(s, maker, appended);
691 break :zl;
692 },
693 else => {},
694 }
695 const path: Path = .{
696 .root_dir = graph.zig_lib_directory,
697 .sub_path = sub_path_dirname,
698 };
699 try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path));
700 },
701 .local_cache => {
702 const path: Path = .{
703 .root_dir = graph.local_cache_root,
704 .sub_path = sub_path_dirname,
705 };
706 try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path));
707 },
708 .global_cache => {
709 const path: Path = .{
710 .root_dir = graph.global_cache_root,
711 .sub_path = sub_path_dirname,
712 };
713 try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path));
714 },
715 .build_root => {
716 const path: Path = .{
717 .root_dir = graph.build_root_directory,
718 .sub_path = sub_path_dirname,
719 };
720 try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path));
721 },
722 }
723 }
724 },689 },
725 .time_report => if (maker.web_server) |ws| {690 .time_report => if (maker.web_server) |ws| {
726 const tr = body_r.takeStruct(std.zig.Server.Message.TimeReport, .little) catch unreachable;691 const tr = body_r.takeStruct(std.zig.Server.Message.TimeReport, .little) catch unreachable;
lib/compiler/Maker/WebServer.zig+23-1
...@@ -650,15 +650,37 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim...@@ -650,15 +650,37 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
650 const compile_prog_node = ws.root_prog_node.start("Compile WebAssembly Component", 0);650 const compile_prog_node = ws.root_prog_node.start("Compile WebAssembly Component", 0);
651 defer compile_prog_node.end();651 defer compile_prog_node.end();
652652
653 var cache: Cache = .{
654 .gpa = gpa,
655 .io = io,
656 .manifest_dir = try graph.global_cache_root.handle.createDirPathOpen(io, "h", .{}),
657 .cwd = graph.cache.cwd,
658 };
659 defer cache.manifest_dir.close(io);
660 cache.addPrefix(.cwd());
661 cache.addPrefix(graph.zig_lib_directory);
662 cache.addPrefix(graph.global_cache_root);
663
653 const result = try std.zig.buildExeSubprocess(gpa, io, .{664 const result = try std.zig.buildExeSubprocess(gpa, io, .{
654 .argv = argv.items,665 .argv = argv.items,
655 .cache_root = graph.global_cache_root,666 .cache_root = graph.global_cache_root,
667 .cache = &cache,
668 .prefix_map = .{
669 0, // cwd is the same
670 1, // zig lib dir is the same
671 2, // local cache is mapped to global cache
672 2, // global cache is the same
673 0, // build root is not provided
674 },
656 .root_name = root_name,675 .root_name = root_name,
657 .arch_os_abi = arch_os_abi,676 .arch_os_abi = arch_os_abi,
658 .cpu_features = cpu_features,677 .cpu_features = cpu_features,
659 .progress_node = compile_prog_node,678 .progress_node = compile_prog_node,
660 });679 });
661 if (!result.cache_hit) log.info("source changes detected; rebuilt wasm component", .{});680 switch (result.cache_result) {
681 .hit => {},
682 else => log.info("source changes detected: {t}; rebuilt wasm component", .{result.cache_result}),
683 }
662 return result.path;684 return result.path;
663}685}
664686
lib/std/Build/Cache.zig+56-70
...@@ -286,7 +286,7 @@ pub const Manifest = struct {...@@ -286,7 +286,7 @@ pub const Manifest = struct {
286 want_refresh_timestamp: bool = true,286 want_refresh_timestamp: bool = true,
287 /// Uses `Cache.gpa`.287 /// Uses `Cache.gpa`.
288 files: Files = .empty,288 files: Files = .empty,
289 /// Indexes line up with `files`, but only up until `hit` is called. Uses289 /// Indexes line up with `files`, but only up until `check` is called. Uses
290 /// `Cache.gpa`.290 /// `Cache.gpa`.
291 input_paths: std.ArrayList(InputPath) = .empty,291 input_paths: std.ArrayList(InputPath) = .empty,
292 /// Keeps track of the last time we performed a file system write to observe292 /// Keeps track of the last time we performed a file system write to observe
...@@ -693,12 +693,12 @@ pub const Manifest = struct {...@@ -693,12 +693,12 @@ pub const Manifest = struct {
693 };693 };
694694
695 /// Add a file or directory path as a dependency of process being cached.695 /// Add a file or directory path as a dependency of process being cached.
696 /// When `hit` is called, the contents will be checked to ensure696 /// When `check` is called, the contents will be checked to ensure
697 /// that it matches the contents from previous times.697 /// that it matches the contents from previous times.
698 ///698 ///
699 /// The contents of the input file may be requested and subsequently699 /// The contents of the input file may be requested and subsequently
700 /// obtained via methods of the returned `InputPath.Index` after calling700 /// obtained via methods of the returned `InputPath.Index` after calling
701 /// `hit`.701 /// `check`.
702 ///702 ///
703 /// Contents of a directory are considered to be the sorted list of file703 /// Contents of a directory are considered to be the sorted list of file
704 /// names of direct entries, separated by null byte. Each file name is704 /// names of direct entries, separated by null byte. Each file name is
...@@ -910,7 +910,57 @@ pub const Manifest = struct {...@@ -910,7 +910,57 @@ pub const Manifest = struct {
910910
911 var input_digest: BinDigest = undefined;911 var input_digest: BinDigest = undefined;
912 man.hash.hasher.final(&input_digest);912 man.hash.hasher.final(&input_digest);
913 const input_hex_digest = binToHex(input_digest);913
914 try openFile(man, &input_digest, diag);
915
916 man.want_refresh_timestamp = true;
917
918 // We're going to construct a second hash. Its input will begin with the digest we've already computed
919 // (`input_digest`), and then it'll have the digests of each input file, including discovered files (see
920 // `addDiscoveredPath`). If this is a hit, we learn the set of discovered files from the manifest on disk. If
921 // this is a miss, we'll learn those from future calls to `addDiscoveredPath` etc. As such, the state of
922 // `man.hash.hasher` after this function depends on whether this is a hit or a miss.
923 //
924 // If we return `CheckResult.hit`, then `man.hash.hasher` must already include the digests of the discovered
925 // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests
926 // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered
927 // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user
928 // can use `final`, and will at some point `finalize` the file list to disk.
929 man.hash.hasher = hasher_init;
930 man.hash.hasher.update(&input_digest);
931
932 hit: {
933 const miss_result = miss: {
934 const result = try man.checkLocked(diag);
935 if (result == .hit) {
936 break :hit;
937 } else if (!try man.upgradeToExclusiveLock(diag)) {
938 break :miss result;
939 }
940 // Missed with the shared lock, and upgraded to an exclusive lock. However, another process may have
941 // modified the cache directory, so we need to check again before deciding to miss.
942 man.shrinkFilesToInput();
943 const refreshed_result = try man.checkLocked(diag);
944 if (refreshed_result == .hit) break :hit;
945 break :miss refreshed_result;
946 };
947
948 // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a
949 // cache miss, which is needed because they will be used in the manifest digest.
950 man.state = .miss;
951 return miss_result;
952 }
953
954 if (man.want_shared_lock) {
955 man.downgradeToSharedLock() catch |err| return fail(diag, .{ .manifest_lock = err });
956 }
957
958 man.state = .hit;
959 return .hit;
960 }
961
962 pub fn openFile(man: *Manifest, input_digest: *const BinDigest, diag: *CheckDiagnostic) CheckError!void {
963 const input_hex_digest = binToHex(input_digest.*);
914964
915 const manifest_file_path = &input_hex_digest;965 const manifest_file_path = &input_hex_digest;
916 const io = man.cache.io;966 const io = man.cache.io;
...@@ -969,51 +1019,6 @@ pub const Manifest = struct {...@@ -969,51 +1019,6 @@ pub const Manifest = struct {
969 else => |e| return fail(diag, .{ .manifest_create = e }),1019 else => |e| return fail(diag, .{ .manifest_create = e }),
970 }1020 }
971 }1021 }
972
973 man.want_refresh_timestamp = true;
974
975 // We're going to construct a second hash. Its input will begin with the digest we've already computed
976 // (`input_digest`), and then it'll have the digests of each input file, including discovered files (see
977 // `addDiscoveredPath`). If this is a hit, we learn the set of discovered files from the manifest on disk. If
978 // this is a miss, we'll learn those from future calls to `addDiscoveredPath` etc. As such, the state of
979 // `man.hash.hasher` after this function depends on whether this is a hit or a miss.
980 //
981 // If we return `CheckResult.hit`, then `man.hash.hasher` must already include the digests of the discovered
982 // files, so the caller can call `final`. Otherwise, on a cache miss, `man.hash.hasher` will include the digests
983 // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered
984 // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user
985 // can use `final`, and will at some point `finalize` the file list to disk.
986 man.hash.hasher = hasher_init;
987 man.hash.hasher.update(&input_digest);
988
989 hit: {
990 const miss_result = miss: {
991 const result = try man.checkLocked(diag);
992 if (result == .hit) {
993 break :hit;
994 } else if (!try man.upgradeToExclusiveLock(diag)) {
995 break :miss result;
996 }
997 // Missed with the shared lock, and upgraded to an exclusive lock. However, another process may have
998 // modified the cache directory, so we need to check again before deciding to miss.
999 man.shrinkFilesToInput();
1000 const refreshed_result = try man.checkLocked(diag);
1001 if (refreshed_result == .hit) break :hit;
1002 break :miss refreshed_result;
1003 };
1004
1005 // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a
1006 // cache miss, which is needed because they will be used in the manifest digest.
1007 man.state = .miss;
1008 return miss_result;
1009 }
1010
1011 if (man.want_shared_lock) {
1012 man.downgradeToSharedLock() catch |err| return fail(diag, .{ .manifest_lock = err });
1013 }
1014
1015 man.state = .hit;
1016 return .hit;
1017 }1022 }
10181023
1019 fn transitionToMissDiscovered(m: *Manifest) void {1024 fn transitionToMissDiscovered(m: *Manifest) void {
...@@ -1673,7 +1678,7 @@ pub const Manifest = struct {...@@ -1673,7 +1678,7 @@ pub const Manifest = struct {
1673 if (timestamp.nanoseconds < man.recent_problematic_timestamp.nanoseconds)1678 if (timestamp.nanoseconds < man.recent_problematic_timestamp.nanoseconds)
1674 return false;1679 return false;
16751680
1676 // This flag prevents multiple filesystem writes for the same hit() call.1681 // This flag prevents multiple filesystem writes for the same `check` call.
1677 if (man.want_refresh_timestamp) {1682 if (man.want_refresh_timestamp) {
1678 man.want_refresh_timestamp = false;1683 man.want_refresh_timestamp = false;
16791684
...@@ -2120,7 +2125,7 @@ pub const Manifest = struct {...@@ -2120,7 +2125,7 @@ pub const Manifest = struct {
2120 }2125 }
21212126
2122 /// Releases the manifest file and frees any memory the Manifest was using.2127 /// Releases the manifest file and frees any memory the Manifest was using.
2123 /// `Manifest.hit` must be called first.2128 /// `Manifest.check` must be called first.
2124 ///2129 ///
2125 /// Don't forget to call `finalize` before this!2130 /// Don't forget to call `finalize` before this!
2126 pub fn deinit(m: *Manifest) void {2131 pub fn deinit(m: *Manifest) void {
...@@ -2142,25 +2147,6 @@ pub const Manifest = struct {...@@ -2142,25 +2147,6 @@ pub const Manifest = struct {
2142 m.* = undefined;2147 m.* = undefined;
2143 }2148 }
21442149
2145 pub fn populateFileSystemInputs(man: *const Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void {
2146 assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len);
2147 buf.clearRetainingCapacity();
2148 const gpa = man.cache.gpa;
2149 const files = man.files.keys();
2150 if (files.len > 0) {
2151 const contents = man.contents.items;
2152 for (files) |file| {
2153 const file_path = file.path(contents);
2154 try buf.ensureUnusedCapacity(gpa, file_path.len + 2);
2155 buf.appendAssumeCapacity(@as(u8, file.get(contents).flags.prefix) + 1);
2156 buf.appendSliceAssumeCapacity(file_path);
2157 buf.appendAssumeCapacity(0);
2158 }
2159 // The null byte is a separator, not a terminator.
2160 buf.items.len -= 1;
2161 }
2162 }
2163
2164 /// Add the full set of paths from another `Manifest` as a dependency of process being cached, after cache miss2150 /// Add the full set of paths from another `Manifest` as a dependency of process being cached, after cache miss
2165 /// occurs.2151 /// occurs.
2166 ///2152 ///
lib/std/zig.zig+35-29
...@@ -1826,10 +1826,12 @@ pub fn isUpDir(p: []const u8) bool {...@@ -1826,10 +1826,12 @@ pub fn isUpDir(p: []const u8) bool {
1826pub const BuildExeSubprocessOptions = struct {1826pub const BuildExeSubprocessOptions = struct {
1827 argv: []const []const u8,1827 argv: []const []const u8,
1828 cache_root: Cache.Directory,1828 cache_root: Cache.Directory,
1829 cache: *Cache,
1829 root_name: []const u8,1830 root_name: []const u8,
18301831
1831 environ_map: ?*std.process.Environ.Map = null,1832 environ_map: ?*std.process.Environ.Map = null,
1832 cache_manifest: ?*Cache.Manifest = null,1833 cache_manifest: ?*Cache.Manifest = null,
1834 prefix_map: [5]u8 = .{ 0, 1, 2, 3, 4 },
1833 arch_os_abi: ?[]const u8 = null,1835 arch_os_abi: ?[]const u8 = null,
1834 cpu_features: ?[]const u8 = null,1836 cpu_features: ?[]const u8 = null,
1835 progress_node: std.Progress.Node = .none,1837 progress_node: std.Progress.Node = .none,
...@@ -1848,7 +1850,7 @@ pub const BuildExeSubprocessError = error{...@@ -1848,7 +1850,7 @@ pub const BuildExeSubprocessError = error{
18481850
1849pub const BuildExeSubprocessResult = struct {1851pub const BuildExeSubprocessResult = struct {
1850 received_fs_inputs: bool,1852 received_fs_inputs: bool,
1851 cache_hit: bool,1853 cache_result: Cache.Manifest.CheckResult,
1852 path: Cache.Path,1854 path: Cache.Path,
1853};1855};
18541856
...@@ -1911,7 +1913,7 @@ pub fn buildExeSubprocess(...@@ -1911,7 +1913,7 @@ pub fn buildExeSubprocess(
1911 const error_bundle = options.error_bundle orelse &default_error_bundle;1913 const error_bundle = options.error_bundle orelse &default_error_bundle;
19121914
1913 var received_fs_inputs = false;1915 var received_fs_inputs = false;
1914 var cache_hit = false;1916 var cache_result: Cache.Manifest.CheckResult = .incomplete_manifest;
19151917
1916 var eos_err: error{EndOfStream}!void = {};1918 var eos_err: error{EndOfStream}!void = {};
19171919
...@@ -1948,37 +1950,41 @@ pub fn buildExeSubprocess(...@@ -1948,37 +1950,41 @@ pub fn buildExeSubprocess(
1948 else => |e| return e,1950 else => |e| return e,
1949 };1951 };
1950 },1952 },
1951 .emit_digest => {1953 .cache_manifest => {
1952 const EmitDigest = Server.Message.EmitDigest;1954 var body_r: std.Io.Reader = .fixed(body);
1953 const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body);1955 const cache_manifest = body_r.takeStruct(std.zig.Server.Message.CacheManifest, .little) catch unreachable;
1954 cache_hit = ebp_hdr.flags.cache_hit;1956 var diag: Cache.Manifest.CheckDiagnostic = undefined;
1955 const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];1957 var man = options.cache.obtain();
1958 defer man.deinit();
1959 man.openFile(&cache_manifest.input_hash, &diag) catch |err| switch (err) {
1960 else => |e| return e,
1961 error.CacheCheckFailed => {
1962 log.err("opening cache manifest failed: {f}", .{diag.fmt(&man)});
1963 return error.AlreadyReported;
1964 },
1965 };
1966
1967 cache_result = switch (cache_manifest.cache_status.result) {
1968 .hit => .hit,
1969 .incomplete_manifest => .incomplete_manifest,
1970 .invalid_manifest => .invalid_manifest,
1971 .path_deleted => .{ .path_deleted = @fromBackingInt(cache_manifest.cache_status.file_off) },
1972 .directory_status_changed => .{ .directory_status_changed = @fromBackingInt(cache_manifest.cache_status.file_off) },
1973 .metadata_changed => .{ .metadata_changed = @fromBackingInt(cache_manifest.cache_status.file_off) },
1974 .contents_changed => .{ .contents_changed = @fromBackingInt(cache_manifest.cache_status.file_off) },
1975 };
1976
1977 if (options.cache_manifest) |parent_man| {
1978 try parent_man.addDiscoveredManifest(&man, options.prefix_map);
1979 received_fs_inputs = true;
1980 }
1981
1956 if (result) |r| gpa.free(r.sub_path);1982 if (result) |r| gpa.free(r.sub_path);
1957 result = .{1983 result = .{
1958 .root_dir = options.cache_root,1984 .root_dir = options.cache_root,
1959 .sub_path = try Dir.path.join(gpa, &.{ "o", &Cache.binToHex(digest.*) }),1985 .sub_path = try Dir.path.join(gpa, &.{ "o", &Cache.binToHex(cache_manifest.discovered_hash) }),
1960 };1986 };
1961 },1987 },
1962 .file_system_inputs => if (options.cache_manifest) |man| {
1963 received_fs_inputs = true;
1964 var it = mem.splitScalar(u8, body, 0);
1965 while (it.next()) |prefixed_path| {
1966 const prefix: Server.Message.PathPrefix = @fromBackingInt(@intCast(prefixed_path[0] - 1));
1967 const sub_path = prefixed_path[1..];
1968 man.addDiscoveredPath(.{
1969 .discovered_path = .{ .prefixed = .{
1970 .prefix = @intCast(@backingInt(prefix)),
1971 .sub_path = sub_path,
1972 } },
1973 }) catch |err| switch (err) {
1974 error.Canceled, error.OutOfMemory => |e| return e,
1975 else => |e| {
1976 log.err("adding {t} {s} to cache failed: {t}", .{ prefix, sub_path, e });
1977 return error.AlreadyReported;
1978 },
1979 };
1980 }
1981 },
1982 else => {}, // ignore other messages1988 else => {}, // ignore other messages
1983 }1989 }
1984 }1990 }
...@@ -2049,7 +2055,7 @@ pub fn buildExeSubprocess(...@@ -2049,7 +2055,7 @@ pub fn buildExeSubprocess(
2049 defer gpa.free(bin_name);2055 defer gpa.free(bin_name);
2050 return .{2056 return .{
2051 .received_fs_inputs = received_fs_inputs,2057 .received_fs_inputs = received_fs_inputs,
2052 .cache_hit = cache_hit,2058 .cache_result = cache_result,
2053 .path = try base_path.join(gpa, bin_name),2059 .path = try base_path.join(gpa, bin_name),
2054 };2060 };
2055}2061}
lib/std/zig/Server.zig+39-29
...@@ -33,8 +33,6 @@ pub const Message = struct {...@@ -33,8 +33,6 @@ pub const Message = struct {
33 zig_version,33 zig_version,
34 /// Body is an ErrorBundle.34 /// Body is an ErrorBundle.
35 error_bundle,35 error_bundle,
36 /// Body is a EmitDigest.
37 emit_digest,
38 /// Body is a TestMetadata36 /// Body is a TestMetadata
39 test_metadata,37 test_metadata,
40 /// Body is a TestResults38 /// Body is a TestResults
...@@ -45,14 +43,9 @@ pub const Message = struct {...@@ -45,14 +43,9 @@ pub const Message = struct {
45 /// runner to enforce unit test time limits is relatively accurate under extreme system load43 /// runner to enforce unit test time limits is relatively accurate under extreme system load
46 /// (where there may be a non-trivial delay before the test process is scheduled).44 /// (where there may be a non-trivial delay before the test process is scheduled).
47 test_started,45 test_started,
48 /// Body is a series of strings, delimited by null bytes.46 /// All the information needed to place a file watch on file system inputs. Same memory layout as
49 /// Each string is a prefixed file path.47 /// `std.Build.Cache.Manifest.contents`. Body is a `CacheManifest`.
50 /// The first byte indicates the file prefix path (see prefixes fields48 cache_manifest,
51 /// of Cache). This byte is sent over the wire incremented so that null
52 /// bytes are not confused with string terminators.
53 /// The remaining bytes is the file path relative to that prefix.
54 /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)
55 file_system_inputs,
56 /// Body is:49 /// Body is:
57 /// - a u64le that indicates the file path within the cache used50 /// - a u64le that indicates the file path within the cache used
58 /// to store coverage information. The integer is a hash of the PCs51 /// to store coverage information. The integer is a hash of the PCs
...@@ -208,14 +201,33 @@ pub const Message = struct {...@@ -208,14 +201,33 @@ pub const Message = struct {
208 };201 };
209 };202 };
210203
211 /// Trailing:204 pub const CacheManifest = extern struct {
212 /// * the hex digest of the cache directory within the /o/ subdirectory.205 /// Hit or miss, and if miss, what kind of miss, and which file or directory caused it.
213 pub const EmitDigest = extern struct {206 cache_status: CacheStatus,
214 flags: Flags,207 /// Binary digest of the cache inputs. Tells where to locate the manifest file.
215208 input_hash: std.Build.Cache.BinDigest,
216 pub const Flags = packed struct(u8) {209 /// Binary digest of hashed input_hash plus each file's hash. Tells where to locate the cached
217 cache_hit: bool,210 /// artifacts. This is technically denormalized data assuming the cache manifest file is intact.
218 reserved: u7 = 0,211 discovered_hash: std.Build.Cache.BinDigest,
212
213 pub const CacheStatus = packed struct(u32) {
214 result: Result,
215 /// Presence determined by cache_status.
216 file_off: u29,
217
218 pub const Result = enum(u3) {
219 hit,
220 incomplete_manifest,
221 invalid_manifest,
222 /// Has file_off
223 path_deleted,
224 /// Has file_off
225 directory_status_changed,
226 /// Has file_off
227 metadata_changed,
228 /// Has file_off
229 contents_changed,
230 };
219 };231 };
220 };232 };
221233
...@@ -317,30 +329,24 @@ pub fn serveBroadcastFuzzInputMessage(s: *const Server, test_i: u32, bytes: []co...@@ -317,30 +329,24 @@ pub fn serveBroadcastFuzzInputMessage(s: *const Server, test_i: u32, bytes: []co
317 try s.out.flush();329 try s.out.flush();
318}330}
319331
320pub fn serveEmitDigest(332pub fn serveCacheManifestUnflushed(s: *Server, header: OutMessage.CacheManifest) !void {
321 s: *Server,
322 digest: *const [Cache.bin_digest_len]u8,
323 header: OutMessage.EmitDigest,
324) !void {
325 try s.serveMessageHeader(.{333 try s.serveMessageHeader(.{
326 .tag = .emit_digest,334 .tag = .cache_manifest,
327 .bytes_len = @intCast(digest.len + @sizeOf(OutMessage.EmitDigest)),335 .bytes_len = @sizeOf(OutMessage.CacheManifest),
328 });336 });
329 try s.out.writeStruct(header, .little);337 try s.out.writeStruct(header, .little);
330 try s.out.writeAll(digest);
331 try s.out.flush();
332}338}
333339
334pub fn serveTestResults(s: *Server, msg: OutMessage.TestResults) !void {340pub fn serveTestResults(s: *Server, msg: OutMessage.TestResults) !void {
335 try s.serveMessageHeader(.{341 try s.serveMessageHeader(.{
336 .tag = .test_results,342 .tag = .test_results,
337 .bytes_len = @intCast(@sizeOf(OutMessage.TestResults)),343 .bytes_len = @sizeOf(OutMessage.TestResults),
338 });344 });
339 try s.out.writeStruct(msg, .little);345 try s.out.writeStruct(msg, .little);
340 try s.out.flush();346 try s.out.flush();
341}347}
342348
343pub fn serveErrorBundle(s: *Server, tag: Message.Tag, error_bundle: std.zig.ErrorBundle) !void {349pub fn serveErrorBundleUnflushed(s: *Server, tag: Message.Tag, error_bundle: std.zig.ErrorBundle) !void {
344 const eb_hdr: OutMessage.ErrorBundle = .{350 const eb_hdr: OutMessage.ErrorBundle = .{
345 .extra_len = @intCast(error_bundle.extra.len),351 .extra_len = @intCast(error_bundle.extra.len),
346 .string_bytes_len = @intCast(error_bundle.string_bytes.len),352 .string_bytes_len = @intCast(error_bundle.string_bytes.len),
...@@ -354,6 +360,10 @@ pub fn serveErrorBundle(s: *Server, tag: Message.Tag, error_bundle: std.zig.Erro...@@ -354,6 +360,10 @@ pub fn serveErrorBundle(s: *Server, tag: Message.Tag, error_bundle: std.zig.Erro
354 try s.out.writeStruct(eb_hdr, .little);360 try s.out.writeStruct(eb_hdr, .little);
355 try s.out.writeSliceEndian(u32, error_bundle.extra, .little);361 try s.out.writeSliceEndian(u32, error_bundle.extra, .little);
356 try s.out.writeAll(error_bundle.string_bytes);362 try s.out.writeAll(error_bundle.string_bytes);
363}
364
365pub fn serveErrorBundle(s: *Server, tag: Message.Tag, error_bundle: std.zig.ErrorBundle) !void {
366 try serveErrorBundleUnflushed(s, tag, error_bundle);
357 try s.out.flush();367 try s.out.flush();
358}368}
359369
src/Compilation.zig+41-157
...@@ -176,7 +176,14 @@ debug_compile_errors: bool,...@@ -176,7 +176,14 @@ debug_compile_errors: bool,
176/// Do not check this field directly. Instead, use the `debugIncremental` wrapper function.176/// Do not check this field directly. Instead, use the `debugIncremental` wrapper function.
177debug_incremental: bool,177debug_incremental: bool,
178alloc_failure_occurred: bool = false,178alloc_failure_occurred: bool = false,
179last_update_was_cache_hit: bool = false,179
180last_update_cache_status: Cache.Manifest.CheckResult,
181/// Corresponds to .zig-cache/h/* filenames.
182/// Remains stable across updates.
183input_digest: [Cache.bin_digest_len]u8,
184/// Corresponds to .zig-cache/o/* filenames.
185/// May change across updates.
186discovered_digest: [Cache.bin_digest_len]u8,
180187
181c_source_files: []const CSourceFile,188c_source_files: []const CSourceFile,
182rc_source_files: []const RcSourceFile,189rc_source_files: []const RcSourceFile,
...@@ -251,12 +258,6 @@ llvm_opt_bisect_limit: c_int,...@@ -251,12 +258,6 @@ llvm_opt_bisect_limit: c_int,
251258
252time_report: ?TimeReport,259time_report: ?TimeReport,
253260
254file_system_inputs: ?*std.ArrayList(u8),
255
256/// This is the digest of the cache for the current compilation.
257/// This digest will be known after update() is called.
258digest: ?[Cache.bin_digest_len]u8 = null,
259
260/// Non-`null` iff we are emitting a binary.261/// Non-`null` iff we are emitting a binary.
261/// Does not change for the lifetime of this `Compilation`.262/// Does not change for the lifetime of this `Compilation`.
262/// Cwd-relative if `cache_use == .none`. Otherwise, relative to our subdirectory in the cache.263/// Cwd-relative if `cache_use == .none`. Otherwise, relative to our subdirectory in the cache.
...@@ -1600,9 +1601,6 @@ pub const CreateOptions = struct {...@@ -1600,9 +1601,6 @@ pub const CreateOptions = struct {
1600 error_limit: ?Zcu.ErrorInt = null,1601 error_limit: ?Zcu.ErrorInt = null,
1601 global_cc_argv: []const []const u8 = &.{},1602 global_cc_argv: []const []const u8 = &.{},
16021603
1603 /// Tracks all files that can cause the Compilation to be invalidated and need a rebuild.
1604 file_system_inputs: ?*std.ArrayList(u8) = null,
1605
1606 parent_whole_cache: ?ParentWholeCache = null,1604 parent_whole_cache: ?ParentWholeCache = null,
16071605
1608 environ_map: *const std.process.Environ.Map,1606 environ_map: *const std.process.Environ.Map,
...@@ -2147,7 +2145,6 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,...@@ -2147,7 +2145,6 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
2147 .force_undefined_symbols = options.force_undefined_symbols,2145 .force_undefined_symbols = options.force_undefined_symbols,
2148 .link_eh_frame_hdr = link_eh_frame_hdr,2146 .link_eh_frame_hdr = link_eh_frame_hdr,
2149 .global_cc_argv = options.global_cc_argv,2147 .global_cc_argv = options.global_cc_argv,
2150 .file_system_inputs = options.file_system_inputs,
2151 .parent_whole_cache = options.parent_whole_cache,2148 .parent_whole_cache = options.parent_whole_cache,
2152 .link_diags = .init(gpa, io),2149 .link_diags = .init(gpa, io),
2153 .oneshot_prelink_tasks = .empty,2150 .oneshot_prelink_tasks = .empty,
...@@ -2158,6 +2155,11 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,...@@ -2158,6 +2155,11 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
2158 .emit_llvm_bc = try options.emit_llvm_bc.resolve(arena, &options, .llvm_bc),2155 .emit_llvm_bc = try options.emit_llvm_bc.resolve(arena, &options, .llvm_bc),
2159 .emit_docs = try options.emit_docs.resolve(arena, &options, .docs),2156 .emit_docs = try options.emit_docs.resolve(arena, &options, .docs),
2160 .environ_map = options.environ_map,2157 .environ_map = options.environ_map,
2158
2159 // These are populated during `update`.
2160 .last_update_cache_status = undefined,
2161 .input_digest = undefined,
2162 .discovered_digest = undefined,
2161 };2163 };
21622164
2163 errdefer {2165 errdefer {
...@@ -2300,7 +2302,8 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,...@@ -2300,7 +2302,8 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
2300 // In the case of incremental cache mode, this `artifact_directory`2302 // In the case of incremental cache mode, this `artifact_directory`
2301 // is computed based on a hash of non-linker inputs, and it is where all2303 // is computed based on a hash of non-linker inputs, and it is where all
2302 // build artifacts are stored (even while in-progress).2304 // build artifacts are stored (even while in-progress).
2303 comp.digest = hash.peekBin();2305 comp.input_digest = hash.peekBin();
2306 comp.computed_digest = comp.input_digest;
2304 const digest = hash.final();2307 const digest = hash.final();
23052308
2306 const artifact_sub_dir = "o" ++ fs.path.sep_str ++ digest;2309 const artifact_sub_dir = "o" ++ fs.path.sep_str ++ digest;
...@@ -2817,7 +2820,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE...@@ -2817,7 +2820,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
2817 log.debug("{s} whole cache {f} ignore_hit={}", .{ comp.root_name, status.fmt(&man), ignore_hit });2820 log.debug("{s} whole cache {f} ignore_hit={}", .{ comp.root_name, status.fmt(&man), ignore_hit });
2818 if (status == .hit and !ignore_hit) {2821 if (status == .hit and !ignore_hit) {
2819 // In this case the cache hit contains the full set of file system inputs. Nice!2822 // In this case the cache hit contains the full set of file system inputs. Nice!
2820 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
2821 if (comp.parent_whole_cache) |pwc| {2823 if (comp.parent_whole_cache) |pwc| {
2822 try pwc.mutex.lock(io);2824 try pwc.mutex.lock(io);
2823 defer pwc.mutex.unlock(io);2825 defer pwc.mutex.unlock(io);
...@@ -2881,29 +2883,14 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE...@@ -2881,29 +2883,14 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
2881 try comp.c_object_work_queue.ensureUnusedCapacity(gpa, comp.c_objects.items.len);2883 try comp.c_object_work_queue.ensureUnusedCapacity(gpa, comp.c_objects.items.len);
2882 for (comp.c_objects.items) |c_object| {2884 for (comp.c_objects.items) |c_object| {
2883 comp.c_object_work_queue.pushBackAssumeCapacity(c_object);2885 comp.c_object_work_queue.pushBackAssumeCapacity(c_object);
2884 try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{c_object.src.src_path}));
2885 }2886 }
2886 }2887 }
28872888
2888 for (comp.link_inputs) |input| {
2889 const path = input.path();
2890 try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{
2891 path.root_dir.path orelse ".",
2892 path.sub_path,
2893 }));
2894 }
2895
2896 // For compiling Win32 resources, we rely on the cache hash system to avoid duplicating work.2889 // For compiling Win32 resources, we rely on the cache hash system to avoid duplicating work.
2897 // Add a Job for each Win32 resource file.2890 // Add a Job for each Win32 resource file.
2898 try comp.win32_resource_work_queue.ensureUnusedCapacity(gpa, comp.win32_resources.items.len);2891 try comp.win32_resource_work_queue.ensureUnusedCapacity(gpa, comp.win32_resources.items.len);
2899 for (comp.win32_resources.items) |win32_resource| {2892 for (comp.win32_resources.items) |win32_resource| {
2900 comp.win32_resource_work_queue.pushBackAssumeCapacity(win32_resource);2893 comp.win32_resource_work_queue.pushBackAssumeCapacity(win32_resource);
2901 switch (win32_resource.src) {
2902 .rc => |f| {
2903 try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{f.src_path}));
2904 },
2905 .manifest => {},
2906 }
2907 }2894 }
29082895
2909 if (comp.zcu) |zcu| {2896 if (comp.zcu) |zcu| {
...@@ -2911,11 +2898,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE...@@ -2911,11 +2898,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
29112898
2912 zcu.skip_analysis_this_update = false;2899 zcu.skip_analysis_this_update = false;
29132900
2914 // TODO: doing this in `resolveReferences` later could avoid adding inputs for dead embedfiles. Investigate!
2915 for (zcu.embed_table.keys()) |embed_file| {
2916 try comp.appendFileSystemInput(embed_file.path);
2917 }
2918
2919 zcu.analysis_roots_len = 0;2901 zcu.analysis_roots_len = 0;
29202902
2921 zcu.analysis_roots_buffer[zcu.analysis_roots_len] = zcu.std_mod;2903 zcu.analysis_roots_buffer[zcu.analysis_roots_len] = zcu.std_mod;
...@@ -3003,7 +2985,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE...@@ -3003,7 +2985,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
3003 }2985 }
30042986
3005 if (comp.link_depfile) |depfile_path| if (comp.bin_file) |lf| {2987 if (comp.link_depfile) |depfile_path| if (comp.bin_file) |lf| {
3006 assert(comp.file_system_inputs != null);
3007 comp.createDepFile(depfile_path, lf.emit) catch |err|2988 comp.createDepFile(depfile_path, lf.emit) catch |err|
3008 comp.setMiscFailure(.link_depfile, "writing linker dependency file failed: {t}", .{err});2989 comp.setMiscFailure(.link_depfile, "writing linker dependency file failed: {t}", .{err});
3009 };2990 };
...@@ -3024,7 +3005,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE...@@ -3024,7 +3005,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
3024 switch (comp.cache_use) {3005 switch (comp.cache_use) {
3025 .none, .incremental => try flush(comp, arena),3006 .none, .incremental => try flush(comp, arena),
3026 .whole => |whole| {3007 .whole => |whole| {
3027 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
3028 if (comp.parent_whole_cache) |pwc| {3008 if (comp.parent_whole_cache) |pwc| {
3029 try pwc.mutex.lock(io);3009 try pwc.mutex.lock(io);
3030 defer pwc.mutex.unlock(io);3010 defer pwc.mutex.unlock(io);
...@@ -3129,39 +3109,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE...@@ -3129,39 +3109,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
3129 }3109 }
3130}3110}
31313111
3132/// Thread-safe. Assumes that `comp.mutex` is *not* already held by the caller.
3133pub fn appendFileSystemInput(comp: *Compilation, path: Compilation.Path) Allocator.Error!void {
3134 const gpa = comp.gpa;
3135 const io = comp.io;
3136 const fsi = comp.file_system_inputs orelse return;
3137 const prefixes = comp.cache_parent.prefixes();
3138
3139 const want_prefix_dir: Cache.Directory = switch (path.root) {
3140 .zig_lib => comp.dirs.zig_lib,
3141 .global_cache => comp.dirs.global_cache,
3142 .local_cache => comp.dirs.local_cache,
3143 .build_root => comp.dirs.build_root,
3144 .none => .cwd(),
3145 };
3146 const prefix: u8 = for (prefixes, 1..) |prefix_dir, i| {
3147 if (prefix_dir.eql(want_prefix_dir)) {
3148 break @intCast(i);
3149 }
3150 } else std.debug.panic(
3151 "missing prefix directory {t} ('{f}') for {q}",
3152 .{ path.root, want_prefix_dir, path.sub_path },
3153 );
3154
3155 // There may be concurrent calls to this function from C object workers and/or the main thread.
3156 comp.mutex.lockUncancelable(io);
3157 defer comp.mutex.unlock(io);
3158
3159 try fsi.ensureUnusedCapacity(gpa, path.sub_path.len + 3);
3160 if (fsi.items.len > 0) fsi.appendAssumeCapacity(0);
3161 fsi.appendAssumeCapacity(prefix);
3162 fsi.appendSliceAssumeCapacity(path.sub_path);
3163}
3164
3165fn resolveEmitPath(comp: *Compilation, path: []const u8) Cache.Path {3112fn resolveEmitPath(comp: *Compilation, path: []const u8) Cache.Path {
3166 return .{3113 return .{
3167 .root_dir = switch (comp.cache_use) {3114 .root_dir = switch (comp.cache_use) {
...@@ -4649,6 +4596,7 @@ fn writeDepFile(...@@ -4649,6 +4596,7 @@ fn writeDepFile(
4649 w: *std.Io.Writer,4596 w: *std.Io.Writer,
4650) std.Io.Writer.Error!void {4597) std.Io.Writer.Error!void {
4651 const prefixes = comp.cache_parent.prefixes();4598 const prefixes = comp.cache_parent.prefixes();
4599 if (true) @panic("TODO reimplement this based on Cache.Manifest data");
4652 const fsi = comp.file_system_inputs.?.items;4600 const fsi = comp.file_system_inputs.?.items;
46534601
4654 try w.print("{f}:", .{bin_file});4602 try w.print("{f}:", .{bin_file});
...@@ -4667,11 +4615,8 @@ fn writeDepFile(...@@ -4667,11 +4615,8 @@ fn writeDepFile(
4667}4615}
46684616
4669fn workerDocsCopy(comp: *Compilation) void {4617fn workerDocsCopy(comp: *Compilation) void {
4670 docsCopyFallible(comp) catch |err| return comp.lockAndSetMiscFailure(4618 docsCopyFallible(comp) catch |err|
4671 .docs_copy,4619 return comp.lockAndSetMiscFailure(.docs_copy, "unable to copy autodocs artifacts: {t}", .{err});
4672 "unable to copy autodocs artifacts: {s}",
4673 .{@errorName(err)},
4674 );
4675}4620}
46764621
4677fn docsCopyFallible(comp: *Compilation) anyerror!void {4622fn docsCopyFallible(comp: *Compilation) anyerror!void {
...@@ -4680,11 +4625,9 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {...@@ -4680,11 +4625,9 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
46804625
4681 const docs_path = comp.resolveEmitPath(comp.emit_docs.?);4626 const docs_path = comp.resolveEmitPath(comp.emit_docs.?);
4682 var out_dir = docs_path.root_dir.handle.createDirPathOpen(io, docs_path.sub_path, .{}) catch |err| {4627 var out_dir = docs_path.root_dir.handle.createDirPathOpen(io, docs_path.sub_path, .{}) catch |err| {
4683 return comp.lockAndSetMiscFailure(4628 return comp.lockAndSetMiscFailure(.docs_copy, "unable to create output directory {qf}: {t}", .{
4684 .docs_copy,4629 docs_path, err,
4685 "unable to create output directory '{f}': {s}",4630 });
4686 .{ docs_path, @errorName(err) },
4687 );
4688 };4631 };
4689 defer out_dir.close(io);4632 defer out_dir.close(io);
46904633
...@@ -4695,11 +4638,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {...@@ -4695,11 +4638,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4695 }4638 }
46964639
4697 var tar_file = out_dir.createFile(io, "sources.tar", .{}) catch |err| {4640 var tar_file = out_dir.createFile(io, "sources.tar", .{}) catch |err| {
4698 return comp.lockAndSetMiscFailure(4641 return comp.lockAndSetMiscFailure(.docs_copy, "unable to create '{f}/sources.tar': {t}", .{ docs_path, err });
4699 .docs_copy,
4700 "unable to create '{f}/sources.tar': {s}",
4701 .{ docs_path, @errorName(err) },
4702 );
4703 };4642 };
4704 defer tar_file.close(io);4643 defer tar_file.close(io);
47054644
...@@ -5719,64 +5658,17 @@ fn updateCObject(...@@ -5719,64 +5658,17 @@ fn updateCObject(
5719 .sub_path = fs.path.basename(raw_dep_file_path),5658 .sub_path = fs.path.basename(raw_dep_file_path),
5720 };5659 };
57215660
5722 if (comp.file_system_inputs != null) {
5723 // TODO instead of this rely on passing the manifest contents directly
5724
5725 // Use the same file size limit as the cache code does for dependency files.
5726 const dep_file_contents = dep_file_path.root_dir.handle.readFileAlloc(io, dep_file_path.sub_path, gpa, .unlimited) catch |err| switch (err) {
5727 error.OutOfMemory, error.Canceled => |e| return e,
5728 else => |e| return comp.failCObjRetryable(
5729 c_object,
5730 "failed to read '{f}': {t}",
5731 .{ dep_file_path, e },
5732 ),
5733 };
5734 defer gpa.free(dep_file_contents);
5735
5736 var str_buf: std.ArrayList(u8) = .empty;
5737 defer str_buf.deinit(gpa);
5738
5739 var it: std.Build.Cache.DepTokenizer = .{ .bytes = dep_file_contents };
5740 while (it.next()) |token| {
5741 const input_path: Compilation.Path = switch (token) {
5742 .target, .target_must_resolve => continue,
5743 .prereq => |file_path| try .fromUnresolved(arena, comp.dirs, &.{file_path}),
5744 .prereq_must_resolve => p: {
5745 try token.resolve(gpa, &str_buf);
5746 break :p try .fromUnresolved(arena, comp.dirs, &.{str_buf.items});
5747 },
5748 else => |err_token| return comp.failCObjRetryable(
5749 c_object,
5750 "failed parsing '{f}': {f}",
5751 .{ dep_file_path, err_token },
5752 ),
5753 };
5754 try comp.appendFileSystemInput(input_path);
5755 }
5756 }
5757
5758 // Add the files depended on to the cache system.5661 // Add the files depended on to the cache system.
5759 var diags: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined;5662 var diags: Cache.Manifest.AddDiscoveredDepFileDiagnostic = undefined;
5760 man.addDiscoveredDepFile(dep_file_path, &diags) catch |err| switch (err) {5663 man.addDiscoveredDepFile(dep_file_path, &diags) catch |err| switch (err) {
5761 error.OutOfMemory,5664 error.OutOfMemory, error.Canceled => |e| return e,
5762 error.Canceled,5665 error.InvalidDepFile => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{
5763 => |e| return e,5666 dep_file_path, diags.dep_tokenizer,
57645667 }),
5765 error.InvalidDepFile => return comp.failCObjRetryable(5668 error.FileSystemFailure => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{
5766 c_object,5669 dep_file_path, diags.add_discovered_path,
5767 "failed to parse '{f}': {f}",5670 }),
5768 .{ dep_file_path, diags.dep_tokenizer },5671 else => |e| return comp.failCObjRetryable(c_object, "failed to read {qf}: {t}", .{ dep_file_path, e }),
5769 ),
5770 error.FileSystemFailure => return comp.failCObjRetryable(
5771 c_object,
5772 "failed to parse '{f}': {f}",
5773 .{ dep_file_path, diags.add_discovered_path },
5774 ),
5775 else => |e| return comp.failCObjRetryable(
5776 c_object,
5777 "failed to read '{f}': {t}",
5778 .{ dep_file_path, e },
5779 ),
5780 };5672 };
5781 switch (comp.cache_use) {5673 switch (comp.cache_use) {
5782 .whole => |whole| {5674 .whole => |whole| {
...@@ -5784,25 +5676,17 @@ fn updateCObject(...@@ -5784,25 +5676,17 @@ fn updateCObject(
5784 try whole.cache_manifest_mutex.lock(io);5676 try whole.cache_manifest_mutex.lock(io);
5785 defer whole.cache_manifest_mutex.unlock(io);5677 defer whole.cache_manifest_mutex.unlock(io);
5786 whole_cache_manifest.addDiscoveredDepFile(dep_file_path, &diags) catch |err| switch (err) {5678 whole_cache_manifest.addDiscoveredDepFile(dep_file_path, &diags) catch |err| switch (err) {
5787 error.OutOfMemory,5679 error.OutOfMemory, error.Canceled => |e| return e,
5788 error.Canceled,5680
5789 => |e| return e,5681 error.InvalidDepFile => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{
57905682 dep_file_path, diags.dep_tokenizer,
5791 error.InvalidDepFile => return comp.failCObjRetryable(5683 }),
5792 c_object,5684 error.FileSystemFailure => return comp.failCObjRetryable(c_object, "failed to parse {qf}: {f}", .{
5793 "failed to parse '{f}': {f}",5685 dep_file_path, diags.add_discovered_path,
5794 .{ dep_file_path, diags.dep_tokenizer },5686 }),
5795 ),5687 else => |e| return comp.failCObjRetryable(c_object, "failed to read {qf}: {t}", .{
5796 error.FileSystemFailure => return comp.failCObjRetryable(5688 dep_file_path, e,
5797 c_object,5689 }),
5798 "failed to parse '{f}': {f}",
5799 .{ dep_file_path, diags.add_discovered_path },
5800 ),
5801 else => |e| return comp.failCObjRetryable(
5802 c_object,
5803 "failed to read '{f}': {t}",
5804 .{ dep_file_path, e },
5805 ),
5806 };5690 };
5807 }5691 }
5808 },5692 },
src/Zcu/PerThread.zig-4
...@@ -2643,8 +2643,6 @@ fn computeAliveFiles(pt: Zcu.PerThread) Allocator.Error!bool {...@@ -2643,8 +2643,6 @@ fn computeAliveFiles(pt: Zcu.PerThread) Allocator.Error!bool {
2643 .success => {},2643 .success => {},
2644 }2644 }
26452645
2646 try comp.appendFileSystemInput(file.path);
2647
2648 switch (file.getMode()) {2646 switch (file.getMode()) {
2649 .zig => {}, // continue to logic below2647 .zig => {}, // continue to logic below
2650 .zon => continue, // ZON can't import anything2648 .zon => continue, // ZON can't import anything
...@@ -3020,8 +3018,6 @@ fn newEmbedFile(pt: Zcu.PerThread, path: Compilation.Path) EmbedFileError!*Zcu.E...@@ -3020,8 +3018,6 @@ fn newEmbedFile(pt: Zcu.PerThread, path: Compilation.Path) EmbedFileError!*Zcu.E
3020 var opt_ip_str: ?InternPool.String = null;3018 var opt_ip_str: ?InternPool.String = null;
3021 try pt.updateEmbedFile(new_file, &opt_ip_str);3019 try pt.updateEmbedFile(new_file, &opt_ip_str);
30223020
3023 try comp.appendFileSystemInput(path);
3024
3025 // Add the file contents to the `whole` cache manifest if necessary.3021 // Add the file contents to the `whole` cache manifest if necessary.
3026 cache: {3022 cache: {
3027 const whole = switch (zcu.comp.cache_use) {3023 const whole = switch (zcu.comp.cache_use) {
src/main.zig+18-57
...@@ -3627,9 +3627,6 @@ fn buildOutputType(...@@ -3627,9 +3627,6 @@ fn buildOutputType(
3627 break :b .whole;3627 break :b .whole;
3628 };3628 };
36293629
3630 var file_system_inputs: std.ArrayList(u8) = .empty;
3631 defer file_system_inputs.deinit(gpa);
3632
3633 // Deduplicate rpath entries3630 // Deduplicate rpath entries
3634 var rpath_dedup = std.array_hash_map.String(void){};3631 var rpath_dedup = std.array_hash_map.String(void){};
3635 for (create_module.rpath_list.items) |rpath| {3632 for (create_module.rpath_list.items) |rpath| {
...@@ -3762,7 +3759,6 @@ fn buildOutputType(...@@ -3762,7 +3759,6 @@ fn buildOutputType(
3762 // than to any particular module. This feature can greatly reduce CLI3759 // than to any particular module. This feature can greatly reduce CLI
3763 // noise when --search-prefix and -M are combined.3760 // noise when --search-prefix and -M are combined.
3764 .global_cc_argv = try cc_argv.toOwnedSlice(arena),3761 .global_cc_argv = try cc_argv.toOwnedSlice(arena),
3765 .file_system_inputs = &file_system_inputs,
3766 .debug_compiler_runtime_libs = debug_compiler_runtime_libs,3762 .debug_compiler_runtime_libs = debug_compiler_runtime_libs,
3767 .environ_map = environ_map,3763 .environ_map = environ_map,
3768 }) catch |err| switch (err) {3764 }) catch |err| switch (err) {
...@@ -3818,7 +3814,6 @@ fn buildOutputType(...@@ -3818,7 +3814,6 @@ fn buildOutputType(
3818 arg_mode,3814 arg_mode,
3819 all_args,3815 all_args,
3820 runtime_args_start,3816 runtime_args_start,
3821 environ_map,
3822 );3817 );
3823 return cleanExit(io);3818 return cleanExit(io);
3824 },3819 },
...@@ -3845,7 +3840,6 @@ fn buildOutputType(...@@ -3845,7 +3840,6 @@ fn buildOutputType(
3845 arg_mode,3840 arg_mode,
3846 all_args,3841 all_args,
3847 runtime_args_start,3842 runtime_args_start,
3848 environ_map,
3849 );3843 );
3850 return cleanExit(io);3844 return cleanExit(io);
3851 },3845 },
...@@ -3858,7 +3852,7 @@ fn buildOutputType(...@@ -3858,7 +3852,7 @@ fn buildOutputType(
3858 defer root_prog_node.end();3852 defer root_prog_node.end();
38593853
3860 if (arg_mode == .translate_c) {3854 if (arg_mode == .translate_c) {
3861 return cmdTranslateC(comp, arena, null, null, root_prog_node, environ_map);3855 return cmdTranslateC(comp, arena, null, root_prog_node, environ_map);
3862 }3856 }
38633857
3864 updateModule(comp, color, root_prog_node) catch |err| switch (err) {3858 updateModule(comp, color, root_prog_node) catch |err| switch (err) {
...@@ -4400,7 +4394,6 @@ fn serve(...@@ -4400,7 +4394,6 @@ fn serve(
4400 arg_mode: ArgMode,4394 arg_mode: ArgMode,
4401 all_args: []const []const u8,4395 all_args: []const []const u8,
4402 runtime_args_start: ?usize,4396 runtime_args_start: ?usize,
4403 environ_map: *process.Environ.Map,
4404) !void {4397) !void {
4405 const gpa = comp.gpa;4398 const gpa = comp.gpa;
4406 const io = comp.io;4399 const io = comp.io;
...@@ -4422,8 +4415,6 @@ fn serve(...@@ -4422,8 +4415,6 @@ fn serve(
4422 const main_progress_node = std.Progress.start(io, .{});4415 const main_progress_node = std.Progress.start(io, .{});
4423 defer main_progress_node.end();4416 defer main_progress_node.end();
44244417
4425 const file_system_inputs = comp.file_system_inputs.?;
4426
4427 const IncrementalDebugServer = if (build_options.enable_debug_extensions and !builtin.single_threaded)4418 const IncrementalDebugServer = if (build_options.enable_debug_extensions and !builtin.single_threaded)
4428 @import("IncrementalDebugServer.zig")4419 @import("IncrementalDebugServer.zig")
4429 else4420 else
...@@ -4446,29 +4437,8 @@ fn serve(...@@ -4446,29 +4437,8 @@ fn serve(
4446 switch (hdr.tag) {4437 switch (hdr.tag) {
4447 .exit => return cleanExit(io),4438 .exit => return cleanExit(io),
4448 .update => {4439 .update => {
4449 file_system_inputs.clearRetainingCapacity();
4450
4451 if (arg_mode == .translate_c) {4440 if (arg_mode == .translate_c) {
4452 var arena_instance = std.heap.ArenaAllocator.init(gpa);4441 @panic("TODO delete translate-c");
4453 defer arena_instance.deinit();
4454 const arena = arena_instance.allocator();
4455 var output: Compilation.TranslateCResult = undefined;
4456 try cmdTranslateC(comp, arena, &output, file_system_inputs, main_progress_node, environ_map);
4457 defer output.deinit(gpa);
4458
4459 if (file_system_inputs.items.len != 0) {
4460 try server.serveStringMessage(.file_system_inputs, file_system_inputs.items);
4461 }
4462
4463 if (output.errors.errorMessageCount() != 0) {
4464 try server.serveErrorBundle(.error_bundle, output.errors);
4465 } else {
4466 try server.serveEmitDigest(&output.digest, .{
4467 .flags = .{ .cache_hit = output.cache_hit },
4468 });
4469 }
4470
4471 continue;
4472 }4442 }
44734443
4474 try comp.makeBinFileWritable();4444 try comp.makeBinFileWritable();
...@@ -4496,7 +4466,6 @@ fn serve(...@@ -4496,7 +4466,6 @@ fn serve(
4496 //);4466 //);
4497 },4467 },
4498 .hot_update => {4468 .hot_update => {
4499 file_system_inputs.clearRetainingCapacity();
4500 if (child_pid) |pid| {4469 if (child_pid) |pid| {
4501 try comp.hotCodeSwap(main_progress_node, pid);4470 try comp.hotCodeSwap(main_progress_node, pid);
4502 try serveUpdateResults(&server, comp);4471 try serveUpdateResults(&server, comp);
...@@ -4530,14 +4499,6 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {...@@ -4530,14 +4499,6 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
4530 var error_bundle = try comp.getAllErrorsAlloc();4499 var error_bundle = try comp.getAllErrorsAlloc();
4531 defer error_bundle.deinit(gpa);4500 defer error_bundle.deinit(gpa);
45324501
4533 if (comp.file_system_inputs) |file_system_inputs| {
4534 if (file_system_inputs.items.len == 0) {
4535 assert(error_bundle.errorMessageCount() > 0);
4536 } else {
4537 try s.serveStringMessage(.file_system_inputs, file_system_inputs.items);
4538 }
4539 }
4540
4541 if (comp.time_report) |*tr| {4502 if (comp.time_report) |*tr| {
4542 var decls_len: u32 = 0;4503 var decls_len: u32 = 0;
45434504
...@@ -4613,22 +4574,25 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {...@@ -4613,22 +4574,25 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
4613 },4574 },
4614 });4575 });
4615 try s.out.writeVecAll(&slices);4576 try s.out.writeVecAll(&slices);
4616 try s.out.flush();
4617 }4577 }
46184578
4619 if (error_bundle.errorMessageCount() > 0) {4579 try s.serveCacheManifestUnflushed(.{
4620 try s.serveErrorBundle(.error_bundle, error_bundle);4580 .cache_status = switch (comp.last_update_cache_status) {
4621 return;4581 .hit => .{ .result = .hit, .file_off = 0 },
4622 }4582 .incomplete_manifest => .{ .result = .incomplete_manifest, .file_off = 0 },
46234583 .invalid_manifest => .{ .result = .invalid_manifest, .file_off = 0 },
4624 if (comp.digest) |digest| {4584 .path_deleted => |off| .{ .result = .path_deleted, .file_off = off },
4625 try s.serveEmitDigest(&digest, .{4585 .directory_status_changed => |off| .{ .result = .directory_status_changed, .file_off = off },
4626 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },4586 .metadata_changed => |off| .{ .result = .metadata_changed, .file_off = off },
4627 });4587 .contents_changed => |off| .{ .result = .contents_changed, .file_off = off },
4628 }4588 },
4589 .input_hash = comp.input_hash,
4590 .discovered_hash = comp.discovered_hash,
4591 });
46294592
4630 // Serve empty error bundle to indicate the update is done.4593 // Even if error bundle is empty, serve it to indicate the update is done.
4631 try s.serveErrorBundle(.error_bundle, std.zig.ErrorBundle.empty);4594 try s.serveErrorBundleUnflushed(.error_bundle, error_bundle);
4595 try s.out.flush();
4632}4596}
46334597
4634fn runOrTest(4598fn runOrTest(
...@@ -4889,7 +4853,6 @@ fn cmdTranslateC(...@@ -4889,7 +4853,6 @@ fn cmdTranslateC(
4889 comp: *Compilation,4853 comp: *Compilation,
4890 arena: Allocator,4854 arena: Allocator,
4891 fancy_output: ?*Compilation.TranslateCResult,4855 fancy_output: ?*Compilation.TranslateCResult,
4892 file_system_inputs: ?*std.ArrayList(u8),
4893 prog_node: std.Progress.Node,4856 prog_node: std.Progress.Node,
4894 environ_map: *process.Environ.Map,4857 environ_map: *process.Environ.Map,
4895) !void {4858) !void {
...@@ -4934,7 +4897,6 @@ fn cmdTranslateC(...@@ -4934,7 +4897,6 @@ fn cmdTranslateC(
49344897
4935 if (result.errors.errorMessageCount() != 0) {4898 if (result.errors.errorMessageCount() != 0) {
4936 if (fancy_output) |p| {4899 if (fancy_output) |p| {
4937 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
4938 p.* = result;4900 p.* = result;
4939 return;4901 return;
4940 } else {4902 } else {
...@@ -4948,7 +4910,6 @@ fn cmdTranslateC(...@@ -4948,7 +4910,6 @@ fn cmdTranslateC(
4948 break :result result;4910 break :result result;
4949 };4911 };
49504912
4951 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
4952 if (fancy_output) |p| {4913 if (fancy_output) |p| {
4953 p.* = result;4914 p.* = result;
4954 } else {4915 } else {