authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-24 19:40:54-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-24 19:40:54-07:00
log94f4f9c4ef31570589e5572c86947eb4a2fd58ac
tree9c1da6927b1bd385ed94425355f7fc8110d1cb76
parent4135cc9d0bd6e70808b92d3ab0cce41c9eb87689

add sub-compilation cache inputs to parents in whole mode

closes #20782

2 files changed, 69 insertions(+), 1 deletions(-)

lib/std/Build/Cache.zig+31
......@@ -1024,6 +1024,37 @@ pub const Manifest = struct {
10241024 buf.items.len -= 1;
10251025 }
10261026 }
1027
1028 pub fn populateOtherManifest(man: *Manifest, other: *Manifest, prefix_map: [4]u8) Allocator.Error!void {
1029 const gpa = other.cache.gpa;
1030 assert(@typeInfo(std.zig.Server.Message.PathPrefix).Enum.fields.len == man.cache.prefixes_len);
1031 assert(man.cache.prefixes_len == 4);
1032 for (man.files.keys()) |file| {
1033 const prefixed_path: PrefixedPath = .{
1034 .prefix = prefix_map[file.prefixed_path.prefix],
1035 .sub_path = try gpa.dupe(u8, file.prefixed_path.sub_path),
1036 };
1037 errdefer gpa.free(prefixed_path.sub_path);
1038
1039 const gop = try other.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
1040 errdefer _ = other.files.pop();
1041
1042 if (gop.found_existing) {
1043 gpa.free(prefixed_path.sub_path);
1044 continue;
1045 }
1046
1047 gop.key_ptr.* = .{
1048 .prefixed_path = prefixed_path,
1049 .max_file_size = file.max_file_size,
1050 .stat = file.stat,
1051 .bin_digest = file.bin_digest,
1052 .contents = null,
1053 };
1054
1055 other.hash.hasher.update(&gop.key_ptr.bin_digest);
1056 }
1057 }
10271058};
10281059
10291060/// On operating systems that support symlinks, does a readlink. On other operating systems,
src/Compilation.zig+38-1
......@@ -201,6 +201,9 @@ c_source_files: []const CSourceFile,
201201rc_source_files: []const RcSourceFile,
202202global_cc_argv: []const []const u8,
203203cache_parent: *Cache,
204/// Populated when a sub-Compilation is created during the `update` of its parent.
205/// In this case the child must additionally add file system inputs to this object.
206parent_whole_cache: ?ParentWholeCache,
204207/// Path to own executable for invoking `zig clang`.
205208self_exe_path: ?[]const u8,
206209zig_lib_directory: Directory,
......@@ -970,6 +973,12 @@ pub const SystemLib = link.SystemLib;
970973
971974pub const CacheMode = enum { incremental, whole };
972975
976pub const ParentWholeCache = struct {
977 manifest: *Cache.Manifest,
978 mutex: *std.Thread.Mutex,
979 prefix_map: [4]u8,
980};
981
973982const CacheUse = union(CacheMode) {
974983 incremental: *Incremental,
975984 whole: *Whole,
......@@ -1207,6 +1216,8 @@ pub const CreateOptions = struct {
12071216 /// Tracks all files that can cause the Compilation to be invalidated and need a rebuild.
12081217 file_system_inputs: ?*std.ArrayListUnmanaged(u8) = null,
12091218
1219 parent_whole_cache: ?ParentWholeCache = null,
1220
12101221 pub const Entry = link.File.OpenOptions.Entry;
12111222};
12121223
......@@ -1563,6 +1574,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
15631574 .link_eh_frame_hdr = link_eh_frame_hdr,
15641575 .global_cc_argv = options.global_cc_argv,
15651576 .file_system_inputs = options.file_system_inputs,
1577 .parent_whole_cache = options.parent_whole_cache,
15661578 };
15671579
15681580 // Prevent some footguns by making the "any" fields of config reflect
......@@ -2106,6 +2118,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21062118 if (is_hit) {
21072119 // In this case the cache hit contains the full set of file system inputs. Nice!
21082120 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
2121 if (comp.parent_whole_cache) |pwc| {
2122 pwc.mutex.lock();
2123 defer pwc.mutex.unlock();
2124 try man.populateOtherManifest(pwc.manifest, pwc.prefix_map);
2125 }
21092126
21102127 comp.last_update_was_cache_hit = true;
21112128 log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name});
......@@ -2303,6 +2320,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
23032320 switch (comp.cache_use) {
23042321 .whole => |whole| {
23052322 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
2323 if (comp.parent_whole_cache) |pwc| {
2324 pwc.mutex.lock();
2325 defer pwc.mutex.unlock();
2326 try man.populateOtherManifest(pwc.manifest, pwc.prefix_map);
2327 }
23062328
23072329 const digest = man.final();
23082330
......@@ -6420,14 +6442,29 @@ fn buildOutputFromZig(
64206442 .output_mode = output_mode,
64216443 });
64226444
6445 const parent_whole_cache: ?ParentWholeCache = switch (comp.cache_use) {
6446 .whole => |whole| .{
6447 .manifest = whole.cache_manifest.?,
6448 .mutex = &whole.cache_manifest_mutex,
6449 .prefix_map = .{
6450 0, // cwd is the same
6451 1, // zig lib dir is the same
6452 3, // local cache is mapped to global cache
6453 3, // global cache is the same
6454 },
6455 },
6456 .incremental => null,
6457 };
6458
64236459 const sub_compilation = try Compilation.create(gpa, arena, .{
64246460 .global_cache_directory = comp.global_cache_directory,
64256461 .local_cache_directory = comp.global_cache_directory,
64266462 .zig_lib_directory = comp.zig_lib_directory,
6463 .cache_mode = .whole,
6464 .parent_whole_cache = parent_whole_cache,
64276465 .self_exe_path = comp.self_exe_path,
64286466 .config = config,
64296467 .root_mod = root_mod,
6430 .cache_mode = .whole,
64316468 .root_name = root_name,
64326469 .thread_pool = comp.thread_pool,
64336470 .libc_installation = comp.libc_installation,