authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-25 07:12:39-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-07-25 07:12:39-07:00
logb22357b88b44e47a9c0347cb47d2be2ed3c461a1
tree06f0b86b53d99bce9a11a5f76eee90d3daaa6899
parent18685e928dcf179271a18cec3d8d16ed0547d3b5
parent94f4f9c4ef31570589e5572c86947eb4a2fd58ac
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #20783 from ziglang/cache-fix

build compiler_rt and fuzzer in parallel; fix false positive cache hits

3 files changed, 90 insertions(+), 27 deletions(-)

lib/std/Build/Cache.zig+31
......@@ -1024,6 +1024,37 @@ pub const Manifest = struct {
10241024 buf.items.len -= 1;
10251025 }
10261026 }
1027
1028 pub fn populateOtherManifest(man: *Manifest, other: *Manifest, prefix_map: [4]u8) Allocator.Error!void {
1029 const gpa = other.cache.gpa;
1030 assert(@typeInfo(std.zig.Server.Message.PathPrefix).Enum.fields.len == man.cache.prefixes_len);
1031 assert(man.cache.prefixes_len == 4);
1032 for (man.files.keys()) |file| {
1033 const prefixed_path: PrefixedPath = .{
1034 .prefix = prefix_map[file.prefixed_path.prefix],
1035 .sub_path = try gpa.dupe(u8, file.prefixed_path.sub_path),
1036 };
1037 errdefer gpa.free(prefixed_path.sub_path);
1038
1039 const gop = try other.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
1040 errdefer _ = other.files.pop();
1041
1042 if (gop.found_existing) {
1043 gpa.free(prefixed_path.sub_path);
1044 continue;
1045 }
1046
1047 gop.key_ptr.* = .{
1048 .prefixed_path = prefixed_path,
1049 .max_file_size = file.max_file_size,
1050 .stat = file.stat,
1051 .bin_digest = file.bin_digest,
1052 .contents = null,
1053 };
1054
1055 other.hash.hasher.update(&gop.key_ptr.bin_digest);
1056 }
1057 }
10271058};
10281059
10291060/// On operating systems that support symlinks, does a readlink. On other operating systems,
src/Compilation.zig+56-26
......@@ -201,6 +201,9 @@ c_source_files: []const CSourceFile,
201201rc_source_files: []const RcSourceFile,
202202global_cc_argv: []const []const u8,
203203cache_parent: *Cache,
204/// Populated when a sub-Compilation is created during the `update` of its parent.
205/// In this case the child must additionally add file system inputs to this object.
206parent_whole_cache: ?ParentWholeCache,
204207/// Path to own executable for invoking `zig clang`.
205208self_exe_path: ?[]const u8,
206209zig_lib_directory: Directory,
......@@ -262,9 +265,6 @@ emit_asm: ?EmitLoc,
262265emit_llvm_ir: ?EmitLoc,
263266emit_llvm_bc: ?EmitLoc,
264267
265work_queue_wait_group: WaitGroup = .{},
266astgen_wait_group: WaitGroup = .{},
267
268268llvm_opt_bisect_limit: c_int,
269269
270270file_system_inputs: ?*std.ArrayListUnmanaged(u8),
......@@ -973,6 +973,12 @@ pub const SystemLib = link.SystemLib;
973973
974974pub const CacheMode = enum { incremental, whole };
975975
976pub const ParentWholeCache = struct {
977 manifest: *Cache.Manifest,
978 mutex: *std.Thread.Mutex,
979 prefix_map: [4]u8,
980};
981
976982const CacheUse = union(CacheMode) {
977983 incremental: *Incremental,
978984 whole: *Whole,
......@@ -1210,6 +1216,8 @@ pub const CreateOptions = struct {
12101216 /// Tracks all files that can cause the Compilation to be invalidated and need a rebuild.
12111217 file_system_inputs: ?*std.ArrayListUnmanaged(u8) = null,
12121218
1219 parent_whole_cache: ?ParentWholeCache = null,
1220
12131221 pub const Entry = link.File.OpenOptions.Entry;
12141222};
12151223
......@@ -1566,6 +1574,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
15661574 .link_eh_frame_hdr = link_eh_frame_hdr,
15671575 .global_cc_argv = options.global_cc_argv,
15681576 .file_system_inputs = options.file_system_inputs,
1577 .parent_whole_cache = options.parent_whole_cache,
15691578 };
15701579
15711580 // Prevent some footguns by making the "any" fields of config reflect
......@@ -2109,6 +2118,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21092118 if (is_hit) {
21102119 // In this case the cache hit contains the full set of file system inputs. Nice!
21112120 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
2121 if (comp.parent_whole_cache) |pwc| {
2122 pwc.mutex.lock();
2123 defer pwc.mutex.unlock();
2124 try man.populateOtherManifest(pwc.manifest, pwc.prefix_map);
2125 }
21122126
21132127 comp.last_update_was_cache_hit = true;
21142128 log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name});
......@@ -2306,6 +2320,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
23062320 switch (comp.cache_use) {
23072321 .whole => |whole| {
23082322 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
2323 if (comp.parent_whole_cache) |pwc| {
2324 pwc.mutex.lock();
2325 defer pwc.mutex.unlock();
2326 try man.populateOtherManifest(pwc.manifest, pwc.prefix_map);
2327 }
23092328
23102329 const digest = man.final();
23112330
......@@ -3535,13 +3554,13 @@ fn performAllTheWorkInner(
35353554 // (at least for now) single-threaded main work queue. However, C object compilation
35363555 // only needs to be finished by the end of this function.
35373556
3538 comp.work_queue_wait_group.reset();
3539 defer comp.work_queue_wait_group.wait();
3557 var work_queue_wait_group: WaitGroup = .{};
3558 defer work_queue_wait_group.wait();
35403559
35413560 if (comp.docs_emit != null) {
35423561 dev.check(.docs_emit);
3543 comp.thread_pool.spawnWg(&comp.work_queue_wait_group, workerDocsCopy, .{comp});
3544 comp.work_queue_wait_group.spawnManager(workerDocsWasm, .{ comp, main_progress_node });
3562 comp.thread_pool.spawnWg(&work_queue_wait_group, workerDocsCopy, .{comp});
3563 work_queue_wait_group.spawnManager(workerDocsWasm, .{ comp, main_progress_node });
35453564 }
35463565
35473566 {
......@@ -3551,8 +3570,8 @@ fn performAllTheWorkInner(
35513570 const zir_prog_node = main_progress_node.start("AST Lowering", 0);
35523571 defer zir_prog_node.end();
35533572
3554 comp.astgen_wait_group.reset();
3555 defer comp.astgen_wait_group.wait();
3573 var astgen_wait_group: WaitGroup = .{};
3574 defer astgen_wait_group.wait();
35563575
35573576 // builtin.zig is handled specially for two reasons:
35583577 // 1. to avoid race condition of zig processes truncating each other's builtin.zig files
......@@ -3574,7 +3593,7 @@ fn performAllTheWorkInner(
35743593
35753594 const file = mod.builtin_file orelse continue;
35763595
3577 comp.thread_pool.spawnWg(&comp.astgen_wait_group, workerUpdateBuiltinZigFile, .{
3596 comp.thread_pool.spawnWg(&astgen_wait_group, workerUpdateBuiltinZigFile, .{
35783597 comp, mod, file,
35793598 });
35803599 }
......@@ -3594,32 +3613,36 @@ fn performAllTheWorkInner(
35943613 const path_digest = zcu.filePathDigest(file_index);
35953614 const root_decl = zcu.fileRootDecl(file_index);
35963615 const file = zcu.fileByIndex(file_index);
3597 comp.thread_pool.spawnWgId(&comp.astgen_wait_group, workerAstGenFile, .{
3598 comp, file, file_index, path_digest, root_decl, zir_prog_node, &comp.astgen_wait_group, .root,
3616 comp.thread_pool.spawnWgId(&astgen_wait_group, workerAstGenFile, .{
3617 comp, file, file_index, path_digest, root_decl, zir_prog_node, &astgen_wait_group, .root,
35993618 });
36003619 }
36013620 }
36023621
36033622 while (comp.embed_file_work_queue.readItem()) |embed_file| {
3604 comp.thread_pool.spawnWg(&comp.astgen_wait_group, workerCheckEmbedFile, .{
3623 comp.thread_pool.spawnWg(&astgen_wait_group, workerCheckEmbedFile, .{
36053624 comp, embed_file,
36063625 });
36073626 }
36083627 }
36093628
36103629 while (comp.c_object_work_queue.readItem()) |c_object| {
3611 comp.thread_pool.spawnWg(&comp.work_queue_wait_group, workerUpdateCObject, .{
3630 comp.thread_pool.spawnWg(&work_queue_wait_group, workerUpdateCObject, .{
36123631 comp, c_object, main_progress_node,
36133632 });
36143633 }
36153634
36163635 while (comp.win32_resource_work_queue.readItem()) |win32_resource| {
3617 comp.thread_pool.spawnWg(&comp.work_queue_wait_group, workerUpdateWin32Resource, .{
3636 comp.thread_pool.spawnWg(&work_queue_wait_group, workerUpdateWin32Resource, .{
36183637 comp, win32_resource, main_progress_node,
36193638 });
36203639 }
36213640 }
36223641
3642 if (comp.job_queued_compiler_rt_lib) work_queue_wait_group.spawnManager(buildRt, .{ comp, "compiler_rt.zig", .compiler_rt, .Lib, &comp.compiler_rt_lib, main_progress_node });
3643 if (comp.job_queued_compiler_rt_obj) work_queue_wait_group.spawnManager(buildRt, .{ comp, "compiler_rt.zig", .compiler_rt, .Obj, &comp.compiler_rt_obj, main_progress_node });
3644 if (comp.job_queued_fuzzer_lib) work_queue_wait_group.spawnManager(buildRt, .{ comp, "fuzzer.zig", .libfuzzer, .Lib, &comp.fuzzer_lib, main_progress_node });
3645
36233646 if (comp.module) |zcu| {
36243647 const pt: Zcu.PerThread = .{ .zcu = zcu, .tid = .main };
36253648 if (comp.incremental) {
......@@ -3633,7 +3656,7 @@ fn performAllTheWorkInner(
36333656 zcu.codegen_prog_node = main_progress_node.start("Code Generation", 0);
36343657 }
36353658
3636 if (!InternPool.single_threaded) comp.thread_pool.spawnWgId(&comp.work_queue_wait_group, codegenThread, .{comp});
3659 if (!InternPool.single_threaded) comp.thread_pool.spawnWgId(&work_queue_wait_group, codegenThread, .{comp});
36373660 defer if (!InternPool.single_threaded) {
36383661 {
36393662 comp.codegen_work.mutex.lock();
......@@ -3661,10 +3684,6 @@ fn performAllTheWorkInner(
36613684 }
36623685 break;
36633686 }
3664
3665 buildCompilerRtOneShot(comp, &comp.job_queued_compiler_rt_lib, "compiler_rt.zig", .compiler_rt, .Lib, &comp.compiler_rt_lib, main_progress_node);
3666 buildCompilerRtOneShot(comp, &comp.job_queued_compiler_rt_obj, "compiler_rt.zig", .compiler_rt, .Obj, &comp.compiler_rt_obj, main_progress_node);
3667 buildCompilerRtOneShot(comp, &comp.job_queued_fuzzer_lib, "fuzzer.zig", .libfuzzer, .Lib, &comp.fuzzer_lib, main_progress_node);
36683687}
36693688
36703689const JobError = Allocator.Error;
......@@ -4668,18 +4687,14 @@ fn workerUpdateWin32Resource(
46684687 };
46694688}
46704689
4671fn buildCompilerRtOneShot(
4690fn buildRt(
46724691 comp: *Compilation,
4673 job_queued: *bool,
46744692 root_source_name: []const u8,
46754693 misc_task: MiscTask,
46764694 output_mode: std.builtin.OutputMode,
46774695 out: *?CRTFile,
46784696 prog_node: std.Progress.Node,
46794697) void {
4680 if (!job_queued.*) return;
4681 job_queued.* = false;
4682
46834698 comp.buildOutputFromZig(
46844699 root_source_name,
46854700 output_mode,
......@@ -6427,14 +6442,29 @@ fn buildOutputFromZig(
64276442 .output_mode = output_mode,
64286443 });
64296444
6445 const parent_whole_cache: ?ParentWholeCache = switch (comp.cache_use) {
6446 .whole => |whole| .{
6447 .manifest = whole.cache_manifest.?,
6448 .mutex = &whole.cache_manifest_mutex,
6449 .prefix_map = .{
6450 0, // cwd is the same
6451 1, // zig lib dir is the same
6452 3, // local cache is mapped to global cache
6453 3, // global cache is the same
6454 },
6455 },
6456 .incremental => null,
6457 };
6458
64306459 const sub_compilation = try Compilation.create(gpa, arena, .{
64316460 .global_cache_directory = comp.global_cache_directory,
64326461 .local_cache_directory = comp.global_cache_directory,
64336462 .zig_lib_directory = comp.zig_lib_directory,
6463 .cache_mode = .whole,
6464 .parent_whole_cache = parent_whole_cache,
64346465 .self_exe_path = comp.self_exe_path,
64356466 .config = config,
64366467 .root_mod = root_mod,
6437 .cache_mode = .whole,
64386468 .root_name = root_name,
64396469 .thread_pool = comp.thread_pool,
64406470 .libc_installation = comp.libc_installation,
src/main.zig+3-1
......@@ -5309,7 +5309,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
53095309 const term = t: {
53105310 std.debug.lockStdErr();
53115311 defer std.debug.unlockStdErr();
5312 break :t try child.spawnAndWait();
5312 break :t child.spawnAndWait() catch |err| {
5313 fatal("unable to spawn {s}: {s}", .{ child_argv.items[0], @errorName(err) });
5314 };
53135315 };
53145316
53155317 switch (term) {