authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-24 18:24:00-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-24 18:24:00-07:00
log3844b91db72d5c3c98f2b2d97043a0c3de1a3d86
treee89fa901487e0a467bc1ea894ac769fbb9489d2c
parentf4f5b2bc41e09ed68caf3c839433e44bd9b33249

Compilation: build compiler_rt and fuzzer in parallel

With the rest of the pipeline. Tracked by #9188

1 files changed, 18 insertions(+), 25 deletions(-)

src/Compilation.zig+18-25
......@@ -262,9 +262,6 @@ emit_asm: ?EmitLoc,
262262emit_llvm_ir: ?EmitLoc,
263263emit_llvm_bc: ?EmitLoc,
264264
265work_queue_wait_group: WaitGroup = .{},
266astgen_wait_group: WaitGroup = .{},
267
268265llvm_opt_bisect_limit: c_int,
269266
270267file_system_inputs: ?*std.ArrayListUnmanaged(u8),
......@@ -3535,13 +3532,13 @@ fn performAllTheWorkInner(
35353532 // (at least for now) single-threaded main work queue. However, C object compilation
35363533 // only needs to be finished by the end of this function.
35373534
3538 comp.work_queue_wait_group.reset();
3539 defer comp.work_queue_wait_group.wait();
3535 var work_queue_wait_group: WaitGroup = .{};
3536 defer work_queue_wait_group.wait();
35403537
35413538 if (comp.docs_emit != null) {
35423539 dev.check(.docs_emit);
3543 comp.thread_pool.spawnWg(&comp.work_queue_wait_group, workerDocsCopy, .{comp});
3544 comp.work_queue_wait_group.spawnManager(workerDocsWasm, .{ comp, main_progress_node });
3540 comp.thread_pool.spawnWg(&work_queue_wait_group, workerDocsCopy, .{comp});
3541 work_queue_wait_group.spawnManager(workerDocsWasm, .{ comp, main_progress_node });
35453542 }
35463543
35473544 {
......@@ -3551,8 +3548,8 @@ fn performAllTheWorkInner(
35513548 const zir_prog_node = main_progress_node.start("AST Lowering", 0);
35523549 defer zir_prog_node.end();
35533550
3554 comp.astgen_wait_group.reset();
3555 defer comp.astgen_wait_group.wait();
3551 var astgen_wait_group: WaitGroup = .{};
3552 defer astgen_wait_group.wait();
35563553
35573554 // builtin.zig is handled specially for two reasons:
35583555 // 1. to avoid race condition of zig processes truncating each other's builtin.zig files
......@@ -3574,7 +3571,7 @@ fn performAllTheWorkInner(
35743571
35753572 const file = mod.builtin_file orelse continue;
35763573
3577 comp.thread_pool.spawnWg(&comp.astgen_wait_group, workerUpdateBuiltinZigFile, .{
3574 comp.thread_pool.spawnWg(&astgen_wait_group, workerUpdateBuiltinZigFile, .{
35783575 comp, mod, file,
35793576 });
35803577 }
......@@ -3594,32 +3591,36 @@ fn performAllTheWorkInner(
35943591 const path_digest = zcu.filePathDigest(file_index);
35953592 const root_decl = zcu.fileRootDecl(file_index);
35963593 const file = zcu.fileByIndex(file_index);
3597 comp.thread_pool.spawnWgId(&comp.astgen_wait_group, workerAstGenFile, .{
3598 comp, file, file_index, path_digest, root_decl, zir_prog_node, &comp.astgen_wait_group, .root,
3594 comp.thread_pool.spawnWgId(&astgen_wait_group, workerAstGenFile, .{
3595 comp, file, file_index, path_digest, root_decl, zir_prog_node, &astgen_wait_group, .root,
35993596 });
36003597 }
36013598 }
36023599
36033600 while (comp.embed_file_work_queue.readItem()) |embed_file| {
3604 comp.thread_pool.spawnWg(&comp.astgen_wait_group, workerCheckEmbedFile, .{
3601 comp.thread_pool.spawnWg(&astgen_wait_group, workerCheckEmbedFile, .{
36053602 comp, embed_file,
36063603 });
36073604 }
36083605 }
36093606
36103607 while (comp.c_object_work_queue.readItem()) |c_object| {
3611 comp.thread_pool.spawnWg(&comp.work_queue_wait_group, workerUpdateCObject, .{
3608 comp.thread_pool.spawnWg(&work_queue_wait_group, workerUpdateCObject, .{
36123609 comp, c_object, main_progress_node,
36133610 });
36143611 }
36153612
36163613 while (comp.win32_resource_work_queue.readItem()) |win32_resource| {
3617 comp.thread_pool.spawnWg(&comp.work_queue_wait_group, workerUpdateWin32Resource, .{
3614 comp.thread_pool.spawnWg(&work_queue_wait_group, workerUpdateWin32Resource, .{
36183615 comp, win32_resource, main_progress_node,
36193616 });
36203617 }
36213618 }
36223619
3620 if (comp.job_queued_compiler_rt_lib) work_queue_wait_group.spawnManager(buildRt, .{ comp, "compiler_rt.zig", .compiler_rt, .Lib, &comp.compiler_rt_lib, main_progress_node });
3621 if (comp.job_queued_compiler_rt_obj) work_queue_wait_group.spawnManager(buildRt, .{ comp, "compiler_rt.zig", .compiler_rt, .Obj, &comp.compiler_rt_obj, main_progress_node });
3622 if (comp.job_queued_fuzzer_lib) work_queue_wait_group.spawnManager(buildRt, .{ comp, "fuzzer.zig", .libfuzzer, .Lib, &comp.fuzzer_lib, main_progress_node });
3623
36233624 if (comp.module) |zcu| {
36243625 const pt: Zcu.PerThread = .{ .zcu = zcu, .tid = .main };
36253626 if (comp.incremental) {
......@@ -3633,7 +3634,7 @@ fn performAllTheWorkInner(
36333634 zcu.codegen_prog_node = main_progress_node.start("Code Generation", 0);
36343635 }
36353636
3636 if (!InternPool.single_threaded) comp.thread_pool.spawnWgId(&comp.work_queue_wait_group, codegenThread, .{comp});
3637 if (!InternPool.single_threaded) comp.thread_pool.spawnWgId(&work_queue_wait_group, codegenThread, .{comp});
36373638 defer if (!InternPool.single_threaded) {
36383639 {
36393640 comp.codegen_work.mutex.lock();
......@@ -3661,10 +3662,6 @@ fn performAllTheWorkInner(
36613662 }
36623663 break;
36633664 }
3664
3665 buildCompilerRtOneShot(comp, &comp.job_queued_compiler_rt_lib, "compiler_rt.zig", .compiler_rt, .Lib, &comp.compiler_rt_lib, main_progress_node);
3666 buildCompilerRtOneShot(comp, &comp.job_queued_compiler_rt_obj, "compiler_rt.zig", .compiler_rt, .Obj, &comp.compiler_rt_obj, main_progress_node);
3667 buildCompilerRtOneShot(comp, &comp.job_queued_fuzzer_lib, "fuzzer.zig", .libfuzzer, .Lib, &comp.fuzzer_lib, main_progress_node);
36683665}
36693666
36703667const JobError = Allocator.Error;
......@@ -4668,18 +4665,14 @@ fn workerUpdateWin32Resource(
46684665 };
46694666}
46704667
4671fn buildCompilerRtOneShot(
4668fn buildRt(
46724669 comp: *Compilation,
4673 job_queued: *bool,
46744670 root_source_name: []const u8,
46754671 misc_task: MiscTask,
46764672 output_mode: std.builtin.OutputMode,
46774673 out: *?CRTFile,
46784674 prog_node: std.Progress.Node,
46794675) void {
4680 if (!job_queued.*) return;
4681 job_queued.* = false;
4682
46834676 comp.buildOutputFromZig(
46844677 root_source_name,
46854678 output_mode,