authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-24 02:11:59-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-24 12:22:00-08:00
logd3f4e9d4ac9772d0916fc6c33145789054532ea4
tree95c0b4c933cf3b8dcf7e552e87ff0be7b793e1aa
parentb3d9b0e3f6bab5d662ec385cf754872f8a90607f

Compilation pipeline: repeat failed prelink tasks

and remove faulty assertion. When a prelink task fails, the completed_prelink_tasks counter will not decrement. A future improvement will be needed to make the pipeline fully robust and handle failed prelink tasks, followed by updates in which those tasks succeed, and compilation proceeds like normal. Currently if a prelink task fails, the Compilation will be left in a state unrecoverable by an incremental update.

1 files changed, 56 insertions(+), 38 deletions(-)

src/Compilation.zig+56-38
......@@ -3692,65 +3692,67 @@ fn performAllTheWorkInner(
36923692 work_queue_wait_group.spawnManager(workerDocsWasm, .{ comp, main_progress_node });
36933693 }
36943694
3695 if (testAndClear(&comp.queued_jobs.compiler_rt_lib)) {
3695 // In case it failed last time, try again. `clearMiscFailures` was already
3696 // called at the start of `update`.
3697 if (comp.queued_jobs.compiler_rt_lib and comp.compiler_rt_lib == null) {
36963698 comp.link_task_wait_group.spawnManager(buildRt, .{ comp, "compiler_rt.zig", .compiler_rt, .Lib, &comp.compiler_rt_lib, main_progress_node });
36973699 }
36983700
3699 if (testAndClear(&comp.queued_jobs.compiler_rt_obj)) {
3701 if (comp.queued_jobs.compiler_rt_obj and comp.compiler_rt_obj == null) {
37003702 comp.link_task_wait_group.spawnManager(buildRt, .{ comp, "compiler_rt.zig", .compiler_rt, .Obj, &comp.compiler_rt_obj, main_progress_node });
37013703 }
37023704
3703 if (testAndClear(&comp.queued_jobs.fuzzer_lib)) {
3705 if (comp.queued_jobs.fuzzer_lib and comp.fuzzer_lib == null) {
37043706 comp.link_task_wait_group.spawnManager(buildRt, .{ comp, "fuzzer.zig", .libfuzzer, .Lib, &comp.fuzzer_lib, main_progress_node });
37053707 }
37063708
3707 if (testAndClear(&comp.queued_jobs.glibc_shared_objects)) {
3709 if (comp.queued_jobs.glibc_shared_objects) {
37083710 comp.link_task_wait_group.spawnManager(buildGlibcSharedObjects, .{ comp, main_progress_node });
37093711 }
37103712
3711 if (testAndClear(&comp.queued_jobs.libunwind)) {
3713 if (comp.queued_jobs.libunwind) {
37123714 comp.link_task_wait_group.spawnManager(buildLibUnwind, .{ comp, main_progress_node });
37133715 }
37143716
3715 if (testAndClear(&comp.queued_jobs.libcxx)) {
3717 if (comp.queued_jobs.libcxx) {
37163718 comp.link_task_wait_group.spawnManager(buildLibCxx, .{ comp, main_progress_node });
37173719 }
37183720
3719 if (testAndClear(&comp.queued_jobs.libcxxabi)) {
3721 if (comp.queued_jobs.libcxxabi) {
37203722 comp.link_task_wait_group.spawnManager(buildLibCxxAbi, .{ comp, main_progress_node });
37213723 }
37223724
3723 if (testAndClear(&comp.queued_jobs.libtsan)) {
3725 if (comp.queued_jobs.libtsan) {
37243726 comp.link_task_wait_group.spawnManager(buildLibTsan, .{ comp, main_progress_node });
37253727 }
37263728
3727 if (testAndClear(&comp.queued_jobs.zig_libc)) {
3729 if (comp.queued_jobs.zig_libc and comp.libc_static_lib == null) {
37283730 comp.link_task_wait_group.spawnManager(buildZigLibc, .{ comp, main_progress_node });
37293731 }
37303732
37313733 for (0..@typeInfo(musl.CrtFile).@"enum".fields.len) |i| {
3732 if (testAndClear(&comp.queued_jobs.musl_crt_file[i])) {
3734 if (comp.queued_jobs.musl_crt_file[i]) {
37333735 const tag: musl.CrtFile = @enumFromInt(i);
37343736 comp.link_task_wait_group.spawnManager(buildMuslCrtFile, .{ comp, tag, main_progress_node });
37353737 }
37363738 }
37373739
37383740 for (0..@typeInfo(glibc.CrtFile).@"enum".fields.len) |i| {
3739 if (testAndClear(&comp.queued_jobs.glibc_crt_file[i])) {
3741 if (comp.queued_jobs.glibc_crt_file[i]) {
37403742 const tag: glibc.CrtFile = @enumFromInt(i);
37413743 comp.link_task_wait_group.spawnManager(buildGlibcCrtFile, .{ comp, tag, main_progress_node });
37423744 }
37433745 }
37443746
37453747 for (0..@typeInfo(wasi_libc.CrtFile).@"enum".fields.len) |i| {
3746 if (testAndClear(&comp.queued_jobs.wasi_libc_crt_file[i])) {
3748 if (comp.queued_jobs.wasi_libc_crt_file[i]) {
37473749 const tag: wasi_libc.CrtFile = @enumFromInt(i);
37483750 comp.link_task_wait_group.spawnManager(buildWasiLibcCrtFile, .{ comp, tag, main_progress_node });
37493751 }
37503752 }
37513753
37523754 for (0..@typeInfo(mingw.CrtFile).@"enum".fields.len) |i| {
3753 if (testAndClear(&comp.queued_jobs.mingw_crt_file[i])) {
3755 if (comp.queued_jobs.mingw_crt_file[i]) {
37543756 const tag: mingw.CrtFile = @enumFromInt(i);
37553757 comp.link_task_wait_group.spawnManager(buildMingwCrtFile, .{ comp, tag, main_progress_node });
37563758 }
......@@ -3850,7 +3852,10 @@ fn performAllTheWorkInner(
38503852 comp.link_task_wait_group.wait();
38513853 comp.link_task_wait_group.reset();
38523854 std.log.scoped(.link).debug("finished waiting for link_task_wait_group", .{});
3853 assert(comp.remaining_prelink_tasks == 0);
3855 if (comp.remaining_prelink_tasks > 0) {
3856 // Indicates an error occurred preventing prelink phase from completing.
3857 return;
3858 }
38543859 }
38553860
38563861 work: while (true) {
......@@ -4648,76 +4653,95 @@ fn buildRt(
46484653}
46494654
46504655fn buildMuslCrtFile(comp: *Compilation, crt_file: musl.CrtFile, prog_node: std.Progress.Node) void {
4651 musl.buildCrtFile(comp, crt_file, prog_node) catch |err| switch (err) {
4656 if (musl.buildCrtFile(comp, crt_file, prog_node)) |_| {
4657 comp.queued_jobs.musl_crt_file[@intFromEnum(crt_file)] = false;
4658 } else |err| switch (err) {
46524659 error.SubCompilationFailed => return, // error reported already
46534660 else => comp.lockAndSetMiscFailure(.musl_crt_file, "unable to build musl {s}: {s}", .{
46544661 @tagName(crt_file), @errorName(err),
46554662 }),
4656 };
4663 }
46574664}
46584665
46594666fn buildGlibcCrtFile(comp: *Compilation, crt_file: glibc.CrtFile, prog_node: std.Progress.Node) void {
4660 glibc.buildCrtFile(comp, crt_file, prog_node) catch |err| switch (err) {
4667 if (glibc.buildCrtFile(comp, crt_file, prog_node)) |_| {
4668 comp.queued_jobs.glibc_crt_file[@intFromEnum(crt_file)] = false;
4669 } else |err| switch (err) {
46614670 error.SubCompilationFailed => return, // error reported already
46624671 else => comp.lockAndSetMiscFailure(.glibc_crt_file, "unable to build glibc {s}: {s}", .{
46634672 @tagName(crt_file), @errorName(err),
46644673 }),
4665 };
4674 }
46664675}
46674676
46684677fn buildGlibcSharedObjects(comp: *Compilation, prog_node: std.Progress.Node) void {
4669 glibc.buildSharedObjects(comp, prog_node) catch |err| switch (err) {
4678 if (glibc.buildSharedObjects(comp, prog_node)) |_| {
4679 // The job should no longer be queued up since it succeeded.
4680 comp.queued_jobs.glibc_shared_objects = false;
4681 } else |err| switch (err) {
46704682 error.SubCompilationFailed => return, // error reported already
46714683 else => comp.lockAndSetMiscFailure(.glibc_shared_objects, "unable to build glibc shared objects: {s}", .{
46724684 @errorName(err),
46734685 }),
4674 };
4686 }
46754687}
46764688
46774689fn buildMingwCrtFile(comp: *Compilation, crt_file: mingw.CrtFile, prog_node: std.Progress.Node) void {
4678 mingw.buildCrtFile(comp, crt_file, prog_node) catch |err| switch (err) {
4690 if (mingw.buildCrtFile(comp, crt_file, prog_node)) |_| {
4691 comp.queued_jobs.mingw_crt_file[@intFromEnum(crt_file)] = false;
4692 } else |err| switch (err) {
46794693 error.SubCompilationFailed => return, // error reported already
46804694 else => comp.lockAndSetMiscFailure(.mingw_crt_file, "unable to build mingw-w64 {s}: {s}", .{
46814695 @tagName(crt_file), @errorName(err),
46824696 }),
4683 };
4697 }
46844698}
46854699
46864700fn buildWasiLibcCrtFile(comp: *Compilation, crt_file: wasi_libc.CrtFile, prog_node: std.Progress.Node) void {
4687 wasi_libc.buildCrtFile(comp, crt_file, prog_node) catch |err| switch (err) {
4701 if (wasi_libc.buildCrtFile(comp, crt_file, prog_node)) |_| {
4702 comp.queued_jobs.wasi_libc_crt_file[@intFromEnum(crt_file)] = false;
4703 } else |err| switch (err) {
46884704 error.SubCompilationFailed => return, // error reported already
46894705 else => comp.lockAndSetMiscFailure(.wasi_libc_crt_file, "unable to build WASI libc {s}: {s}", .{
46904706 @tagName(crt_file), @errorName(err),
46914707 }),
4692 };
4708 }
46934709}
46944710
46954711fn buildLibUnwind(comp: *Compilation, prog_node: std.Progress.Node) void {
4696 libunwind.buildStaticLib(comp, prog_node) catch |err| switch (err) {
4712 if (libunwind.buildStaticLib(comp, prog_node)) |_| {
4713 comp.queued_jobs.libunwind = false;
4714 } else |err| switch (err) {
46974715 error.SubCompilationFailed => return, // error reported already
46984716 else => comp.lockAndSetMiscFailure(.libunwind, "unable to build libunwind: {s}", .{@errorName(err)}),
4699 };
4717 }
47004718}
47014719
47024720fn buildLibCxx(comp: *Compilation, prog_node: std.Progress.Node) void {
4703 libcxx.buildLibCxx(comp, prog_node) catch |err| switch (err) {
4721 if (libcxx.buildLibCxx(comp, prog_node)) |_| {
4722 comp.queued_jobs.libcxx = false;
4723 } else |err| switch (err) {
47044724 error.SubCompilationFailed => return, // error reported already
47054725 else => comp.lockAndSetMiscFailure(.libcxx, "unable to build libcxx: {s}", .{@errorName(err)}),
4706 };
4726 }
47074727}
47084728
47094729fn buildLibCxxAbi(comp: *Compilation, prog_node: std.Progress.Node) void {
4710 libcxx.buildLibCxxAbi(comp, prog_node) catch |err| switch (err) {
4730 if (libcxx.buildLibCxxAbi(comp, prog_node)) |_| {
4731 comp.queued_jobs.libcxxabi = false;
4732 } else |err| switch (err) {
47114733 error.SubCompilationFailed => return, // error reported already
47124734 else => comp.lockAndSetMiscFailure(.libcxxabi, "unable to build libcxxabi: {s}", .{@errorName(err)}),
4713 };
4735 }
47144736}
47154737
47164738fn buildLibTsan(comp: *Compilation, prog_node: std.Progress.Node) void {
4717 libtsan.buildTsan(comp, prog_node) catch |err| switch (err) {
4739 if (libtsan.buildTsan(comp, prog_node)) |_| {
4740 comp.queued_jobs.libtsan = false;
4741 } else |err| switch (err) {
47184742 error.SubCompilationFailed => return, // error reported already
47194743 else => comp.lockAndSetMiscFailure(.libtsan, "unable to build TSAN library: {s}", .{@errorName(err)}),
4720 };
4744 }
47214745}
47224746
47234747fn buildZigLibc(comp: *Compilation, prog_node: std.Progress.Node) void {
......@@ -6779,9 +6803,3 @@ pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
67796803pub fn compilerRtStrip(comp: Compilation) bool {
67806804 return comp.root_mod.strip;
67816805}
6782
6783fn testAndClear(b: *bool) bool {
6784 const result = b.*;
6785 b.* = false;
6786 return result;
6787}