authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-02-04 03:00:13+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-02-04 18:38:40+00:00
log269c1ae649017836f15313d1d4977402be11eed5
treec38b9e06df7c14f90db02cb6afa0ee42c7e7fbfa
parenta0004cebc255405764e889effb25a42fe07d8463
signaturelock-open Commit is signed but in an unrecognized format.

Zcu: incremental compilation improvements

* Mark root Decls for re-analysis separately * Check for re-analysis of root Decls * Remove `outdated` entry when analyzing fn body * Remove legacy `outdated` field from Decl analysis state

2 files changed, 160 insertions(+), 136 deletions(-)

src/Compilation.zig-1
......@@ -3548,7 +3548,6 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v
35483548 switch (decl.analysis) {
35493549 .unreferenced => unreachable,
35503550 .in_progress => unreachable,
3551 .outdated => unreachable,
35523551
35533552 .file_failure,
35543553 .sema_failure,
src/Module.zig+160-135
......@@ -156,7 +156,14 @@ potentially_outdated: std.AutoArrayHashMapUnmanaged(InternPool.Depender, u32) =
156156outdated: std.AutoArrayHashMapUnmanaged(InternPool.Depender, u32) = .{},
157157/// This contains all `Depender`s in `outdated` whose PO dependency count is 0.
158158/// Such `Depender`s are ready for immediate re-analysis.
159/// See `findOutdatedToAnalyze` for details.
159160outdated_ready: std.AutoArrayHashMapUnmanaged(InternPool.Depender, void) = .{},
161/// This contains a set of Decls which may not be in `outdated`, but are the
162/// root Decls of files which have updated source and thus must be re-analyzed.
163/// If such a Decl is only in this set, the struct type index may be preserved
164/// (only the namespace might change). If such a Decl is also `outdated`, the
165/// struct type index must be recreated.
166outdated_file_root: std.AutoArrayHashMapUnmanaged(Decl.Index, void) = .{},
160167
161168stage1_flags: packed struct {
162169 have_winmain: bool = false,
......@@ -431,13 +438,9 @@ pub const Decl = struct {
431438 /// This indicates the failure was something like running out of disk space,
432439 /// and attempting codegen again may succeed.
433440 codegen_failure_retryable,
434 /// Everything is done. During an update, this Decl may be out of date, depending
435 /// on its dependencies. The `generation` field can be used to determine if this
436 /// completion status occurred before or after a given update.
441 /// Sematic analysis of this Decl has succeeded. However, the Decl may
442 /// be outdated due to an incomplete update!
437443 complete,
438 /// A Module update is in progress, and this Decl has been flagged as being known
439 /// to require re-analysis.
440 outdated,
441444 },
442445 /// Whether `typed_value`, `align`, `linksection` and `addrspace` are populated.
443446 has_tv: bool,
......@@ -735,8 +738,7 @@ pub const Namespace = struct {
735738 file_scope: *File,
736739 /// Will be a struct, enum, union, or opaque.
737740 ty: Type,
738 /// Direct children of the namespace. Used during an update to detect
739 /// which decls have been added/removed from source.
741 /// Direct children of the namespace.
740742 /// Declaration order is preserved via entry order.
741743 /// These are only declarations named directly by the AST; anonymous
742744 /// declarations are not stored here.
......@@ -2492,6 +2494,7 @@ pub fn deinit(zcu: *Zcu) void {
24922494 zcu.potentially_outdated.deinit(gpa);
24932495 zcu.outdated.deinit(gpa);
24942496 zcu.outdated_ready.deinit(gpa);
2497 zcu.outdated_file_root.deinit(gpa);
24952498
24962499 zcu.test_functions.deinit(gpa);
24972500
......@@ -2858,27 +2861,13 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
28582861 file.prev_zir = null;
28592862 }
28602863
2861 if (file.root_decl.unwrap()) |root_decl| mark_outdated: {
2864 if (file.root_decl.unwrap()) |root_decl| {
28622865 // The root of this file must be re-analyzed, since the file has changed.
28632866 comp.mutex.lock();
28642867 defer comp.mutex.unlock();
28652868
2866 const root_decl_depender = InternPool.Depender.wrap(.{ .decl = root_decl });
2867
2868 const gop = try mod.outdated.getOrPut(gpa, root_decl_depender);
2869 // If this Decl is already marked as outdated, nothing needs to be done.
2870 if (gop.found_existing) break :mark_outdated;
2871
2872 log.debug("outdated: {} (root Decl)", .{root_decl});
2873
2874 // If it's already PO, forward its existing PO dependency count.
2875 // Otherwise, it has no PO dependencies yet.
2876 if (mod.potentially_outdated.fetchSwapRemove(root_decl_depender)) |kv| {
2877 gop.value_ptr.* = kv.value;
2878 } else {
2879 gop.value_ptr.* = 0;
2880 try mod.outdated_ready.put(mod.gpa, root_decl_depender, {});
2881 }
2869 log.debug("outdated root Decl: {}", .{root_decl});
2870 try mod.outdated_file_root.put(gpa, root_decl, {});
28822871 }
28832872}
28842873
......@@ -3187,6 +3176,26 @@ pub fn findOutdatedToAnalyze(zcu: *Zcu) Allocator.Error!?InternPool.Depender {
31873176 return zcu.outdated_ready.keys()[0];
31883177 }
31893178
3179 // Next, we will see if there is any outdated file root which was not in
3180 // `outdated`. This set will be small (number of files changed in this
3181 // update), so it's alright for us to just iterate here.
3182 for (zcu.outdated_file_root.keys()) |file_decl| {
3183 const decl_depender = InternPool.Depender.wrap(.{ .decl = file_decl });
3184 if (zcu.outdated.contains(decl_depender)) {
3185 // Since we didn't hit this in the first loop, this Decl must have
3186 // pending dependencies, so is ineligible.
3187 continue;
3188 }
3189 if (zcu.potentially_outdated.contains(decl_depender)) {
3190 // This Decl's struct may or may not need to be recreated depending
3191 // on whether it is outdated. If we analyzed it now, we would have
3192 // to assume it was outdated and recreate it!
3193 continue;
3194 }
3195 log.debug("findOutdatedToAnalyze: outdated file root decl '{d}'", .{file_decl});
3196 return decl_depender;
3197 }
3198
31903199 // There is no single Depender which is ready for re-analysis. Instead, we
31913200 // must assume that some Decl with PO dependencies is outdated - e.g. in the
31923201 // above example we arbitrarily pick one of A or B. We should select a Decl,
......@@ -3407,26 +3416,27 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl_index: Decl.Index) SemaError!void {
34073416 .in_progress => unreachable,
34083417
34093418 .file_failure,
3410 .sema_failure,
3411 .sema_failure_retryable,
34123419 .liveness_failure,
34133420 .codegen_failure,
3414 .dependency_failure,
34153421 .codegen_failure_retryable,
3422 .dependency_failure,
34163423 => return error.AnalysisFail,
34173424
3418 .complete => if (was_outdated) {
3419 if (build_options.only_c) unreachable;
3420 // The exports this Decl performs will be re-discovered, so we remove them here
3421 // prior to re-analysis.
3422 try mod.deleteDeclExports(decl_index);
3423 } else return,
3425 .sema_failure,
3426 .sema_failure_retryable,
3427 => if (!was_outdated) return error.AnalysisFail,
34243428
3425 .outdated => unreachable, // TODO: remove this field
3429 .complete => if (!was_outdated) return,
34263430
34273431 .unreferenced => {},
34283432 }
34293433
3434 if (was_outdated) {
3435 // The exports this Decl performs will be re-discovered, so we remove them here
3436 // prior to re-analysis.
3437 try mod.deleteDeclExports(decl_index);
3438 }
3439
34303440 var decl_prog_node = mod.sema_prog_node.start("", 0);
34313441 decl_prog_node.activate();
34323442 defer decl_prog_node.end();
......@@ -3493,7 +3503,6 @@ pub fn ensureFuncBodyAnalyzed(zcu: *Zcu, func_index: InternPool.Index) SemaError
34933503 switch (decl.analysis) {
34943504 .unreferenced => unreachable,
34953505 .in_progress => unreachable,
3496 .outdated => unreachable,
34973506
34983507 .file_failure,
34993508 .sema_failure,
......@@ -3503,109 +3512,117 @@ pub fn ensureFuncBodyAnalyzed(zcu: *Zcu, func_index: InternPool.Index) SemaError
35033512 .sema_failure_retryable,
35043513 => return error.AnalysisFail,
35053514
3506 .complete, .codegen_failure_retryable => {
3507 switch (func.analysis(ip).state) {
3508 .sema_failure, .dependency_failure => return error.AnalysisFail,
3509 .none, .queued => {},
3510 .in_progress => unreachable,
3511 .inline_only => unreachable, // don't queue work for this
3512 .success => return,
3513 }
3515 .complete, .codegen_failure_retryable => {},
3516 }
35143517
3515 const gpa = zcu.gpa;
3518 const func_as_depender = InternPool.Depender.wrap(.{ .func = func_index });
3519 const was_outdated = zcu.outdated.swapRemove(func_as_depender) or
3520 zcu.potentially_outdated.swapRemove(func_as_depender);
35163521
3517 var tmp_arena = std.heap.ArenaAllocator.init(gpa);
3518 defer tmp_arena.deinit();
3519 const sema_arena = tmp_arena.allocator();
3522 if (was_outdated) {
3523 _ = zcu.outdated_ready.swapRemove(func_as_depender);
3524 }
35203525
3521 var air = zcu.analyzeFnBody(func_index, sema_arena) catch |err| switch (err) {
3522 error.AnalysisFail => {
3523 if (func.analysis(ip).state == .in_progress) {
3524 // If this decl caused the compile error, the analysis field would
3525 // be changed to indicate it was this Decl's fault. Because this
3526 // did not happen, we infer here that it was a dependency failure.
3527 func.analysis(ip).state = .dependency_failure;
3528 }
3529 return error.AnalysisFail;
3530 },
3531 error.OutOfMemory => return error.OutOfMemory,
3532 };
3533 defer air.deinit(gpa);
3526 switch (func.analysis(ip).state) {
3527 .sema_failure, .dependency_failure => if (!was_outdated) return error.AnalysisFail,
3528 .none, .queued => {},
3529 .in_progress => unreachable,
3530 .inline_only => unreachable, // don't queue work for this
3531 .success => if (!was_outdated) return,
3532 }
35343533
3535 const comp = zcu.comp;
3534 const gpa = zcu.gpa;
35363535
3537 const dump_air = builtin.mode == .Debug and comp.verbose_air;
3538 const dump_llvm_ir = builtin.mode == .Debug and (comp.verbose_llvm_ir != null or comp.verbose_llvm_bc != null);
3536 var tmp_arena = std.heap.ArenaAllocator.init(gpa);
3537 defer tmp_arena.deinit();
3538 const sema_arena = tmp_arena.allocator();
35393539
3540 if (comp.bin_file == null and zcu.llvm_object == null and !dump_air and !dump_llvm_ir) {
3541 return;
3540 var air = zcu.analyzeFnBody(func_index, sema_arena) catch |err| switch (err) {
3541 error.AnalysisFail => {
3542 if (func.analysis(ip).state == .in_progress) {
3543 // If this decl caused the compile error, the analysis field would
3544 // be changed to indicate it was this Decl's fault. Because this
3545 // did not happen, we infer here that it was a dependency failure.
3546 func.analysis(ip).state = .dependency_failure;
35423547 }
3548 return error.AnalysisFail;
3549 },
3550 error.OutOfMemory => return error.OutOfMemory,
3551 };
3552 defer air.deinit(gpa);
35433553
3544 var liveness = try Liveness.analyze(gpa, air, ip);
3545 defer liveness.deinit(gpa);
3554 const comp = zcu.comp;
35463555
3547 if (dump_air) {
3548 const fqn = try decl.getFullyQualifiedName(zcu);
3549 std.debug.print("# Begin Function AIR: {}:\n", .{fqn.fmt(ip)});
3550 @import("print_air.zig").dump(zcu, air, liveness);
3551 std.debug.print("# End Function AIR: {}\n\n", .{fqn.fmt(ip)});
3552 }
3556 const dump_air = builtin.mode == .Debug and comp.verbose_air;
3557 const dump_llvm_ir = builtin.mode == .Debug and (comp.verbose_llvm_ir != null or comp.verbose_llvm_bc != null);
35533558
3554 if (std.debug.runtime_safety) {
3555 var verify = Liveness.Verify{
3556 .gpa = gpa,
3557 .air = air,
3558 .liveness = liveness,
3559 .intern_pool = ip,
3560 };
3561 defer verify.deinit();
3562
3563 verify.verify() catch |err| switch (err) {
3564 error.OutOfMemory => return error.OutOfMemory,
3565 else => {
3566 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);
3567 zcu.failed_decls.putAssumeCapacityNoClobber(
3568 decl_index,
3569 try Module.ErrorMsg.create(
3570 gpa,
3571 decl.srcLoc(zcu),
3572 "invalid liveness: {s}",
3573 .{@errorName(err)},
3574 ),
3575 );
3576 decl.analysis = .liveness_failure;
3577 return error.AnalysisFail;
3578 },
3579 };
3580 }
3559 if (comp.bin_file == null and zcu.llvm_object == null and !dump_air and !dump_llvm_ir) {
3560 return;
3561 }
35813562
3582 if (comp.bin_file) |lf| {
3583 lf.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {
3584 error.OutOfMemory => return error.OutOfMemory,
3585 error.AnalysisFail => {
3586 decl.analysis = .codegen_failure;
3587 },
3588 else => {
3589 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);
3590 zcu.failed_decls.putAssumeCapacityNoClobber(decl_index, try Module.ErrorMsg.create(
3591 gpa,
3592 decl.srcLoc(zcu),
3593 "unable to codegen: {s}",
3594 .{@errorName(err)},
3595 ));
3596 decl.analysis = .codegen_failure_retryable;
3597 },
3598 };
3599 } else if (zcu.llvm_object) |llvm_object| {
3600 if (build_options.only_c) unreachable;
3601 llvm_object.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {
3602 error.OutOfMemory => return error.OutOfMemory,
3603 error.AnalysisFail => {
3604 decl.analysis = .codegen_failure;
3605 },
3606 };
3607 }
3608 },
3563 var liveness = try Liveness.analyze(gpa, air, ip);
3564 defer liveness.deinit(gpa);
3565
3566 if (dump_air) {
3567 const fqn = try decl.getFullyQualifiedName(zcu);
3568 std.debug.print("# Begin Function AIR: {}:\n", .{fqn.fmt(ip)});
3569 @import("print_air.zig").dump(zcu, air, liveness);
3570 std.debug.print("# End Function AIR: {}\n\n", .{fqn.fmt(ip)});
3571 }
3572
3573 if (std.debug.runtime_safety) {
3574 var verify = Liveness.Verify{
3575 .gpa = gpa,
3576 .air = air,
3577 .liveness = liveness,
3578 .intern_pool = ip,
3579 };
3580 defer verify.deinit();
3581
3582 verify.verify() catch |err| switch (err) {
3583 error.OutOfMemory => return error.OutOfMemory,
3584 else => {
3585 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);
3586 zcu.failed_decls.putAssumeCapacityNoClobber(
3587 decl_index,
3588 try Module.ErrorMsg.create(
3589 gpa,
3590 decl.srcLoc(zcu),
3591 "invalid liveness: {s}",
3592 .{@errorName(err)},
3593 ),
3594 );
3595 decl.analysis = .liveness_failure;
3596 return error.AnalysisFail;
3597 },
3598 };
3599 }
3600
3601 if (comp.bin_file) |lf| {
3602 lf.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {
3603 error.OutOfMemory => return error.OutOfMemory,
3604 error.AnalysisFail => {
3605 decl.analysis = .codegen_failure;
3606 },
3607 else => {
3608 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);
3609 zcu.failed_decls.putAssumeCapacityNoClobber(decl_index, try Module.ErrorMsg.create(
3610 gpa,
3611 decl.srcLoc(zcu),
3612 "unable to codegen: {s}",
3613 .{@errorName(err)},
3614 ));
3615 decl.analysis = .codegen_failure_retryable;
3616 },
3617 };
3618 } else if (zcu.llvm_object) |llvm_object| {
3619 if (build_options.only_c) unreachable;
3620 llvm_object.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {
3621 error.OutOfMemory => return error.OutOfMemory,
3622 error.AnalysisFail => {
3623 decl.analysis = .codegen_failure;
3624 },
3625 };
36093626 }
36103627}
36113628
......@@ -3625,7 +3642,6 @@ pub fn ensureFuncBodyAnalysisQueued(mod: *Module, func_index: InternPool.Index)
36253642 switch (decl.analysis) {
36263643 .unreferenced => unreachable,
36273644 .in_progress => unreachable,
3628 .outdated => unreachable,
36293645
36303646 .file_failure,
36313647 .sema_failure,
......@@ -3813,6 +3829,15 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !SemaDeclResult {
38133829 return error.AnalysisFail;
38143830 }
38153831
3832 if (mod.declIsRoot(decl_index)) {
3833 // This comes from an `analyze_decl` job on an incremental update where
3834 // this file changed.
3835 @panic("TODO: update root Decl of modified file");
3836 } else if (decl.owns_tv) {
3837 // We are re-analyzing an owner Decl (for a function or a namespace type).
3838 @panic("TODO: update owner Decl");
3839 }
3840
38163841 const gpa = mod.gpa;
38173842 const zir = decl.getFileScope(mod).zir;
38183843
......@@ -3880,12 +3905,12 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !SemaDeclResult {
38803905 };
38813906 defer sema.deinit();
38823907
3883 // Every Decl other (than file root Decls, which do not have a ZIR index) has a dependency on its own source.
3884 if (decl.zir_decl_index.unwrap()) |zir_decl_index| {
3885 try sema.declareDependency(.{ .src_hash = try ip.trackZir(sema.gpa, decl.getFileScope(mod), zir_decl_index) });
3886 }
3887
3888 assert(!mod.declIsRoot(decl_index));
3908 // Every Decl (other than file root Decls, which do not have a ZIR index) has a dependency on its own source.
3909 try sema.declareDependency(.{ .src_hash = try ip.trackZir(
3910 sema.gpa,
3911 decl.getFileScope(mod),
3912 decl.zir_decl_index.unwrap().?,
3913 ) });
38893914
38903915 var block_scope: Sema.Block = .{
38913916 .parent = null,