authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-04 14:42:09+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-04 16:20:29+00:00
log3ca588bcc6d6640b7faa41a271580dd384963927
treec47ab77e60e00f5c52a32257124d3bc6b642cf77
parent55a2e535fdb663793b84769cb6c3a261bda3fc66
signaturelock-open Commit is signed but in an unrecognized format.

compiler: integrate importing ZON with incremental compilation

The changes from a few commits earlier, where semantic analysis no longer occurs if any Zig files failed to lower to ZIR, mean `file` dependencies are no longer necessary! However, we now need them for ZON files, to be invalidated whenever a ZON file changes.

5 files changed, 48 insertions(+), 22 deletions(-)

src/Compilation.zig+12-4
......@@ -2903,10 +2903,12 @@ pub fn makeBinFileWritable(comp: *Compilation) !void {
29032903const Header = extern struct {
29042904 intern_pool: extern struct {
29052905 thread_count: u32,
2906 file_deps_len: u32,
29072906 src_hash_deps_len: u32,
29082907 nav_val_deps_len: u32,
29092908 nav_ty_deps_len: u32,
2909 interned_deps_len: u32,
2910 zon_file_deps_len: u32,
2911 embed_file_deps_len: u32,
29102912 namespace_deps_len: u32,
29112913 namespace_name_deps_len: u32,
29122914 first_dependency_len: u32,
......@@ -2947,10 +2949,12 @@ pub fn saveState(comp: *Compilation) !void {
29472949 const header: Header = .{
29482950 .intern_pool = .{
29492951 .thread_count = @intCast(ip.locals.len),
2950 .file_deps_len = @intCast(ip.file_deps.count()),
29512952 .src_hash_deps_len = @intCast(ip.src_hash_deps.count()),
29522953 .nav_val_deps_len = @intCast(ip.nav_val_deps.count()),
29532954 .nav_ty_deps_len = @intCast(ip.nav_ty_deps.count()),
2955 .interned_deps_len = @intCast(ip.interned_deps.count()),
2956 .zon_file_deps_len = @intCast(ip.zon_file_deps.count()),
2957 .embed_file_deps_len = @intCast(ip.embed_file_deps.count()),
29542958 .namespace_deps_len = @intCast(ip.namespace_deps.count()),
29552959 .namespace_name_deps_len = @intCast(ip.namespace_name_deps.count()),
29562960 .first_dependency_len = @intCast(ip.first_dependency.count()),
......@@ -2975,14 +2979,18 @@ pub fn saveState(comp: *Compilation) !void {
29752979 addBuf(&bufs, mem.asBytes(&header));
29762980 addBuf(&bufs, mem.sliceAsBytes(pt_headers.items));
29772981
2978 addBuf(&bufs, mem.sliceAsBytes(ip.file_deps.keys()));
2979 addBuf(&bufs, mem.sliceAsBytes(ip.file_deps.values()));
29802982 addBuf(&bufs, mem.sliceAsBytes(ip.src_hash_deps.keys()));
29812983 addBuf(&bufs, mem.sliceAsBytes(ip.src_hash_deps.values()));
29822984 addBuf(&bufs, mem.sliceAsBytes(ip.nav_val_deps.keys()));
29832985 addBuf(&bufs, mem.sliceAsBytes(ip.nav_val_deps.values()));
29842986 addBuf(&bufs, mem.sliceAsBytes(ip.nav_ty_deps.keys()));
29852987 addBuf(&bufs, mem.sliceAsBytes(ip.nav_ty_deps.values()));
2988 addBuf(&bufs, mem.sliceAsBytes(ip.interned_deps.keys()));
2989 addBuf(&bufs, mem.sliceAsBytes(ip.interned_deps.values()));
2990 addBuf(&bufs, mem.sliceAsBytes(ip.zon_file_deps.keys()));
2991 addBuf(&bufs, mem.sliceAsBytes(ip.zon_file_deps.values()));
2992 addBuf(&bufs, mem.sliceAsBytes(ip.embed_file_deps.keys()));
2993 addBuf(&bufs, mem.sliceAsBytes(ip.embed_file_deps.values()));
29862994 addBuf(&bufs, mem.sliceAsBytes(ip.namespace_deps.keys()));
29872995 addBuf(&bufs, mem.sliceAsBytes(ip.namespace_deps.values()));
29882996 addBuf(&bufs, mem.sliceAsBytes(ip.namespace_name_deps.keys()));
src/InternPool.zig+8-12
......@@ -17,13 +17,6 @@ tid_shift_31: if (single_threaded) u0 else std.math.Log2Int(u32),
1717/// Cached shift amount to put a `tid` in the top bits of a 32-bit value.
1818tid_shift_32: if (single_threaded) u0 else std.math.Log2Int(u32),
1919
20/// Dependencies on whether an entire file gets past AstGen.
21/// These are triggered by `@import`, so that:
22/// * if a file initially fails AstGen, triggering a transitive failure, when a future update
23/// causes it to succeed AstGen, the `@import` is re-analyzed, allowing analysis to proceed
24/// * if a file initially succeds AstGen, but a future update causes the file to fail it,
25/// the `@import` is re-analyzed, registering a transitive failure
26file_deps: std.AutoArrayHashMapUnmanaged(FileIndex, DepEntry.Index),
2720/// Dependencies on the source code hash associated with a ZIR instruction.
2821/// * For a `declaration`, this is the entire declaration body.
2922/// * For a `struct_decl`, `union_decl`, etc, this is the source of the fields (but not declarations).
......@@ -42,6 +35,9 @@ nav_ty_deps: std.AutoArrayHashMapUnmanaged(Nav.Index, DepEntry.Index),
4235/// * a container type requiring resolution (invalidated when the type must be recreated at a new index)
4336/// Value is index into `dep_entries` of the first dependency on this interned value.
4437interned_deps: std.AutoArrayHashMapUnmanaged(Index, DepEntry.Index),
38/// Dependencies on a ZON file. Triggered by `@import` of ZON.
39/// Value is index into `dep_entries` of the first dependency on this ZON file.
40zon_file_deps: std.AutoArrayHashMapUnmanaged(FileIndex, DepEntry.Index),
4541/// Dependencies on an embedded file.
4642/// Introduced by `@embedFile`; invalidated when the file changes.
4743/// Value is index into `dep_entries` of the first dependency on this `Zcu.EmbedFile`.
......@@ -89,11 +85,11 @@ pub const empty: InternPool = .{
8985 .tid_shift_30 = if (single_threaded) 0 else 31,
9086 .tid_shift_31 = if (single_threaded) 0 else 31,
9187 .tid_shift_32 = if (single_threaded) 0 else 31,
92 .file_deps = .empty,
9388 .src_hash_deps = .empty,
9489 .nav_val_deps = .empty,
9590 .nav_ty_deps = .empty,
9691 .interned_deps = .empty,
92 .zon_file_deps = .empty,
9793 .embed_file_deps = .empty,
9894 .namespace_deps = .empty,
9995 .namespace_name_deps = .empty,
......@@ -824,11 +820,11 @@ pub const Nav = struct {
824820};
825821
826822pub const Dependee = union(enum) {
827 file: FileIndex,
828823 src_hash: TrackedInst.Index,
829824 nav_val: Nav.Index,
830825 nav_ty: Nav.Index,
831826 interned: Index,
827 zon_file: FileIndex,
832828 embed_file: Zcu.EmbedFile.Index,
833829 namespace: TrackedInst.Index,
834830 namespace_name: NamespaceNameKey,
......@@ -876,11 +872,11 @@ pub const DependencyIterator = struct {
876872
877873pub fn dependencyIterator(ip: *const InternPool, dependee: Dependee) DependencyIterator {
878874 const first_entry = switch (dependee) {
879 .file => |x| ip.file_deps.get(x),
880875 .src_hash => |x| ip.src_hash_deps.get(x),
881876 .nav_val => |x| ip.nav_val_deps.get(x),
882877 .nav_ty => |x| ip.nav_ty_deps.get(x),
883878 .interned => |x| ip.interned_deps.get(x),
879 .zon_file => |x| ip.zon_file_deps.get(x),
884880 .embed_file => |x| ip.embed_file_deps.get(x),
885881 .namespace => |x| ip.namespace_deps.get(x),
886882 .namespace_name => |x| ip.namespace_name_deps.get(x),
......@@ -947,11 +943,11 @@ pub fn addDependency(ip: *InternPool, gpa: Allocator, depender: AnalUnit, depend
947943 },
948944 inline else => |dependee_payload, tag| new_index: {
949945 const gop = try switch (tag) {
950 .file => ip.file_deps,
951946 .src_hash => ip.src_hash_deps,
952947 .nav_val => ip.nav_val_deps,
953948 .nav_ty => ip.nav_ty_deps,
954949 .interned => ip.interned_deps,
950 .zon_file => ip.zon_file_deps,
955951 .embed_file => ip.embed_file_deps,
956952 .namespace => ip.namespace_deps,
957953 .namespace_name => ip.namespace_name_deps,
......@@ -6688,11 +6684,11 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
66886684pub fn deinit(ip: *InternPool, gpa: Allocator) void {
66896685 if (debug_state.enable_checks) std.debug.assert(debug_state.intern_pool == null);
66906686
6691 ip.file_deps.deinit(gpa);
66926687 ip.src_hash_deps.deinit(gpa);
66936688 ip.nav_val_deps.deinit(gpa);
66946689 ip.nav_ty_deps.deinit(gpa);
66956690 ip.interned_deps.deinit(gpa);
6691 ip.zon_file_deps.deinit(gpa);
66966692 ip.embed_file_deps.deinit(gpa);
66976693 ip.namespace_deps.deinit(gpa);
66986694 ip.namespace_name_deps.deinit(gpa);
src/Sema.zig+1-2
......@@ -6143,7 +6143,6 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr
61436143 pt.updateFile(result.file, path_digest) catch |err|
61446144 return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)});
61456145
6146 try sema.declareDependency(.{ .file = result.file_index });
61476146 try pt.ensureFileAnalyzed(result.file_index);
61486147 const ty = zcu.fileRootType(result.file_index);
61496148 try sema.declareDependency(.{ .interned = ty });
......@@ -13986,7 +13985,6 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
1398613985 };
1398713986 switch (result.file.getMode()) {
1398813987 .zig => {
13989 try sema.declareDependency(.{ .file = result.file_index });
1399013988 try pt.ensureFileAnalyzed(result.file_index);
1399113989 const ty = zcu.fileRootType(result.file_index);
1399213990 try sema.declareDependency(.{ .interned = ty });
......@@ -14003,6 +14001,7 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
1400314001 return sema.fail(block, operand_src, "'@import' of ZON must have a known result type", .{});
1400414002 }
1400514003
14004 try sema.declareDependency(.{ .zon_file = result.file_index });
1400614005 const interned = try LowerZon.run(
1400714006 sema,
1400814007 result.file,
src/Zcu.zig+12-4
......@@ -705,6 +705,14 @@ pub const File = struct {
705705 /// field is populated with that old ZIR.
706706 prev_zir: ?*Zir = null,
707707
708 /// This field serves a similar purpose to `prev_zir`, but for ZOIR. However, since we do not
709 /// need to map old ZOIR to new ZOIR -- instead only invalidating dependencies if the ZOIR
710 /// changed -- this field is just a simple boolean.
711 ///
712 /// When `zoir` is updated, this field is set to `true`. In `updateZirRefs`, if this is `true`,
713 /// we invalidate the corresponding `zon_file` dependency, and reset it to `false`.
714 zoir_invalidated: bool = false,
715
708716 /// A single reference to a file.
709717 pub const Reference = union(enum) {
710718 /// The file is imported directly (i.e. not as a package) with @import.
......@@ -4074,10 +4082,6 @@ fn formatDependee(data: struct { dependee: InternPool.Dependee, zcu: *Zcu }, com
40744082 const zcu = data.zcu;
40754083 const ip = &zcu.intern_pool;
40764084 switch (data.dependee) {
4077 .file => |file| {
4078 const file_path = zcu.fileByIndex(file).sub_file_path;
4079 return writer.print("file('{s}')", .{file_path});
4080 },
40814085 .src_hash => |ti| {
40824086 const info = ti.resolveFull(ip) orelse {
40834087 return writer.writeAll("inst(<lost>)");
......@@ -4098,6 +4102,10 @@ fn formatDependee(data: struct { dependee: InternPool.Dependee, zcu: *Zcu }, com
40984102 .func => |f| return writer.print("ies('{}')", .{ip.getNav(f.owner_nav).fqn.fmt(ip)}),
40994103 else => unreachable,
41004104 },
4105 .zon_file => |file| {
4106 const file_path = zcu.fileByIndex(file).sub_file_path;
4107 return writer.print("zon_file('{s}')", .{file_path});
4108 },
41014109 .embed_file => |ef_idx| {
41024110 const ef = ef_idx.get(zcu);
41034111 return writer.print("embed_file('{s}')", .{std.fs.path.fmtJoin(&.{
src/Zcu/PerThread.zig+15
......@@ -145,6 +145,9 @@ pub fn updateFile(
145145 file.zir = null;
146146 }
147147
148 // If ZOIR is changing, then we need to invalidate dependencies on it
149 if (file.zoir != null) file.zoir_invalidated = true;
150
148151 // We're going to re-load everything, so unload source, AST, ZIR, ZOIR.
149152 file.unload(gpa);
150153
......@@ -380,11 +383,23 @@ pub fn updateZirRefs(pt: Zcu.PerThread) Allocator.Error!void {
380383 const gpa = zcu.gpa;
381384
382385 // We need to visit every updated File for every TrackedInst in InternPool.
386 // This only includes Zig files; ZON files are omitted.
383387 var updated_files: std.AutoArrayHashMapUnmanaged(Zcu.File.Index, UpdatedFile) = .empty;
384388 defer cleanupUpdatedFiles(gpa, &updated_files);
389
385390 for (zcu.import_table.values()) |file_index| {
386391 const file = zcu.fileByIndex(file_index);
387392 assert(file.status == .success);
393 switch (file.getMode()) {
394 .zig => {}, // logic below
395 .zon => {
396 if (file.zoir_invalidated) {
397 try zcu.markDependeeOutdated(.not_marked_po, .{ .zon_file = file_index });
398 file.zoir_invalidated = false;
399 }
400 continue;
401 },
402 }
388403 const old_zir = file.prev_zir orelse continue;
389404 const new_zir = file.zir.?;
390405 const gop = try updated_files.getOrPut(gpa, file_index);