authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-04 14:03:40+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-04 16:20:29+00:00
log55a2e535fdb663793b84769cb6c3a261bda3fc66
treef13faecbc0fc66c49791165cce0189382026f428
parent0907432fffd2d9f5319022332a60fc88d3aacf4e
signaturelock-open Commit is signed but in an unrecognized format.

compiler: integrate ZON with the ZIR caching system

This came with a big cleanup to `Zcu.PerThread.updateFile` (formerly `astGenFile`). Also, change how the cache manifest works for files in the import table. Instead of being added to the manifest when we call `semaFile` on them, we iterate the import table after running the AstGen workers and add all the files to the cache manifest then. The downside is that this is a bit more eager to include files in the manifest; in particular, files which are imported but not actually referenced are now included in analysis. So, for instance, modifying any standard library file will invalidate all Zig compilations using that standard library, even if they don't use that file. The original motivation here was simply that the old logic in `semaFile` didn't translate nicely to ZON. However, it turns out to actually be necessary for correctness. Because `@import("foo.zig")` is an AstGen-level error if `foo.zig` does not exist, we need to invalidate the cache when an imported but unreferenced file is removed to make sure this error is triggered when it needs to be. Resolves: #22746

6 files changed, 405 insertions(+), 187 deletions(-)

lib/std/zig/Zoir.zig+25
...@@ -10,6 +10,31 @@ string_bytes: []u8,...@@ -10,6 +10,31 @@ string_bytes: []u8,
10compile_errors: []Zoir.CompileError,10compile_errors: []Zoir.CompileError,
11error_notes: []Zoir.CompileError.Note,11error_notes: []Zoir.CompileError.Note,
1212
13/// The data stored at byte offset 0 when ZOIR is stored in a file.
14pub const Header = extern struct {
15 nodes_len: u32,
16 extra_len: u32,
17 limbs_len: u32,
18 string_bytes_len: u32,
19 compile_errors_len: u32,
20 error_notes_len: u32,
21
22 /// We could leave this as padding, however it triggers a Valgrind warning because
23 /// we read and write undefined bytes to the file system. This is harmless, but
24 /// it's essentially free to have a zero field here and makes the warning go away,
25 /// making it more likely that following Valgrind warnings will be taken seriously.
26 unused: u64 = 0,
27
28 stat_inode: std.fs.File.INode,
29 stat_size: u64,
30 stat_mtime: i128,
31
32 comptime {
33 // Check that `unused` is working as expected
34 assert(std.meta.hasUniqueRepresentation(Header));
35 }
36};
37
13pub fn hasCompileErrors(zoir: Zoir) bool {38pub fn hasCompileErrors(zoir: Zoir) bool {
14 if (zoir.compile_errors.len > 0) {39 if (zoir.compile_errors.len > 0) {
15 assert(zoir.nodes.len == 0);40 assert(zoir.nodes.len == 0);
src/Compilation.zig+38-8
...@@ -2220,10 +2220,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2220,10 +2220,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2220 try comp.astgen_work_queue.ensureUnusedCapacity(zcu.import_table.count());2220 try comp.astgen_work_queue.ensureUnusedCapacity(zcu.import_table.count());
2221 for (zcu.import_table.values()) |file_index| {2221 for (zcu.import_table.values()) |file_index| {
2222 if (zcu.fileByIndex(file_index).mod.isBuiltin()) continue;2222 if (zcu.fileByIndex(file_index).mod.isBuiltin()) continue;
2223 const file = zcu.fileByIndex(file_index);2223 comp.astgen_work_queue.writeItemAssumeCapacity(file_index);
2224 if (file.getMode() == .zig) {
2225 comp.astgen_work_queue.writeItemAssumeCapacity(file_index);
2226 }
2227 }2224 }
2228 if (comp.file_system_inputs) |fsi| {2225 if (comp.file_system_inputs) |fsi| {
2229 for (zcu.import_table.values()) |file_index| {2226 for (zcu.import_table.values()) |file_index| {
...@@ -3810,11 +3807,40 @@ fn performAllTheWorkInner(...@@ -3810,11 +3807,40 @@ fn performAllTheWorkInner(
3810 const pt: Zcu.PerThread = .activate(zcu, .main);3807 const pt: Zcu.PerThread = .activate(zcu, .main);
3811 defer pt.deactivate();3808 defer pt.deactivate();
38123809
3810 // If the cache mode is `whole`, then add every source file to the cache manifest.
3811 switch (comp.cache_use) {
3812 .whole => |whole| if (whole.cache_manifest) |man| {
3813 const gpa = zcu.gpa;
3814 for (zcu.import_table.values()) |file_index| {
3815 const file = zcu.fileByIndex(file_index);
3816 const source = file.getSource(gpa) catch |err| {
3817 try pt.reportRetryableFileError(file_index, "unable to load source: {s}", .{@errorName(err)});
3818 continue;
3819 };
3820 const resolved_path = try std.fs.path.resolve(gpa, &.{
3821 file.mod.root.root_dir.path orelse ".",
3822 file.mod.root.sub_path,
3823 file.sub_file_path,
3824 });
3825 errdefer gpa.free(resolved_path);
3826 whole.cache_manifest_mutex.lock();
3827 defer whole.cache_manifest_mutex.unlock();
3828 man.addFilePostContents(resolved_path, source.bytes, source.stat) catch |err| switch (err) {
3829 error.OutOfMemory => |e| return e,
3830 else => {
3831 try pt.reportRetryableFileError(file_index, "unable to update cache: {s}", .{@errorName(err)});
3832 continue;
3833 },
3834 };
3835 }
3836 },
3837 .incremental => {},
3838 }
3839
3813 try reportMultiModuleErrors(pt);3840 try reportMultiModuleErrors(pt);
38143841
3815 const any_fatal_files = for (zcu.import_table.values()) |file_index| {3842 const any_fatal_files = for (zcu.import_table.values()) |file_index| {
3816 const file = zcu.fileByIndex(file_index);3843 const file = zcu.fileByIndex(file_index);
3817 if (file.getMode() == .zon) continue;
3818 switch (file.status) {3844 switch (file.status) {
3819 .never_loaded => unreachable, // everything is loaded by the workers3845 .never_loaded => unreachable, // everything is loaded by the workers
3820 .retryable_failure, .astgen_failure => break true,3846 .retryable_failure, .astgen_failure => break true,
...@@ -3822,7 +3848,7 @@ fn performAllTheWorkInner(...@@ -3822,7 +3848,7 @@ fn performAllTheWorkInner(
3822 }3848 }
3823 } else false;3849 } else false;
38243850
3825 if (any_fatal_files) {3851 if (any_fatal_files or comp.alloc_failure_occurred) {
3826 // We give up right now! No updating of ZIR refs, no nothing. The idea is that this prevents3852 // We give up right now! No updating of ZIR refs, no nothing. The idea is that this prevents
3827 // us from invalidating lots of incremental dependencies due to files with e.g. parse errors.3853 // us from invalidating lots of incremental dependencies due to files with e.g. parse errors.
3828 // However, this means our analysis data is invalid, so we want to omit all analysis errors.3854 // However, this means our analysis data is invalid, so we want to omit all analysis errors.
...@@ -4290,7 +4316,6 @@ fn workerUpdateFile(...@@ -4290,7 +4316,6 @@ fn workerUpdateFile(
4290 wg: *WaitGroup,4316 wg: *WaitGroup,
4291 src: Zcu.AstGenSrc,4317 src: Zcu.AstGenSrc,
4292) void {4318) void {
4293 assert(file.getMode() == .zig);
4294 const child_prog_node = prog_node.start(file.sub_file_path, 0);4319 const child_prog_node = prog_node.start(file.sub_file_path, 0);
4295 defer child_prog_node.end();4320 defer child_prog_node.end();
42964321
...@@ -4310,6 +4335,11 @@ fn workerUpdateFile(...@@ -4310,6 +4335,11 @@ fn workerUpdateFile(
4310 },4335 },
4311 };4336 };
43124337
4338 switch (file.getMode()) {
4339 .zig => {}, // continue to logic below
4340 .zon => return, // ZON can't import anything so we're done
4341 }
4342
4313 // Pre-emptively look for `@import` paths and queue them up.4343 // Pre-emptively look for `@import` paths and queue them up.
4314 // If we experience an error preemptively fetching the4344 // If we experience an error preemptively fetching the
4315 // file, just ignore it and let it happen again later during Sema.4345 // file, just ignore it and let it happen again later during Sema.
...@@ -4344,7 +4374,7 @@ fn workerUpdateFile(...@@ -4344,7 +4374,7 @@ fn workerUpdateFile(
4344 const imported_path_digest = pt.zcu.filePathDigest(res.file_index);4374 const imported_path_digest = pt.zcu.filePathDigest(res.file_index);
4345 break :blk .{ res, imported_path_digest };4375 break :blk .{ res, imported_path_digest };
4346 };4376 };
4347 if (import_result.is_new and import_result.file.getMode() == .zig) {4377 if (import_result.is_new) {
4348 log.debug("AstGen of {s} has import '{s}'; queuing AstGen of {s}", .{4378 log.debug("AstGen of {s} has import '{s}'; queuing AstGen of {s}", .{
4349 file.sub_file_path, import_path, import_result.file.sub_file_path,4379 file.sub_file_path, import_path, import_result.file.sub_file_path,
4350 });4380 });
src/Sema.zig-6
...@@ -13994,12 +13994,6 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air....@@ -13994,12 +13994,6 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
13994 return Air.internedToRef(ty);13994 return Air.internedToRef(ty);
13995 },13995 },
13996 .zon => {13996 .zon => {
13997 _ = result.file.getTree(zcu.gpa) catch |err| {
13998 // TODO: these errors are file system errors; make sure an update() will
13999 // retry this and not cache the file system error, which may be transient.
14000 return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ result.file.sub_file_path, @errorName(err) });
14001 };
14002
14003 if (extra.res_ty == .none) {13997 if (extra.res_ty == .none) {
14004 return sema.fail(block, operand_src, "'@import' of ZON must have a known result type", .{});13998 return sema.fail(block, operand_src, "'@import' of ZON must have a known result type", .{});
14005 }13999 }
src/Sema/LowerZon.zig-2
...@@ -39,8 +39,6 @@ pub fn run(...@@ -39,8 +39,6 @@ pub fn run(
39) CompileError!InternPool.Index {39) CompileError!InternPool.Index {
40 const pt = sema.pt;40 const pt = sema.pt;
4141
42 _ = try file.getZoir(pt.zcu);
43
44 const tracked_inst = try pt.zcu.intern_pool.trackZir(pt.zcu.gpa, pt.tid, .{42 const tracked_inst = try pt.zcu.intern_pool.trackZir(pt.zcu.gpa, pt.tid, .{
45 .file = file_index,43 .file = file_index,
46 .inst = .main_struct_inst, // this is the only trackable instruction in a ZON file44 .inst = .main_struct_inst, // this is the only trackable instruction in a ZON file
src/Zcu.zig+183
...@@ -2643,6 +2643,189 @@ pub fn loadZirCacheBody(gpa: Allocator, header: Zir.Header, cache_file: std.fs.F...@@ -2643,6 +2643,189 @@ pub fn loadZirCacheBody(gpa: Allocator, header: Zir.Header, cache_file: std.fs.F
2643 return zir;2643 return zir;
2644}2644}
26452645
2646pub fn saveZirCache(gpa: Allocator, cache_file: std.fs.File, stat: std.fs.File.Stat, zir: Zir) (std.fs.File.WriteError || Allocator.Error)!void {
2647 const safety_buffer = if (data_has_safety_tag)
2648 try gpa.alloc([8]u8, zir.instructions.len)
2649 else
2650 undefined;
2651 defer if (data_has_safety_tag) gpa.free(safety_buffer);
2652
2653 const data_ptr: [*]const u8 = if (data_has_safety_tag)
2654 if (zir.instructions.len == 0)
2655 undefined
2656 else
2657 @ptrCast(safety_buffer.ptr)
2658 else
2659 @ptrCast(zir.instructions.items(.data).ptr);
2660
2661 if (data_has_safety_tag) {
2662 // The `Data` union has a safety tag but in the file format we store it without.
2663 for (zir.instructions.items(.data), 0..) |*data, i| {
2664 const as_struct: *const HackDataLayout = @ptrCast(data);
2665 safety_buffer[i] = as_struct.data;
2666 }
2667 }
2668
2669 const header: Zir.Header = .{
2670 .instructions_len = @intCast(zir.instructions.len),
2671 .string_bytes_len = @intCast(zir.string_bytes.len),
2672 .extra_len = @intCast(zir.extra.len),
2673
2674 .stat_size = stat.size,
2675 .stat_inode = stat.inode,
2676 .stat_mtime = stat.mtime,
2677 };
2678 var iovecs: [5]std.posix.iovec_const = .{
2679 .{
2680 .base = @ptrCast(&header),
2681 .len = @sizeOf(Zir.Header),
2682 },
2683 .{
2684 .base = @ptrCast(zir.instructions.items(.tag).ptr),
2685 .len = zir.instructions.len,
2686 },
2687 .{
2688 .base = data_ptr,
2689 .len = zir.instructions.len * 8,
2690 },
2691 .{
2692 .base = zir.string_bytes.ptr,
2693 .len = zir.string_bytes.len,
2694 },
2695 .{
2696 .base = @ptrCast(zir.extra.ptr),
2697 .len = zir.extra.len * 4,
2698 },
2699 };
2700 try cache_file.writevAll(&iovecs);
2701}
2702
2703pub fn saveZoirCache(cache_file: std.fs.File, stat: std.fs.File.Stat, zoir: Zoir) std.fs.File.WriteError!void {
2704 const header: Zoir.Header = .{
2705 .nodes_len = @intCast(zoir.nodes.len),
2706 .extra_len = @intCast(zoir.extra.len),
2707 .limbs_len = @intCast(zoir.limbs.len),
2708 .string_bytes_len = @intCast(zoir.string_bytes.len),
2709 .compile_errors_len = @intCast(zoir.compile_errors.len),
2710 .error_notes_len = @intCast(zoir.error_notes.len),
2711
2712 .stat_size = stat.size,
2713 .stat_inode = stat.inode,
2714 .stat_mtime = stat.mtime,
2715 };
2716 var iovecs: [9]std.posix.iovec_const = .{
2717 .{
2718 .base = @ptrCast(&header),
2719 .len = @sizeOf(Zoir.Header),
2720 },
2721 .{
2722 .base = @ptrCast(zoir.nodes.items(.tag)),
2723 .len = zoir.nodes.len * @sizeOf(Zoir.Node.Repr.Tag),
2724 },
2725 .{
2726 .base = @ptrCast(zoir.nodes.items(.data)),
2727 .len = zoir.nodes.len * 4,
2728 },
2729 .{
2730 .base = @ptrCast(zoir.nodes.items(.ast_node)),
2731 .len = zoir.nodes.len * 4,
2732 },
2733 .{
2734 .base = @ptrCast(zoir.extra),
2735 .len = zoir.extra.len * 4,
2736 },
2737 .{
2738 .base = @ptrCast(zoir.limbs),
2739 .len = zoir.limbs.len * 4,
2740 },
2741 .{
2742 .base = zoir.string_bytes.ptr,
2743 .len = zoir.string_bytes.len,
2744 },
2745 .{
2746 .base = @ptrCast(zoir.compile_errors),
2747 .len = zoir.compile_errors.len * @sizeOf(Zoir.CompileError),
2748 },
2749 .{
2750 .base = @ptrCast(zoir.error_notes),
2751 .len = zoir.error_notes.len * @sizeOf(Zoir.CompileError.Note),
2752 },
2753 };
2754 try cache_file.writevAll(&iovecs);
2755}
2756
2757pub fn loadZoirCacheBody(gpa: Allocator, header: Zoir.Header, cache_file: std.fs.File) !Zoir {
2758 var zoir: Zoir = .{
2759 .nodes = .empty,
2760 .extra = &.{},
2761 .limbs = &.{},
2762 .string_bytes = &.{},
2763 .compile_errors = &.{},
2764 .error_notes = &.{},
2765 };
2766 errdefer zoir.deinit(gpa);
2767
2768 zoir.nodes = nodes: {
2769 var nodes: std.MultiArrayList(Zoir.Node.Repr) = .empty;
2770 defer nodes.deinit(gpa);
2771 try nodes.setCapacity(gpa, header.nodes_len);
2772 nodes.len = header.nodes_len;
2773 break :nodes nodes.toOwnedSlice();
2774 };
2775
2776 zoir.extra = try gpa.alloc(u32, header.extra_len);
2777 zoir.limbs = try gpa.alloc(std.math.big.Limb, header.limbs_len);
2778 zoir.string_bytes = try gpa.alloc(u8, header.string_bytes_len);
2779
2780 zoir.compile_errors = try gpa.alloc(Zoir.CompileError, header.compile_errors_len);
2781 zoir.error_notes = try gpa.alloc(Zoir.CompileError.Note, header.error_notes_len);
2782
2783 var iovecs: [8]std.posix.iovec = .{
2784 .{
2785 .base = @ptrCast(zoir.nodes.items(.tag)),
2786 .len = header.nodes_len * @sizeOf(Zoir.Node.Repr.Tag),
2787 },
2788 .{
2789 .base = @ptrCast(zoir.nodes.items(.data)),
2790 .len = header.nodes_len * 4,
2791 },
2792 .{
2793 .base = @ptrCast(zoir.nodes.items(.ast_node)),
2794 .len = header.nodes_len * 4,
2795 },
2796 .{
2797 .base = @ptrCast(zoir.extra),
2798 .len = header.extra_len * 4,
2799 },
2800 .{
2801 .base = @ptrCast(zoir.limbs),
2802 .len = header.limbs_len * @sizeOf(std.math.big.Limb),
2803 },
2804 .{
2805 .base = zoir.string_bytes.ptr,
2806 .len = header.string_bytes_len,
2807 },
2808 .{
2809 .base = @ptrCast(zoir.compile_errors),
2810 .len = header.compile_errors_len * @sizeOf(Zoir.CompileError),
2811 },
2812 .{
2813 .base = @ptrCast(zoir.error_notes),
2814 .len = header.error_notes_len * @sizeOf(Zoir.CompileError.Note),
2815 },
2816 };
2817
2818 const bytes_expected = expected: {
2819 var n: usize = 0;
2820 for (iovecs) |v| n += v.len;
2821 break :expected n;
2822 };
2823
2824 const bytes_read = try cache_file.readvAll(&iovecs);
2825 if (bytes_read != bytes_expected) return error.UnexpectedFileSize;
2826 return zoir;
2827}
2828
2646pub fn markDependeeOutdated(2829pub fn markDependeeOutdated(
2647 zcu: *Zcu,2830 zcu: *Zcu,
2648 /// When we are diffing ZIR and marking things as outdated, we won't yet have marked the dependencies as PO.2831 /// When we are diffing ZIR and marking things as outdated, we won't yet have marked the dependencies as PO.
src/Zcu/PerThread.zig+159-171
...@@ -26,6 +26,8 @@ const Type = @import("../Type.zig");...@@ -26,6 +26,8 @@ const Type = @import("../Type.zig");
26const Value = @import("../Value.zig");26const Value = @import("../Value.zig");
27const Zcu = @import("../Zcu.zig");27const Zcu = @import("../Zcu.zig");
28const Zir = std.zig.Zir;28const Zir = std.zig.Zir;
29const Zoir = std.zig.Zoir;
30const ZonGen = std.zig.ZonGen;
2931
30zcu: *Zcu,32zcu: *Zcu,
3133
...@@ -73,6 +75,8 @@ pub fn destroyFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) void {...@@ -73,6 +75,8 @@ pub fn destroyFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) void {
73 if (!is_builtin) gpa.destroy(file);75 if (!is_builtin) gpa.destroy(file);
74}76}
7577
78/// Ensures that `file` has up-to-date ZIR. If not, loads the ZIR cache or runs
79/// AstGen as needed. Also updates `file.status`.
76pub fn updateFile(80pub fn updateFile(
77 pt: Zcu.PerThread,81 pt: Zcu.PerThread,
78 file: *Zcu.File,82 file: *Zcu.File,
...@@ -126,6 +130,24 @@ pub fn updateFile(...@@ -126,6 +130,24 @@ pub fn updateFile(
126 },130 },
127 };131 };
128132
133 // The old compile error, if any, is no longer relevant.
134 pt.lockAndClearFileCompileError(file);
135
136 // If `zir` is not null, and `prev_zir` is null, then `TrackedInst`s are associated with `zir`.
137 // We need to keep it around!
138 // As an optimization, also check `loweringFailed`; if true, but `prev_zir == null`, then this
139 // file has never passed AstGen, so we actually need not cache the old ZIR.
140 if (file.zir != null and file.prev_zir == null and !file.zir.?.loweringFailed()) {
141 assert(file.prev_zir == null);
142 const prev_zir_ptr = try gpa.create(Zir);
143 file.prev_zir = prev_zir_ptr;
144 prev_zir_ptr.* = file.zir.?;
145 file.zir = null;
146 }
147
148 // We're going to re-load everything, so unload source, AST, ZIR, ZOIR.
149 file.unload(gpa);
150
129 // We ask for a lock in order to coordinate with other zig processes.151 // We ask for a lock in order to coordinate with other zig processes.
130 // If another process is already working on this file, we will get the cached152 // If another process is already working on this file, we will get the cached
131 // version. Likewise if we're working on AstGen and another process asks for153 // version. Likewise if we're working on AstGen and another process asks for
...@@ -180,175 +202,164 @@ pub fn updateFile(...@@ -180,175 +202,164 @@ pub fn updateFile(
180 };202 };
181 defer cache_file.close();203 defer cache_file.close();
182204
183 while (true) {205 const need_update = while (true) {
184 update: {206 const result = switch (file.getMode()) {
185 // First we read the header to determine the lengths of arrays.207 inline else => |mode| try loadZirZoirCache(zcu, cache_file, stat, file, mode),
186 const header = cache_file.reader().readStruct(Zir.Header) catch |err| switch (err) {208 };
187 // This can happen if Zig bails out of this function between creating209 switch (result) {
188 // the cached file and writing it.210 .success => {
189 error.EndOfStream => break :update,211 log.debug("AstGen cached success: {s}", .{file.sub_file_path});
190 else => |e| return e,212 break false;
191 };213 },
192 const unchanged_metadata =214 .invalid => {},
193 stat.size == header.stat_size and215 .truncated => log.warn("unexpected EOF reading cached ZIR for {s}", .{file.sub_file_path}),
194 stat.mtime == header.stat_mtime and216 .stale => log.debug("AstGen cache stale: {s}", .{file.sub_file_path}),
195 stat.inode == header.stat_inode;
196
197 if (!unchanged_metadata) {
198 log.debug("AstGen cache stale: {s}", .{file.sub_file_path});
199 break :update;
200 }
201 log.debug("AstGen cache hit: {s} instructions_len={d}", .{
202 file.sub_file_path, header.instructions_len,
203 });
204
205 file.zir = Zcu.loadZirCacheBody(gpa, header, cache_file) catch |err| switch (err) {
206 error.UnexpectedFileSize => {
207 log.warn("unexpected EOF reading cached ZIR for {s}", .{file.sub_file_path});
208 break :update;
209 },
210 else => |e| return e,
211 };
212 file.stat = .{
213 .size = header.stat_size,
214 .inode = header.stat_inode,
215 .mtime = header.stat_mtime,
216 };
217 file.status = .success;
218 log.debug("AstGen cached success: {s}", .{file.sub_file_path});
219
220 if (file.zir.?.hasCompileErrors()) {
221 comp.mutex.lock();
222 defer comp.mutex.unlock();
223 try zcu.failed_files.putNoClobber(gpa, file, null);
224 }
225 if (file.zir.?.loweringFailed()) {
226 file.status = .astgen_failure;
227 return error.AnalysisFail;
228 }
229 return;
230 }217 }
231218
232 // If we already have the exclusive lock then it is our job to update.219 // If we already have the exclusive lock then it is our job to update.
233 if (builtin.os.tag == .wasi or lock == .exclusive) break;220 if (builtin.os.tag == .wasi or lock == .exclusive) break true;
234 // Otherwise, unlock to give someone a chance to get the exclusive lock221 // Otherwise, unlock to give someone a chance to get the exclusive lock
235 // and then upgrade to an exclusive lock.222 // and then upgrade to an exclusive lock.
236 cache_file.unlock();223 cache_file.unlock();
237 lock = .exclusive;224 lock = .exclusive;
238 try cache_file.lock(lock);225 try cache_file.lock(lock);
239 }226 };
240227
241 // The cache is definitely stale so delete the contents to avoid an underwrite later.228 if (need_update) {
242 cache_file.setEndPos(0) catch |err| switch (err) {229 // The cache is definitely stale so delete the contents to avoid an underwrite later.
243 error.FileTooBig => unreachable, // 0 is not too big230 cache_file.setEndPos(0) catch |err| switch (err) {
231 error.FileTooBig => unreachable, // 0 is not too big
232 else => |e| return e,
233 };
244234
245 else => |e| return e,235 if (stat.size > std.math.maxInt(u32))
246 };236 return error.FileTooBig;
247237
248 pt.lockAndClearFileCompileError(file);238 const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
239 defer if (file.source == null) gpa.free(source);
240 const amt = try source_file.readAll(source);
241 if (amt != stat.size)
242 return error.UnexpectedEndOfFile;
249243
250 // If `zir` is not null, and `prev_zir` is null, then `TrackedInst`s are associated with `zir`.244 file.source = source;
251 // We need to keep it around!245
252 // As an optimization, also check `loweringFailed`; if true, but `prev_zir == null`, then this246 // Any potential AST errors are converted to ZIR errors when we run AstGen/ZonGen.
253 // file has never passed AstGen, so we actually need not cache the old ZIR.247 file.tree = try Ast.parse(gpa, source, file.getMode());
254 if (file.zir != null and file.prev_zir == null and !file.zir.?.loweringFailed()) {
255 assert(file.prev_zir == null);
256 const prev_zir_ptr = try gpa.create(Zir);
257 file.prev_zir = prev_zir_ptr;
258 prev_zir_ptr.* = file.zir.?;
259 file.zir = null;
260 }
261 file.unload(gpa);
262248
263 if (stat.size > std.math.maxInt(u32))249 switch (file.getMode()) {
264 return error.FileTooBig;250 .zig => {
251 file.zir = try AstGen.generate(gpa, file.tree.?);
252 Zcu.saveZirCache(gpa, cache_file, stat, file.zir.?) catch |err| switch (err) {
253 error.OutOfMemory => |e| return e,
254 else => log.warn("unable to write cached ZIR code for {}{s} to {}{s}: {s}", .{
255 file.mod.root, file.sub_file_path, cache_directory, &hex_digest, @errorName(err),
256 }),
257 };
258 },
259 .zon => {
260 file.zoir = try ZonGen.generate(gpa, file.tree.?, .{});
261 Zcu.saveZoirCache(cache_file, stat, file.zoir.?) catch |err| {
262 log.warn("unable to write cached ZOIR code for {}{s} to {}{s}: {s}", .{
263 file.mod.root, file.sub_file_path, cache_directory, &hex_digest, @errorName(err),
264 });
265 };
266 },
267 }
265268
266 const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);269 log.debug("AstGen fresh success: {s}", .{file.sub_file_path});
267 defer if (file.source == null) gpa.free(source);270 }
268 const amt = try source_file.readAll(source);
269 if (amt != stat.size)
270 return error.UnexpectedEndOfFile;
271271
272 file.stat = .{272 file.stat = .{
273 .size = stat.size,273 .size = stat.size,
274 .inode = stat.inode,274 .inode = stat.inode,
275 .mtime = stat.mtime,275 .mtime = stat.mtime,
276 };276 };
277 file.source = source;
278277
279 file.tree = try Ast.parse(gpa, source, .zig);278 // Now, `zir` or `zoir` is definitely populated and up-to-date.
279 // Mark file successes/failures as needed.
280280
281 // Any potential AST errors are converted to ZIR errors here.281 switch (file.getMode()) {
282 file.zir = try AstGen.generate(gpa, file.tree.?);282 .zig => {
283 file.status = .success;283 if (file.zir.?.hasCompileErrors()) {
284 log.debug("AstGen fresh success: {s}", .{file.sub_file_path});284 comp.mutex.lock();
285 defer comp.mutex.unlock();
286 try zcu.failed_files.putNoClobber(gpa, file, null);
287 }
288 if (file.zir.?.loweringFailed()) {
289 file.status = .astgen_failure;
290 } else {
291 file.status = .success;
292 }
293 },
294 .zon => {
295 if (file.zoir.?.hasCompileErrors()) {
296 file.status = .astgen_failure;
297 comp.mutex.lock();
298 defer comp.mutex.unlock();
299 try zcu.failed_files.putNoClobber(gpa, file, null);
300 } else {
301 file.status = .success;
302 }
303 },
304 }
285305
286 const safety_buffer = if (Zcu.data_has_safety_tag)306 switch (file.status) {
287 try gpa.alloc([8]u8, file.zir.?.instructions.len)307 .never_loaded => unreachable,
288 else308 .retryable_failure => unreachable,
289 undefined;309 .astgen_failure => return error.AnalysisFail,
290 defer if (Zcu.data_has_safety_tag) gpa.free(safety_buffer);310 .success => return,
291 const data_ptr = if (Zcu.data_has_safety_tag)
292 if (file.zir.?.instructions.len == 0)
293 @as([*]const u8, undefined)
294 else
295 @as([*]const u8, @ptrCast(safety_buffer.ptr))
296 else
297 @as([*]const u8, @ptrCast(file.zir.?.instructions.items(.data).ptr));
298 if (Zcu.data_has_safety_tag) {
299 // The `Data` union has a safety tag but in the file format we store it without.
300 for (file.zir.?.instructions.items(.data), 0..) |*data, i| {
301 const as_struct: *const Zcu.HackDataLayout = @ptrCast(data);
302 safety_buffer[i] = as_struct.data;
303 }
304 }311 }
312}
305313
306 const header: Zir.Header = .{314fn loadZirZoirCache(
307 .instructions_len = @as(u32, @intCast(file.zir.?.instructions.len)),315 zcu: *Zcu,
308 .string_bytes_len = @as(u32, @intCast(file.zir.?.string_bytes.len)),316 cache_file: std.fs.File,
309 .extra_len = @as(u32, @intCast(file.zir.?.extra.len)),317 stat: std.fs.File.Stat,
318 file: *Zcu.File,
319 comptime mode: Ast.Mode,
320) !enum { success, invalid, truncated, stale } {
321 assert(file.getMode() == mode);
310322
311 .stat_size = stat.size,323 const gpa = zcu.gpa;
312 .stat_inode = stat.inode,324
313 .stat_mtime = stat.mtime,325 const Header = switch (mode) {
314 };326 .zig => Zir.Header,
315 var iovecs = [_]std.posix.iovec_const{327 .zon => Zoir.Header,
316 .{
317 .base = @as([*]const u8, @ptrCast(&header)),
318 .len = @sizeOf(Zir.Header),
319 },
320 .{
321 .base = @as([*]const u8, @ptrCast(file.zir.?.instructions.items(.tag).ptr)),
322 .len = file.zir.?.instructions.len,
323 },
324 .{
325 .base = data_ptr,
326 .len = file.zir.?.instructions.len * 8,
327 },
328 .{
329 .base = file.zir.?.string_bytes.ptr,
330 .len = file.zir.?.string_bytes.len,
331 },
332 .{
333 .base = @as([*]const u8, @ptrCast(file.zir.?.extra.ptr)),
334 .len = file.zir.?.extra.len * 4,
335 },
336 };328 };
337 cache_file.writevAll(&iovecs) catch |err| {329
338 log.warn("unable to write cached ZIR code for {}{s} to {}{s}: {s}", .{330 // First we read the header to determine the lengths of arrays.
339 file.mod.root, file.sub_file_path, cache_directory, &hex_digest, @errorName(err),331 const header = cache_file.reader().readStruct(Header) catch |err| switch (err) {
340 });332 // This can happen if Zig bails out of this function between creating
333 // the cached file and writing it.
334 error.EndOfStream => return .invalid,
335 else => |e| return e,
341 };336 };
342337
343 if (file.zir.?.hasCompileErrors()) {338 const unchanged_metadata =
344 comp.mutex.lock();339 stat.size == header.stat_size and
345 defer comp.mutex.unlock();340 stat.mtime == header.stat_mtime and
346 try zcu.failed_files.putNoClobber(gpa, file, null);341 stat.inode == header.stat_inode;
342
343 if (!unchanged_metadata) {
344 return .stale;
347 }345 }
348 if (file.zir.?.loweringFailed()) {346
349 file.status = .astgen_failure;347 switch (mode) {
350 return error.AnalysisFail;348 .zig => {
349 file.zir = Zcu.loadZirCacheBody(gpa, header, cache_file) catch |err| switch (err) {
350 error.UnexpectedFileSize => return .truncated,
351 else => |e| return e,
352 };
353 },
354 .zon => {
355 file.zoir = Zcu.loadZoirCacheBody(gpa, header, cache_file) catch |err| switch (err) {
356 error.UnexpectedFileSize => return .truncated,
357 else => |e| return e,
358 };
359 },
351 }360 }
361
362 return .success;
352}363}
353364
354const UpdatedFile = struct {365const UpdatedFile = struct {
...@@ -1819,7 +1830,6 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void {...@@ -1819,7 +1830,6 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void {
1819 defer tracy.end();1830 defer tracy.end();
18201831
1821 const zcu = pt.zcu;1832 const zcu = pt.zcu;
1822 const gpa = zcu.gpa;
1823 const file = zcu.fileByIndex(file_index);1833 const file = zcu.fileByIndex(file_index);
1824 assert(file.getMode() == .zig);1834 assert(file.getMode() == .zig);
1825 assert(zcu.fileRootType(file_index) == .none);1835 assert(zcu.fileRootType(file_index) == .none);
...@@ -1834,36 +1844,6 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void {...@@ -1834,36 +1844,6 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void {
1834 });1844 });
1835 const struct_ty = try pt.createFileRootStruct(file_index, new_namespace_index, false);1845 const struct_ty = try pt.createFileRootStruct(file_index, new_namespace_index, false);
1836 errdefer zcu.intern_pool.remove(pt.tid, struct_ty);1846 errdefer zcu.intern_pool.remove(pt.tid, struct_ty);
1837
1838 switch (zcu.comp.cache_use) {
1839 .whole => |whole| if (whole.cache_manifest) |man| {
1840 const source = file.getSource(gpa) catch |err| {
1841 try pt.reportRetryableFileError(file_index, "unable to load source: {s}", .{@errorName(err)});
1842 return error.AnalysisFail;
1843 };
1844
1845 const resolved_path = std.fs.path.resolve(gpa, &.{
1846 file.mod.root.root_dir.path orelse ".",
1847 file.mod.root.sub_path,
1848 file.sub_file_path,
1849 }) catch |err| {
1850 try pt.reportRetryableFileError(file_index, "unable to resolve path: {s}", .{@errorName(err)});
1851 return error.AnalysisFail;
1852 };
1853 errdefer gpa.free(resolved_path);
1854
1855 whole.cache_manifest_mutex.lock();
1856 defer whole.cache_manifest_mutex.unlock();
1857 man.addFilePostContents(resolved_path, source.bytes, source.stat) catch |err| switch (err) {
1858 error.OutOfMemory => |e| return e,
1859 else => {
1860 try pt.reportRetryableFileError(file_index, "unable to update cache: {s}", .{@errorName(err)});
1861 return error.AnalysisFail;
1862 },
1863 };
1864 },
1865 .incremental => {},
1866 }
1867}1847}
18681848
1869pub fn importPkg(pt: Zcu.PerThread, mod: *Module) Allocator.Error!Zcu.ImportFileResult {1849pub fn importPkg(pt: Zcu.PerThread, mod: *Module) Allocator.Error!Zcu.ImportFileResult {
...@@ -2800,8 +2780,16 @@ pub fn getErrorValueFromSlice(pt: Zcu.PerThread, name: []const u8) Allocator.Err...@@ -2800,8 +2780,16 @@ pub fn getErrorValueFromSlice(pt: Zcu.PerThread, name: []const u8) Allocator.Err
2800/// Removes any entry from `Zcu.failed_files` associated with `file`. Acquires `Compilation.mutex` as needed.2780/// Removes any entry from `Zcu.failed_files` associated with `file`. Acquires `Compilation.mutex` as needed.
2801/// `file.zir` must be unchanged from the last update, as it is used to determine if there is such an entry.2781/// `file.zir` must be unchanged from the last update, as it is used to determine if there is such an entry.
2802fn lockAndClearFileCompileError(pt: Zcu.PerThread, file: *Zcu.File) void {2782fn lockAndClearFileCompileError(pt: Zcu.PerThread, file: *Zcu.File) void {
2803 const zir = file.zir orelse return;2783 switch (file.getMode()) {
2804 if (!zir.hasCompileErrors()) return;2784 .zig => {
2785 const zir = file.zir orelse return;
2786 if (!zir.hasCompileErrors()) return;
2787 },
2788 .zon => {
2789 const zoir = file.zoir orelse return;
2790 if (!zoir.hasCompileErrors()) return;
2791 },
2792 }
28052793
2806 pt.zcu.comp.mutex.lock();2794 pt.zcu.comp.mutex.lock();
2807 defer pt.zcu.comp.mutex.unlock();2795 defer pt.zcu.comp.mutex.unlock();