authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-01-26 01:41:56+00:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-01-26 01:41:56+00:00
log3767b08039b86c798dc7f43a1659ad10b65b248f
tree1cab306aab6df4977f2b9ce6b789fe9013fe0476
parent8fa47bb904c888dadf20af5eb72b9643eed2bfea
parentfcf8d5ada28c02722575cc78d41171692a227061
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22602 from mlugg/incr-embedfile

incremental: handle `@embedFile`

19 files changed, 467 insertions(+), 208 deletions(-)

lib/std/fs/path.zig+30-1
...@@ -18,7 +18,6 @@ const debug = std.debug;...@@ -18,7 +18,6 @@ const debug = std.debug;
18const assert = debug.assert;18const assert = debug.assert;
19const testing = std.testing;19const testing = std.testing;
20const mem = std.mem;20const mem = std.mem;
21const fmt = std.fmt;
22const ascii = std.ascii;21const ascii = std.ascii;
23const Allocator = mem.Allocator;22const Allocator = mem.Allocator;
24const math = std.math;23const math = std.math;
...@@ -147,6 +146,36 @@ pub fn joinZ(allocator: Allocator, paths: []const []const u8) ![:0]u8 {...@@ -147,6 +146,36 @@ pub fn joinZ(allocator: Allocator, paths: []const []const u8) ![:0]u8 {
147 return out[0 .. out.len - 1 :0];146 return out[0 .. out.len - 1 :0];
148}147}
149148
149pub fn fmtJoin(paths: []const []const u8) std.fmt.Formatter(formatJoin) {
150 return .{ .data = paths };
151}
152
153fn formatJoin(paths: []const []const u8, comptime fmt: []const u8, options: std.fmt.FormatOptions, w: anytype) !void {
154 _ = fmt;
155 _ = options;
156
157 const first_path_idx = for (paths, 0..) |p, idx| {
158 if (p.len != 0) break idx;
159 } else return;
160
161 try w.writeAll(paths[first_path_idx]); // first component
162 var prev_path = paths[first_path_idx];
163 for (paths[first_path_idx + 1 ..]) |this_path| {
164 if (this_path.len == 0) continue; // skip empty components
165 const prev_sep = isSep(prev_path[prev_path.len - 1]);
166 const this_sep = isSep(this_path[0]);
167 if (!prev_sep and !this_sep) {
168 try w.writeByte(sep);
169 }
170 if (prev_sep and this_sep) {
171 try w.writeAll(this_path[1..]); // skip redundant separator
172 } else {
173 try w.writeAll(this_path);
174 }
175 prev_path = this_path;
176 }
177}
178
150fn testJoinMaybeZUefi(paths: []const []const u8, expected: []const u8, zero: bool) !void {179fn testJoinMaybeZUefi(paths: []const []const u8, expected: []const u8, zero: bool) !void {
151 const uefiIsSep = struct {180 const uefiIsSep = struct {
152 fn isSep(byte: u8) bool {181 fn isSep(byte: u8) bool {
src/Compilation.zig+23-60
...@@ -154,10 +154,6 @@ win32_resource_work_queue: if (dev.env.supports(.win32_resource)) std.fifo.Linea...@@ -154,10 +154,6 @@ win32_resource_work_queue: if (dev.env.supports(.win32_resource)) std.fifo.Linea
154/// since the last compilation, as well as scan for `@import` and queue up154/// since the last compilation, as well as scan for `@import` and queue up
155/// additional jobs corresponding to those new files.155/// additional jobs corresponding to those new files.
156astgen_work_queue: std.fifo.LinearFifo(Zcu.File.Index, .Dynamic),156astgen_work_queue: std.fifo.LinearFifo(Zcu.File.Index, .Dynamic),
157/// These jobs are to inspect the file system stat() and if the embedded file has changed
158/// on disk, mark the corresponding Decl outdated and queue up an `analyze_decl`
159/// task for it.
160embed_file_work_queue: std.fifo.LinearFifo(*Zcu.EmbedFile, .Dynamic),
161157
162/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.158/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
163/// This data is accessed by multiple threads and is protected by `mutex`.159/// This data is accessed by multiple threads and is protected by `mutex`.
...@@ -1465,7 +1461,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -1465,7 +1461,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
1465 .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),1461 .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
1466 .win32_resource_work_queue = if (dev.env.supports(.win32_resource)) std.fifo.LinearFifo(*Win32Resource, .Dynamic).init(gpa) else .{},1462 .win32_resource_work_queue = if (dev.env.supports(.win32_resource)) std.fifo.LinearFifo(*Win32Resource, .Dynamic).init(gpa) else .{},
1467 .astgen_work_queue = std.fifo.LinearFifo(Zcu.File.Index, .Dynamic).init(gpa),1463 .astgen_work_queue = std.fifo.LinearFifo(Zcu.File.Index, .Dynamic).init(gpa),
1468 .embed_file_work_queue = std.fifo.LinearFifo(*Zcu.EmbedFile, .Dynamic).init(gpa),
1469 .c_source_files = options.c_source_files,1464 .c_source_files = options.c_source_files,
1470 .rc_source_files = options.rc_source_files,1465 .rc_source_files = options.rc_source_files,
1471 .cache_parent = cache,1466 .cache_parent = cache,
...@@ -1920,7 +1915,6 @@ pub fn destroy(comp: *Compilation) void {...@@ -1920,7 +1915,6 @@ pub fn destroy(comp: *Compilation) void {
1920 comp.c_object_work_queue.deinit();1915 comp.c_object_work_queue.deinit();
1921 comp.win32_resource_work_queue.deinit();1916 comp.win32_resource_work_queue.deinit();
1922 comp.astgen_work_queue.deinit();1917 comp.astgen_work_queue.deinit();
1923 comp.embed_file_work_queue.deinit();
19241918
1925 comp.windows_libs.deinit(gpa);1919 comp.windows_libs.deinit(gpa);
19261920
...@@ -2235,11 +2229,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2235,11 +2229,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2235 }2229 }
2236 }2230 }
22372231
2238 // Put a work item in for checking if any files used with `@embedFile` changed.
2239 try comp.embed_file_work_queue.ensureUnusedCapacity(zcu.embed_table.count());
2240 for (zcu.embed_table.values()) |embed_file| {
2241 comp.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
2242 }
2243 if (comp.file_system_inputs) |fsi| {2232 if (comp.file_system_inputs) |fsi| {
2244 const ip = &zcu.intern_pool;2233 const ip = &zcu.intern_pool;
2245 for (zcu.embed_table.values()) |embed_file| {2234 for (zcu.embed_table.values()) |embed_file| {
...@@ -3223,9 +3212,6 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle {...@@ -3223,9 +3212,6 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle {
3223 try addZirErrorMessages(&bundle, file);3212 try addZirErrorMessages(&bundle, file);
3224 }3213 }
3225 }3214 }
3226 for (zcu.failed_embed_files.values()) |error_msg| {
3227 try addModuleErrorMsg(zcu, &bundle, error_msg.*);
3228 }
3229 var sorted_failed_analysis: std.AutoArrayHashMapUnmanaged(InternPool.AnalUnit, *Zcu.ErrorMsg).DataList.Slice = s: {3215 var sorted_failed_analysis: std.AutoArrayHashMapUnmanaged(InternPool.AnalUnit, *Zcu.ErrorMsg).DataList.Slice = s: {
3230 const SortOrder = struct {3216 const SortOrder = struct {
3231 zcu: *Zcu,3217 zcu: *Zcu,
...@@ -3804,9 +3790,10 @@ fn performAllTheWorkInner(...@@ -3804,9 +3790,10 @@ fn performAllTheWorkInner(
3804 }3790 }
3805 }3791 }
38063792
3807 while (comp.embed_file_work_queue.readItem()) |embed_file| {3793 for (0.., zcu.embed_table.values()) |ef_index_usize, ef| {
3808 comp.thread_pool.spawnWg(&astgen_wait_group, workerCheckEmbedFile, .{3794 const ef_index: Zcu.EmbedFile.Index = @enumFromInt(ef_index_usize);
3809 comp, embed_file,3795 comp.thread_pool.spawnWgId(&astgen_wait_group, workerCheckEmbedFile, .{
3796 comp, ef_index, ef,
3810 });3797 });
3811 }3798 }
3812 }3799 }
...@@ -4369,33 +4356,33 @@ fn workerUpdateBuiltinZigFile(...@@ -4369,33 +4356,33 @@ fn workerUpdateBuiltinZigFile(
4369 };4356 };
4370}4357}
43714358
4372fn workerCheckEmbedFile(comp: *Compilation, embed_file: *Zcu.EmbedFile) void {4359fn workerCheckEmbedFile(tid: usize, comp: *Compilation, ef_index: Zcu.EmbedFile.Index, ef: *Zcu.EmbedFile) void {
4373 comp.detectEmbedFileUpdate(embed_file) catch |err| {4360 comp.detectEmbedFileUpdate(@enumFromInt(tid), ef_index, ef) catch |err| switch (err) {
4374 comp.reportRetryableEmbedFileError(embed_file, err) catch |oom| switch (oom) {4361 error.OutOfMemory => {
4375 // Swallowing this error is OK because it's implied to be OOM when4362 comp.mutex.lock();
4376 // there is a missing `failed_embed_files` error message.4363 defer comp.mutex.unlock();
4377 error.OutOfMemory => {},4364 comp.setAllocFailure();
4378 };4365 },
4379 return;
4380 };4366 };
4381}4367}
43824368
4383fn detectEmbedFileUpdate(comp: *Compilation, embed_file: *Zcu.EmbedFile) !void {4369fn detectEmbedFileUpdate(comp: *Compilation, tid: Zcu.PerThread.Id, ef_index: Zcu.EmbedFile.Index, ef: *Zcu.EmbedFile) !void {
4384 const zcu = comp.zcu.?;4370 const zcu = comp.zcu.?;
4385 const ip = &zcu.intern_pool;4371 const pt: Zcu.PerThread = .activate(zcu, tid);
4386 var file = try embed_file.owner.root.openFile(embed_file.sub_file_path.toSlice(ip), .{});4372 defer pt.deactivate();
4387 defer file.close();4373
4374 const old_val = ef.val;
4375 const old_err = ef.err;
43884376
4389 const stat = try file.stat();4377 try pt.updateEmbedFile(ef, null);
43904378
4391 const unchanged_metadata =4379 if (ef.val != .none and ef.val == old_val) return; // success, value unchanged
4392 stat.size == embed_file.stat.size and4380 if (ef.val == .none and old_val == .none and ef.err == old_err) return; // failure, error unchanged
4393 stat.mtime == embed_file.stat.mtime and
4394 stat.inode == embed_file.stat.inode;
43954381
4396 if (unchanged_metadata) return;4382 comp.mutex.lock();
4383 defer comp.mutex.unlock();
43974384
4398 @panic("TODO: handle embed file incremental update");4385 try zcu.markDependeeOutdated(.not_marked_po, .{ .embed_file = ef_index });
4399}4386}
44004387
4401pub fn obtainCObjectCacheManifest(4388pub fn obtainCObjectCacheManifest(
...@@ -4797,30 +4784,6 @@ fn reportRetryableWin32ResourceError(...@@ -4797,30 +4784,6 @@ fn reportRetryableWin32ResourceError(
4797 }4784 }
4798}4785}
47994786
4800fn reportRetryableEmbedFileError(
4801 comp: *Compilation,
4802 embed_file: *Zcu.EmbedFile,
4803 err: anyerror,
4804) error{OutOfMemory}!void {
4805 const zcu = comp.zcu.?;
4806 const gpa = zcu.gpa;
4807 const src_loc = embed_file.src_loc;
4808 const ip = &zcu.intern_pool;
4809 const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}/{s}': {s}", .{
4810 embed_file.owner.root,
4811 embed_file.sub_file_path.toSlice(ip),
4812 @errorName(err),
4813 });
4814
4815 errdefer err_msg.destroy(gpa);
4816
4817 {
4818 comp.mutex.lock();
4819 defer comp.mutex.unlock();
4820 try zcu.failed_embed_files.putNoClobber(gpa, embed_file, err_msg);
4821 }
4822}
4823
4824fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Progress.Node) !void {4787fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Progress.Node) !void {
4825 if (comp.config.c_frontend == .aro) {4788 if (comp.config.c_frontend == .aro) {
4826 return comp.failCObj(c_object, "aro does not support compiling C objects yet", .{});4789 return comp.failCObj(c_object, "aro does not support compiling C objects yet", .{});
src/InternPool.zig+9
...@@ -42,6 +42,10 @@ nav_ty_deps: std.AutoArrayHashMapUnmanaged(Nav.Index, DepEntry.Index),...@@ -42,6 +42,10 @@ nav_ty_deps: std.AutoArrayHashMapUnmanaged(Nav.Index, DepEntry.Index),
42/// * a container type requiring resolution (invalidated when the type must be recreated at a new index)42/// * a container type requiring resolution (invalidated when the type must be recreated at a new index)
43/// Value is index into `dep_entries` of the first dependency on this interned value.43/// Value is index into `dep_entries` of the first dependency on this interned value.
44interned_deps: std.AutoArrayHashMapUnmanaged(Index, DepEntry.Index),44interned_deps: std.AutoArrayHashMapUnmanaged(Index, DepEntry.Index),
45/// Dependencies on an embedded file.
46/// Introduced by `@embedFile`; invalidated when the file changes.
47/// Value is index into `dep_entries` of the first dependency on this `Zcu.EmbedFile`.
48embed_file_deps: std.AutoArrayHashMapUnmanaged(Zcu.EmbedFile.Index, DepEntry.Index),
45/// Dependencies on the full set of names in a ZIR namespace.49/// Dependencies on the full set of names in a ZIR namespace.
46/// Key refers to a `struct_decl`, `union_decl`, etc.50/// Key refers to a `struct_decl`, `union_decl`, etc.
47/// Value is index into `dep_entries` of the first dependency on this namespace.51/// Value is index into `dep_entries` of the first dependency on this namespace.
...@@ -90,6 +94,7 @@ pub const empty: InternPool = .{...@@ -90,6 +94,7 @@ pub const empty: InternPool = .{
90 .nav_val_deps = .empty,94 .nav_val_deps = .empty,
91 .nav_ty_deps = .empty,95 .nav_ty_deps = .empty,
92 .interned_deps = .empty,96 .interned_deps = .empty,
97 .embed_file_deps = .empty,
93 .namespace_deps = .empty,98 .namespace_deps = .empty,
94 .namespace_name_deps = .empty,99 .namespace_name_deps = .empty,
95 .memoized_state_main_deps = .none,100 .memoized_state_main_deps = .none,
...@@ -824,6 +829,7 @@ pub const Dependee = union(enum) {...@@ -824,6 +829,7 @@ pub const Dependee = union(enum) {
824 nav_val: Nav.Index,829 nav_val: Nav.Index,
825 nav_ty: Nav.Index,830 nav_ty: Nav.Index,
826 interned: Index,831 interned: Index,
832 embed_file: Zcu.EmbedFile.Index,
827 namespace: TrackedInst.Index,833 namespace: TrackedInst.Index,
828 namespace_name: NamespaceNameKey,834 namespace_name: NamespaceNameKey,
829 memoized_state: MemoizedStateStage,835 memoized_state: MemoizedStateStage,
...@@ -875,6 +881,7 @@ pub fn dependencyIterator(ip: *const InternPool, dependee: Dependee) DependencyI...@@ -875,6 +881,7 @@ pub fn dependencyIterator(ip: *const InternPool, dependee: Dependee) DependencyI
875 .nav_val => |x| ip.nav_val_deps.get(x),881 .nav_val => |x| ip.nav_val_deps.get(x),
876 .nav_ty => |x| ip.nav_ty_deps.get(x),882 .nav_ty => |x| ip.nav_ty_deps.get(x),
877 .interned => |x| ip.interned_deps.get(x),883 .interned => |x| ip.interned_deps.get(x),
884 .embed_file => |x| ip.embed_file_deps.get(x),
878 .namespace => |x| ip.namespace_deps.get(x),885 .namespace => |x| ip.namespace_deps.get(x),
879 .namespace_name => |x| ip.namespace_name_deps.get(x),886 .namespace_name => |x| ip.namespace_name_deps.get(x),
880 .memoized_state => |stage| switch (stage) {887 .memoized_state => |stage| switch (stage) {
...@@ -945,6 +952,7 @@ pub fn addDependency(ip: *InternPool, gpa: Allocator, depender: AnalUnit, depend...@@ -945,6 +952,7 @@ pub fn addDependency(ip: *InternPool, gpa: Allocator, depender: AnalUnit, depend
945 .nav_val => ip.nav_val_deps,952 .nav_val => ip.nav_val_deps,
946 .nav_ty => ip.nav_ty_deps,953 .nav_ty => ip.nav_ty_deps,
947 .interned => ip.interned_deps,954 .interned => ip.interned_deps,
955 .embed_file => ip.embed_file_deps,
948 .namespace => ip.namespace_deps,956 .namespace => ip.namespace_deps,
949 .namespace_name => ip.namespace_name_deps,957 .namespace_name => ip.namespace_name_deps,
950 .memoized_state => comptime unreachable,958 .memoized_state => comptime unreachable,
...@@ -6682,6 +6690,7 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {...@@ -6682,6 +6690,7 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {
6682 ip.nav_val_deps.deinit(gpa);6690 ip.nav_val_deps.deinit(gpa);
6683 ip.nav_ty_deps.deinit(gpa);6691 ip.nav_ty_deps.deinit(gpa);
6684 ip.interned_deps.deinit(gpa);6692 ip.interned_deps.deinit(gpa);
6693 ip.embed_file_deps.deinit(gpa);
6685 ip.namespace_deps.deinit(gpa);6694 ip.namespace_deps.deinit(gpa);
6686 ip.namespace_name_deps.deinit(gpa);6695 ip.namespace_name_deps.deinit(gpa);
66876696
src/Sema.zig+14-6
...@@ -13964,6 +13964,8 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -13964,6 +13964,8 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
13964 defer tracy.end();13964 defer tracy.end();
1396513965
13966 const pt = sema.pt;13966 const pt = sema.pt;
13967 const zcu = pt.zcu;
13968
13967 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;13969 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;
13968 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);13970 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);
13969 const name = try sema.resolveConstString(block, operand_src, inst_data.operand, .{ .simple = .operand_embedFile });13971 const name = try sema.resolveConstString(block, operand_src, inst_data.operand, .{ .simple = .operand_embedFile });
...@@ -13972,18 +13974,24 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -13972,18 +13974,24 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
13972 return sema.fail(block, operand_src, "file path name cannot be empty", .{});13974 return sema.fail(block, operand_src, "file path name cannot be empty", .{});
13973 }13975 }
1397413976
13975 const val = pt.embedFile(block.getFileScope(pt.zcu), name, operand_src) catch |err| switch (err) {13977 const ef_idx = pt.embedFile(block.getFileScope(zcu), name) catch |err| switch (err) {
13976 error.ImportOutsideModulePath => {13978 error.ImportOutsideModulePath => {
13977 return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});13979 return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});
13978 },13980 },
13979 else => {13981 error.CurrentWorkingDirectoryUnlinked => {
13980 // TODO: these errors are file system errors; make sure an update() will13982 // TODO: this should be some kind of retryable failure, in case the cwd is put back
13981 // retry this and not cache the file system error, which may be transient.13983 return sema.fail(block, operand_src, "unable to resolve '{s}': working directory has been unlinked", .{name});
13982 return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(err) });
13983 },13984 },
13985 error.OutOfMemory => |e| return e,
13984 };13986 };
13987 try sema.declareDependency(.{ .embed_file = ef_idx });
13988
13989 const result = ef_idx.get(zcu);
13990 if (result.val == .none) {
13991 return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(result.err.?) });
13992 }
1398513993
13986 return Air.internedToRef(val);13994 return Air.internedToRef(result.val);
13987}13995}
1398813996
13989fn zirRetErrValueCode(sema: *Sema, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {13997fn zirRetErrValueCode(sema: *Sema, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
src/Zcu.zig+22-11
...@@ -143,8 +143,6 @@ compile_log_sources: std.AutoArrayHashMapUnmanaged(AnalUnit, extern struct {...@@ -143,8 +143,6 @@ compile_log_sources: std.AutoArrayHashMapUnmanaged(AnalUnit, extern struct {
143/// Using a map here for consistency with the other fields here.143/// Using a map here for consistency with the other fields here.
144/// The ErrorMsg memory is owned by the `File`, using Module's general purpose allocator.144/// The ErrorMsg memory is owned by the `File`, using Module's general purpose allocator.
145failed_files: std.AutoArrayHashMapUnmanaged(*File, ?*ErrorMsg) = .empty,145failed_files: std.AutoArrayHashMapUnmanaged(*File, ?*ErrorMsg) = .empty,
146/// The ErrorMsg memory is owned by the `EmbedFile`, using Module's general purpose allocator.
147failed_embed_files: std.AutoArrayHashMapUnmanaged(*EmbedFile, *ErrorMsg) = .empty,
148failed_exports: std.AutoArrayHashMapUnmanaged(Export.Index, *ErrorMsg) = .empty,146failed_exports: std.AutoArrayHashMapUnmanaged(Export.Index, *ErrorMsg) = .empty,
149/// If analysis failed due to a cimport error, the corresponding Clang errors147/// If analysis failed due to a cimport error, the corresponding Clang errors
150/// are stored here.148/// are stored here.
...@@ -898,13 +896,23 @@ pub const File = struct {...@@ -898,13 +896,23 @@ pub const File = struct {
898};896};
899897
900pub const EmbedFile = struct {898pub const EmbedFile = struct {
901 /// Relative to the owning module's root directory.
902 sub_file_path: InternPool.NullTerminatedString,
903 /// Module that this file is a part of, managed externally.899 /// Module that this file is a part of, managed externally.
904 owner: *Package.Module,900 owner: *Package.Module,
905 stat: Cache.File.Stat,901 /// Relative to the owning module's root directory.
902 sub_file_path: InternPool.NullTerminatedString,
903
904 /// `.none` means the file was not loaded, so `stat` is undefined.
906 val: InternPool.Index,905 val: InternPool.Index,
907 src_loc: LazySrcLoc,906 /// If this is `null` and `val` is `.none`, the file has never been loaded.
907 err: ?(std.fs.File.OpenError || std.fs.File.StatError || std.fs.File.ReadError || error{UnexpectedEof}),
908 stat: Cache.File.Stat,
909
910 pub const Index = enum(u32) {
911 _,
912 pub fn get(idx: Index, zcu: *const Zcu) *EmbedFile {
913 return zcu.embed_table.values()[@intFromEnum(idx)];
914 }
915 };
908};916};
909917
910/// This struct holds data necessary to construct API-facing `AllErrors.Message`.918/// This struct holds data necessary to construct API-facing `AllErrors.Message`.
...@@ -2464,11 +2472,6 @@ pub fn deinit(zcu: *Zcu) void {...@@ -2464,11 +2472,6 @@ pub fn deinit(zcu: *Zcu) void {
2464 }2472 }
2465 zcu.failed_files.deinit(gpa);2473 zcu.failed_files.deinit(gpa);
24662474
2467 for (zcu.failed_embed_files.values()) |msg| {
2468 msg.destroy(gpa);
2469 }
2470 zcu.failed_embed_files.deinit(gpa);
2471
2472 for (zcu.failed_exports.values()) |value| {2475 for (zcu.failed_exports.values()) |value| {
2473 value.destroy(gpa);2476 value.destroy(gpa);
2474 }2477 }
...@@ -3887,6 +3890,14 @@ fn formatDependee(data: struct { dependee: InternPool.Dependee, zcu: *Zcu }, com...@@ -3887,6 +3890,14 @@ fn formatDependee(data: struct { dependee: InternPool.Dependee, zcu: *Zcu }, com
3887 .func => |f| return writer.print("ies('{}')", .{ip.getNav(f.owner_nav).fqn.fmt(ip)}),3890 .func => |f| return writer.print("ies('{}')", .{ip.getNav(f.owner_nav).fqn.fmt(ip)}),
3888 else => unreachable,3891 else => unreachable,
3889 },3892 },
3893 .embed_file => |ef_idx| {
3894 const ef = ef_idx.get(zcu);
3895 return writer.print("embed_file('{s}')", .{std.fs.path.fmtJoin(&.{
3896 ef.owner.root.root_dir.path orelse "",
3897 ef.owner.root.sub_path,
3898 ef.sub_file_path.toSlice(ip),
3899 })});
3900 },
3890 .namespace => |ti| {3901 .namespace => |ti| {
3891 const info = ti.resolveFull(ip) orelse {3902 const info = ti.resolveFull(ip) orelse {
3892 return writer.writeAll("namespace(<lost>)");3903 return writer.writeAll("namespace(<lost>)");
src/Zcu/PerThread.zig+143-90
...@@ -2117,32 +2117,32 @@ pub fn embedFile(...@@ -2117,32 +2117,32 @@ pub fn embedFile(
2117 pt: Zcu.PerThread,2117 pt: Zcu.PerThread,
2118 cur_file: *Zcu.File,2118 cur_file: *Zcu.File,
2119 import_string: []const u8,2119 import_string: []const u8,
2120 src_loc: Zcu.LazySrcLoc,2120) error{
2121) !InternPool.Index {2121 OutOfMemory,
2122 ImportOutsideModulePath,
2123 CurrentWorkingDirectoryUnlinked,
2124}!Zcu.EmbedFile.Index {
2122 const zcu = pt.zcu;2125 const zcu = pt.zcu;
2123 const gpa = zcu.gpa;2126 const gpa = zcu.gpa;
21242127
2125 if (cur_file.mod.deps.get(import_string)) |pkg| {2128 if (cur_file.mod.deps.get(import_string)) |mod| {
2126 const resolved_path = try std.fs.path.resolve(gpa, &.{2129 const resolved_path = try std.fs.path.resolve(gpa, &.{
2127 pkg.root.root_dir.path orelse ".",2130 mod.root.root_dir.path orelse ".",
2128 pkg.root.sub_path,2131 mod.root.sub_path,
2129 pkg.root_src_path,2132 mod.root_src_path,
2130 });2133 });
2131 var keep_resolved_path = false;2134 errdefer gpa.free(resolved_path);
2132 defer if (!keep_resolved_path) gpa.free(resolved_path);
21332135
2134 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);2136 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);
2135 errdefer {2137 errdefer assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2136 assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2137 keep_resolved_path = false;
2138 }
2139 if (gop.found_existing) return gop.value_ptr.*.val;
2140 keep_resolved_path = true;
21412138
2142 const sub_file_path = try gpa.dupe(u8, pkg.root_src_path);2139 if (gop.found_existing) {
2143 errdefer gpa.free(sub_file_path);2140 gpa.free(resolved_path); // we're not using this key
2141 return @enumFromInt(gop.index);
2142 }
21442143
2145 return pt.newEmbedFile(pkg, sub_file_path, resolved_path, gop.value_ptr, src_loc);2144 gop.value_ptr.* = try pt.newEmbedFile(mod, mod.root_src_path, resolved_path);
2145 return @enumFromInt(gop.index);
2146 }2146 }
21472147
2148 // The resolved path is used as the key in the table, to detect if a file2148 // The resolved path is used as the key in the table, to detect if a file
...@@ -2154,17 +2154,15 @@ pub fn embedFile(...@@ -2154,17 +2154,15 @@ pub fn embedFile(
2154 "..",2154 "..",
2155 import_string,2155 import_string,
2156 });2156 });
21572157 errdefer gpa.free(resolved_path);
2158 var keep_resolved_path = false;
2159 defer if (!keep_resolved_path) gpa.free(resolved_path);
21602158
2161 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);2159 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);
2162 errdefer {2160 errdefer assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2163 assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));2161
2164 keep_resolved_path = false;2162 if (gop.found_existing) {
2163 gpa.free(resolved_path); // we're not using this key
2164 return @enumFromInt(gop.index);
2165 }2165 }
2166 if (gop.found_existing) return gop.value_ptr.*.val;
2167 keep_resolved_path = true;
21682166
2169 const resolved_root_path = try std.fs.path.resolve(gpa, &.{2167 const resolved_root_path = try std.fs.path.resolve(gpa, &.{
2170 cur_file.mod.root.root_dir.path orelse ".",2168 cur_file.mod.root.root_dir.path orelse ".",
...@@ -2172,101 +2170,156 @@ pub fn embedFile(...@@ -2172,101 +2170,156 @@ pub fn embedFile(
2172 });2170 });
2173 defer gpa.free(resolved_root_path);2171 defer gpa.free(resolved_root_path);
21742172
2175 const sub_file_path = p: {2173 const sub_file_path = std.fs.path.relative(gpa, resolved_root_path, resolved_path) catch |err| switch (err) {
2176 const relative = try std.fs.path.relative(gpa, resolved_root_path, resolved_path);2174 error.Unexpected => unreachable,
2177 errdefer gpa.free(relative);2175 else => |e| return e,
2178
2179 if (!isUpDir(relative) and !std.fs.path.isAbsolute(relative)) {
2180 break :p relative;
2181 }
2182 return error.ImportOutsideModulePath;
2183 };2176 };
2184 defer gpa.free(sub_file_path);2177 defer gpa.free(sub_file_path);
21852178
2186 return pt.newEmbedFile(cur_file.mod, sub_file_path, resolved_path, gop.value_ptr, src_loc);2179 if (isUpDir(sub_file_path) or std.fs.path.isAbsolute(sub_file_path)) {
2180 return error.ImportOutsideModulePath;
2181 }
2182
2183 gop.value_ptr.* = try pt.newEmbedFile(cur_file.mod, sub_file_path, resolved_path);
2184 return @enumFromInt(gop.index);
2187}2185}
21882186
2189/// https://github.com/ziglang/zig/issues/143072187pub fn updateEmbedFile(
2190fn newEmbedFile(
2191 pt: Zcu.PerThread,2188 pt: Zcu.PerThread,
2192 pkg: *Module,2189 ef: *Zcu.EmbedFile,
2193 sub_file_path: []const u8,2190 /// If not `null`, the interned file data is stored here, if it was loaded.
2194 resolved_path: []const u8,2191 /// `newEmbedFile` uses this to add the file to the `whole` cache manifest.
2195 result: **Zcu.EmbedFile,2192 ip_str_out: ?*?InternPool.String,
2196 src_loc: Zcu.LazySrcLoc,2193) Allocator.Error!void {
2197) !InternPool.Index {2194 pt.updateEmbedFileInner(ef, ip_str_out) catch |err| switch (err) {
2195 error.OutOfMemory => |e| return e,
2196 else => |e| {
2197 ef.val = .none;
2198 ef.err = e;
2199 ef.stat = undefined;
2200 },
2201 };
2202}
2203
2204fn updateEmbedFileInner(
2205 pt: Zcu.PerThread,
2206 ef: *Zcu.EmbedFile,
2207 ip_str_out: ?*?InternPool.String,
2208) !void {
2209 const tid = pt.tid;
2198 const zcu = pt.zcu;2210 const zcu = pt.zcu;
2199 const gpa = zcu.gpa;2211 const gpa = zcu.gpa;
2200 const ip = &zcu.intern_pool;2212 const ip = &zcu.intern_pool;
22012213
2202 const new_file = try gpa.create(Zcu.EmbedFile);2214 var file = try ef.owner.root.openFile(ef.sub_file_path.toSlice(ip), .{});
2203 errdefer gpa.destroy(new_file);
2204
2205 var file = try pkg.root.openFile(sub_file_path, .{});
2206 defer file.close();2215 defer file.close();
22072216
2208 const actual_stat = try file.stat();2217 const stat: Cache.File.Stat = .fromFs(try file.stat());
2209 const stat: Cache.File.Stat = .{
2210 .size = actual_stat.size,
2211 .inode = actual_stat.inode,
2212 .mtime = actual_stat.mtime,
2213 };
2214 const size = std.math.cast(usize, actual_stat.size) orelse return error.Overflow;
2215
2216 const strings = ip.getLocal(pt.tid).getMutableStrings(gpa);
2217 const bytes = try strings.addManyAsSlice(try std.math.add(usize, size, 1));
2218 const actual_read = try file.readAll(bytes[0][0..size]);
2219 if (actual_read != size) return error.UnexpectedEndOfFile;
2220 bytes[0][size] = 0;
22212218
2222 const comp = zcu.comp;2219 if (ef.val != .none) {
2223 switch (comp.cache_use) {2220 const old_stat = ef.stat;
2224 .whole => |whole| if (whole.cache_manifest) |man| {2221 const unchanged_metadata =
2225 const copied_resolved_path = try gpa.dupe(u8, resolved_path);2222 stat.size == old_stat.size and
2226 errdefer gpa.free(copied_resolved_path);2223 stat.mtime == old_stat.mtime and
2227 whole.cache_manifest_mutex.lock();2224 stat.inode == old_stat.inode;
2228 defer whole.cache_manifest_mutex.unlock();2225 if (unchanged_metadata) return;
2229 try man.addFilePostContents(copied_resolved_path, bytes[0][0..size], stat);
2230 },
2231 .incremental => {},
2232 }2226 }
22332227
2234 const array_ty = try pt.intern(.{ .array_type = .{2228 const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig;
2229 const size_plus_one = std.math.add(usize, size, 1) catch return error.FileTooBig;
2230
2231 // The loaded bytes of the file, including a sentinel 0 byte.
2232 const ip_str: InternPool.String = str: {
2233 const strings = ip.getLocal(tid).getMutableStrings(gpa);
2234 const old_len = strings.mutate.len;
2235 errdefer strings.shrinkRetainingCapacity(old_len);
2236 const bytes = (try strings.addManyAsSlice(size_plus_one))[0];
2237 const actual_read = try file.readAll(bytes[0..size]);
2238 if (actual_read != size) return error.UnexpectedEof;
2239 bytes[size] = 0;
2240 break :str try ip.getOrPutTrailingString(gpa, tid, @intCast(bytes.len), .maybe_embedded_nulls);
2241 };
2242 if (ip_str_out) |p| p.* = ip_str;
2243
2244 const array_ty = try pt.arrayType(.{
2235 .len = size,2245 .len = size,
2236 .sentinel = .zero_u8,2246 .sentinel = .zero_u8,
2237 .child = .u8_type,2247 .child = .u8_type,
2238 } });2248 });
2249 const ptr_ty = try pt.singleConstPtrType(array_ty);
2250
2239 const array_val = try pt.intern(.{ .aggregate = .{2251 const array_val = try pt.intern(.{ .aggregate = .{
2240 .ty = array_ty,2252 .ty = array_ty.toIntern(),
2241 .storage = .{ .bytes = try ip.getOrPutTrailingString(gpa, pt.tid, @intCast(bytes[0].len), .maybe_embedded_nulls) },2253 .storage = .{ .bytes = ip_str },
2242 } });2254 } });
2243
2244 const ptr_ty = (try pt.ptrType(.{
2245 .child = array_ty,
2246 .flags = .{
2247 .alignment = .none,
2248 .is_const = true,
2249 .address_space = .generic,
2250 },
2251 })).toIntern();
2252 const ptr_val = try pt.intern(.{ .ptr = .{2255 const ptr_val = try pt.intern(.{ .ptr = .{
2253 .ty = ptr_ty,2256 .ty = ptr_ty.toIntern(),
2254 .base_addr = .{ .uav = .{2257 .base_addr = .{ .uav = .{
2255 .val = array_val,2258 .val = array_val,
2256 .orig_ty = ptr_ty,2259 .orig_ty = ptr_ty.toIntern(),
2257 } },2260 } },
2258 .byte_offset = 0,2261 .byte_offset = 0,
2259 } });2262 } });
22602263
2261 result.* = new_file;2264 ef.val = ptr_val;
2265 ef.err = null;
2266 ef.stat = stat;
2267}
2268
2269fn newEmbedFile(
2270 pt: Zcu.PerThread,
2271 mod: *Module,
2272 /// The path of the file to embed relative to the root of `mod`.
2273 sub_file_path: []const u8,
2274 /// The resolved path of the file to embed.
2275 resolved_path: []const u8,
2276) !*Zcu.EmbedFile {
2277 const zcu = pt.zcu;
2278 const comp = zcu.comp;
2279 const gpa = zcu.gpa;
2280 const ip = &zcu.intern_pool;
2281
2282 if (comp.file_system_inputs) |fsi|
2283 try comp.appendFileSystemInput(fsi, mod.root, sub_file_path);
2284
2285 const new_file = try gpa.create(Zcu.EmbedFile);
2286 errdefer gpa.destroy(new_file);
2287
2262 new_file.* = .{2288 new_file.* = .{
2289 .owner = mod,
2263 .sub_file_path = try ip.getOrPutString(gpa, pt.tid, sub_file_path, .no_embedded_nulls),2290 .sub_file_path = try ip.getOrPutString(gpa, pt.tid, sub_file_path, .no_embedded_nulls),
2264 .owner = pkg,2291 .val = .none,
2265 .stat = stat,2292 .err = null,
2266 .val = ptr_val,2293 .stat = undefined,
2267 .src_loc = src_loc,
2268 };2294 };
2269 return ptr_val;2295
2296 var opt_ip_str: ?InternPool.String = null;
2297 try pt.updateEmbedFile(new_file, &opt_ip_str);
2298
2299 // Add the file contents to the `whole` cache manifest if necessary.
2300 cache: {
2301 const whole = switch (zcu.comp.cache_use) {
2302 .whole => |whole| whole,
2303 .incremental => break :cache,
2304 };
2305 const man = whole.cache_manifest orelse break :cache;
2306 const ip_str = opt_ip_str orelse break :cache;
2307
2308 const copied_resolved_path = try gpa.dupe(u8, resolved_path);
2309 errdefer gpa.free(copied_resolved_path);
2310
2311 const array_len = Value.fromInterned(new_file.val).typeOf(zcu).childType(zcu).arrayLen(zcu);
2312
2313 whole.cache_manifest_mutex.lock();
2314 defer whole.cache_manifest_mutex.unlock();
2315
2316 man.addFilePostContents(copied_resolved_path, ip_str.toSlice(array_len, ip), new_file.stat) catch |err| switch (err) {
2317 error.Unexpected => unreachable,
2318 else => |e| return e,
2319 };
2320 }
2321
2322 return new_file;
2270}2323}
22712324
2272pub fn scanNamespace(2325pub fn scanNamespace(
test/incremental/add_decl+1-1
...@@ -39,7 +39,7 @@ pub fn main() !void {...@@ -39,7 +39,7 @@ pub fn main() !void {
39}39}
40const foo = "good morning\n";40const foo = "good morning\n";
41const bar = "good evening\n";41const bar = "good evening\n";
42#expect_error=ignored42#expect_error=main.zig:3:37: error: use of undeclared identifier 'qux'
4343
44#update=add missing declaration44#update=add missing declaration
45#file=main.zig45#file=main.zig
test/incremental/add_decl_namespaced+2-1
...@@ -39,7 +39,8 @@ pub fn main() !void {...@@ -39,7 +39,8 @@ pub fn main() !void {
39}39}
40const foo = "good morning\n";40const foo = "good morning\n";
41const bar = "good evening\n";41const bar = "good evening\n";
42#expect_error=ignored42#expect_error=main.zig:3:44: error: root source file struct 'main' has no member named 'qux'
43#expect_error=main.zig:1:1: note: struct declared here
4344
44#update=add missing declaration45#update=add missing declaration
45#file=main.zig46#file=main.zig
test/incremental/change_embed_file created+46
...@@ -0,0 +1,46 @@
1#target=x86_64-linux-selfhosted
2#target=x86_64-linux-cbe
3#target=x86_64-windows-cbe
4#target=wasm32-wasi-selfhosted
5#update=initial version
6#file=main.zig
7const std = @import("std");
8const string = @embedFile("string.txt");
9pub fn main() !void {
10 try std.io.getStdOut().writeAll(string);
11}
12#file=string.txt
13Hello, World!
14#expect_stdout="Hello, World!\n"
15
16#update=change file contents
17#file=string.txt
18Hello again, World!
19#expect_stdout="Hello again, World!\n"
20
21#update=delete file
22#rm_file=string.txt
23#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound
24
25#update=remove reference to file
26#file=main.zig
27const std = @import("std");
28const string = @embedFile("string.txt");
29pub fn main() !void {
30 try std.io.getStdOut().writeAll("a hardcoded string\n");
31}
32#expect_stdout="a hardcoded string\n"
33
34#update=re-introduce reference to file
35#file=main.zig
36const std = @import("std");
37const string = @embedFile("string.txt");
38pub fn main() !void {
39 try std.io.getStdOut().writeAll(string);
40}
41#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound
42
43#update=recreate file
44#file=string.txt
45We're back, World!
46#expect_stdout="We're back, World!\n"
test/incremental/change_enum_tag_type+1-1
...@@ -39,7 +39,7 @@ comptime {...@@ -39,7 +39,7 @@ comptime {
39 std.debug.assert(@TypeOf(@intFromEnum(Foo.e)) == Tag);39 std.debug.assert(@TypeOf(@intFromEnum(Foo.e)) == Tag);
40}40}
41const std = @import("std");41const std = @import("std");
42#expect_error=ignored42#expect_error=main.zig:7:5: error: enumeration value '4' too large for type 'u2'
43#update=increase tag size43#update=increase tag size
44#file=main.zig44#file=main.zig
45const Tag = u3;45const Tag = u3;
test/incremental/compile_error_then_log+5-2
...@@ -4,19 +4,22 @@...@@ -4,19 +4,22 @@
4#target=wasm32-wasi-selfhosted4#target=wasm32-wasi-selfhosted
5#update=initial version with compile error5#update=initial version with compile error
6#file=main.zig6#file=main.zig
7pub fn main() void {}
7comptime {8comptime {
8 @compileError("this is an error");9 @compileError("this is an error");
9}10}
10comptime {11comptime {
11 @compileLog("this is a log");12 @compileLog("this is a log");
12}13}
13#expect_error=ignored14#expect_error=main.zig:3:5: error: this is an error
15
14#update=remove the compile error16#update=remove the compile error
15#file=main.zig17#file=main.zig
18pub fn main() void {}
16comptime {19comptime {
17 //@compileError("this is an error");20 //@compileError("this is an error");
18}21}
19comptime {22comptime {
20 @compileLog("this is a log");23 @compileLog("this is a log");
21}24}
22#expect_error=ignored25#expect_error=main.zig:6:5: error: found compile log statement
test/incremental/delete_comptime_decls+5-2
...@@ -26,7 +26,10 @@ comptime {...@@ -26,7 +26,10 @@ comptime {
26 const slice = array[3..2];26 const slice = array[3..2];
27 _ = slice;27 _ = slice;
28}28}
29#expect_error=ignored29#expect_error=main.zig:5:32: error: end index 6 out of bounds for slice of length 4 +1 (sentinel)
30#expect_error=main.zig:10:28: error: end index 6 out of bounds for array of length 4 +1 (sentinel)
31#expect_error=main.zig:15:28: error: end index 5 out of bounds for array of length 4
32#expect_error=main.zig:20:25: error: start index 3 is larger than end index 2
3033
31#update=delete and modify comptime decls34#update=delete and modify comptime decls
32#file=main.zig35#file=main.zig
...@@ -38,4 +41,4 @@ comptime {...@@ -38,4 +41,4 @@ comptime {
38 const y = x[0..runtime_len];41 const y = x[0..runtime_len];
39 _ = y;42 _ = y;
40}43}
41#expect_error=ignored44#expect_error=main.zig:6:16: error: slice of null pointer
test/incremental/fix_astgen_failure+2-2
...@@ -11,7 +11,7 @@ pub fn main() !void {...@@ -11,7 +11,7 @@ pub fn main() !void {
11pub fn hello() !void {11pub fn hello() !void {
12 try std.io.getStdOut().writeAll("Hello, World!\n");12 try std.io.getStdOut().writeAll("Hello, World!\n");
13}13}
14#expect_error=ignored14#expect_error=foo.zig:2:9: error: use of undeclared identifier 'std'
15#update=fix the error15#update=fix the error
16#file=foo.zig16#file=foo.zig
17const std = @import("std");17const std = @import("std");
...@@ -25,7 +25,7 @@ const std = @import("std");...@@ -25,7 +25,7 @@ const std = @import("std");
25pub fn hello() !void {25pub fn hello() !void {
26 try std.io.getStdOut().writeAll(hello_str);26 try std.io.getStdOut().writeAll(hello_str);
27}27}
28#expect_error=ignored28#expect_error=foo.zig:3:37: error: use of undeclared identifier 'hello_str'
29#update=fix the new error29#update=fix the new error
30#file=foo.zig30#file=foo.zig
31const std = @import("std");31const std = @import("std");
test/incremental/fix_many_errors+20-1
...@@ -24,7 +24,26 @@ export fn f6() void { @compileError("f6"); }...@@ -24,7 +24,26 @@ export fn f6() void { @compileError("f6"); }
24export fn f7() void { @compileError("f7"); }24export fn f7() void { @compileError("f7"); }
25export fn f8() void { @compileError("f8"); }25export fn f8() void { @compileError("f8"); }
26export fn f9() void { @compileError("f9"); }26export fn f9() void { @compileError("f9"); }
27#expect_error=ignored27#expect_error=main.zig:2:12: error: c0
28#expect_error=main.zig:3:12: error: c1
29#expect_error=main.zig:4:12: error: c2
30#expect_error=main.zig:5:12: error: c3
31#expect_error=main.zig:6:12: error: c4
32#expect_error=main.zig:7:12: error: c5
33#expect_error=main.zig:8:12: error: c6
34#expect_error=main.zig:9:12: error: c7
35#expect_error=main.zig:10:12: error: c8
36#expect_error=main.zig:11:12: error: c9
37#expect_error=main.zig:12:23: error: f0
38#expect_error=main.zig:13:23: error: f1
39#expect_error=main.zig:14:23: error: f2
40#expect_error=main.zig:15:23: error: f3
41#expect_error=main.zig:16:23: error: f4
42#expect_error=main.zig:17:23: error: f5
43#expect_error=main.zig:18:23: error: f6
44#expect_error=main.zig:19:23: error: f7
45#expect_error=main.zig:20:23: error: f8
46#expect_error=main.zig:21:23: error: f9
28#update=fix all the errors47#update=fix all the errors
29#file=main.zig48#file=main.zig
30pub fn main() !void {}49pub fn main() !void {}
test/incremental/remove_enum_field+2-1
...@@ -23,4 +23,5 @@ pub fn main() !void {...@@ -23,4 +23,5 @@ pub fn main() !void {
23 try std.io.getStdOut().writer().print("{}\n", .{@intFromEnum(MyEnum.foo)});23 try std.io.getStdOut().writer().print("{}\n", .{@intFromEnum(MyEnum.foo)});
24}24}
25const std = @import("std");25const std = @import("std");
26#expect_error=ignored26#expect_error=main.zig:6:73: error: enum 'main.MyEnum' has no member named 'foo'
27#expect_error=main.zig:1:16: note: enum declared here
test/incremental/remove_invalid_union_backing_enum+2-1
...@@ -15,7 +15,8 @@ pub fn main() void {...@@ -15,7 +15,8 @@ pub fn main() void {
15 const u: U = .{ .a = 123 };15 const u: U = .{ .a = 123 };
16 _ = u;16 _ = u;
17}17}
18#expect_error=ignored18#expect_error=main.zig:6:5: error: no field named 'd' in enum 'main.E'
19#expect_error=main.zig:1:11: note: enum declared here
19#update=remove invalid backing enum20#update=remove invalid backing enum
20#file=main.zig21#file=main.zig
21const U = union {22const U = union {
test/incremental/temporary_parse_error+1-1
...@@ -11,7 +11,7 @@ pub fn main() !void {}...@@ -11,7 +11,7 @@ pub fn main() !void {}
11#update=introduce parse error11#update=introduce parse error
12#file=main.zig12#file=main.zig
13pub fn main() !void {13pub fn main() !void {
14#expect_error=ignored14#expect_error=main.zig:2:1: error: expected statement, found 'EOF'
1515
16#update=fix parse error16#update=fix parse error
17#file=main.zig17#file=main.zig
test/incremental/unreferenced_error+1-1
...@@ -18,7 +18,7 @@ pub fn main() !void {...@@ -18,7 +18,7 @@ pub fn main() !void {
18 try std.io.getStdOut().writeAll(a);18 try std.io.getStdOut().writeAll(a);
19}19}
20const a = @compileError("bad a");20const a = @compileError("bad a");
21#expect_error=ignored21#expect_error=main.zig:5:11: error: bad a
2222
23#update=remove error reference23#update=remove error reference
24#file=main.zig24#file=main.zig
tools/incr-check.zig+138-26
...@@ -340,19 +340,63 @@ const Eval = struct {...@@ -340,19 +340,63 @@ const Eval = struct {
340 }340 }
341341
342 fn checkErrorOutcome(eval: *Eval, update: Case.Update, error_bundle: std.zig.ErrorBundle) !void {342 fn checkErrorOutcome(eval: *Eval, update: Case.Update, error_bundle: std.zig.ErrorBundle) !void {
343 switch (update.outcome) {343 const expected_errors = switch (update.outcome) {
344 .unknown => return,344 .unknown => return,
345 .compile_errors => |expected_errors| {345 .compile_errors => |expected_errors| expected_errors,
346 for (expected_errors) |expected_error| {
347 _ = expected_error;
348 @panic("TODO check if the expected error matches the compile errors");
349 }
350 },
351 .stdout, .exit_code => {346 .stdout, .exit_code => {
352 const color: std.zig.Color = .auto;347 const color: std.zig.Color = .auto;
353 error_bundle.renderToStdErr(color.renderOptions());348 error_bundle.renderToStdErr(color.renderOptions());
354 eval.fatal("update '{s}': unexpected compile errors", .{update.name});349 eval.fatal("update '{s}': unexpected compile errors", .{update.name});
355 },350 },
351 };
352
353 var expected_idx: usize = 0;
354
355 for (error_bundle.getMessages()) |err_idx| {
356 if (expected_idx == expected_errors.len) {
357 const color: std.zig.Color = .auto;
358 error_bundle.renderToStdErr(color.renderOptions());
359 eval.fatal("update '{s}': more errors than expected", .{update.name});
360 }
361 eval.checkOneError(update, error_bundle, expected_errors[expected_idx], false, err_idx);
362 expected_idx += 1;
363
364 for (error_bundle.getNotes(err_idx)) |note_idx| {
365 if (expected_idx == expected_errors.len) {
366 const color: std.zig.Color = .auto;
367 error_bundle.renderToStdErr(color.renderOptions());
368 eval.fatal("update '{s}': more error notes than expected", .{update.name});
369 }
370 eval.checkOneError(update, error_bundle, expected_errors[expected_idx], true, note_idx);
371 expected_idx += 1;
372 }
373 }
374 }
375
376 fn checkOneError(
377 eval: *Eval,
378 update: Case.Update,
379 eb: std.zig.ErrorBundle,
380 expected: Case.ExpectedError,
381 is_note: bool,
382 err_idx: std.zig.ErrorBundle.MessageIndex,
383 ) void {
384 const err = eb.getErrorMessage(err_idx);
385 if (err.src_loc == .none) @panic("TODO error message with no source location");
386 if (err.count != 1) @panic("TODO error message with count>1");
387 const msg = eb.nullTerminatedString(err.msg);
388 const src = eb.getSourceLocation(err.src_loc);
389 const filename = eb.nullTerminatedString(src.src_path);
390
391 if (expected.is_note != is_note or
392 !std.mem.eql(u8, expected.filename, filename) or
393 expected.line != src.line + 1 or
394 expected.column != src.column + 1 or
395 !std.mem.eql(u8, expected.msg, msg))
396 {
397 const color: std.zig.Color = .auto;
398 eb.renderToStdErr(color.renderOptions());
399 eval.fatal("update '{s}': compile error did not match expected error", .{update.name});
356 }400 }
357 }401 }
358402
...@@ -595,11 +639,11 @@ const Case = struct {...@@ -595,11 +639,11 @@ const Case = struct {
595 };639 };
596640
597 const ExpectedError = struct {641 const ExpectedError = struct {
598 file_name: ?[]const u8 = null,642 is_note: bool,
599 line: ?u32 = null,643 filename: []const u8,
600 column: ?u32 = null,644 line: u32,
601 msg_exact: ?[]const u8 = null,645 column: u32,
602 msg_substring: ?[]const u8 = null,646 msg: []const u8,
603 };647 };
604648
605 fn parse(arena: Allocator, bytes: []const u8) !Case {649 fn parse(arena: Allocator, bytes: []const u8) !Case {
...@@ -608,6 +652,7 @@ const Case = struct {...@@ -608,6 +652,7 @@ const Case = struct {
608 var targets: std.ArrayListUnmanaged(Target) = .empty;652 var targets: std.ArrayListUnmanaged(Target) = .empty;
609 var updates: std.ArrayListUnmanaged(Update) = .empty;653 var updates: std.ArrayListUnmanaged(Update) = .empty;
610 var changes: std.ArrayListUnmanaged(FullContents) = .empty;654 var changes: std.ArrayListUnmanaged(FullContents) = .empty;
655 var deletes: std.ArrayListUnmanaged([]const u8) = .empty;
611 var it = std.mem.splitScalar(u8, bytes, '\n');656 var it = std.mem.splitScalar(u8, bytes, '\n');
612 var line_n: usize = 1;657 var line_n: usize = 1;
613 var root_source_file: ?[]const u8 = null;658 var root_source_file: ?[]const u8 = null;
...@@ -647,33 +692,42 @@ const Case = struct {...@@ -647,33 +692,42 @@ const Case = struct {
647 if (updates.items.len > 0) {692 if (updates.items.len > 0) {
648 const last_update = &updates.items[updates.items.len - 1];693 const last_update = &updates.items[updates.items.len - 1];
649 last_update.changes = try changes.toOwnedSlice(arena);694 last_update.changes = try changes.toOwnedSlice(arena);
695 last_update.deletes = try deletes.toOwnedSlice(arena);
650 }696 }
651 try updates.append(arena, .{697 try updates.append(arena, .{
652 .name = val,698 .name = val,
653 .outcome = .unknown,699 .outcome = .unknown,
654 });700 });
655 } else if (std.mem.eql(u8, key, "file")) {701 } else if (std.mem.eql(u8, key, "file")) {
656 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});702 if (updates.items.len == 0) fatal("line {d}: file directive before update", .{line_n});
657703
658 if (root_source_file == null)704 if (root_source_file == null)
659 root_source_file = val;705 root_source_file = val;
660706
661 const start_index = it.index.?;707 // Because Windows is so excellent, we need to convert CRLF to LF, so
662 const src = while (true) : (line_n += 1) {708 // can't just slice into the input here. How delightful!
663 const old = it;709 var src: std.ArrayListUnmanaged(u8) = .empty;
664 const next_line = it.next() orelse fatal("line {d}: unexpected EOF", .{line_n});710
665 if (std.mem.startsWith(u8, next_line, "#")) {711 while (true) {
666 const end_index = old.index.?;712 const next_line_raw = it.peek() orelse fatal("line {d}: unexpected EOF", .{line_n});
667 const src = bytes[start_index..end_index];713 const next_line = std.mem.trimRight(u8, next_line_raw, "\r");
668 it = old;714 if (std.mem.startsWith(u8, next_line, "#")) break;
669 break src;715
670 }716 _ = it.next();
671 };717 line_n += 1;
718
719 try src.ensureUnusedCapacity(arena, next_line.len + 1);
720 src.appendSliceAssumeCapacity(next_line);
721 src.appendAssumeCapacity('\n');
722 }
672723
673 try changes.append(arena, .{724 try changes.append(arena, .{
674 .name = val,725 .name = val,
675 .bytes = src,726 .bytes = src.items,
676 });727 });
728 } else if (std.mem.eql(u8, key, "rm_file")) {
729 if (updates.items.len == 0) fatal("line {d}: rm_file directive before update", .{line_n});
730 try deletes.append(arena, val);
677 } else if (std.mem.eql(u8, key, "expect_stdout")) {731 } else if (std.mem.eql(u8, key, "expect_stdout")) {
678 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});732 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});
679 const last_update = &updates.items[updates.items.len - 1];733 const last_update = &updates.items[updates.items.len - 1];
...@@ -687,7 +741,24 @@ const Case = struct {...@@ -687,7 +741,24 @@ const Case = struct {
687 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});741 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});
688 const last_update = &updates.items[updates.items.len - 1];742 const last_update = &updates.items[updates.items.len - 1];
689 if (last_update.outcome != .unknown) fatal("line {d}: conflicting expect directive", .{line_n});743 if (last_update.outcome != .unknown) fatal("line {d}: conflicting expect directive", .{line_n});
690 last_update.outcome = .{ .compile_errors = &.{} };744
745 var errors: std.ArrayListUnmanaged(ExpectedError) = .empty;
746 try errors.append(arena, parseExpectedError(val, line_n));
747 while (true) {
748 const next_line = it.peek() orelse break;
749 if (!std.mem.startsWith(u8, next_line, "#")) break;
750 var new_line_it = std.mem.splitScalar(u8, next_line, '=');
751 const new_key = new_line_it.first()[1..];
752 const new_val = std.mem.trimRight(u8, new_line_it.rest(), "\r");
753 if (new_val.len == 0) break;
754 if (!std.mem.eql(u8, new_key, "expect_error")) break;
755
756 _ = it.next();
757 line_n += 1;
758 try errors.append(arena, parseExpectedError(new_val, line_n));
759 }
760
761 last_update.outcome = .{ .compile_errors = errors.items };
691 } else {762 } else {
692 fatal("line {d}: unrecognized key '{s}'", .{ line_n, key });763 fatal("line {d}: unrecognized key '{s}'", .{ line_n, key });
693 }764 }
...@@ -701,6 +772,7 @@ const Case = struct {...@@ -701,6 +772,7 @@ const Case = struct {
701 if (changes.items.len > 0) {772 if (changes.items.len > 0) {
702 const last_update = &updates.items[updates.items.len - 1];773 const last_update = &updates.items[updates.items.len - 1];
703 last_update.changes = changes.items; // arena so no need for toOwnedSlice774 last_update.changes = changes.items; // arena so no need for toOwnedSlice
775 last_update.deletes = deletes.items;
704 }776 }
705777
706 return .{778 return .{
...@@ -736,3 +808,43 @@ fn waitChild(child: *std.process.Child, eval: *Eval) void {...@@ -736,3 +808,43 @@ fn waitChild(child: *std.process.Child, eval: *Eval) void {
736 .Signal, .Stopped, .Unknown => eval.fatal("compiler terminated unexpectedly", .{}),808 .Signal, .Stopped, .Unknown => eval.fatal("compiler terminated unexpectedly", .{}),
737 }809 }
738}810}
811
812fn parseExpectedError(str: []const u8, l: usize) Case.ExpectedError {
813 // #expect_error=foo.zig:1:2: error: the error message
814 // #expect_error=foo.zig:1:2: note: and a note
815
816 const fatal = std.process.fatal;
817
818 var it = std.mem.splitScalar(u8, str, ':');
819 const filename = it.first();
820 const line_str = it.next() orelse fatal("line {d}: incomplete error specification", .{l});
821 const column_str = it.next() orelse fatal("line {d}: incomplete error specification", .{l});
822 const error_or_note_str = std.mem.trim(
823 u8,
824 it.next() orelse fatal("line {d}: incomplete error specification", .{l}),
825 " ",
826 );
827 const message = std.mem.trim(u8, it.rest(), " ");
828 if (filename.len == 0) fatal("line {d}: empty filename", .{l});
829 if (message.len == 0) fatal("line {d}: empty error message", .{l});
830 const is_note = if (std.mem.eql(u8, error_or_note_str, "error"))
831 false
832 else if (std.mem.eql(u8, error_or_note_str, "note"))
833 true
834 else
835 fatal("line {d}: expeted 'error' or 'note', found '{s}'", .{ l, error_or_note_str });
836
837 const line = std.fmt.parseInt(u32, line_str, 10) catch
838 fatal("line {d}: invalid line number '{s}'", .{ l, line_str });
839
840 const column = std.fmt.parseInt(u32, column_str, 10) catch
841 fatal("line {d}: invalid column number '{s}'", .{ l, column_str });
842
843 return .{
844 .is_note = is_note,
845 .filename = filename,
846 .line = line,
847 .column = column,
848 .msg = message,
849 };
850}