authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-01-25 04:48:16+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-01-25 06:07:08+00:00
logf47b8de2ad706616b648c52f5036102cb804e65d
tree20246773622e31440107d87455d5a01ac1cd8dcf
parent5202c977d95bf65775e733264b7d37ed940a2997
signaturelock-open Commit is signed but in an unrecognized format.

incremental: handle `@embedFile`

Uses of `@embedFile` register dependencies on the corresponding `Zcu.EmbedFile`. At the start of every update, we iterate all embedded files and update them if necessary, and invalidate the dependencies if they changed. In order to properly integrate with the lazy analysis model, failed embed files are now reported by the `AnalUnit` which actually used `@embedFile`; the filesystem error is stored in the `Zcu.EmbedFile`. An incremental test is added covering incremental updates to embedded files, and I have verified locally that dependency invalidation is working correctly.

7 files changed, 264 insertions(+), 168 deletions(-)

src/Compilation.zig+23-60
...@@ -154,10 +154,6 @@ win32_resource_work_queue: if (dev.env.supports(.win32_resource)) std.fifo.Linea...@@ -154,10 +154,6 @@ win32_resource_work_queue: if (dev.env.supports(.win32_resource)) std.fifo.Linea
154/// since the last compilation, as well as scan for `@import` and queue up154/// since the last compilation, as well as scan for `@import` and queue up
155/// additional jobs corresponding to those new files.155/// additional jobs corresponding to those new files.
156astgen_work_queue: std.fifo.LinearFifo(Zcu.File.Index, .Dynamic),156astgen_work_queue: std.fifo.LinearFifo(Zcu.File.Index, .Dynamic),
157/// These jobs are to inspect the file system stat() and if the embedded file has changed
158/// on disk, mark the corresponding Decl outdated and queue up an `analyze_decl`
159/// task for it.
160embed_file_work_queue: std.fifo.LinearFifo(*Zcu.EmbedFile, .Dynamic),
161157
162/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.158/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
163/// This data is accessed by multiple threads and is protected by `mutex`.159/// This data is accessed by multiple threads and is protected by `mutex`.
...@@ -1465,7 +1461,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -1465,7 +1461,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
1465 .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),1461 .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
1466 .win32_resource_work_queue = if (dev.env.supports(.win32_resource)) std.fifo.LinearFifo(*Win32Resource, .Dynamic).init(gpa) else .{},1462 .win32_resource_work_queue = if (dev.env.supports(.win32_resource)) std.fifo.LinearFifo(*Win32Resource, .Dynamic).init(gpa) else .{},
1467 .astgen_work_queue = std.fifo.LinearFifo(Zcu.File.Index, .Dynamic).init(gpa),1463 .astgen_work_queue = std.fifo.LinearFifo(Zcu.File.Index, .Dynamic).init(gpa),
1468 .embed_file_work_queue = std.fifo.LinearFifo(*Zcu.EmbedFile, .Dynamic).init(gpa),
1469 .c_source_files = options.c_source_files,1464 .c_source_files = options.c_source_files,
1470 .rc_source_files = options.rc_source_files,1465 .rc_source_files = options.rc_source_files,
1471 .cache_parent = cache,1466 .cache_parent = cache,
...@@ -1932,7 +1927,6 @@ pub fn destroy(comp: *Compilation) void {...@@ -1932,7 +1927,6 @@ pub fn destroy(comp: *Compilation) void {
1932 comp.c_object_work_queue.deinit();1927 comp.c_object_work_queue.deinit();
1933 comp.win32_resource_work_queue.deinit();1928 comp.win32_resource_work_queue.deinit();
1934 comp.astgen_work_queue.deinit();1929 comp.astgen_work_queue.deinit();
1935 comp.embed_file_work_queue.deinit();
19361930
1937 comp.windows_libs.deinit(gpa);1931 comp.windows_libs.deinit(gpa);
19381932
...@@ -2247,11 +2241,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2247,11 +2241,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2247 }2241 }
2248 }2242 }
22492243
2250 // Put a work item in for checking if any files used with `@embedFile` changed.
2251 try comp.embed_file_work_queue.ensureUnusedCapacity(zcu.embed_table.count());
2252 for (zcu.embed_table.values()) |embed_file| {
2253 comp.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
2254 }
2255 if (comp.file_system_inputs) |fsi| {2244 if (comp.file_system_inputs) |fsi| {
2256 const ip = &zcu.intern_pool;2245 const ip = &zcu.intern_pool;
2257 for (zcu.embed_table.values()) |embed_file| {2246 for (zcu.embed_table.values()) |embed_file| {
...@@ -3235,9 +3224,6 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle {...@@ -3235,9 +3224,6 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle {
3235 try addZirErrorMessages(&bundle, file);3224 try addZirErrorMessages(&bundle, file);
3236 }3225 }
3237 }3226 }
3238 for (zcu.failed_embed_files.values()) |error_msg| {
3239 try addModuleErrorMsg(zcu, &bundle, error_msg.*);
3240 }
3241 var sorted_failed_analysis: std.AutoArrayHashMapUnmanaged(InternPool.AnalUnit, *Zcu.ErrorMsg).DataList.Slice = s: {3227 var sorted_failed_analysis: std.AutoArrayHashMapUnmanaged(InternPool.AnalUnit, *Zcu.ErrorMsg).DataList.Slice = s: {
3242 const SortOrder = struct {3228 const SortOrder = struct {
3243 zcu: *Zcu,3229 zcu: *Zcu,
...@@ -3812,9 +3798,10 @@ fn performAllTheWorkInner(...@@ -3812,9 +3798,10 @@ fn performAllTheWorkInner(
3812 }3798 }
3813 }3799 }
38143800
3815 while (comp.embed_file_work_queue.readItem()) |embed_file| {3801 for (0.., zcu.embed_table.values()) |ef_index_usize, ef| {
3816 comp.thread_pool.spawnWg(&astgen_wait_group, workerCheckEmbedFile, .{3802 const ef_index: Zcu.EmbedFile.Index = @enumFromInt(ef_index_usize);
3817 comp, embed_file,3803 comp.thread_pool.spawnWgId(&astgen_wait_group, workerCheckEmbedFile, .{
3804 comp, ef_index, ef,
3818 });3805 });
3819 }3806 }
3820 }3807 }
...@@ -4377,33 +4364,33 @@ fn workerUpdateBuiltinZigFile(...@@ -4377,33 +4364,33 @@ fn workerUpdateBuiltinZigFile(
4377 };4364 };
4378}4365}
43794366
4380fn workerCheckEmbedFile(comp: *Compilation, embed_file: *Zcu.EmbedFile) void {4367fn workerCheckEmbedFile(tid: usize, comp: *Compilation, ef_index: Zcu.EmbedFile.Index, ef: *Zcu.EmbedFile) void {
4381 comp.detectEmbedFileUpdate(embed_file) catch |err| {4368 comp.detectEmbedFileUpdate(@enumFromInt(tid), ef_index, ef) catch |err| switch (err) {
4382 comp.reportRetryableEmbedFileError(embed_file, err) catch |oom| switch (oom) {4369 error.OutOfMemory => {
4383 // Swallowing this error is OK because it's implied to be OOM when4370 comp.mutex.lock();
4384 // there is a missing `failed_embed_files` error message.4371 defer comp.mutex.unlock();
4385 error.OutOfMemory => {},4372 comp.setAllocFailure();
4386 };4373 },
4387 return;
4388 };4374 };
4389}4375}
43904376
4391fn detectEmbedFileUpdate(comp: *Compilation, embed_file: *Zcu.EmbedFile) !void {4377fn detectEmbedFileUpdate(comp: *Compilation, tid: Zcu.PerThread.Id, ef_index: Zcu.EmbedFile.Index, ef: *Zcu.EmbedFile) !void {
4392 const zcu = comp.zcu.?;4378 const zcu = comp.zcu.?;
4393 const ip = &zcu.intern_pool;4379 const pt: Zcu.PerThread = .activate(zcu, tid);
4394 var file = try embed_file.owner.root.openFile(embed_file.sub_file_path.toSlice(ip), .{});4380 defer pt.deactivate();
4395 defer file.close();4381
4382 const old_val = ef.val;
4383 const old_err = ef.err;
43964384
4397 const stat = try file.stat();4385 try pt.updateEmbedFile(ef, null);
43984386
4399 const unchanged_metadata =4387 if (ef.val != .none and ef.val == old_val) return; // success, value unchanged
4400 stat.size == embed_file.stat.size and4388 if (ef.val == .none and old_val == .none and ef.err == old_err) return; // failure, error unchanged
4401 stat.mtime == embed_file.stat.mtime and
4402 stat.inode == embed_file.stat.inode;
44034389
4404 if (unchanged_metadata) return;4390 comp.mutex.lock();
4391 defer comp.mutex.unlock();
44054392
4406 @panic("TODO: handle embed file incremental update");4393 try zcu.markDependeeOutdated(.not_marked_po, .{ .embed_file = ef_index });
4407}4394}
44084395
4409pub fn obtainCObjectCacheManifest(4396pub fn obtainCObjectCacheManifest(
...@@ -4802,30 +4789,6 @@ fn reportRetryableWin32ResourceError(...@@ -4802,30 +4789,6 @@ fn reportRetryableWin32ResourceError(
4802 }4789 }
4803}4790}
48044791
4805fn reportRetryableEmbedFileError(
4806 comp: *Compilation,
4807 embed_file: *Zcu.EmbedFile,
4808 err: anyerror,
4809) error{OutOfMemory}!void {
4810 const zcu = comp.zcu.?;
4811 const gpa = zcu.gpa;
4812 const src_loc = embed_file.src_loc;
4813 const ip = &zcu.intern_pool;
4814 const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}/{s}': {s}", .{
4815 embed_file.owner.root,
4816 embed_file.sub_file_path.toSlice(ip),
4817 @errorName(err),
4818 });
4819
4820 errdefer err_msg.destroy(gpa);
4821
4822 {
4823 comp.mutex.lock();
4824 defer comp.mutex.unlock();
4825 try zcu.failed_embed_files.putNoClobber(gpa, embed_file, err_msg);
4826 }
4827}
4828
4829fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Progress.Node) !void {4792fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Progress.Node) !void {
4830 if (comp.config.c_frontend == .aro) {4793 if (comp.config.c_frontend == .aro) {
4831 return comp.failCObj(c_object, "aro does not support compiling C objects yet", .{});4794 return comp.failCObj(c_object, "aro does not support compiling C objects yet", .{});
src/InternPool.zig+9
...@@ -42,6 +42,10 @@ nav_ty_deps: std.AutoArrayHashMapUnmanaged(Nav.Index, DepEntry.Index),...@@ -42,6 +42,10 @@ nav_ty_deps: std.AutoArrayHashMapUnmanaged(Nav.Index, DepEntry.Index),
42/// * a container type requiring resolution (invalidated when the type must be recreated at a new index)42/// * a container type requiring resolution (invalidated when the type must be recreated at a new index)
43/// Value is index into `dep_entries` of the first dependency on this interned value.43/// Value is index into `dep_entries` of the first dependency on this interned value.
44interned_deps: std.AutoArrayHashMapUnmanaged(Index, DepEntry.Index),44interned_deps: std.AutoArrayHashMapUnmanaged(Index, DepEntry.Index),
45/// Dependencies on an embedded file.
46/// Introduced by `@embedFile`; invalidated when the file changes.
47/// Value is index into `dep_entries` of the first dependency on this `Zcu.EmbedFile`.
48embed_file_deps: std.AutoArrayHashMapUnmanaged(Zcu.EmbedFile.Index, DepEntry.Index),
45/// Dependencies on the full set of names in a ZIR namespace.49/// Dependencies on the full set of names in a ZIR namespace.
46/// Key refers to a `struct_decl`, `union_decl`, etc.50/// Key refers to a `struct_decl`, `union_decl`, etc.
47/// Value is index into `dep_entries` of the first dependency on this namespace.51/// Value is index into `dep_entries` of the first dependency on this namespace.
...@@ -90,6 +94,7 @@ pub const empty: InternPool = .{...@@ -90,6 +94,7 @@ pub const empty: InternPool = .{
90 .nav_val_deps = .empty,94 .nav_val_deps = .empty,
91 .nav_ty_deps = .empty,95 .nav_ty_deps = .empty,
92 .interned_deps = .empty,96 .interned_deps = .empty,
97 .embed_file_deps = .empty,
93 .namespace_deps = .empty,98 .namespace_deps = .empty,
94 .namespace_name_deps = .empty,99 .namespace_name_deps = .empty,
95 .memoized_state_main_deps = .none,100 .memoized_state_main_deps = .none,
...@@ -824,6 +829,7 @@ pub const Dependee = union(enum) {...@@ -824,6 +829,7 @@ pub const Dependee = union(enum) {
824 nav_val: Nav.Index,829 nav_val: Nav.Index,
825 nav_ty: Nav.Index,830 nav_ty: Nav.Index,
826 interned: Index,831 interned: Index,
832 embed_file: Zcu.EmbedFile.Index,
827 namespace: TrackedInst.Index,833 namespace: TrackedInst.Index,
828 namespace_name: NamespaceNameKey,834 namespace_name: NamespaceNameKey,
829 memoized_state: MemoizedStateStage,835 memoized_state: MemoizedStateStage,
...@@ -875,6 +881,7 @@ pub fn dependencyIterator(ip: *const InternPool, dependee: Dependee) DependencyI...@@ -875,6 +881,7 @@ pub fn dependencyIterator(ip: *const InternPool, dependee: Dependee) DependencyI
875 .nav_val => |x| ip.nav_val_deps.get(x),881 .nav_val => |x| ip.nav_val_deps.get(x),
876 .nav_ty => |x| ip.nav_ty_deps.get(x),882 .nav_ty => |x| ip.nav_ty_deps.get(x),
877 .interned => |x| ip.interned_deps.get(x),883 .interned => |x| ip.interned_deps.get(x),
884 .embed_file => |x| ip.embed_file_deps.get(x),
878 .namespace => |x| ip.namespace_deps.get(x),885 .namespace => |x| ip.namespace_deps.get(x),
879 .namespace_name => |x| ip.namespace_name_deps.get(x),886 .namespace_name => |x| ip.namespace_name_deps.get(x),
880 .memoized_state => |stage| switch (stage) {887 .memoized_state => |stage| switch (stage) {
...@@ -945,6 +952,7 @@ pub fn addDependency(ip: *InternPool, gpa: Allocator, depender: AnalUnit, depend...@@ -945,6 +952,7 @@ pub fn addDependency(ip: *InternPool, gpa: Allocator, depender: AnalUnit, depend
945 .nav_val => ip.nav_val_deps,952 .nav_val => ip.nav_val_deps,
946 .nav_ty => ip.nav_ty_deps,953 .nav_ty => ip.nav_ty_deps,
947 .interned => ip.interned_deps,954 .interned => ip.interned_deps,
955 .embed_file => ip.embed_file_deps,
948 .namespace => ip.namespace_deps,956 .namespace => ip.namespace_deps,
949 .namespace_name => ip.namespace_name_deps,957 .namespace_name => ip.namespace_name_deps,
950 .memoized_state => comptime unreachable,958 .memoized_state => comptime unreachable,
...@@ -6612,6 +6620,7 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {...@@ -6612,6 +6620,7 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {
6612 ip.nav_val_deps.deinit(gpa);6620 ip.nav_val_deps.deinit(gpa);
6613 ip.nav_ty_deps.deinit(gpa);6621 ip.nav_ty_deps.deinit(gpa);
6614 ip.interned_deps.deinit(gpa);6622 ip.interned_deps.deinit(gpa);
6623 ip.embed_file_deps.deinit(gpa);
6615 ip.namespace_deps.deinit(gpa);6624 ip.namespace_deps.deinit(gpa);
6616 ip.namespace_name_deps.deinit(gpa);6625 ip.namespace_name_deps.deinit(gpa);
66176626
src/Sema.zig+14-6
...@@ -13949,6 +13949,8 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -13949,6 +13949,8 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
13949 defer tracy.end();13949 defer tracy.end();
1395013950
13951 const pt = sema.pt;13951 const pt = sema.pt;
13952 const zcu = pt.zcu;
13953
13952 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;13954 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;
13953 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);13955 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);
13954 const name = try sema.resolveConstString(block, operand_src, inst_data.operand, .{ .simple = .operand_embedFile });13956 const name = try sema.resolveConstString(block, operand_src, inst_data.operand, .{ .simple = .operand_embedFile });
...@@ -13957,18 +13959,24 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -13957,18 +13959,24 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
13957 return sema.fail(block, operand_src, "file path name cannot be empty", .{});13959 return sema.fail(block, operand_src, "file path name cannot be empty", .{});
13958 }13960 }
1395913961
13960 const val = pt.embedFile(block.getFileScope(pt.zcu), name, operand_src) catch |err| switch (err) {13962 const ef_idx = pt.embedFile(block.getFileScope(zcu), name) catch |err| switch (err) {
13961 error.ImportOutsideModulePath => {13963 error.ImportOutsideModulePath => {
13962 return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});13964 return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});
13963 },13965 },
13964 else => {13966 error.CurrentWorkingDirectoryUnlinked => {
13965 // TODO: these errors are file system errors; make sure an update() will13967 // TODO: this should be some kind of retryable failure, in case the cwd is put back
13966 // retry this and not cache the file system error, which may be transient.13968 return sema.fail(block, operand_src, "unable to resolve '{s}': working directory has been unlinked", .{name});
13967 return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(err) });
13968 },13969 },
13970 error.OutOfMemory => |e| return e,
13969 };13971 };
13972 try sema.declareDependency(.{ .embed_file = ef_idx });
13973
13974 const result = ef_idx.get(zcu);
13975 if (result.val == .none) {
13976 return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(result.err.?) });
13977 }
1397013978
13971 return Air.internedToRef(val);13979 return Air.internedToRef(result.val);
13972}13980}
1397313981
13974fn zirRetErrValueCode(sema: *Sema, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {13982fn zirRetErrValueCode(sema: *Sema, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
src/Zcu.zig+22-11
...@@ -143,8 +143,6 @@ compile_log_sources: std.AutoArrayHashMapUnmanaged(AnalUnit, extern struct {...@@ -143,8 +143,6 @@ compile_log_sources: std.AutoArrayHashMapUnmanaged(AnalUnit, extern struct {
143/// Using a map here for consistency with the other fields here.143/// Using a map here for consistency with the other fields here.
144/// The ErrorMsg memory is owned by the `File`, using Module's general purpose allocator.144/// The ErrorMsg memory is owned by the `File`, using Module's general purpose allocator.
145failed_files: std.AutoArrayHashMapUnmanaged(*File, ?*ErrorMsg) = .empty,145failed_files: std.AutoArrayHashMapUnmanaged(*File, ?*ErrorMsg) = .empty,
146/// The ErrorMsg memory is owned by the `EmbedFile`, using Module's general purpose allocator.
147failed_embed_files: std.AutoArrayHashMapUnmanaged(*EmbedFile, *ErrorMsg) = .empty,
148failed_exports: std.AutoArrayHashMapUnmanaged(Export.Index, *ErrorMsg) = .empty,146failed_exports: std.AutoArrayHashMapUnmanaged(Export.Index, *ErrorMsg) = .empty,
149/// If analysis failed due to a cimport error, the corresponding Clang errors147/// If analysis failed due to a cimport error, the corresponding Clang errors
150/// are stored here.148/// are stored here.
...@@ -893,13 +891,23 @@ pub const File = struct {...@@ -893,13 +891,23 @@ pub const File = struct {
893};891};
894892
895pub const EmbedFile = struct {893pub const EmbedFile = struct {
896 /// Relative to the owning module's root directory.
897 sub_file_path: InternPool.NullTerminatedString,
898 /// Module that this file is a part of, managed externally.894 /// Module that this file is a part of, managed externally.
899 owner: *Package.Module,895 owner: *Package.Module,
900 stat: Cache.File.Stat,896 /// Relative to the owning module's root directory.
897 sub_file_path: InternPool.NullTerminatedString,
898
899 /// `.none` means the file was not loaded, so `stat` is undefined.
901 val: InternPool.Index,900 val: InternPool.Index,
902 src_loc: LazySrcLoc,901 /// If this is `null` and `val` is `.none`, the file has never been loaded.
902 err: ?(std.fs.File.OpenError || std.fs.File.StatError || std.fs.File.ReadError || error{UnexpectedEof}),
903 stat: Cache.File.Stat,
904
905 pub const Index = enum(u32) {
906 _,
907 pub fn get(idx: Index, zcu: *const Zcu) *EmbedFile {
908 return zcu.embed_table.values()[@intFromEnum(idx)];
909 }
910 };
903};911};
904912
905/// This struct holds data necessary to construct API-facing `AllErrors.Message`.913/// This struct holds data necessary to construct API-facing `AllErrors.Message`.
...@@ -2459,11 +2467,6 @@ pub fn deinit(zcu: *Zcu) void {...@@ -2459,11 +2467,6 @@ pub fn deinit(zcu: *Zcu) void {
2459 }2467 }
2460 zcu.failed_files.deinit(gpa);2468 zcu.failed_files.deinit(gpa);
24612469
2462 for (zcu.failed_embed_files.values()) |msg| {
2463 msg.destroy(gpa);
2464 }
2465 zcu.failed_embed_files.deinit(gpa);
2466
2467 for (zcu.failed_exports.values()) |value| {2470 for (zcu.failed_exports.values()) |value| {
2468 value.destroy(gpa);2471 value.destroy(gpa);
2469 }2472 }
...@@ -3882,6 +3885,14 @@ fn formatDependee(data: struct { dependee: InternPool.Dependee, zcu: *Zcu }, com...@@ -3882,6 +3885,14 @@ fn formatDependee(data: struct { dependee: InternPool.Dependee, zcu: *Zcu }, com
3882 .func => |f| return writer.print("ies('{}')", .{ip.getNav(f.owner_nav).fqn.fmt(ip)}),3885 .func => |f| return writer.print("ies('{}')", .{ip.getNav(f.owner_nav).fqn.fmt(ip)}),
3883 else => unreachable,3886 else => unreachable,
3884 },3887 },
3888 .embed_file => |ef_idx| {
3889 const ef = ef_idx.get(zcu);
3890 return writer.print("embed_file('{s}')", .{std.fs.path.fmtJoin(&.{
3891 ef.owner.root.root_dir.path orelse "",
3892 ef.owner.root.sub_path,
3893 ef.sub_file_path.toSlice(ip),
3894 })});
3895 },
3885 .namespace => |ti| {3896 .namespace => |ti| {
3886 const info = ti.resolveFull(ip) orelse {3897 const info = ti.resolveFull(ip) orelse {
3887 return writer.writeAll("namespace(<lost>)");3898 return writer.writeAll("namespace(<lost>)");
src/Zcu/PerThread.zig+143-90
...@@ -2117,32 +2117,32 @@ pub fn embedFile(...@@ -2117,32 +2117,32 @@ pub fn embedFile(
2117 pt: Zcu.PerThread,2117 pt: Zcu.PerThread,
2118 cur_file: *Zcu.File,2118 cur_file: *Zcu.File,
2119 import_string: []const u8,2119 import_string: []const u8,
2120 src_loc: Zcu.LazySrcLoc,2120) error{
2121) !InternPool.Index {2121 OutOfMemory,
2122 ImportOutsideModulePath,
2123 CurrentWorkingDirectoryUnlinked,
2124}!Zcu.EmbedFile.Index {
2122 const zcu = pt.zcu;2125 const zcu = pt.zcu;
2123 const gpa = zcu.gpa;2126 const gpa = zcu.gpa;
21242127
2125 if (cur_file.mod.deps.get(import_string)) |pkg| {2128 if (cur_file.mod.deps.get(import_string)) |mod| {
2126 const resolved_path = try std.fs.path.resolve(gpa, &.{2129 const resolved_path = try std.fs.path.resolve(gpa, &.{
2127 pkg.root.root_dir.path orelse ".",2130 mod.root.root_dir.path orelse ".",
2128 pkg.root.sub_path,2131 mod.root.sub_path,
2129 pkg.root_src_path,2132 mod.root_src_path,
2130 });2133 });
2131 var keep_resolved_path = false;2134 errdefer gpa.free(resolved_path);
2132 defer if (!keep_resolved_path) gpa.free(resolved_path);
21332135
2134 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);2136 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);
2135 errdefer {2137 errdefer assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2136 assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2137 keep_resolved_path = false;
2138 }
2139 if (gop.found_existing) return gop.value_ptr.*.val;
2140 keep_resolved_path = true;
21412138
2142 const sub_file_path = try gpa.dupe(u8, pkg.root_src_path);2139 if (gop.found_existing) {
2143 errdefer gpa.free(sub_file_path);2140 gpa.free(resolved_path); // we're not using this key
2141 return @enumFromInt(gop.index);
2142 }
21442143
2145 return pt.newEmbedFile(pkg, sub_file_path, resolved_path, gop.value_ptr, src_loc);2144 gop.value_ptr.* = try pt.newEmbedFile(mod, mod.root_src_path, resolved_path);
2145 return @enumFromInt(gop.index);
2146 }2146 }
21472147
2148 // The resolved path is used as the key in the table, to detect if a file2148 // The resolved path is used as the key in the table, to detect if a file
...@@ -2154,17 +2154,15 @@ pub fn embedFile(...@@ -2154,17 +2154,15 @@ pub fn embedFile(
2154 "..",2154 "..",
2155 import_string,2155 import_string,
2156 });2156 });
21572157 errdefer gpa.free(resolved_path);
2158 var keep_resolved_path = false;
2159 defer if (!keep_resolved_path) gpa.free(resolved_path);
21602158
2161 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);2159 const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);
2162 errdefer {2160 errdefer assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2163 assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));2161
2164 keep_resolved_path = false;2162 if (gop.found_existing) {
2163 gpa.free(resolved_path); // we're not using this key
2164 return @enumFromInt(gop.index);
2165 }2165 }
2166 if (gop.found_existing) return gop.value_ptr.*.val;
2167 keep_resolved_path = true;
21682166
2169 const resolved_root_path = try std.fs.path.resolve(gpa, &.{2167 const resolved_root_path = try std.fs.path.resolve(gpa, &.{
2170 cur_file.mod.root.root_dir.path orelse ".",2168 cur_file.mod.root.root_dir.path orelse ".",
...@@ -2172,101 +2170,156 @@ pub fn embedFile(...@@ -2172,101 +2170,156 @@ pub fn embedFile(
2172 });2170 });
2173 defer gpa.free(resolved_root_path);2171 defer gpa.free(resolved_root_path);
21742172
2175 const sub_file_path = p: {2173 const sub_file_path = std.fs.path.relative(gpa, resolved_root_path, resolved_path) catch |err| switch (err) {
2176 const relative = try std.fs.path.relative(gpa, resolved_root_path, resolved_path);2174 error.Unexpected => unreachable,
2177 errdefer gpa.free(relative);2175 else => |e| return e,
2178
2179 if (!isUpDir(relative) and !std.fs.path.isAbsolute(relative)) {
2180 break :p relative;
2181 }
2182 return error.ImportOutsideModulePath;
2183 };2176 };
2184 defer gpa.free(sub_file_path);2177 defer gpa.free(sub_file_path);
21852178
2186 return pt.newEmbedFile(cur_file.mod, sub_file_path, resolved_path, gop.value_ptr, src_loc);2179 if (isUpDir(sub_file_path) or std.fs.path.isAbsolute(sub_file_path)) {
2180 return error.ImportOutsideModulePath;
2181 }
2182
2183 gop.value_ptr.* = try pt.newEmbedFile(cur_file.mod, sub_file_path, resolved_path);
2184 return @enumFromInt(gop.index);
2187}2185}
21882186
2189/// https://github.com/ziglang/zig/issues/143072187pub fn updateEmbedFile(
2190fn newEmbedFile(
2191 pt: Zcu.PerThread,2188 pt: Zcu.PerThread,
2192 pkg: *Module,2189 ef: *Zcu.EmbedFile,
2193 sub_file_path: []const u8,2190 /// If not `null`, the interned file data is stored here, if it was loaded.
2194 resolved_path: []const u8,2191 /// `newEmbedFile` uses this to add the file to the `whole` cache manifest.
2195 result: **Zcu.EmbedFile,2192 ip_str_out: ?*?InternPool.String,
2196 src_loc: Zcu.LazySrcLoc,2193) Allocator.Error!void {
2197) !InternPool.Index {2194 pt.updateEmbedFileInner(ef, ip_str_out) catch |err| switch (err) {
2195 error.OutOfMemory => |e| return e,
2196 else => |e| {
2197 ef.val = .none;
2198 ef.err = e;
2199 ef.stat = undefined;
2200 },
2201 };
2202}
2203
2204fn updateEmbedFileInner(
2205 pt: Zcu.PerThread,
2206 ef: *Zcu.EmbedFile,
2207 ip_str_out: ?*?InternPool.String,
2208) !void {
2209 const tid = pt.tid;
2198 const zcu = pt.zcu;2210 const zcu = pt.zcu;
2199 const gpa = zcu.gpa;2211 const gpa = zcu.gpa;
2200 const ip = &zcu.intern_pool;2212 const ip = &zcu.intern_pool;
22012213
2202 const new_file = try gpa.create(Zcu.EmbedFile);2214 var file = try ef.owner.root.openFile(ef.sub_file_path.toSlice(ip), .{});
2203 errdefer gpa.destroy(new_file);
2204
2205 var file = try pkg.root.openFile(sub_file_path, .{});
2206 defer file.close();2215 defer file.close();
22072216
2208 const actual_stat = try file.stat();2217 const stat: Cache.File.Stat = .fromFs(try file.stat());
2209 const stat: Cache.File.Stat = .{
2210 .size = actual_stat.size,
2211 .inode = actual_stat.inode,
2212 .mtime = actual_stat.mtime,
2213 };
2214 const size = std.math.cast(usize, actual_stat.size) orelse return error.Overflow;
2215
2216 const strings = ip.getLocal(pt.tid).getMutableStrings(gpa);
2217 const bytes = try strings.addManyAsSlice(try std.math.add(usize, size, 1));
2218 const actual_read = try file.readAll(bytes[0][0..size]);
2219 if (actual_read != size) return error.UnexpectedEndOfFile;
2220 bytes[0][size] = 0;
22212218
2222 const comp = zcu.comp;2219 if (ef.val != .none) {
2223 switch (comp.cache_use) {2220 const old_stat = ef.stat;
2224 .whole => |whole| if (whole.cache_manifest) |man| {2221 const unchanged_metadata =
2225 const copied_resolved_path = try gpa.dupe(u8, resolved_path);2222 stat.size == old_stat.size and
2226 errdefer gpa.free(copied_resolved_path);2223 stat.mtime == old_stat.mtime and
2227 whole.cache_manifest_mutex.lock();2224 stat.inode == old_stat.inode;
2228 defer whole.cache_manifest_mutex.unlock();2225 if (unchanged_metadata) return;
2229 try man.addFilePostContents(copied_resolved_path, bytes[0][0..size], stat);
2230 },
2231 .incremental => {},
2232 }2226 }
22332227
2234 const array_ty = try pt.intern(.{ .array_type = .{2228 const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig;
2229 const size_plus_one = std.math.add(usize, size, 1) catch return error.FileTooBig;
2230
2231 // The loaded bytes of the file, including a sentinel 0 byte.
2232 const ip_str: InternPool.String = str: {
2233 const strings = ip.getLocal(tid).getMutableStrings(gpa);
2234 const old_len = strings.mutate.len;
2235 errdefer strings.shrinkRetainingCapacity(old_len);
2236 const bytes = (try strings.addManyAsSlice(size_plus_one))[0];
2237 const actual_read = try file.readAll(bytes[0..size]);
2238 if (actual_read != size) return error.UnexpectedEof;
2239 bytes[size] = 0;
2240 break :str try ip.getOrPutTrailingString(gpa, tid, @intCast(bytes.len), .maybe_embedded_nulls);
2241 };
2242 if (ip_str_out) |p| p.* = ip_str;
2243
2244 const array_ty = try pt.arrayType(.{
2235 .len = size,2245 .len = size,
2236 .sentinel = .zero_u8,2246 .sentinel = .zero_u8,
2237 .child = .u8_type,2247 .child = .u8_type,
2238 } });2248 });
2249 const ptr_ty = try pt.singleConstPtrType(array_ty);
2250
2239 const array_val = try pt.intern(.{ .aggregate = .{2251 const array_val = try pt.intern(.{ .aggregate = .{
2240 .ty = array_ty,2252 .ty = array_ty.toIntern(),
2241 .storage = .{ .bytes = try ip.getOrPutTrailingString(gpa, pt.tid, @intCast(bytes[0].len), .maybe_embedded_nulls) },2253 .storage = .{ .bytes = ip_str },
2242 } });2254 } });
2243
2244 const ptr_ty = (try pt.ptrType(.{
2245 .child = array_ty,
2246 .flags = .{
2247 .alignment = .none,
2248 .is_const = true,
2249 .address_space = .generic,
2250 },
2251 })).toIntern();
2252 const ptr_val = try pt.intern(.{ .ptr = .{2255 const ptr_val = try pt.intern(.{ .ptr = .{
2253 .ty = ptr_ty,2256 .ty = ptr_ty.toIntern(),
2254 .base_addr = .{ .uav = .{2257 .base_addr = .{ .uav = .{
2255 .val = array_val,2258 .val = array_val,
2256 .orig_ty = ptr_ty,2259 .orig_ty = ptr_ty.toIntern(),
2257 } },2260 } },
2258 .byte_offset = 0,2261 .byte_offset = 0,
2259 } });2262 } });
22602263
2261 result.* = new_file;2264 ef.val = ptr_val;
2265 ef.err = null;
2266 ef.stat = stat;
2267}
2268
2269fn newEmbedFile(
2270 pt: Zcu.PerThread,
2271 mod: *Module,
2272 /// The path of the file to embed relative to the root of `mod`.
2273 sub_file_path: []const u8,
2274 /// The resolved path of the file to embed.
2275 resolved_path: []const u8,
2276) !*Zcu.EmbedFile {
2277 const zcu = pt.zcu;
2278 const comp = zcu.comp;
2279 const gpa = zcu.gpa;
2280 const ip = &zcu.intern_pool;
2281
2282 if (comp.file_system_inputs) |fsi|
2283 try comp.appendFileSystemInput(fsi, mod.root, sub_file_path);
2284
2285 const new_file = try gpa.create(Zcu.EmbedFile);
2286 errdefer gpa.destroy(new_file);
2287
2262 new_file.* = .{2288 new_file.* = .{
2289 .owner = mod,
2263 .sub_file_path = try ip.getOrPutString(gpa, pt.tid, sub_file_path, .no_embedded_nulls),2290 .sub_file_path = try ip.getOrPutString(gpa, pt.tid, sub_file_path, .no_embedded_nulls),
2264 .owner = pkg,2291 .val = .none,
2265 .stat = stat,2292 .err = null,
2266 .val = ptr_val,2293 .stat = undefined,
2267 .src_loc = src_loc,
2268 };2294 };
2269 return ptr_val;2295
2296 var opt_ip_str: ?InternPool.String = null;
2297 try pt.updateEmbedFile(new_file, &opt_ip_str);
2298
2299 // Add the file contents to the `whole` cache manifest if necessary.
2300 cache: {
2301 const whole = switch (zcu.comp.cache_use) {
2302 .whole => |whole| whole,
2303 .incremental => break :cache,
2304 };
2305 const man = whole.cache_manifest orelse break :cache;
2306 const ip_str = opt_ip_str orelse break :cache;
2307
2308 const copied_resolved_path = try gpa.dupe(u8, resolved_path);
2309 errdefer gpa.free(copied_resolved_path);
2310
2311 const array_len = Value.fromInterned(new_file.val).typeOf(zcu).childType(zcu).arrayLen(zcu);
2312
2313 whole.cache_manifest_mutex.lock();
2314 defer whole.cache_manifest_mutex.unlock();
2315
2316 man.addFilePostContents(copied_resolved_path, ip_str.toSlice(array_len, ip), new_file.stat) catch |err| switch (err) {
2317 error.Unexpected => unreachable,
2318 else => |e| return e,
2319 };
2320 }
2321
2322 return new_file;
2270}2323}
22712324
2272pub fn scanNamespace(2325pub fn scanNamespace(
test/incremental/change_embed_file created+46
...@@ -0,0 +1,46 @@
1#target=x86_64-linux-selfhosted
2#target=x86_64-linux-cbe
3#target=x86_64-windows-cbe
4#target=wasm32-wasi-selfhosted
5#update=initial version
6#file=main.zig
7const std = @import("std");
8const string = @embedFile("string.txt");
9pub fn main() !void {
10 try std.io.getStdOut().writeAll(string);
11}
12#file=string.txt
13Hello, World!
14#expect_stdout="Hello, World!\n"
15
16#update=change file contents
17#file=string.txt
18Hello again, World!
19#expect_stdout="Hello again, World!\n"
20
21#update=delete file
22#rm_file=string.txt
23#expect_error=ignored
24
25#update=remove reference to file
26#file=main.zig
27const std = @import("std");
28const string = @embedFile("string.txt");
29pub fn main() !void {
30 try std.io.getStdOut().writeAll("a hardcoded string\n");
31}
32#expect_stdout="a hardcoded string\n"
33
34#update=re-introduce reference to file
35#file=main.zig
36const std = @import("std");
37const string = @embedFile("string.txt");
38pub fn main() !void {
39 try std.io.getStdOut().writeAll(string);
40}
41#expect_error=ignore
42
43#update=recreate file
44#file=string.txt
45We're back, World!
46#expect_stdout="We're back, World!\n"
tools/incr-check.zig+7-1
...@@ -608,6 +608,7 @@ const Case = struct {...@@ -608,6 +608,7 @@ const Case = struct {
608 var targets: std.ArrayListUnmanaged(Target) = .empty;608 var targets: std.ArrayListUnmanaged(Target) = .empty;
609 var updates: std.ArrayListUnmanaged(Update) = .empty;609 var updates: std.ArrayListUnmanaged(Update) = .empty;
610 var changes: std.ArrayListUnmanaged(FullContents) = .empty;610 var changes: std.ArrayListUnmanaged(FullContents) = .empty;
611 var deletes: std.ArrayListUnmanaged([]const u8) = .empty;
611 var it = std.mem.splitScalar(u8, bytes, '\n');612 var it = std.mem.splitScalar(u8, bytes, '\n');
612 var line_n: usize = 1;613 var line_n: usize = 1;
613 var root_source_file: ?[]const u8 = null;614 var root_source_file: ?[]const u8 = null;
...@@ -647,13 +648,14 @@ const Case = struct {...@@ -647,13 +648,14 @@ const Case = struct {
647 if (updates.items.len > 0) {648 if (updates.items.len > 0) {
648 const last_update = &updates.items[updates.items.len - 1];649 const last_update = &updates.items[updates.items.len - 1];
649 last_update.changes = try changes.toOwnedSlice(arena);650 last_update.changes = try changes.toOwnedSlice(arena);
651 last_update.deletes = try deletes.toOwnedSlice(arena);
650 }652 }
651 try updates.append(arena, .{653 try updates.append(arena, .{
652 .name = val,654 .name = val,
653 .outcome = .unknown,655 .outcome = .unknown,
654 });656 });
655 } else if (std.mem.eql(u8, key, "file")) {657 } else if (std.mem.eql(u8, key, "file")) {
656 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});658 if (updates.items.len == 0) fatal("line {d}: file directive before update", .{line_n});
657659
658 if (root_source_file == null)660 if (root_source_file == null)
659 root_source_file = val;661 root_source_file = val;
...@@ -674,6 +676,9 @@ const Case = struct {...@@ -674,6 +676,9 @@ const Case = struct {
674 .name = val,676 .name = val,
675 .bytes = src,677 .bytes = src,
676 });678 });
679 } else if (std.mem.eql(u8, key, "rm_file")) {
680 if (updates.items.len == 0) fatal("line {d}: rm_file directive before update", .{line_n});
681 try deletes.append(arena, val);
677 } else if (std.mem.eql(u8, key, "expect_stdout")) {682 } else if (std.mem.eql(u8, key, "expect_stdout")) {
678 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});683 if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});
679 const last_update = &updates.items[updates.items.len - 1];684 const last_update = &updates.items[updates.items.len - 1];
...@@ -701,6 +706,7 @@ const Case = struct {...@@ -701,6 +706,7 @@ const Case = struct {
701 if (changes.items.len > 0) {706 if (changes.items.len > 0) {
702 const last_update = &updates.items[updates.items.len - 1];707 const last_update = &updates.items[updates.items.len - 1];
703 last_update.changes = changes.items; // arena so no need for toOwnedSlice708 last_update.changes = changes.items; // arena so no need for toOwnedSlice
709 last_update.deletes = deletes.items;
704 }710 }
705711
706 return .{712 return .{