Commit f47b8de2ad706616b648c52f5036102cb804e65d | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2025-01-25 04:48:16+00:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2025-01-25 06:07:08+00:00
log	f47b8de2ad706616b648c52f5036102cb804e65d
tree	20246773622e31440107d87455d5a01ac1cd8dcf
parent	5202c977d95bf65775e733264b7d37ed940a2997
signature	Commit is signed but in an unrecognized format.

incremental: handle `@embedFile`

Uses of `@embedFile` register dependencies on the corresponding
`Zcu.EmbedFile`. At the start of every update, we iterate all embedded
files and update them if necessary, and invalidate the dependencies if
they changed.

In order to properly integrate with the lazy analysis model, failed
embed files are now reported by the `AnalUnit` which actually used
`@embedFile`; the filesystem error is stored in the `Zcu.EmbedFile`.

An incremental test is added covering incremental updates to embedded
files, and I have verified locally that dependency invalidation is
working correctly.

7 files changed, 264 insertions(+), 168 deletions(-)

src
test
- incremental
  - change_embed_file
tools
- incr-check.zig

src/Compilation.zig+23-60

...	...	@@ -154,10 +154,6 @@ win32_resource_work_queue: if (dev.env.supports(.win32_resource)) std.fifo.Linea
154	154	/// since the last compilation, as well as scan for `@import` and queue up
155	155	/// additional jobs corresponding to those new files.
156	156	astgen_work_queue: std.fifo.LinearFifo(Zcu.File.Index, .Dynamic),
157		/// These jobs are to inspect the file system stat() and if the embedded file has changed
158		/// on disk, mark the corresponding Decl outdated and queue up an `analyze_decl`
159		/// task for it.
160		embed_file_work_queue: std.fifo.LinearFifo(*Zcu.EmbedFile, .Dynamic),
161	157
162	158	/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
163	159	/// This data is accessed by multiple threads and is protected by `mutex`.
...	...	@@ -1465,7 +1461,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
1465	1461	.c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
1466	1462	.win32_resource_work_queue = if (dev.env.supports(.win32_resource)) std.fifo.LinearFifo(*Win32Resource, .Dynamic).init(gpa) else .{},
1467	1463	.astgen_work_queue = std.fifo.LinearFifo(Zcu.File.Index, .Dynamic).init(gpa),
1468		.embed_file_work_queue = std.fifo.LinearFifo(*Zcu.EmbedFile, .Dynamic).init(gpa),
1469	1464	.c_source_files = options.c_source_files,
1470	1465	.rc_source_files = options.rc_source_files,
1471	1466	.cache_parent = cache,
...	...	@@ -1932,7 +1927,6 @@ pub fn destroy(comp: *Compilation) void {
1932	1927	comp.c_object_work_queue.deinit();
1933	1928	comp.win32_resource_work_queue.deinit();
1934	1929	comp.astgen_work_queue.deinit();
1935		comp.embed_file_work_queue.deinit();
1936	1930
1937	1931	comp.windows_libs.deinit(gpa);
1938	1932
...	...	@@ -2247,11 +2241,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2247	2241	}
2248	2242	}
2249	2243
2250		// Put a work item in for checking if any files used with `@embedFile` changed.
2251		try comp.embed_file_work_queue.ensureUnusedCapacity(zcu.embed_table.count());
2252		for (zcu.embed_table.values()) \|embed_file\| {
2253		comp.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
2254		}
2255	2244	if (comp.file_system_inputs) \|fsi\| {
2256	2245	const ip = &zcu.intern_pool;
2257	2246	for (zcu.embed_table.values()) \|embed_file\| {
...	...	@@ -3235,9 +3224,6 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle {
3235	3224	try addZirErrorMessages(&bundle, file);
3236	3225	}
3237	3226	}
3238		for (zcu.failed_embed_files.values()) \|error_msg\| {
3239		try addModuleErrorMsg(zcu, &bundle, error_msg.*);
3240		}
3241	3227	var sorted_failed_analysis: std.AutoArrayHashMapUnmanaged(InternPool.AnalUnit, *Zcu.ErrorMsg).DataList.Slice = s: {
3242	3228	const SortOrder = struct {
3243	3229	zcu: *Zcu,
...	...	@@ -3812,9 +3798,10 @@ fn performAllTheWorkInner(
3812	3798	}
3813	3799	}
3814	3800
3815		while (comp.embed_file_work_queue.readItem()) \|embed_file\| {
3816		comp.thread_pool.spawnWg(&astgen_wait_group, workerCheckEmbedFile, .{
3817		comp, embed_file,
	3801	for (0.., zcu.embed_table.values()) \|ef_index_usize, ef\| {
	3802	const ef_index: Zcu.EmbedFile.Index = @enumFromInt(ef_index_usize);
	3803	comp.thread_pool.spawnWgId(&astgen_wait_group, workerCheckEmbedFile, .{
	3804	comp, ef_index, ef,
3818	3805	});
3819	3806	}
3820	3807	}
...	...	@@ -4377,33 +4364,33 @@ fn workerUpdateBuiltinZigFile(
4377	4364	};
4378	4365	}
4379	4366
4380		fn workerCheckEmbedFile(comp: Compilation, embed_file: Zcu.EmbedFile) void {
4381		comp.detectEmbedFileUpdate(embed_file) catch \|err\| {
4382		comp.reportRetryableEmbedFileError(embed_file, err) catch \|oom\| switch (oom) {
4383		// Swallowing this error is OK because it's implied to be OOM when
4384		// there is a missing `failed_embed_files` error message.
4385		error.OutOfMemory => {},
4386		};
4387		return;
	4367	fn workerCheckEmbedFile(tid: usize, comp: Compilation, ef_index: Zcu.EmbedFile.Index, ef: Zcu.EmbedFile) void {
	4368	comp.detectEmbedFileUpdate(@enumFromInt(tid), ef_index, ef) catch \|err\| switch (err) {
	4369	error.OutOfMemory => {
	4370	comp.mutex.lock();
	4371	defer comp.mutex.unlock();
	4372	comp.setAllocFailure();
	4373	},
4388	4374	};
4389	4375	}
4390	4376
4391		fn detectEmbedFileUpdate(comp: Compilation, embed_file: Zcu.EmbedFile) !void {
	4377	fn detectEmbedFileUpdate(comp: Compilation, tid: Zcu.PerThread.Id, ef_index: Zcu.EmbedFile.Index, ef: Zcu.EmbedFile) !void {
4392	4378	const zcu = comp.zcu.?;
4393		const ip = &zcu.intern_pool;
4394		var file = try embed_file.owner.root.openFile(embed_file.sub_file_path.toSlice(ip), .{});
4395		defer file.close();
	4379	const pt: Zcu.PerThread = .activate(zcu, tid);
	4380	defer pt.deactivate();
	4381
	4382	const old_val = ef.val;
	4383	const old_err = ef.err;
4396	4384
4397		const stat = try file.stat();
	4385	try pt.updateEmbedFile(ef, null);
4398	4386
4399		const unchanged_metadata =
4400		stat.size == embed_file.stat.size and
4401		stat.mtime == embed_file.stat.mtime and
4402		stat.inode == embed_file.stat.inode;
	4387	if (ef.val != .none and ef.val == old_val) return; // success, value unchanged
	4388	if (ef.val == .none and old_val == .none and ef.err == old_err) return; // failure, error unchanged
4403	4389
4404		if (unchanged_metadata) return;
	4390	comp.mutex.lock();
	4391	defer comp.mutex.unlock();
4405	4392
4406		@panic("TODO: handle embed file incremental update");
	4393	try zcu.markDependeeOutdated(.not_marked_po, .{ .embed_file = ef_index });
4407	4394	}
4408	4395
4409	4396	pub fn obtainCObjectCacheManifest(
...	...	@@ -4802,30 +4789,6 @@ fn reportRetryableWin32ResourceError(
4802	4789	}
4803	4790	}
4804	4791
4805		fn reportRetryableEmbedFileError(
4806		comp: *Compilation,
4807		embed_file: *Zcu.EmbedFile,
4808		err: anyerror,
4809		) error{OutOfMemory}!void {
4810		const zcu = comp.zcu.?;
4811		const gpa = zcu.gpa;
4812		const src_loc = embed_file.src_loc;
4813		const ip = &zcu.intern_pool;
4814		const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}/{s}': {s}", .{
4815		embed_file.owner.root,
4816		embed_file.sub_file_path.toSlice(ip),
4817		@errorName(err),
4818		});
4819
4820		errdefer err_msg.destroy(gpa);
4821
4822		{
4823		comp.mutex.lock();
4824		defer comp.mutex.unlock();
4825		try zcu.failed_embed_files.putNoClobber(gpa, embed_file, err_msg);
4826		}
4827		}
4828
4829	4792	fn updateCObject(comp: Compilation, c_object: CObject, c_obj_prog_node: std.Progress.Node) !void {
4830	4793	if (comp.config.c_frontend == .aro) {
4831	4794	return comp.failCObj(c_object, "aro does not support compiling C objects yet", .{});

src/InternPool.zig+9

...	...	@@ -42,6 +42,10 @@ nav_ty_deps: std.AutoArrayHashMapUnmanaged(Nav.Index, DepEntry.Index),
42	42	/// * a container type requiring resolution (invalidated when the type must be recreated at a new index)
43	43	/// Value is index into `dep_entries` of the first dependency on this interned value.
44	44	interned_deps: std.AutoArrayHashMapUnmanaged(Index, DepEntry.Index),
	45	/// Dependencies on an embedded file.
	46	/// Introduced by `@embedFile`; invalidated when the file changes.
	47	/// Value is index into `dep_entries` of the first dependency on this `Zcu.EmbedFile`.
	48	embed_file_deps: std.AutoArrayHashMapUnmanaged(Zcu.EmbedFile.Index, DepEntry.Index),
45	49	/// Dependencies on the full set of names in a ZIR namespace.
46	50	/// Key refers to a `struct_decl`, `union_decl`, etc.
47	51	/// Value is index into `dep_entries` of the first dependency on this namespace.
...	...	@@ -90,6 +94,7 @@ pub const empty: InternPool = .{
90	94	.nav_val_deps = .empty,
91	95	.nav_ty_deps = .empty,
92	96	.interned_deps = .empty,
	97	.embed_file_deps = .empty,
93	98	.namespace_deps = .empty,
94	99	.namespace_name_deps = .empty,
95	100	.memoized_state_main_deps = .none,
...	...	@@ -824,6 +829,7 @@ pub const Dependee = union(enum) {
824	829	nav_val: Nav.Index,
825	830	nav_ty: Nav.Index,
826	831	interned: Index,
	832	embed_file: Zcu.EmbedFile.Index,
827	833	namespace: TrackedInst.Index,
828	834	namespace_name: NamespaceNameKey,
829	835	memoized_state: MemoizedStateStage,
...	...	@@ -875,6 +881,7 @@ pub fn dependencyIterator(ip: *const InternPool, dependee: Dependee) DependencyI
875	881	.nav_val => \|x\| ip.nav_val_deps.get(x),
876	882	.nav_ty => \|x\| ip.nav_ty_deps.get(x),
877	883	.interned => \|x\| ip.interned_deps.get(x),
	884	.embed_file => \|x\| ip.embed_file_deps.get(x),
878	885	.namespace => \|x\| ip.namespace_deps.get(x),
879	886	.namespace_name => \|x\| ip.namespace_name_deps.get(x),
880	887	.memoized_state => \|stage\| switch (stage) {
...	...	@@ -945,6 +952,7 @@ pub fn addDependency(ip: *InternPool, gpa: Allocator, depender: AnalUnit, depend
945	952	.nav_val => ip.nav_val_deps,
946	953	.nav_ty => ip.nav_ty_deps,
947	954	.interned => ip.interned_deps,
	955	.embed_file => ip.embed_file_deps,
948	956	.namespace => ip.namespace_deps,
949	957	.namespace_name => ip.namespace_name_deps,
950	958	.memoized_state => comptime unreachable,
...	...	@@ -6612,6 +6620,7 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {
6612	6620	ip.nav_val_deps.deinit(gpa);
6613	6621	ip.nav_ty_deps.deinit(gpa);
6614	6622	ip.interned_deps.deinit(gpa);
	6623	ip.embed_file_deps.deinit(gpa);
6615	6624	ip.namespace_deps.deinit(gpa);
6616	6625	ip.namespace_name_deps.deinit(gpa);
6617	6626

src/Sema.zig+14-6

...	...	@@ -13949,6 +13949,8 @@ fn zirEmbedFile(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!A
13949	13949	defer tracy.end();
13950	13950
13951	13951	const pt = sema.pt;
	13952	const zcu = pt.zcu;
	13953
13952	13954	const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;
13953	13955	const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);
13954	13956	const name = try sema.resolveConstString(block, operand_src, inst_data.operand, .{ .simple = .operand_embedFile });
...	...	@@ -13957,18 +13959,24 @@ fn zirEmbedFile(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!A
13957	13959	return sema.fail(block, operand_src, "file path name cannot be empty", .{});
13958	13960	}
13959	13961
13960		const val = pt.embedFile(block.getFileScope(pt.zcu), name, operand_src) catch \|err\| switch (err) {
	13962	const ef_idx = pt.embedFile(block.getFileScope(zcu), name) catch \|err\| switch (err) {
13961	13963	error.ImportOutsideModulePath => {
13962	13964	return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});
13963	13965	},
13964		else => {
13965		// TODO: these errors are file system errors; make sure an update() will
13966		// retry this and not cache the file system error, which may be transient.
13967		return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(err) });
	13966	error.CurrentWorkingDirectoryUnlinked => {
	13967	// TODO: this should be some kind of retryable failure, in case the cwd is put back
	13968	return sema.fail(block, operand_src, "unable to resolve '{s}': working directory has been unlinked", .{name});
13968	13969	},
	13970	error.OutOfMemory => \|e\| return e,
13969	13971	};
	13972	try sema.declareDependency(.{ .embed_file = ef_idx });
	13973
	13974	const result = ef_idx.get(zcu);
	13975	if (result.val == .none) {
	13976	return sema.fail(block, operand_src, "unable to open '{s}': {s}", .{ name, @errorName(result.err.?) });
	13977	}
13970	13978
13971		return Air.internedToRef(val);
	13979	return Air.internedToRef(result.val);
13972	13980	}
13973	13981
13974	13982	fn zirRetErrValueCode(sema: *Sema, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {

src/Zcu.zig+22-11

...	...	@@ -143,8 +143,6 @@ compile_log_sources: std.AutoArrayHashMapUnmanaged(AnalUnit, extern struct {
143	143	/// Using a map here for consistency with the other fields here.
144	144	/// The ErrorMsg memory is owned by the `File`, using Module's general purpose allocator.
145	145	failed_files: std.AutoArrayHashMapUnmanaged(File, ?ErrorMsg) = .empty,
146		/// The ErrorMsg memory is owned by the `EmbedFile`, using Module's general purpose allocator.
147		failed_embed_files: std.AutoArrayHashMapUnmanaged(EmbedFile, ErrorMsg) = .empty,
148	146	failed_exports: std.AutoArrayHashMapUnmanaged(Export.Index, *ErrorMsg) = .empty,
149	147	/// If analysis failed due to a cimport error, the corresponding Clang errors
150	148	/// are stored here.
...	...	@@ -893,13 +891,23 @@ pub const File = struct {
893	891	};
894	892
895	893	pub const EmbedFile = struct {
896		/// Relative to the owning module's root directory.
897		sub_file_path: InternPool.NullTerminatedString,
898	894	/// Module that this file is a part of, managed externally.
899	895	owner: *Package.Module,
900		stat: Cache.File.Stat,
	896	/// Relative to the owning module's root directory.
	897	sub_file_path: InternPool.NullTerminatedString,
	898
	899	/// `.none` means the file was not loaded, so `stat` is undefined.
901	900	val: InternPool.Index,
902		src_loc: LazySrcLoc,
	901	/// If this is `null` and `val` is `.none`, the file has never been loaded.
	902	err: ?(std.fs.File.OpenError \|\| std.fs.File.StatError \|\| std.fs.File.ReadError \|\| error{UnexpectedEof}),
	903	stat: Cache.File.Stat,
	904
	905	pub const Index = enum(u32) {
	906	_,
	907	pub fn get(idx: Index, zcu: const Zcu) EmbedFile {
	908	return zcu.embed_table.values()[@intFromEnum(idx)];
	909	}
	910	};
903	911	};
904	912
905	913	/// This struct holds data necessary to construct API-facing `AllErrors.Message`.
...	...	@@ -2459,11 +2467,6 @@ pub fn deinit(zcu: *Zcu) void {
2459	2467	}
2460	2468	zcu.failed_files.deinit(gpa);
2461	2469
2462		for (zcu.failed_embed_files.values()) \|msg\| {
2463		msg.destroy(gpa);
2464		}
2465		zcu.failed_embed_files.deinit(gpa);
2466
2467	2470	for (zcu.failed_exports.values()) \|value\| {
2468	2471	value.destroy(gpa);
2469	2472	}
...	...	@@ -3882,6 +3885,14 @@ fn formatDependee(data: struct { dependee: InternPool.Dependee, zcu: *Zcu }, com
3882	3885	.func => \|f\| return writer.print("ies('{}')", .{ip.getNav(f.owner_nav).fqn.fmt(ip)}),
3883	3886	else => unreachable,
3884	3887	},
	3888	.embed_file => \|ef_idx\| {
	3889	const ef = ef_idx.get(zcu);
	3890	return writer.print("embed_file('{s}')", .{std.fs.path.fmtJoin(&.{
	3891	ef.owner.root.root_dir.path orelse "",
	3892	ef.owner.root.sub_path,
	3893	ef.sub_file_path.toSlice(ip),
	3894	})});
	3895	},
3885	3896	.namespace => \|ti\| {
3886	3897	const info = ti.resolveFull(ip) orelse {
3887	3898	return writer.writeAll("namespace(<lost>)");

src/Zcu/PerThread.zig+143-90

...	...	@@ -2117,32 +2117,32 @@ pub fn embedFile(
2117	2117	pt: Zcu.PerThread,
2118	2118	cur_file: *Zcu.File,
2119	2119	import_string: []const u8,
2120		src_loc: Zcu.LazySrcLoc,
2121		) !InternPool.Index {
	2120	) error{
	2121	OutOfMemory,
	2122	ImportOutsideModulePath,
	2123	CurrentWorkingDirectoryUnlinked,
	2124	}!Zcu.EmbedFile.Index {
2122	2125	const zcu = pt.zcu;
2123	2126	const gpa = zcu.gpa;
2124	2127
2125		if (cur_file.mod.deps.get(import_string)) \|pkg\| {
	2128	if (cur_file.mod.deps.get(import_string)) \|mod\| {
2126	2129	const resolved_path = try std.fs.path.resolve(gpa, &.{
2127		pkg.root.root_dir.path orelse ".",
2128		pkg.root.sub_path,
2129		pkg.root_src_path,
	2130	mod.root.root_dir.path orelse ".",
	2131	mod.root.sub_path,
	2132	mod.root_src_path,
2130	2133	});
2131		var keep_resolved_path = false;
2132		defer if (!keep_resolved_path) gpa.free(resolved_path);
	2134	errdefer gpa.free(resolved_path);
2133	2135
2134	2136	const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);
2135		errdefer {
2136		assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2137		keep_resolved_path = false;
2138		}
2139		if (gop.found_existing) return gop.value_ptr.*.val;
2140		keep_resolved_path = true;
	2137	errdefer assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2141	2138
2142		const sub_file_path = try gpa.dupe(u8, pkg.root_src_path);
2143		errdefer gpa.free(sub_file_path);
	2139	if (gop.found_existing) {
	2140	gpa.free(resolved_path); // we're not using this key
	2141	return @enumFromInt(gop.index);
	2142	}
2144	2143
2145		return pt.newEmbedFile(pkg, sub_file_path, resolved_path, gop.value_ptr, src_loc);
	2144	gop.value_ptr.* = try pt.newEmbedFile(mod, mod.root_src_path, resolved_path);
	2145	return @enumFromInt(gop.index);
2146	2146	}
2147	2147
2148	2148	// The resolved path is used as the key in the table, to detect if a file
...	...	@@ -2154,17 +2154,15 @@ pub fn embedFile(
2154	2154	"..",
2155	2155	import_string,
2156	2156	});
2157
2158		var keep_resolved_path = false;
2159		defer if (!keep_resolved_path) gpa.free(resolved_path);
	2157	errdefer gpa.free(resolved_path);
2160	2158
2161	2159	const gop = try zcu.embed_table.getOrPut(gpa, resolved_path);
2162		errdefer {
2163		assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
2164		keep_resolved_path = false;
	2160	errdefer assert(std.mem.eql(u8, zcu.embed_table.pop().key, resolved_path));
	2161
	2162	if (gop.found_existing) {
	2163	gpa.free(resolved_path); // we're not using this key
	2164	return @enumFromInt(gop.index);
2165	2165	}
2166		if (gop.found_existing) return gop.value_ptr.*.val;
2167		keep_resolved_path = true;
2168	2166
2169	2167	const resolved_root_path = try std.fs.path.resolve(gpa, &.{
2170	2168	cur_file.mod.root.root_dir.path orelse ".",
...	...	@@ -2172,101 +2170,156 @@ pub fn embedFile(
2172	2170	});
2173	2171	defer gpa.free(resolved_root_path);
2174	2172
2175		const sub_file_path = p: {
2176		const relative = try std.fs.path.relative(gpa, resolved_root_path, resolved_path);
2177		errdefer gpa.free(relative);
2178
2179		if (!isUpDir(relative) and !std.fs.path.isAbsolute(relative)) {
2180		break :p relative;
2181		}
2182		return error.ImportOutsideModulePath;
	2173	const sub_file_path = std.fs.path.relative(gpa, resolved_root_path, resolved_path) catch \|err\| switch (err) {
	2174	error.Unexpected => unreachable,
	2175	else => \|e\| return e,
2183	2176	};
2184	2177	defer gpa.free(sub_file_path);
2185	2178
2186		return pt.newEmbedFile(cur_file.mod, sub_file_path, resolved_path, gop.value_ptr, src_loc);
	2179	if (isUpDir(sub_file_path) or std.fs.path.isAbsolute(sub_file_path)) {
	2180	return error.ImportOutsideModulePath;
	2181	}
	2182
	2183	gop.value_ptr.* = try pt.newEmbedFile(cur_file.mod, sub_file_path, resolved_path);
	2184	return @enumFromInt(gop.index);
2187	2185	}
2188	2186
2189		/// https://github.com/ziglang/zig/issues/14307
2190		fn newEmbedFile(
	2187	pub fn updateEmbedFile(
2191	2188	pt: Zcu.PerThread,
2192		pkg: *Module,
2193		sub_file_path: []const u8,
2194		resolved_path: []const u8,
2195		result: **Zcu.EmbedFile,
2196		src_loc: Zcu.LazySrcLoc,
2197		) !InternPool.Index {
	2189	ef: *Zcu.EmbedFile,
	2190	/// If not `null`, the interned file data is stored here, if it was loaded.
	2191	/// `newEmbedFile` uses this to add the file to the `whole` cache manifest.
	2192	ip_str_out: ?*?InternPool.String,
	2193	) Allocator.Error!void {
	2194	pt.updateEmbedFileInner(ef, ip_str_out) catch \|err\| switch (err) {
	2195	error.OutOfMemory => \|e\| return e,
	2196	else => \|e\| {
	2197	ef.val = .none;
	2198	ef.err = e;
	2199	ef.stat = undefined;
	2200	},
	2201	};
	2202	}
	2203
	2204	fn updateEmbedFileInner(
	2205	pt: Zcu.PerThread,
	2206	ef: *Zcu.EmbedFile,
	2207	ip_str_out: ?*?InternPool.String,
	2208	) !void {
	2209	const tid = pt.tid;
2198	2210	const zcu = pt.zcu;
2199	2211	const gpa = zcu.gpa;
2200	2212	const ip = &zcu.intern_pool;
2201	2213
2202		const new_file = try gpa.create(Zcu.EmbedFile);
2203		errdefer gpa.destroy(new_file);
2204
2205		var file = try pkg.root.openFile(sub_file_path, .{});
	2214	var file = try ef.owner.root.openFile(ef.sub_file_path.toSlice(ip), .{});
2206	2215	defer file.close();
2207	2216
2208		const actual_stat = try file.stat();
2209		const stat: Cache.File.Stat = .{
2210		.size = actual_stat.size,
2211		.inode = actual_stat.inode,
2212		.mtime = actual_stat.mtime,
2213		};
2214		const size = std.math.cast(usize, actual_stat.size) orelse return error.Overflow;
2215
2216		const strings = ip.getLocal(pt.tid).getMutableStrings(gpa);
2217		const bytes = try strings.addManyAsSlice(try std.math.add(usize, size, 1));
2218		const actual_read = try file.readAll(bytes[0][0..size]);
2219		if (actual_read != size) return error.UnexpectedEndOfFile;
2220		bytes[0][size] = 0;
	2217	const stat: Cache.File.Stat = .fromFs(try file.stat());
2221	2218
2222		const comp = zcu.comp;
2223		switch (comp.cache_use) {
2224		.whole => \|whole\| if (whole.cache_manifest) \|man\| {
2225		const copied_resolved_path = try gpa.dupe(u8, resolved_path);
2226		errdefer gpa.free(copied_resolved_path);
2227		whole.cache_manifest_mutex.lock();
2228		defer whole.cache_manifest_mutex.unlock();
2229		try man.addFilePostContents(copied_resolved_path, bytes[0][0..size], stat);
2230		},
2231		.incremental => {},
	2219	if (ef.val != .none) {
	2220	const old_stat = ef.stat;
	2221	const unchanged_metadata =
	2222	stat.size == old_stat.size and
	2223	stat.mtime == old_stat.mtime and
	2224	stat.inode == old_stat.inode;
	2225	if (unchanged_metadata) return;
2232	2226	}
2233	2227
2234		const array_ty = try pt.intern(.{ .array_type = .{
	2228	const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig;
	2229	const size_plus_one = std.math.add(usize, size, 1) catch return error.FileTooBig;
	2230
	2231	// The loaded bytes of the file, including a sentinel 0 byte.
	2232	const ip_str: InternPool.String = str: {
	2233	const strings = ip.getLocal(tid).getMutableStrings(gpa);
	2234	const old_len = strings.mutate.len;
	2235	errdefer strings.shrinkRetainingCapacity(old_len);
	2236	const bytes = (try strings.addManyAsSlice(size_plus_one))[0];
	2237	const actual_read = try file.readAll(bytes[0..size]);
	2238	if (actual_read != size) return error.UnexpectedEof;
	2239	bytes[size] = 0;
	2240	break :str try ip.getOrPutTrailingString(gpa, tid, @intCast(bytes.len), .maybe_embedded_nulls);
	2241	};
	2242	if (ip_str_out) \|p\| p.* = ip_str;
	2243
	2244	const array_ty = try pt.arrayType(.{
2235	2245	.len = size,
2236	2246	.sentinel = .zero_u8,
2237	2247	.child = .u8_type,
2238		} });
	2248	});
	2249	const ptr_ty = try pt.singleConstPtrType(array_ty);
	2250
2239	2251	const array_val = try pt.intern(.{ .aggregate = .{
2240		.ty = array_ty,
2241		.storage = .{ .bytes = try ip.getOrPutTrailingString(gpa, pt.tid, @intCast(bytes[0].len), .maybe_embedded_nulls) },
	2252	.ty = array_ty.toIntern(),
	2253	.storage = .{ .bytes = ip_str },
2242	2254	} });
2243
2244		const ptr_ty = (try pt.ptrType(.{
2245		.child = array_ty,
2246		.flags = .{
2247		.alignment = .none,
2248		.is_const = true,
2249		.address_space = .generic,
2250		},
2251		})).toIntern();
2252	2255	const ptr_val = try pt.intern(.{ .ptr = .{
2253		.ty = ptr_ty,
	2256	.ty = ptr_ty.toIntern(),
2254	2257	.base_addr = .{ .uav = .{
2255	2258	.val = array_val,
2256		.orig_ty = ptr_ty,
	2259	.orig_ty = ptr_ty.toIntern(),
2257	2260	} },
2258	2261	.byte_offset = 0,
2259	2262	} });
2260	2263
2261		result.* = new_file;
	2264	ef.val = ptr_val;
	2265	ef.err = null;
	2266	ef.stat = stat;
	2267	}
	2268
	2269	fn newEmbedFile(
	2270	pt: Zcu.PerThread,
	2271	mod: *Module,
	2272	/// The path of the file to embed relative to the root of `mod`.
	2273	sub_file_path: []const u8,
	2274	/// The resolved path of the file to embed.
	2275	resolved_path: []const u8,
	2276	) !*Zcu.EmbedFile {
	2277	const zcu = pt.zcu;
	2278	const comp = zcu.comp;
	2279	const gpa = zcu.gpa;
	2280	const ip = &zcu.intern_pool;
	2281
	2282	if (comp.file_system_inputs) \|fsi\|
	2283	try comp.appendFileSystemInput(fsi, mod.root, sub_file_path);
	2284
	2285	const new_file = try gpa.create(Zcu.EmbedFile);
	2286	errdefer gpa.destroy(new_file);
	2287
2262	2288	new_file.* = .{
	2289	.owner = mod,
2263	2290	.sub_file_path = try ip.getOrPutString(gpa, pt.tid, sub_file_path, .no_embedded_nulls),
2264		.owner = pkg,
2265		.stat = stat,
2266		.val = ptr_val,
2267		.src_loc = src_loc,
	2291	.val = .none,
	2292	.err = null,
	2293	.stat = undefined,
2268	2294	};
2269		return ptr_val;
	2295
	2296	var opt_ip_str: ?InternPool.String = null;
	2297	try pt.updateEmbedFile(new_file, &opt_ip_str);
	2298
	2299	// Add the file contents to the `whole` cache manifest if necessary.
	2300	cache: {
	2301	const whole = switch (zcu.comp.cache_use) {
	2302	.whole => \|whole\| whole,
	2303	.incremental => break :cache,
	2304	};
	2305	const man = whole.cache_manifest orelse break :cache;
	2306	const ip_str = opt_ip_str orelse break :cache;
	2307
	2308	const copied_resolved_path = try gpa.dupe(u8, resolved_path);
	2309	errdefer gpa.free(copied_resolved_path);
	2310
	2311	const array_len = Value.fromInterned(new_file.val).typeOf(zcu).childType(zcu).arrayLen(zcu);
	2312
	2313	whole.cache_manifest_mutex.lock();
	2314	defer whole.cache_manifest_mutex.unlock();
	2315
	2316	man.addFilePostContents(copied_resolved_path, ip_str.toSlice(array_len, ip), new_file.stat) catch \|err\| switch (err) {
	2317	error.Unexpected => unreachable,
	2318	else => \|e\| return e,
	2319	};
	2320	}
	2321
	2322	return new_file;
2270	2323	}
2271	2324
2272	2325	pub fn scanNamespace(

test/incremental/change_embed_file created+46

...	...	@@ -0,0 +1,46 @@
	1	#target=x86_64-linux-selfhosted
	2	#target=x86_64-linux-cbe
	3	#target=x86_64-windows-cbe
	4	#target=wasm32-wasi-selfhosted
	5	#update=initial version
	6	#file=main.zig
	7	const std = @import("std");
	8	const string = @embedFile("string.txt");
	9	pub fn main() !void {
	10	try std.io.getStdOut().writeAll(string);
	11	}
	12	#file=string.txt
	13	Hello, World!
	14	#expect_stdout="Hello, World!\n"
	15
	16	#update=change file contents
	17	#file=string.txt
	18	Hello again, World!
	19	#expect_stdout="Hello again, World!\n"
	20
	21	#update=delete file
	22	#rm_file=string.txt
	23	#expect_error=ignored
	24
	25	#update=remove reference to file
	26	#file=main.zig
	27	const std = @import("std");
	28	const string = @embedFile("string.txt");
	29	pub fn main() !void {
	30	try std.io.getStdOut().writeAll("a hardcoded string\n");
	31	}
	32	#expect_stdout="a hardcoded string\n"
	33
	34	#update=re-introduce reference to file
	35	#file=main.zig
	36	const std = @import("std");
	37	const string = @embedFile("string.txt");
	38	pub fn main() !void {
	39	try std.io.getStdOut().writeAll(string);
	40	}
	41	#expect_error=ignore
	42
	43	#update=recreate file
	44	#file=string.txt
	45	We're back, World!
	46	#expect_stdout="We're back, World!\n"

tools/incr-check.zig+7-1

...	...	@@ -608,6 +608,7 @@ const Case = struct {
608	608	var targets: std.ArrayListUnmanaged(Target) = .empty;
609	609	var updates: std.ArrayListUnmanaged(Update) = .empty;
610	610	var changes: std.ArrayListUnmanaged(FullContents) = .empty;
	611	var deletes: std.ArrayListUnmanaged([]const u8) = .empty;
611	612	var it = std.mem.splitScalar(u8, bytes, '\n');
612	613	var line_n: usize = 1;
613	614	var root_source_file: ?[]const u8 = null;
...	...	@@ -647,13 +648,14 @@ const Case = struct {
647	648	if (updates.items.len > 0) {
648	649	const last_update = &updates.items[updates.items.len - 1];
649	650	last_update.changes = try changes.toOwnedSlice(arena);
	651	last_update.deletes = try deletes.toOwnedSlice(arena);
650	652	}
651	653	try updates.append(arena, .{
652	654	.name = val,
653	655	.outcome = .unknown,
654	656	});
655	657	} else if (std.mem.eql(u8, key, "file")) {
656		if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});
	658	if (updates.items.len == 0) fatal("line {d}: file directive before update", .{line_n});
657	659
658	660	if (root_source_file == null)
659	661	root_source_file = val;
...	...	@@ -674,6 +676,9 @@ const Case = struct {
674	676	.name = val,
675	677	.bytes = src,
676	678	});
	679	} else if (std.mem.eql(u8, key, "rm_file")) {
	680	if (updates.items.len == 0) fatal("line {d}: rm_file directive before update", .{line_n});
	681	try deletes.append(arena, val);
677	682	} else if (std.mem.eql(u8, key, "expect_stdout")) {
678	683	if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});
679	684	const last_update = &updates.items[updates.items.len - 1];
...	...	@@ -701,6 +706,7 @@ const Case = struct {
701	706	if (changes.items.len > 0) {
702	707	const last_update = &updates.items[updates.items.len - 1];
703	708	last_update.changes = changes.items; // arena so no need for toOwnedSlice
	709	last_update.deletes = deletes.items;
704	710	}
705	711
706	712	return .{