Commit afa66fa392f5a32d16da7f4705c59dad369f6d48 | zig

author	Jacob Young <jacobly@ziglang.org> 2024-07-10 14:33:46-04:00
committer	Jacob Young <jacobly@ziglang.org> 2024-07-10 14:42:50-04:00
log	afa66fa392f5a32d16da7f4705c59dad369f6d48
tree	4872284bc454b34c14f25559ea46ea9bd6bf2deb
parent	f290b54f891a67af456529da0f4f824a1e27b4ef

InternPool: make `tracked_insts` thread-safe

5 files changed, 382 insertions(+), 216 deletions(-)

src/Compilation.zig+22-53

...	...	@@ -2675,7 +2675,10 @@ fn reportMultiModuleErrors(pt: Zcu.PerThread) !void {
2675	2675	.import => \|import\| try Zcu.ErrorMsg.init(
2676	2676	gpa,
2677	2677	.{
2678		.base_node_inst = try ip.trackZir(gpa, import.file, .main_struct_inst),
	2678	.base_node_inst = try ip.trackZir(gpa, pt.tid, .{
	2679	.file = import.file,
	2680	.inst = .main_struct_inst,
	2681	}),
2679	2682	.offset = .{ .token_abs = import.token },
2680	2683	},
2681	2684	"imported from module {s}",
...	...	@@ -2684,7 +2687,10 @@ fn reportMultiModuleErrors(pt: Zcu.PerThread) !void {
2684	2687	.root => \|pkg\| try Zcu.ErrorMsg.init(
2685	2688	gpa,
2686	2689	.{
2687		.base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
	2690	.base_node_inst = try ip.trackZir(gpa, pt.tid, .{
	2691	.file = file_index,
	2692	.inst = .main_struct_inst,
	2693	}),
2688	2694	.offset = .entire_file,
2689	2695	},
2690	2696	"root of module {s}",
...	...	@@ -2698,7 +2704,10 @@ fn reportMultiModuleErrors(pt: Zcu.PerThread) !void {
2698	2704	notes[num_notes] = try Zcu.ErrorMsg.init(
2699	2705	gpa,
2700	2706	.{
2701		.base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
	2707	.base_node_inst = try ip.trackZir(gpa, pt.tid, .{
	2708	.file = file_index,
	2709	.inst = .main_struct_inst,
	2710	}),
2702	2711	.offset = .entire_file,
2703	2712	},
2704	2713	"{} more references omitted",
...	...	@@ -2710,7 +2719,10 @@ fn reportMultiModuleErrors(pt: Zcu.PerThread) !void {
2710	2719	const err = try Zcu.ErrorMsg.create(
2711	2720	gpa,
2712	2721	.{
2713		.base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
	2722	.base_node_inst = try ip.trackZir(gpa, pt.tid, .{
	2723	.file = file_index,
	2724	.inst = .main_struct_inst,
	2725	}),
2714	2726	.offset = .entire_file,
2715	2727	},
2716	2728	"file exists in multiple modules",
...	...	@@ -2776,7 +2788,7 @@ const Header = extern struct {
2776	2788	//extra_len: u32,
2777	2789	//limbs_len: u32,
2778	2790	//string_bytes_len: u32,
2779		tracked_insts_len: u32,
	2791	//tracked_insts_len: u32,
2780	2792	src_hash_deps_len: u32,
2781	2793	decl_val_deps_len: u32,
2782	2794	namespace_deps_len: u32,
...	...	@@ -2805,7 +2817,7 @@ pub fn saveState(comp: *Compilation) !void {
2805	2817	//.extra_len = @intCast(ip.extra.items.len),
2806	2818	//.limbs_len = @intCast(ip.limbs.items.len),
2807	2819	//.string_bytes_len = @intCast(ip.string_bytes.items.len),
2808		.tracked_insts_len = @intCast(ip.tracked_insts.count()),
	2820	//.tracked_insts_len = @intCast(ip.tracked_insts.count()),
2809	2821	.src_hash_deps_len = @intCast(ip.src_hash_deps.count()),
2810	2822	.decl_val_deps_len = @intCast(ip.decl_val_deps.count()),
2811	2823	.namespace_deps_len = @intCast(ip.namespace_deps.count()),
...	...	@@ -2822,7 +2834,7 @@ pub fn saveState(comp: *Compilation) !void {
2822	2834	//addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.items.items(.data)));
2823	2835	//addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.items.items(.tag)));
2824	2836	//addBuf(&bufs_list, &bufs_len, ip.string_bytes.items);
2825		addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.tracked_insts.keys()));
	2837	//addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.tracked_insts.keys()));
2826	2838
2827	2839	addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.src_hash_deps.keys()));
2828	2840	addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.src_hash_deps.values()));
...	...	@@ -4134,14 +4146,6 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) anye
4134	4146	};
4135	4147	}
4136	4148
4137		const AstGenSrc = union(enum) {
4138		root,
4139		import: struct {
4140		importing_file: Zcu.File.Index,
4141		import_tok: std.zig.Ast.TokenIndex,
4142		},
4143		};
4144
4145	4149	fn workerAstGenFile(
4146	4150	tid: usize,
4147	4151	comp: *Compilation,
...	...	@@ -4151,7 +4155,7 @@ fn workerAstGenFile(
4151	4155	root_decl: Zcu.Decl.OptionalIndex,
4152	4156	prog_node: std.Progress.Node,
4153	4157	wg: *WaitGroup,
4154		src: AstGenSrc,
	4158	src: Zcu.AstGenSrc,
4155	4159	) void {
4156	4160	const child_prog_node = prog_node.start(file.sub_file_path, 0);
4157	4161	defer child_prog_node.end();
...	...	@@ -4161,7 +4165,7 @@ fn workerAstGenFile(
4161	4165	error.AnalysisFail => return,
4162	4166	else => {
4163	4167	file.status = .retryable_failure;
4164		comp.reportRetryableAstGenError(src, file_index, err) catch \|oom\| switch (oom) {
	4168	pt.reportRetryableAstGenError(src, file_index, err) catch \|oom\| switch (oom) {
4165	4169	// Swallowing this error is OK because it's implied to be OOM when
4166	4170	// there is a missing `failed_files` error message.
4167	4171	error.OutOfMemory => {},
...	...	@@ -4207,7 +4211,7 @@ fn workerAstGenFile(
4207	4211	log.debug("AstGen of {s} has import '{s}'; queuing AstGen of {s}", .{
4208	4212	file.sub_file_path, import_path, import_result.file.sub_file_path,
4209	4213	});
4210		const sub_src: AstGenSrc = .{ .import = .{
	4214	const sub_src: Zcu.AstGenSrc = .{ .import = .{
4211	4215	.importing_file = file_index,
4212	4216	.import_tok = item.data.token,
4213	4217	} };
...	...	@@ -4560,41 +4564,6 @@ fn reportRetryableWin32ResourceError(
4560	4564	}
4561	4565	}
4562	4566
4563		fn reportRetryableAstGenError(
4564		comp: *Compilation,
4565		src: AstGenSrc,
4566		file_index: Zcu.File.Index,
4567		err: anyerror,
4568		) error{OutOfMemory}!void {
4569		const zcu = comp.module.?;
4570		const gpa = zcu.gpa;
4571
4572		const file = zcu.fileByIndex(file_index);
4573		file.status = .retryable_failure;
4574
4575		const src_loc: Zcu.LazySrcLoc = switch (src) {
4576		.root => .{
4577		.base_node_inst = try zcu.intern_pool.trackZir(gpa, file_index, .main_struct_inst),
4578		.offset = .entire_file,
4579		},
4580		.import => \|info\| .{
4581		.base_node_inst = try zcu.intern_pool.trackZir(gpa, info.importing_file, .main_struct_inst),
4582		.offset = .{ .token_abs = info.import_tok },
4583		},
4584		};
4585
4586		const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{
4587		file.mod.root, file.sub_file_path, @errorName(err),
4588		});
4589		errdefer err_msg.destroy(gpa);
4590
4591		{
4592		comp.mutex.lock();
4593		defer comp.mutex.unlock();
4594		try zcu.failed_files.putNoClobber(gpa, file, err_msg);
4595		}
4596		}
4597
4598	4567	fn reportRetryableEmbedFileError(
4599	4568	comp: *Compilation,
4600	4569	embed_file: *Zcu.EmbedFile,

src/InternPool.zig+156-26

...	...	@@ -20,10 +20,6 @@ tid_shift_32: if (single_threaded) u0 else std.math.Log2Int(u32) = if (single_th
20	20	/// These are not serialized; it is computed upon deserialization.
21	21	maps: std.ArrayListUnmanaged(FieldMap) = .{},
22	22
23		/// An index into `tracked_insts` gives a reference to a single ZIR instruction which
24		/// persists across incremental updates.
25		tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{},
26
27	23	/// Dependencies on the source code hash associated with a ZIR instruction.
28	24	/// * For a `declaration`, this is the entire declaration body.
29	25	/// * For a `struct_decl`, `union_decl`, etc, this is the source of the fields (but not declarations).
...	...	@@ -76,12 +72,15 @@ pub const TrackedInst = extern struct {
76	72	}
77	73	pub const Index = enum(u32) {
78	74	_,
79		pub fn resolveFull(i: TrackedInst.Index, ip: *const InternPool) TrackedInst {
80		return ip.tracked_insts.keys()[@intFromEnum(i)];
	75	pub fn resolveFull(tracked_inst_index: TrackedInst.Index, ip: *const InternPool) TrackedInst {
	76	const tracked_inst_unwrapped = tracked_inst_index.unwrap(ip);
	77	const tracked_insts = ip.getLocalShared(tracked_inst_unwrapped.tid).tracked_insts.acquire();
	78	return tracked_insts.view().items(.@"0")[tracked_inst_unwrapped.index];
81	79	}
82	80	pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Zir.Inst.Index {
83	81	return i.resolveFull(ip).inst;
84	82	}
	83
85	84	pub fn toOptional(i: TrackedInst.Index) Optional {
86	85	return @enumFromInt(@intFromEnum(i));
87	86	}
...	...	@@ -95,21 +94,124 @@ pub const TrackedInst = extern struct {
95	94	};
96	95	}
97	96	};
	97
	98	pub const Unwrapped = struct {
	99	tid: Zcu.PerThread.Id,
	100	index: u32,
	101
	102	pub fn wrap(unwrapped: Unwrapped, ip: *const InternPool) TrackedInst.Index {
	103	assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
	104	assert(unwrapped.index <= ip.getIndexMask(u32));
	105	return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 \|
	106	unwrapped.index);
	107	}
	108	};
	109	pub fn unwrap(tracked_inst_index: TrackedInst.Index, ip: *const InternPool) Unwrapped {
	110	return .{
	111	.tid = @enumFromInt(@intFromEnum(tracked_inst_index) >> ip.tid_shift_32 & ip.getTidMask()),
	112	.index = @intFromEnum(tracked_inst_index) & ip.getIndexMask(u32),
	113	};
	114	}
98	115	};
99	116	};
100	117
101	118	pub fn trackZir(
102	119	ip: *InternPool,
103	120	gpa: Allocator,
104		file: FileIndex,
105		inst: Zir.Inst.Index,
	121	tid: Zcu.PerThread.Id,
	122	key: TrackedInst,
106	123	) Allocator.Error!TrackedInst.Index {
107		const key: TrackedInst = .{
108		.file = file,
109		.inst = inst,
110		};
111		const gop = try ip.tracked_insts.getOrPut(gpa, key);
112		return @enumFromInt(gop.index);
	124	const full_hash = Hash.hash(0, std.mem.asBytes(&key));
	125	const hash: u32 = @truncate(full_hash >> 32);
	126	const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))];
	127	var map = shard.shared.tracked_inst_map.acquire();
	128	const Map = @TypeOf(map);
	129	var map_mask = map.header().mask();
	130	var map_index = hash;
	131	while (true) : (map_index += 1) {
	132	map_index &= map_mask;
	133	const entry = &map.entries[map_index];
	134	const index = entry.acquire().unwrap() orelse break;
	135	if (entry.hash != hash) continue;
	136	if (std.meta.eql(index.resolveFull(ip), key)) return index;
	137	}
	138	shard.mutate.tracked_inst_map.mutex.lock();
	139	defer shard.mutate.tracked_inst_map.mutex.unlock();
	140	if (map.entries != shard.shared.tracked_inst_map.entries) {
	141	shard.mutate.tracked_inst_map.len += 1;
	142	map = shard.shared.tracked_inst_map;
	143	map_mask = map.header().mask();
	144	map_index = hash;
	145	}
	146	while (true) : (map_index += 1) {
	147	map_index &= map_mask;
	148	const entry = &map.entries[map_index];
	149	const index = entry.acquire().unwrap() orelse break;
	150	if (entry.hash != hash) continue;
	151	if (std.meta.eql(index.resolveFull(ip), key)) return index;
	152	}
	153	defer shard.mutate.tracked_inst_map.len += 1;
	154	const local = ip.getLocal(tid);
	155	local.mutate.tracked_insts.mutex.lock();
	156	defer local.mutate.tracked_insts.mutex.unlock();
	157	const list = local.getMutableTrackedInsts(gpa);
	158	try list.ensureUnusedCapacity(1);
	159	const map_header = map.header().*;
	160	if (shard.mutate.tracked_inst_map.len < map_header.capacity * 3 / 5) {
	161	const entry = &map.entries[map_index];
	162	entry.hash = hash;
	163	const index = (TrackedInst.Index.Unwrapped{
	164	.tid = tid,
	165	.index = list.mutate.len,
	166	}).wrap(ip);
	167	list.appendAssumeCapacity(.{key});
	168	entry.release(index.toOptional());
	169	return index;
	170	}
	171	const arena_state = &local.mutate.arena;
	172	var arena = arena_state.promote(gpa);
	173	defer arena_state.* = arena.state;
	174	const new_map_capacity = map_header.capacity * 2;
	175	const new_map_buf = try arena.allocator().alignedAlloc(
	176	u8,
	177	Map.alignment,
	178	Map.entries_offset + new_map_capacity * @sizeOf(Map.Entry),
	179	);
	180	const new_map: Map = .{ .entries = @ptrCast(new_map_buf[Map.entries_offset..].ptr) };
	181	new_map.header().* = .{ .capacity = new_map_capacity };
	182	@memset(new_map.entries[0..new_map_capacity], .{ .value = .none, .hash = undefined });
	183	const new_map_mask = new_map.header().mask();
	184	map_index = 0;
	185	while (map_index < map_header.capacity) : (map_index += 1) {
	186	const entry = &map.entries[map_index];
	187	const index = entry.value.unwrap() orelse continue;
	188	const item_hash = entry.hash;
	189	var new_map_index = item_hash;
	190	while (true) : (new_map_index += 1) {
	191	new_map_index &= new_map_mask;
	192	const new_entry = &new_map.entries[new_map_index];
	193	if (new_entry.value != .none) continue;
	194	new_entry.* = .{
	195	.value = index.toOptional(),
	196	.hash = item_hash,
	197	};
	198	break;
	199	}
	200	}
	201	map = new_map;
	202	map_index = hash;
	203	while (true) : (map_index += 1) {
	204	map_index &= new_map_mask;
	205	if (map.entries[map_index].value == .none) break;
	206	}
	207	const index = (TrackedInst.Index.Unwrapped{
	208	.tid = tid,
	209	.index = list.mutate.len,
	210	}).wrap(ip);
	211	list.appendAssumeCapacity(.{key});
	212	map.entries[map_index] = .{ .value = index.toOptional(), .hash = hash };
	213	shard.shared.tracked_inst_map.release(new_map);
	214	return index;
113	215	}
114	216
115	217	/// Analysis Unit. Represents a single entity which undergoes semantic analysis.
...	...	@@ -324,6 +426,7 @@ const Local = struct {
324	426	extra: ListMutate,
325	427	limbs: ListMutate,
326	428	strings: ListMutate,
	429	tracked_insts: MutexListMutate,
327	430	files: ListMutate,
328	431
329	432	decls: BucketListMutate,
...	...	@@ -335,6 +438,7 @@ const Local = struct {
335	438	extra: Extra,
336	439	limbs: Limbs,
337	440	strings: Strings,
	441	tracked_insts: TrackedInsts,
338	442	files: List(File),
339	443
340	444	decls: Decls,
...	...	@@ -356,6 +460,7 @@ const Local = struct {
356	460	else => @compileError("unsupported host"),
357	461	};
358	462	const Strings = List(struct { u8 });
	463	const TrackedInsts = List(struct { TrackedInst });
359	464
360	465	const decls_bucket_width = 8;
361	466	const decls_bucket_mask = (1 << decls_bucket_width) - 1;
...	...	@@ -375,6 +480,16 @@ const Local = struct {
375	480	};
376	481	};
377	482
	483	const MutexListMutate = struct {
	484	mutex: std.Thread.Mutex,
	485	list: ListMutate,
	486
	487	const empty: MutexListMutate = .{
	488	.mutex = .{},
	489	.list = ListMutate.empty,
	490	};
	491	};
	492
378	493	const BucketListMutate = struct {
379	494	last_bucket_len: u32,
380	495	buckets_list: ListMutate,
...	...	@@ -396,7 +511,7 @@ const Local = struct {
396	511
397	512	const ListSelf = @This();
398	513	const Mutable = struct {
399		gpa: std.mem.Allocator,
	514	gpa: Allocator,
400	515	arena: *std.heap.ArenaAllocator.State,
401	516	mutate: *ListMutate,
402	517	list: *ListSelf,
...	...	@@ -564,7 +679,7 @@ const Local = struct {
564	679	mutable.list.release(new_list);
565	680	}
566	681
567		fn view(mutable: Mutable) View {
	682	pub fn view(mutable: Mutable) View {
568	683	const capacity = mutable.list.header().capacity;
569	684	assert(capacity > 0); // optimizes `MultiArrayList.Slice.items`
570	685	return .{
...	...	@@ -614,7 +729,7 @@ const Local = struct {
614	729	};
615	730	}
616	731
617		pub fn getMutableItems(local: *Local, gpa: std.mem.Allocator) List(Item).Mutable {
	732	pub fn getMutableItems(local: *Local, gpa: Allocator) List(Item).Mutable {
618	733	return .{
619	734	.gpa = gpa,
620	735	.arena = &local.mutate.arena,
...	...	@@ -623,7 +738,7 @@ const Local = struct {
623	738	};
624	739	}
625	740
626		pub fn getMutableExtra(local: *Local, gpa: std.mem.Allocator) Extra.Mutable {
	741	pub fn getMutableExtra(local: *Local, gpa: Allocator) Extra.Mutable {
627	742	return .{
628	743	.gpa = gpa,
629	744	.arena = &local.mutate.arena,
...	...	@@ -636,7 +751,7 @@ const Local = struct {
636	751	/// On 64-bit systems, this array is used for big integers and associated metadata.
637	752	/// Use the helper methods instead of accessing this directly in order to not
638	753	/// violate the above mechanism.
639		pub fn getMutableLimbs(local: *Local, gpa: std.mem.Allocator) Limbs.Mutable {
	754	pub fn getMutableLimbs(local: *Local, gpa: Allocator) Limbs.Mutable {
640	755	return switch (@sizeOf(Limb)) {
641	756	@sizeOf(u32) => local.getMutableExtra(gpa),
642	757	@sizeOf(u64) => .{
...	...	@@ -654,7 +769,7 @@ const Local = struct {
654	769	/// is referencing the data here whether they want to store both index and length,
655	770	/// thus allowing null bytes, or store only index, and use null-termination. The
656	771	/// `strings` array is agnostic to either usage.
657		pub fn getMutableStrings(local: *Local, gpa: std.mem.Allocator) Strings.Mutable {
	772	pub fn getMutableStrings(local: *Local, gpa: Allocator) Strings.Mutable {
658	773	return .{
659	774	.gpa = gpa,
660	775	.arena = &local.mutate.arena,
...	...	@@ -663,6 +778,17 @@ const Local = struct {
663	778	};
664	779	}
665	780
	781	/// An index into `tracked_insts` gives a reference to a single ZIR instruction which
	782	/// persists across incremental updates.
	783	pub fn getMutableTrackedInsts(local: *Local, gpa: Allocator) TrackedInsts.Mutable {
	784	return .{
	785	.gpa = gpa,
	786	.arena = &local.mutate.arena,
	787	.mutate = &local.mutate.tracked_insts.list,
	788	.list = &local.shared.tracked_insts,
	789	};
	790	}
	791
666	792	/// Elements are ordered identically to the `import_table` field of `Zcu`.
667	793	///
668	794	/// Unlike `import_table`, this data is serialized as part of incremental
...	...	@@ -672,7 +798,7 @@ const Local = struct {
672	798	/// `InternPool.TrackedInst`.
673	799	///
674	800	/// Value is the `Decl` of the struct that represents this `File`.
675		pub fn getMutableFiles(local: *Local, gpa: std.mem.Allocator) List(File).Mutable {
	801	pub fn getMutableFiles(local: *Local, gpa: Allocator) List(File).Mutable {
676	802	return .{
677	803	.gpa = gpa,
678	804	.arena = &local.mutate.arena,
...	...	@@ -691,7 +817,7 @@ const Local = struct {
691	817	/// serialization trivial.
692	818	/// * It provides a unique integer to be used for anonymous symbol names, avoiding
693	819	/// multi-threaded contention on an atomic counter.
694		pub fn getMutableDecls(local: *Local, gpa: std.mem.Allocator) Decls.Mutable {
	820	pub fn getMutableDecls(local: *Local, gpa: Allocator) Decls.Mutable {
695	821	return .{
696	822	.gpa = gpa,
697	823	.arena = &local.mutate.arena,
...	...	@@ -701,7 +827,7 @@ const Local = struct {
701	827	}
702	828
703	829	/// Same pattern as with `getMutableDecls`.
704		pub fn getMutableNamespaces(local: *Local, gpa: std.mem.Allocator) Namespaces.Mutable {
	830	pub fn getMutableNamespaces(local: *Local, gpa: Allocator) Namespaces.Mutable {
705	831	return .{
706	832	.gpa = gpa,
707	833	.arena = &local.mutate.arena,
...	...	@@ -723,11 +849,13 @@ const Shard = struct {
723	849	shared: struct {
724	850	map: Map(Index),
725	851	string_map: Map(OptionalNullTerminatedString),
	852	tracked_inst_map: Map(TrackedInst.Index.Optional),
726	853	} align(std.atomic.cache_line),
727	854	mutate: struct {
728	855	// TODO: measure cost of sharing unrelated mutate state
729	856	map: Mutate align(std.atomic.cache_line),
730	857	string_map: Mutate align(std.atomic.cache_line),
	858	tracked_inst_map: Mutate align(std.atomic.cache_line),
731	859	},
732	860
733	861	const Mutate = struct {
...	...	@@ -5240,6 +5368,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
5240	5368	.extra = Local.Extra.empty,
5241	5369	.limbs = Local.Limbs.empty,
5242	5370	.strings = Local.Strings.empty,
	5371	.tracked_insts = Local.TrackedInsts.empty,
5243	5372	.files = Local.List(File).empty,
5244	5373
5245	5374	.decls = Local.Decls.empty,
...	...	@@ -5252,6 +5381,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
5252	5381	.extra = Local.ListMutate.empty,
5253	5382	.limbs = Local.ListMutate.empty,
5254	5383	.strings = Local.ListMutate.empty,
	5384	.tracked_insts = Local.MutexListMutate.empty,
5255	5385	.files = Local.ListMutate.empty,
5256	5386
5257	5387	.decls = Local.BucketListMutate.empty,
...	...	@@ -5267,10 +5397,12 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
5267	5397	.shared = .{
5268	5398	.map = Shard.Map(Index).empty,
5269	5399	.string_map = Shard.Map(OptionalNullTerminatedString).empty,
	5400	.tracked_inst_map = Shard.Map(TrackedInst.Index.Optional).empty,
5270	5401	},
5271	5402	.mutate = .{
5272	5403	.map = Shard.Mutate.empty,
5273	5404	.string_map = Shard.Mutate.empty,
	5405	.tracked_inst_map = Shard.Mutate.empty,
5274	5406	},
5275	5407	});
5276	5408
...	...	@@ -5311,8 +5443,6 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {
5311	5443	for (ip.maps.items) \|*map\| map.deinit(gpa);
5312	5444	ip.maps.deinit(gpa);
5313	5445
5314		ip.tracked_insts.deinit(gpa);
5315
5316	5446	ip.src_hash_deps.deinit(gpa);
5317	5447	ip.decl_val_deps.deinit(gpa);
5318	5448	ip.func_ies_deps.deinit(gpa);
...	...	@@ -9887,7 +10017,7 @@ pub fn getOrPutTrailingString(
9887	10017	}
9888	10018	const key: []const u8 = strings.view().items(.@"0")[start..];
9889	10019	const value: embedded_nulls.StringType() =
9890		@enumFromInt(@as(u32, @intFromEnum(tid)) << ip.tid_shift_32 \| start);
	10020	@enumFromInt(@intFromEnum((String.Unwrapped{ .tid = tid, .index = start }).wrap(ip)));
9891	10021	const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
9892	10022	switch (embedded_nulls) {
9893	10023	.no_embedded_nulls => assert(!has_embedded_null),

src/Sema.zig+5-6

...	...	@@ -835,12 +835,11 @@ pub const Block = struct {
835	835	}
836	836
837	837	fn trackZir(block: *Block, inst: Zir.Inst.Index) Allocator.Error!InternPool.TrackedInst.Index {
838		const sema = block.sema;
839		const gpa = sema.gpa;
840		const zcu = sema.pt.zcu;
841		const ip = &zcu.intern_pool;
842		const file_index = block.getFileScopeIndex(zcu);
843		return ip.trackZir(gpa, file_index, inst);
	838	const pt = block.sema.pt;
	839	return pt.zcu.intern_pool.trackZir(pt.zcu.gpa, pt.tid, .{
	840	.file = block.getFileScopeIndex(pt.zcu),
	841	.inst = inst,
	842	});
844	843	}
845	844	};
846	845

src/Zcu.zig+8-35

...	...	@@ -1018,6 +1018,14 @@ pub const ErrorMsg = struct {
1018	1018	}
1019	1019	};
1020	1020
	1021	pub const AstGenSrc = union(enum) {
	1022	root,
	1023	import: struct {
	1024	importing_file: Zcu.File.Index,
	1025	import_tok: std.zig.Ast.TokenIndex,
	1026	},
	1027	};
	1028
1021	1029	/// Canonical reference to a position within a source file.
1022	1030	pub const SrcLoc = struct {
1023	1031	file_scope: *File,
...	...	@@ -3186,41 +3194,6 @@ pub fn handleUpdateExports(
3186	3194	};
3187	3195	}
3188	3196
3189		pub fn reportRetryableFileError(
3190		zcu: *Zcu,
3191		file_index: File.Index,
3192		comptime format: []const u8,
3193		args: anytype,
3194		) error{OutOfMemory}!void {
3195		const gpa = zcu.gpa;
3196		const ip = &zcu.intern_pool;
3197
3198		const file = zcu.fileByIndex(file_index);
3199		file.status = .retryable_failure;
3200
3201		const err_msg = try ErrorMsg.create(
3202		gpa,
3203		.{
3204		.base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
3205		.offset = .entire_file,
3206		},
3207		format,
3208		args,
3209		);
3210		errdefer err_msg.destroy(gpa);
3211
3212		zcu.comp.mutex.lock();
3213		defer zcu.comp.mutex.unlock();
3214
3215		const gop = try zcu.failed_files.getOrPut(gpa, file);
3216		if (gop.found_existing) {
3217		if (gop.value_ptr.*) \|old_err_msg\| {
3218		old_err_msg.destroy(gpa);
3219		}
3220		}
3221		gop.value_ptr.* = err_msg;
3222		}
3223
3224	3197	pub fn addGlobalAssembly(mod: *Module, decl_index: Decl.Index, source: []const u8) !void {
3225	3198	const gop = try mod.global_assembly.getOrPut(mod.gpa, decl_index);
3226	3199	if (gop.found_existing) {

src/Zcu/PerThread.zig+191-96

...	...	@@ -342,6 +342,7 @@ pub fn astGenFile(
342	342	/// the Compilation mutex when acting on shared state.
343	343	fn updateZirRefs(pt: Zcu.PerThread, file: *Zcu.File, file_index: Zcu.File.Index, old_zir: Zir) !void {
344	344	const zcu = pt.zcu;
	345	const ip = &zcu.intern_pool;
345	346	const gpa = zcu.gpa;
346	347	const new_zir = file.zir;
347	348
...	...	@@ -355,109 +356,117 @@ fn updateZirRefs(pt: Zcu.PerThread, file: *Zcu.File, file_index: Zcu.File.Index,
355	356
356	357	// TODO: this should be done after all AstGen workers complete, to avoid
357	358	// iterating over this full set for every updated file.
358		for (zcu.intern_pool.tracked_insts.keys(), 0..) \|*ti, idx_raw\| {
359		const ti_idx: InternPool.TrackedInst.Index = @enumFromInt(idx_raw);
360		if (ti.file != file_index) continue;
361		const old_inst = ti.inst;
362		ti.inst = inst_map.get(ti.inst) orelse {
363		// Tracking failed for this instruction. Invalidate associated `src_hash` deps.
364		zcu.comp.mutex.lock();
365		defer zcu.comp.mutex.unlock();
366		log.debug("tracking failed for %{d}", .{old_inst});
367		try zcu.markDependeeOutdated(.{ .src_hash = ti_idx });
368		continue;
369		};
	359	for (ip.locals, 0..) \|*local, tid\| {
	360	local.mutate.tracked_insts.mutex.lock();
	361	defer local.mutate.tracked_insts.mutex.unlock();
	362	const tracked_insts_list = local.getMutableTrackedInsts(gpa);
	363	for (tracked_insts_list.view().items(.@"0"), 0..) \|*tracked_inst, tracked_inst_unwrapped_index\| {
	364	if (tracked_inst.file != file_index) continue;
	365	const old_inst = tracked_inst.inst;
	366	const tracked_inst_index = (InternPool.TrackedInst.Index.Unwrapped{
	367	.tid = @enumFromInt(tid),
	368	.index = @intCast(tracked_inst_unwrapped_index),
	369	}).wrap(ip);
	370	tracked_inst.inst = inst_map.get(old_inst) orelse {
	371	// Tracking failed for this instruction. Invalidate associated `src_hash` deps.
	372	zcu.comp.mutex.lock();
	373	defer zcu.comp.mutex.unlock();
	374	log.debug("tracking failed for %{d}", .{old_inst});
	375	try zcu.markDependeeOutdated(.{ .src_hash = tracked_inst_index });
	376	continue;
	377	};
370	378
371		if (old_zir.getAssociatedSrcHash(old_inst)) \|old_hash\| hash_changed: {
372		if (new_zir.getAssociatedSrcHash(ti.inst)) \|new_hash\| {
373		if (std.zig.srcHashEql(old_hash, new_hash)) {
374		break :hash_changed;
	379	if (old_zir.getAssociatedSrcHash(old_inst)) \|old_hash\| hash_changed: {
	380	if (new_zir.getAssociatedSrcHash(tracked_inst.inst)) \|new_hash\| {
	381	if (std.zig.srcHashEql(old_hash, new_hash)) {
	382	break :hash_changed;
	383	}
	384	log.debug("hash for (%{d} -> %{d}) changed: {} -> {}", .{
	385	old_inst,
	386	tracked_inst.inst,
	387	std.fmt.fmtSliceHexLower(&old_hash),
	388	std.fmt.fmtSliceHexLower(&new_hash),
	389	});
375	390	}
376		log.debug("hash for (%{d} -> %{d}) changed: {} -> {}", .{
377		old_inst,
378		ti.inst,
379		std.fmt.fmtSliceHexLower(&old_hash),
380		std.fmt.fmtSliceHexLower(&new_hash),
381		});
	391	// The source hash associated with this instruction changed - invalidate relevant dependencies.
	392	zcu.comp.mutex.lock();
	393	defer zcu.comp.mutex.unlock();
	394	try zcu.markDependeeOutdated(.{ .src_hash = tracked_inst_index });
382	395	}
383		// The source hash associated with this instruction changed - invalidate relevant dependencies.
384		zcu.comp.mutex.lock();
385		defer zcu.comp.mutex.unlock();
386		try zcu.markDependeeOutdated(.{ .src_hash = ti_idx });
387		}
388	396
389		// If this is a `struct_decl` etc, we must invalidate any outdated namespace dependencies.
390		const has_namespace = switch (old_tag[@intFromEnum(old_inst)]) {
391		.extended => switch (old_data[@intFromEnum(old_inst)].extended.opcode) {
392		.struct_decl, .union_decl, .opaque_decl, .enum_decl => true,
	397	// If this is a `struct_decl` etc, we must invalidate any outdated namespace dependencies.
	398	const has_namespace = switch (old_tag[@intFromEnum(old_inst)]) {
	399	.extended => switch (old_data[@intFromEnum(old_inst)].extended.opcode) {
	400	.struct_decl, .union_decl, .opaque_decl, .enum_decl => true,
	401	else => false,
	402	},
393	403	else => false,
394		},
395		else => false,
396		};
397		if (!has_namespace) continue;
398
399		var old_names: std.AutoArrayHashMapUnmanaged(InternPool.NullTerminatedString, void) = .{};
400		defer old_names.deinit(zcu.gpa);
401		{
402		var it = old_zir.declIterator(old_inst);
403		while (it.next()) \|decl_inst\| {
404		const decl_name = old_zir.getDeclaration(decl_inst)[0].name;
405		switch (decl_name) {
406		.@"comptime", .@"usingnamespace", .unnamed_test, .decltest => continue,
407		_ => if (decl_name.isNamedTest(old_zir)) continue,
	404	};
	405	if (!has_namespace) continue;
	406
	407	var old_names: std.AutoArrayHashMapUnmanaged(InternPool.NullTerminatedString, void) = .{};
	408	defer old_names.deinit(zcu.gpa);
	409	{
	410	var it = old_zir.declIterator(old_inst);
	411	while (it.next()) \|decl_inst\| {
	412	const decl_name = old_zir.getDeclaration(decl_inst)[0].name;
	413	switch (decl_name) {
	414	.@"comptime", .@"usingnamespace", .unnamed_test, .decltest => continue,
	415	_ => if (decl_name.isNamedTest(old_zir)) continue,
	416	}
	417	const name_zir = decl_name.toString(old_zir).?;
	418	const name_ip = try zcu.intern_pool.getOrPutString(
	419	zcu.gpa,
	420	pt.tid,
	421	old_zir.nullTerminatedString(name_zir),
	422	.no_embedded_nulls,
	423	);
	424	try old_names.put(zcu.gpa, name_ip, {});
408	425	}
409		const name_zir = decl_name.toString(old_zir).?;
410		const name_ip = try zcu.intern_pool.getOrPutString(
411		zcu.gpa,
412		pt.tid,
413		old_zir.nullTerminatedString(name_zir),
414		.no_embedded_nulls,
415		);
416		try old_names.put(zcu.gpa, name_ip, {});
417	426	}
418		}
419		var any_change = false;
420		{
421		var it = new_zir.declIterator(ti.inst);
422		while (it.next()) \|decl_inst\| {
423		const decl_name = old_zir.getDeclaration(decl_inst)[0].name;
424		switch (decl_name) {
425		.@"comptime", .@"usingnamespace", .unnamed_test, .decltest => continue,
426		_ => if (decl_name.isNamedTest(old_zir)) continue,
	427	var any_change = false;
	428	{
	429	var it = new_zir.declIterator(tracked_inst.inst);
	430	while (it.next()) \|decl_inst\| {
	431	const decl_name = old_zir.getDeclaration(decl_inst)[0].name;
	432	switch (decl_name) {
	433	.@"comptime", .@"usingnamespace", .unnamed_test, .decltest => continue,
	434	_ => if (decl_name.isNamedTest(old_zir)) continue,
	435	}
	436	const name_zir = decl_name.toString(old_zir).?;
	437	const name_ip = try zcu.intern_pool.getOrPutString(
	438	zcu.gpa,
	439	pt.tid,
	440	old_zir.nullTerminatedString(name_zir),
	441	.no_embedded_nulls,
	442	);
	443	if (!old_names.swapRemove(name_ip)) continue;
	444	// Name added
	445	any_change = true;
	446	zcu.comp.mutex.lock();
	447	defer zcu.comp.mutex.unlock();
	448	try zcu.markDependeeOutdated(.{ .namespace_name = .{
	449	.namespace = tracked_inst_index,
	450	.name = name_ip,
	451	} });
427	452	}
428		const name_zir = decl_name.toString(old_zir).?;
429		const name_ip = try zcu.intern_pool.getOrPutString(
430		zcu.gpa,
431		pt.tid,
432		old_zir.nullTerminatedString(name_zir),
433		.no_embedded_nulls,
434		);
435		if (!old_names.swapRemove(name_ip)) continue;
436		// Name added
	453	}
	454	// The only elements remaining in `old_names` now are any names which were removed.
	455	for (old_names.keys()) \|name_ip\| {
437	456	any_change = true;
438	457	zcu.comp.mutex.lock();
439	458	defer zcu.comp.mutex.unlock();
440	459	try zcu.markDependeeOutdated(.{ .namespace_name = .{
441		.namespace = ti_idx,
	460	.namespace = tracked_inst_index,
442	461	.name = name_ip,
443	462	} });
444	463	}
445		}
446		// The only elements remaining in `old_names` now are any names which were removed.
447		for (old_names.keys()) \|name_ip\| {
448		any_change = true;
449		zcu.comp.mutex.lock();
450		defer zcu.comp.mutex.unlock();
451		try zcu.markDependeeOutdated(.{ .namespace_name = .{
452		.namespace = ti_idx,
453		.name = name_ip,
454		} });
455		}
456	464
457		if (any_change) {
458		zcu.comp.mutex.lock();
459		defer zcu.comp.mutex.unlock();
460		try zcu.markDependeeOutdated(.{ .namespace = ti_idx });
	465	if (any_change) {
	466	zcu.comp.mutex.lock();
	467	defer zcu.comp.mutex.unlock();
	468	try zcu.markDependeeOutdated(.{ .namespace = tracked_inst_index });
	469	}
461	470	}
462	471	}
463	472	}
...	...	@@ -854,7 +863,10 @@ fn getFileRootStruct(
854	863	const decls = file.zir.bodySlice(extra_index, decls_len);
855	864	extra_index += decls_len;
856	865
857		const tracked_inst = try ip.trackZir(gpa, file_index, .main_struct_inst);
	866	const tracked_inst = try ip.trackZir(gpa, pt.tid, .{
	867	.file = file_index,
	868	.inst = .main_struct_inst,
	869	});
858	870	const wip_ty = switch (try ip.getStructType(gpa, pt.tid, .{
859	871	.layout = .auto,
860	872	.fields_len = fields_len,
...	...	@@ -1015,7 +1027,7 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void {
1015	1027	switch (zcu.comp.cache_use) {
1016	1028	.whole => \|whole\| if (whole.cache_manifest) \|man\| {
1017	1029	const source = file.getSource(gpa) catch \|err\| {
1018		try Zcu.reportRetryableFileError(zcu, file_index, "unable to load source: {s}", .{@errorName(err)});
	1030	try pt.reportRetryableFileError(file_index, "unable to load source: {s}", .{@errorName(err)});
1019	1031	return error.AnalysisFail;
1020	1032	};
1021	1033
...	...	@@ -1024,7 +1036,7 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void {
1024	1036	file.mod.root.sub_path,
1025	1037	file.sub_file_path,
1026	1038	}) catch \|err\| {
1027		try Zcu.reportRetryableFileError(zcu, file_index, "unable to resolve path: {s}", .{@errorName(err)});
	1039	try pt.reportRetryableFileError(file_index, "unable to resolve path: {s}", .{@errorName(err)});
1028	1040	return error.AnalysisFail;
1029	1041	};
1030	1042	errdefer gpa.free(resolved_path);
...	...	@@ -1148,11 +1160,10 @@ fn semaDecl(pt: Zcu.PerThread, decl_index: Zcu.Decl.Index) !Zcu.SemaDeclResult {
1148	1160	defer sema.deinit();
1149	1161
1150	1162	// Every Decl (other than file root Decls, which do not have a ZIR index) has a dependency on its own source.
1151		try sema.declareDependency(.{ .src_hash = try ip.trackZir(
1152		gpa,
1153		decl.getFileScopeIndex(zcu),
1154		decl_inst,
1155		) });
	1163	try sema.declareDependency(.{ .src_hash = try ip.trackZir(gpa, pt.tid, .{
	1164	.file = decl.getFileScopeIndex(zcu),
	1165	.inst = decl_inst,
	1166	}) });
1156	1167
1157	1168	var block_scope: Sema.Block = .{
1158	1169	.parent = null,
...	...	@@ -1890,7 +1901,10 @@ const ScanDeclIter = struct {
1890	1901	}
1891	1902
1892	1903	const parent_file_scope_index = iter.parent_decl.getFileScopeIndex(zcu);
1893		const tracked_inst = try ip.trackZir(gpa, parent_file_scope_index, decl_inst);
	1904	const tracked_inst = try ip.trackZir(gpa, pt.tid, .{
	1905	.file = parent_file_scope_index,
	1906	.inst = decl_inst,
	1907	});
1894	1908
1895	1909	// We create a Decl for it regardless of analysis status.
1896	1910
...	...	@@ -2611,6 +2625,87 @@ pub fn linkerUpdateDecl(pt: Zcu.PerThread, decl_index: Zcu.Decl.Index) !void {
2611	2625	}
2612	2626	}
2613	2627
	2628	pub fn reportRetryableAstGenError(
	2629	pt: Zcu.PerThread,
	2630	src: Zcu.AstGenSrc,
	2631	file_index: Zcu.File.Index,
	2632	err: anyerror,
	2633	) error{OutOfMemory}!void {
	2634	const zcu = pt.zcu;
	2635	const gpa = zcu.gpa;
	2636	const ip = &zcu.intern_pool;
	2637
	2638	const file = zcu.fileByIndex(file_index);
	2639	file.status = .retryable_failure;
	2640
	2641	const src_loc: Zcu.LazySrcLoc = switch (src) {
	2642	.root => .{
	2643	.base_node_inst = try ip.trackZir(gpa, pt.tid, .{
	2644	.file = file_index,
	2645	.inst = .main_struct_inst,
	2646	}),
	2647	.offset = .entire_file,
	2648	},
	2649	.import => \|info\| .{
	2650	.base_node_inst = try ip.trackZir(gpa, pt.tid, .{
	2651	.file = info.importing_file,
	2652	.inst = .main_struct_inst,
	2653	}),
	2654	.offset = .{ .token_abs = info.import_tok },
	2655	},
	2656	};
	2657
	2658	const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{
	2659	file.mod.root, file.sub_file_path, @errorName(err),
	2660	});
	2661	errdefer err_msg.destroy(gpa);
	2662
	2663	{
	2664	zcu.comp.mutex.lock();
	2665	defer zcu.comp.mutex.unlock();
	2666	try zcu.failed_files.putNoClobber(gpa, file, err_msg);
	2667	}
	2668	}
	2669
	2670	pub fn reportRetryableFileError(
	2671	pt: Zcu.PerThread,
	2672	file_index: Zcu.File.Index,
	2673	comptime format: []const u8,
	2674	args: anytype,
	2675	) error{OutOfMemory}!void {
	2676	const zcu = pt.zcu;
	2677	const gpa = zcu.gpa;
	2678	const ip = &zcu.intern_pool;
	2679
	2680	const file = zcu.fileByIndex(file_index);
	2681	file.status = .retryable_failure;
	2682
	2683	const err_msg = try Zcu.ErrorMsg.create(
	2684	gpa,
	2685	.{
	2686	.base_node_inst = try ip.trackZir(gpa, pt.tid, .{
	2687	.file = file_index,
	2688	.inst = .main_struct_inst,
	2689	}),
	2690	.offset = .entire_file,
	2691	},
	2692	format,
	2693	args,
	2694	);
	2695	errdefer err_msg.destroy(gpa);
	2696
	2697	zcu.comp.mutex.lock();
	2698	defer zcu.comp.mutex.unlock();
	2699
	2700	const gop = try zcu.failed_files.getOrPut(gpa, file);
	2701	if (gop.found_existing) {
	2702	if (gop.value_ptr.*) \|old_err_msg\| {
	2703	old_err_msg.destroy(gpa);
	2704	}
	2705	}
	2706	gop.value_ptr.* = err_msg;
	2707	}
	2708
2614	2709	/// Shortcut for calling `intern_pool.get`.
2615	2710	pub fn intern(pt: Zcu.PerThread, key: InternPool.Key) Allocator.Error!InternPool.Index {
2616	2711	return pt.zcu.intern_pool.get(pt.zcu.gpa, pt.tid, key);