Commit 488d804a1c205583e22aee4076e41f497b3ef1b0 | zig

author	Jacob Young <jacobly@ziglang.org> 2023-04-08 07:36:35-04:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2023-04-20 20:49:34+01:00
log	488d804a1c205583e22aee4076e41f497b3ef1b0
tree	d3a71ecda95b19f428b6c1f675b2746eacb2ec50
parent	6f09a7041ecf328f761df60fae007c800ee1e2ec
signature	Commit is signed but in an unrecognized format.

x86_64: rewrite inst tracking

3 files changed, 495 insertions(+), 667 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
- register_manager.zig
test
- behavior
  - for.zig

src/arch/x86_64/CodeGen.zig+491-666

...	...	@@ -79,14 +79,8 @@ end_di_column: u32,
79	79	/// which is a relative jump, based on the address following the reloc.
80	80	exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
81	81
82		/// Whenever there is a runtime branch, we push a Branch onto this stack,
83		/// and pop it off when the runtime branch joins. This provides an "overlay"
84		/// of the table of mappings from instructions to `MCValue` from within the branch.
85		/// This way we can modify the `MCValue` for an instruction in different ways
86		/// within different branches. Special consideration is needed when a branch
87		/// joins with its parent, to make sure all instructions have the same MCValue
88		/// across each runtime branch upon joining.
89		branch_stack: *std.ArrayList(Branch),
	82	const_tracking: InstTrackingMap = .{},
	83	inst_tracking: InstTrackingMap = .{},
90	84
91	85	// Key is the block instruction
92	86	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
...	...	@@ -95,6 +89,9 @@ register_manager: RegisterManager = .{},
95	89	/// Maps offset to what is stored there.
96	90	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
97	91
	92	/// Index of the current scope.
	93	scope_index: u32 = 0,
	94
98	95	/// Offset from the stack base, representing the end of the stack frame.
99	96	max_end_stack: u32 = 0,
100	97	/// Represents the current end stack offset. If there is no existing slot
...	...	@@ -105,10 +102,12 @@ next_stack_offset: u32 = 0,
105	102	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
106	103
107	104	/// For mir debug info, maps a mir index to a air index
108		mir_to_air_map: if (builtin.mode == .Debug) std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index) else void,
	105	mir_to_air_map: @TypeOf(mir_to_air_map_init) = mir_to_air_map_init,
109	106
110	107	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
111	108
	109	const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged(Mir.Inst.Index, Air.Inst.Index){} else {};
	110
112	111	pub const MCValue = union(enum) {
113	112	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
114	113	/// TODO Look into deleting this tag and using `dead` instead, since every use
...	...	@@ -117,7 +116,8 @@ pub const MCValue = union(enum) {
117	116	/// Control flow will not allow this value to be observed.
118	117	unreach,
119	118	/// No more references to this value remain.
120		dead,
	119	/// The payload is the value of scope_index at the point where the death occurred
	120	dead: u32,
121	121	/// The value is undefined.
122	122	undef,
123	123	/// A pointer-sized integer that fits in a register.
...	...	@@ -183,47 +183,92 @@ pub const MCValue = union(enum) {
183	183	}
184	184	};
185	185
186		const Branch = struct {
187		inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .{},
	186	const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);
	187	const InstTracking = struct {
	188	long: MCValue,
	189	short: MCValue,
188	190
189		fn deinit(self: *Branch, gpa: Allocator) void {
190		self.inst_table.deinit(gpa);
191		self.* = undefined;
	191	fn init(result: MCValue) InstTracking {
	192	return .{ .long = result, .short = result };
	193	}
	194
	195	fn getReg(self: InstTracking) ?Register {
	196	return switch (self.short) {
	197	.register => \|reg\| reg,
	198	.register_overflow => \|ro\| ro.reg,
	199	else => null,
	200	};
	201	}
	202
	203	fn getCondition(self: InstTracking) ?Condition {
	204	return switch (self.short) {
	205	.eflags => \|eflags\| eflags,
	206	.register_overflow => \|ro\| ro.eflags,
	207	else => null,
	208	};
	209	}
	210
	211	fn spill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
	212	switch (self.long) {
	213	.none,
	214	.dead,
	215	.unreach,
	216	=> unreachable,
	217	.register,
	218	.register_overflow,
	219	.eflags,
	220	=> self.long = try function.allocRegOrMem(inst, self.short == .eflags),
	221	.stack_offset => {},
	222	.undef,
	223	.immediate,
	224	.memory,
	225	.linker_load,
	226	.tlv_reloc,
	227	.ptr_stack_offset,
	228	=> return, // these can be rematerialized without using a stack slot
	229	}
	230	log.debug("spilling %{d} from {} to {}", .{ inst, self.short, self.long });
	231	const ty = function.air.typeOfIndex(inst);
	232	try function.setRegOrMem(ty, self.long, self.short);
192	233	}
193	234
194		const FormatContext = struct {
195		insts: []const Air.Inst.Index,
196		mcvs: []const MCValue,
197		};
198
199		fn fmt(
200		ctx: FormatContext,
201		comptime unused_format_string: []const u8,
202		options: std.fmt.FormatOptions,
203		writer: anytype,
204		) @TypeOf(writer).Error!void {
205		_ = options;
206		comptime assert(unused_format_string.len == 0);
207		try writer.writeAll("Branch {\n");
208		for (ctx.insts, ctx.mcvs) \|inst, mcv\| {
209		try writer.print(" %{d} => {}\n", .{ inst, mcv });
	235	fn trackSpill(self: InstTracking, function: Self) void {
	236	if (self.getReg()) \|reg\| function.register_manager.freeReg(reg);
	237	switch (self.short) {
	238	.none, .dead, .unreach => unreachable,
	239	else => {},
210	240	}
211		try writer.writeAll("}");
	241	self.short = self.long;
212	242	}
213	243
214		fn format(branch: Branch, comptime unused_format_string: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void {
215		_ = branch;
216		_ = unused_format_string;
217		_ = options;
218		_ = writer;
219		@compileError("do not format Branch directly; use ty.fmtDebug()");
	244	fn materialize(self: InstTracking, function: Self, inst: Air.Inst.Index, reg: Register) !void {
	245	const ty = function.air.typeOfIndex(inst);
	246	try function.genSetReg(ty, reg, self.long);
220	247	}
221	248
222		fn fmtDebug(self: @This()) std.fmt.Formatter(fmt) {
223		return .{ .data = .{
224		.insts = self.inst_table.keys(),
225		.mcvs = self.inst_table.values(),
226		} };
	249	fn trackMaterialize(self: InstTracking, function: Self, inst: Air.Inst.Index, reg: Register) void {
	250	assert(inst == function.register_manager.registers[
	251	RegisterManager.indexOfRegIntoTracked(reg).?
	252	]);
	253	self.short = .{ .register = reg };
	254	}
	255
	256	fn resurrect(self: *InstTracking, scope_index: u32) void {
	257	switch (self.short) {
	258	.dead => \|die_index\| if (die_index >= scope_index) {
	259	self.short = self.long;
	260	},
	261	else => {},
	262	}
	263	}
	264
	265	fn die(self: InstTracking, function: Self) void {
	266	function.freeValue(self.short);
	267	self.reuse(function);
	268	}
	269
	270	fn reuse(self: InstTracking, function: Self) void {
	271	self.short = .{ .dead = function.scope_index };
227	272	}
228	273	};
229	274
...	...	@@ -235,39 +280,14 @@ const StackAllocation = struct {
235	280
236	281	const BlockData = struct {
237	282	relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
238		branch: Branch = .{},
239		branch_depth: u32,
	283	state: State,
240	284
241	285	fn deinit(self: *BlockData, gpa: Allocator) void {
242		self.branch.deinit(gpa);
243	286	self.relocs.deinit(gpa);
244	287	self.* = undefined;
245	288	}
246	289	};
247	290
248		const BigTomb = struct {
249		function: *Self,
250		inst: Air.Inst.Index,
251		lbt: Liveness.BigTomb,
252
253		fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
254		const dies = bt.lbt.feed();
255		const op_index = Air.refToIndex(op_ref) orelse return;
256		if (!dies) return;
257		bt.function.processDeath(op_index);
258		}
259
260		fn finishAir(bt: *BigTomb, result: MCValue) void {
261		const is_used = !bt.function.liveness.isUnused(bt.inst);
262		if (is_used) {
263		log.debug(" (saving %{d} => {})", .{ bt.inst, result });
264		const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
265		branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
266		}
267		bt.function.finishAirBookkeeping();
268		}
269		};
270
271	291	const Self = @This();
272	292
273	293	pub fn generate(
...	...	@@ -294,19 +314,9 @@ pub fn generate(
294	314	stderr.writeAll(":\n") catch {};
295	315	}
296	316
297		var branch_stack = std.ArrayList(Branch).init(bin_file.allocator);
298		try branch_stack.ensureUnusedCapacity(2);
299		// The outermost branch is used for constants only.
300		branch_stack.appendAssumeCapacity(.{});
301		branch_stack.appendAssumeCapacity(.{});
302		defer {
303		assert(branch_stack.items.len == 2);
304		for (branch_stack.items) \|*branch\| branch.deinit(bin_file.allocator);
305		branch_stack.deinit();
306		}
307
	317	const gpa = bin_file.allocator;
308	318	var function = Self{
309		.gpa = bin_file.allocator,
	319	.gpa = gpa,
310	320	.air = air,
311	321	.liveness = liveness,
312	322	.target = &bin_file.options.target,
...	...	@@ -318,21 +328,21 @@ pub fn generate(
318	328	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
319	329	.fn_type = fn_type,
320	330	.arg_index = 0,
321		.branch_stack = &branch_stack,
322	331	.src_loc = src_loc,
323	332	.stack_align = undefined,
324	333	.end_di_line = module_fn.rbrace_line,
325	334	.end_di_column = module_fn.rbrace_column,
326		.mir_to_air_map = if (builtin.mode == .Debug)
327		std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator)
328		else {},
329	335	};
330		defer function.stack.deinit(bin_file.allocator);
331		defer function.blocks.deinit(bin_file.allocator);
332		defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
333		defer function.mir_instructions.deinit(bin_file.allocator);
334		defer function.mir_extra.deinit(bin_file.allocator);
335		defer if (builtin.mode == .Debug) function.mir_to_air_map.deinit();
	336	defer {
	337	function.stack.deinit(gpa);
	338	function.blocks.deinit(gpa);
	339	function.inst_tracking.deinit(gpa);
	340	function.const_tracking.deinit(gpa);
	341	function.exitlude_jump_relocs.deinit(gpa);
	342	function.mir_instructions.deinit(gpa);
	343	function.mir_extra.deinit(gpa);
	344	if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);
	345	}
336	346
337	347	var call_info = function.resolveCallingConventionValues(fn_type, &.{}) catch \|err\| switch (err) {
338	348	error.CodegenFail => return Result{ .fail = function.err_msg.? },
...	...	@@ -911,9 +921,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
911	921	}
912	922
913	923	const old_air_bookkeeping = self.air_bookkeeping;
914		try self.ensureProcessDeathCapacity(Liveness.bpi);
	924	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
915	925	if (builtin.mode == .Debug) {
916		try self.mir_to_air_map.put(@intCast(Mir.Inst.Index, self.mir_instructions.len), inst);
	926	const mir_inst = @intCast(Mir.Inst.Index, self.mir_instructions.len);
	927	try self.mir_to_air_map.put(self.gpa, mir_inst, inst);
917	928	}
918	929	if (debug_wip_mir) @import("../../print_air.zig").dumpInst(
919	930	inst,
...	...	@@ -1085,7 +1096,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1085	1096
1086	1097	.field_parent_ptr => try self.airFieldParentPtr(inst),
1087	1098
1088		.switch_br => try self.airSwitch(inst),
	1099	.switch_br => try self.airSwitchBr(inst),
1089	1100	.slice_ptr => try self.airSlicePtr(inst),
1090	1101	.slice_len => try self.airSliceLen(inst),
1091	1102
...	...	@@ -1171,8 +1182,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1171	1182	var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
1172	1183	while (it.next()) \|index\| {
1173	1184	const tracked_inst = self.register_manager.registers[index];
1174		const tracked_mcv = self.getResolvedInstValue(tracked_inst).?.*;
1175		assert(RegisterManager.indexOfRegIntoTracked(switch (tracked_mcv) {
	1185	const tracking = self.getResolvedInstValue(tracked_inst).?;
	1186	assert(RegisterManager.indexOfRegIntoTracked(switch (tracking.short) {
1176	1187	.register => \|reg\| reg,
1177	1188	.register_overflow => \|ro\| ro.reg,
1178	1189	else => unreachable,
...	...	@@ -1210,16 +1221,16 @@ fn freeValue(self: *Self, value: MCValue) void {
1210	1221	}
1211	1222	}
1212	1223
	1224	fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) void {
	1225	if (bt.feed()) if (Air.refToIndex(operand)) \|inst\| self.processDeath(inst);
	1226	}
	1227
1213	1228	/// Asserts there is already capacity to insert into top branch inst_table.
1214	1229	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
1215	1230	const air_tags = self.air.instructions.items(.tag);
1216		if (air_tags[inst] == .constant) return; // Constants are immortal.
1217		const prev_value = (self.getResolvedInstValue(inst) orelse return).*;
	1231	if (air_tags[inst] == .constant) return;
1218	1232	log.debug("%{d} => {}", .{ inst, MCValue.dead });
1219		// When editing this function, note that the logic must synchronize with `reuseOperand`.
1220		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1221		branch.inst_table.putAssumeCapacity(inst, .dead);
1222		self.freeValue(prev_value);
	1233	if (self.getResolvedInstValue(inst)) \|tracking\| tracking.die(self);
1223	1234	}
1224	1235
1225	1236	/// Called when there are no operands, and the instruction is always unreferenced.
...	...	@@ -1229,6 +1240,21 @@ fn finishAirBookkeeping(self: *Self) void {
1229	1240	}
1230	1241	}
1231	1242
	1243	fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
	1244	if (self.liveness.isUnused(inst)) switch (result) {
	1245	.none, .dead, .unreach => {},
	1246	else => unreachable, // Why didn't the result die?
	1247	} else {
	1248	log.debug("%{d} => {}", .{ inst, result });
	1249	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
	1250	// In some cases, an operand may be reused as the result.
	1251	// If that operand died and was a register, it was freed by
	1252	// processDeath, so we have to "re-allocate" the register.
	1253	self.getValue(result, inst);
	1254	}
	1255	self.finishAirBookkeeping();
	1256	}
	1257
1232	1258	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
1233	1259	var tomb_bits = self.liveness.getTombBits(inst);
1234	1260	for (operands) \|op\| {
...	...	@@ -1240,26 +1266,7 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
1240	1266	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
1241	1267	self.processDeath(op_index);
1242	1268	}
1243		const is_used = @truncate(u1, tomb_bits) == 0;
1244		if (is_used) {
1245		log.debug("%{d} => {}", .{ inst, result });
1246		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1247		branch.inst_table.putAssumeCapacityNoClobber(inst, result);
1248		// In some cases, an operand may be reused as the result.
1249		// If that operand died and was a register, it was freed by
1250		// processDeath, so we have to "re-allocate" the register.
1251		self.getValue(result, inst);
1252		} else switch (result) {
1253		.none, .dead, .unreach => {},
1254		else => unreachable, // Why didn't the result die?
1255		}
1256		self.finishAirBookkeeping();
1257		}
1258
1259		fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
1260		// In addition to the caller's needs, we need enough space to spill every register and eflags.
1261		const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
1262		try table.ensureUnusedCapacity(self.gpa, additional_count + self.register_manager.registers.len + 1);
	1269	self.finishAirResult(inst, result);
1263	1270	}
1264	1271
1265	1272	fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
...	...	@@ -1344,66 +1351,121 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
1344	1351	}
1345	1352
1346	1353	const State = struct {
1347		registers: abi.RegisterManager.TrackedRegisters,
1348		free_registers: abi.RegisterManager.RegisterBitSet,
1349		eflags_inst: ?Air.Inst.Index,
	1354	registers: RegisterManager.TrackedRegisters,
	1355	free_registers: RegisterManager.RegisterBitSet,
	1356	inst_tracking_len: u32,
	1357	scope_index: u32,
1350	1358	};
1351	1359
1352		fn captureState(self: *Self) State {
1353		return State{
1354		.registers = self.register_manager.registers,
1355		.free_registers = self.register_manager.free_registers,
1356		.eflags_inst = self.eflags_inst,
	1360	fn initRetroactiveState(self: *Self) State {
	1361	var state: State = undefined;
	1362	state.inst_tracking_len = @intCast(u32, self.inst_tracking.count());
	1363	state.scope_index = self.scope_index;
	1364	return state;
	1365	}
	1366
	1367	fn saveRetroactiveState(self: Self, state: State, comptime hack_around_liveness_bug: bool) !void {
	1368	try self.spillEflagsIfOccupied();
	1369	state.registers = self.register_manager.registers;
	1370	state.free_registers = self.register_manager.free_registers;
	1371	if (hack_around_liveness_bug) for (0..state.registers.len) \|index\| {
	1372	if (state.free_registers.isSet(index)) continue;
	1373	if (self.inst_tracking.getIndex(state.registers[index]).? < state.inst_tracking_len) continue;
	1374	state.free_registers.set(index);
1357	1375	};
1358	1376	}
1359	1377
1360		fn revertState(self: *Self, state: State) void {
1361		self.eflags_inst = state.eflags_inst;
1362		self.register_manager.free_registers = state.free_registers;
1363		self.register_manager.registers = state.registers;
	1378	fn saveState(self: *Self) !State {
	1379	var state = self.initRetroactiveState();
	1380	try self.saveRetroactiveState(&state, false);
	1381	return state;
1364	1382	}
1365	1383
1366		pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1367		const stack_mcv = try self.allocRegOrMem(inst, false);
1368		log.debug("spilling %{d} to stack mcv {any}", .{ inst, stack_mcv });
1369		const reg_mcv = self.getResolvedInstValue(inst).?.*;
1370		switch (reg_mcv) {
1371		.register => \|other\| {
1372		assert(reg.to64() == other.to64());
1373		},
1374		.register_overflow => \|ro\| {
1375		assert(reg.to64() == ro.reg.to64());
1376		},
1377		else => {},
	1384	fn restoreState(self: *Self, state: State, comptime opts: struct {
	1385	emit_instructions: bool,
	1386	update_tracking: bool,
	1387	resurrect: bool,
	1388	close_scope: bool,
	1389	}) !void {
	1390	if (opts.close_scope) {
	1391	if (std.debug.runtime_safety) {
	1392	for (self.inst_tracking.values()[state.inst_tracking_len..]) \|tracking\| {
	1393	switch (tracking.short) {
	1394	.dead, .unreach => {},
	1395	else => unreachable,
	1396	}
	1397	}
	1398	}
	1399	self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
1378	1400	}
1379		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1380		branch.inst_table.putAssumeCapacity(inst, stack_mcv);
1381		try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
1382		}
1383	1401
1384		pub fn spillEflagsIfOccupied(self: *Self) !void {
1385		if (self.eflags_inst) \|inst_to_save\| {
1386		const mcv = self.getResolvedInstValue(inst_to_save).?.*;
1387		const new_mcv = switch (mcv) {
1388		.register_overflow => try self.allocRegOrMem(inst_to_save, false),
1389		.eflags => try self.allocRegOrMem(inst_to_save, true),
1390		else => unreachable,
1391		};
	1402	if (opts.resurrect)
	1403	for (self.inst_tracking.values()) \|*tracking\| tracking.resurrect(state.scope_index);
1392	1404
1393		try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
1394		log.debug("spilling %{d} to mcv {any}", .{ inst_to_save, new_mcv });
	1405	for (0..state.registers.len) \|index\| {
	1406	const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
	1407	null
	1408	else
	1409	self.register_manager.registers[index];
	1410	const target_maybe_inst = if (state.free_registers.isSet(index))
	1411	null
	1412	else
	1413	state.registers[index];
	1414	if (std.debug.runtime_safety) if (target_maybe_inst) \|target_inst\|
	1415	assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
	1416	if (current_maybe_inst == target_maybe_inst) continue;
	1417	const reg = RegisterManager.regAtTrackedIndex(
	1418	@intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
	1419	);
	1420	if (opts.emit_instructions) {
	1421	if (current_maybe_inst) \|current_inst\| {
	1422	try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
	1423	}
	1424	if (target_maybe_inst) \|target_inst\| {
	1425	try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg);
	1426	}
	1427	}
	1428	if (opts.update_tracking) {
	1429	if (current_maybe_inst) \|current_inst\| {
	1430	self.inst_tracking.getPtr(current_inst).?.trackSpill(self);
	1431	}
	1432	self.register_manager.freeReg(reg);
	1433	self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
	1434	if (target_maybe_inst) \|target_inst\| {
	1435	self.inst_tracking.getPtr(target_inst).?.trackMaterialize(self, target_inst, reg);
	1436	}
	1437	}
	1438	}
	1439	if (opts.emit_instructions) if (self.eflags_inst) \|inst\|
	1440	try self.inst_tracking.getPtr(inst).?.spill(self, inst);
	1441	if (opts.update_tracking) if (self.eflags_inst) \|inst\| {
	1442	self.eflags_inst = null;
	1443	self.inst_tracking.getPtr(inst).?.trackSpill(self);
	1444	};
1395	1445
1396		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1397		branch.inst_table.putAssumeCapacity(inst_to_save, new_mcv);
	1446	if (opts.update_tracking and std.debug.runtime_safety) {
	1447	assert(self.eflags_inst == null);
	1448	assert(self.register_manager.free_registers.eql(state.free_registers));
	1449	var used_reg_it = state.free_registers.iterator(.{ .kind = .unset });
	1450	while (used_reg_it.next()) \|index\|
	1451	assert(self.register_manager.registers[index] == state.registers[index]);
	1452	}
	1453	}
1398	1454
1399		self.eflags_inst = null;
	1455	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
	1456	const tracking = self.inst_tracking.getPtr(inst).?;
	1457	assert(tracking.getReg().?.to64() == reg.to64());
	1458	try tracking.spill(self, inst);
	1459	tracking.trackSpill(self);
	1460	}
1400	1461
1401		// TODO consolidate with register manager and spillInstruction
1402		// this call should really belong in the register manager!
1403		switch (mcv) {
1404		.register_overflow => \|ro\| self.register_manager.freeReg(ro.reg),
1405		else => {},
1406		}
	1462	pub fn spillEflagsIfOccupied(self: *Self) !void {
	1463	if (self.eflags_inst) \|inst\| {
	1464	self.eflags_inst = null;
	1465	const tracking = self.inst_tracking.getPtr(inst).?;
	1466	assert(tracking.getCondition() != null);
	1467	try tracking.spill(self, inst);
	1468	tracking.trackSpill(self);
1407	1469	}
1408	1470	}
1409	1471
...	...	@@ -1448,7 +1510,7 @@ fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Ty
1448	1510
1449	1511	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1450	1512	const result: MCValue = result: {
1451		if (self.liveness.isUnused(inst)) break :result .dead;
	1513	if (self.liveness.isUnused(inst)) break :result .unreach;
1452	1514
1453	1515	const stack_offset = try self.allocMemPtr(inst);
1454	1516	break :result .{ .ptr_stack_offset = @intCast(i32, stack_offset) };
...	...	@@ -1458,7 +1520,7 @@ fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1458	1520
1459	1521	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1460	1522	const result: MCValue = result: {
1461		if (self.liveness.isUnused(inst)) break :result .dead;
	1523	if (self.liveness.isUnused(inst)) break :result .unreach;
1462	1524
1463	1525	const stack_offset = try self.allocMemPtr(inst);
1464	1526	break :result .{ .ptr_stack_offset = @intCast(i32, stack_offset) };
...	...	@@ -1482,7 +1544,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
1482	1544
1483	1545	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1484	1546	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1485		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	1547	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
1486	1548	const src_ty = self.air.typeOf(ty_op.operand);
1487	1549	const src_int_info = src_ty.intInfo(self.target.*);
1488	1550	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
...	...	@@ -1560,7 +1622,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1560	1622
1561	1623	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1562	1624	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1563		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	1625	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
1564	1626	const dst_ty = self.air.typeOfIndex(inst);
1565	1627	const dst_abi_size = dst_ty.abiSize(self.target.*);
1566	1628	if (dst_abi_size > 8) {
...	...	@@ -1590,7 +1652,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1590	1652	fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
1591	1653	const un_op = self.air.instructions.items(.data)[inst].un_op;
1592	1654	const operand = try self.resolveInst(un_op);
1593		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else operand;
	1655	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else operand;
1594	1656	return self.finishAir(inst, result, .{ un_op, .none, .none });
1595	1657	}
1596	1658
...	...	@@ -1599,7 +1661,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1599	1661	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1600	1662
1601	1663	if (self.liveness.isUnused(inst)) {
1602		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
	1664	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
1603	1665	}
1604	1666
1605	1667	const ptr = try self.resolveInst(bin_op.lhs);
...	...	@@ -1619,7 +1681,7 @@ fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1619	1681	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1620	1682
1621	1683	const result = if (self.liveness.isUnused(inst))
1622		.dead
	1684	.unreach
1623	1685	else
1624	1686	try self.genUnOp(inst, tag, ty_op.operand);
1625	1687	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -1629,7 +1691,7 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1629	1691	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1630	1692
1631	1693	const result = if (self.liveness.isUnused(inst))
1632		.dead
	1694	.unreach
1633	1695	else
1634	1696	try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
1635	1697	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -1640,7 +1702,7 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void
1640	1702	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1641	1703
1642	1704	const result = if (self.liveness.isUnused(inst))
1643		.dead
	1705	.unreach
1644	1706	else
1645	1707	try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
1646	1708	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -1683,7 +1745,7 @@ fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {
1683	1745
1684	1746	fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
1685	1747	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1686		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	1748	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
1687	1749	const tag = self.air.instructions.items(.tag)[inst];
1688	1750	const dst_ty = self.air.typeOfIndex(inst);
1689	1751	if (dst_ty.zigTypeTag() == .Float)
...	...	@@ -1714,7 +1776,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
1714	1776
1715	1777	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1716	1778	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1717		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1779	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1718	1780	const ty = self.air.typeOf(bin_op.lhs);
1719	1781
1720	1782	const lhs_mcv = try self.resolveInst(bin_op.lhs);
...	...	@@ -1767,7 +1829,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1767	1829
1768	1830	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1769	1831	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1770		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1832	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1771	1833	const ty = self.air.typeOf(bin_op.lhs);
1772	1834
1773	1835	const lhs_mcv = try self.resolveInst(bin_op.lhs);
...	...	@@ -1818,7 +1880,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1818	1880
1819	1881	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1820	1882	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1821		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1883	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1822	1884	const ty = self.air.typeOf(bin_op.lhs);
1823	1885
1824	1886	try self.spillRegisters(&.{ .rax, .rdx });
...	...	@@ -1875,7 +1937,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1875	1937	fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1876	1938	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1877	1939	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1878		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1940	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1879	1941	const tag = self.air.instructions.items(.tag)[inst];
1880	1942	const ty = self.air.typeOf(bin_op.lhs);
1881	1943	switch (ty.zigTypeTag()) {
...	...	@@ -1934,7 +1996,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1934	1996	fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1935	1997	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1936	1998	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1937		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1999	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1938	2000	const lhs_ty = self.air.typeOf(bin_op.lhs);
1939	2001	const rhs_ty = self.air.typeOf(bin_op.rhs);
1940	2002	switch (lhs_ty.zigTypeTag()) {
...	...	@@ -2056,7 +2118,7 @@ fn genSetStackTruncatedOverflowCompare(
2056	2118	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2057	2119	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2058	2120	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2059		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2121	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2060	2122	const dst_ty = self.air.typeOf(bin_op.lhs);
2061	2123	switch (dst_ty.zigTypeTag()) {
2062	2124	.Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
...	...	@@ -2246,7 +2308,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
2246	2308	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2247	2309
2248	2310	if (self.liveness.isUnused(inst)) {
2249		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
	2311	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
2250	2312	}
2251	2313
2252	2314	try self.spillRegisters(&.{.rcx});
...	...	@@ -2266,7 +2328,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
2266	2328	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
2267	2329	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2268	2330	const result: MCValue = if (self.liveness.isUnused(inst))
2269		.dead
	2331	.unreach
2270	2332	else
2271	2333	return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
2272	2334	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -2275,7 +2337,7 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
2275	2337	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
2276	2338	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2277	2339	const result: MCValue = result: {
2278		if (self.liveness.isUnused(inst)) break :result .none;
	2340	if (self.liveness.isUnused(inst)) break :result .unreach;
2279	2341
2280	2342	const pl_ty = self.air.typeOfIndex(inst);
2281	2343	const opt_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -2302,7 +2364,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
2302	2364	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2303	2365	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2304	2366	const result: MCValue = result: {
2305		if (self.liveness.isUnused(inst)) break :result .dead;
	2367	if (self.liveness.isUnused(inst)) break :result .unreach;
2306	2368
2307	2369	const dst_ty = self.air.typeOfIndex(inst);
2308	2370	const opt_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -2325,7 +2387,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2325	2387
2326	2388	if (opt_ty.optionalReprIsPayload()) {
2327	2389	break :result if (self.liveness.isUnused(inst))
2328		.dead
	2390	.unreach
2329	2391	else if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2330	2392	src_mcv
2331	2393	else
...	...	@@ -2344,7 +2406,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2344	2406	Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }),
2345	2407	Immediate.u(1),
2346	2408	);
2347		break :result if (self.liveness.isUnused(inst)) .dead else dst_mcv;
	2409	break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
2348	2410	};
2349	2411	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2350	2412	}
...	...	@@ -2352,7 +2414,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2352	2414	fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2353	2415	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2354	2416	if (self.liveness.isUnused(inst)) {
2355		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	2417	return self.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
2356	2418	}
2357	2419	const err_union_ty = self.air.typeOf(ty_op.operand);
2358	2420	const err_ty = err_union_ty.errorUnionSet();
...	...	@@ -2397,7 +2459,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2397	2459	fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2398	2460	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2399	2461	if (self.liveness.isUnused(inst)) {
2400		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	2462	return self.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
2401	2463	}
2402	2464	const err_union_ty = self.air.typeOf(ty_op.operand);
2403	2465	const operand = try self.resolveInst(ty_op.operand);
...	...	@@ -2450,7 +2512,7 @@ fn genUnwrapErrorUnionPayloadMir(
2450	2512	fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2451	2513	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2452	2514	const result: MCValue = result: {
2453		if (self.liveness.isUnused(inst)) break :result .dead;
	2515	if (self.liveness.isUnused(inst)) break :result .unreach;
2454	2516
2455	2517	const src_ty = self.air.typeOf(ty_op.operand);
2456	2518	const src_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -2484,7 +2546,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2484	2546	fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2485	2547	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2486	2548	const result: MCValue = result: {
2487		if (self.liveness.isUnused(inst)) break :result .dead;
	2549	if (self.liveness.isUnused(inst)) break :result .unreach;
2488	2550
2489	2551	const src_ty = self.air.typeOf(ty_op.operand);
2490	2552	const src_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -2540,7 +2602,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2540	2602	Immediate.u(0),
2541	2603	);
2542	2604
2543		if (self.liveness.isUnused(inst)) break :result .dead;
	2605	if (self.liveness.isUnused(inst)) break :result .unreach;
2544	2606
2545	2607	const dst_ty = self.air.typeOfIndex(inst);
2546	2608	const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
...	...	@@ -2564,7 +2626,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2564	2626
2565	2627	fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
2566	2628	const result: MCValue = if (self.liveness.isUnused(inst))
2567		.dead
	2629	.unreach
2568	2630	else
2569	2631	return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
2570	2632	return self.finishAir(inst, result, .{ .none, .none, .none });
...	...	@@ -2583,7 +2645,7 @@ fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {
2583	2645	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2584	2646	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2585	2647	const result: MCValue = result: {
2586		if (self.liveness.isUnused(inst)) break :result .dead;
	2648	if (self.liveness.isUnused(inst)) break :result .unreach;
2587	2649
2588	2650	const pl_ty = self.air.typeOf(ty_op.operand);
2589	2651	if (!pl_ty.hasRuntimeBits()) break :result .{ .immediate = 1 };
...	...	@@ -2630,7 +2692,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2630	2692	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2631	2693
2632	2694	if (self.liveness.isUnused(inst)) {
2633		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	2695	return self.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
2634	2696	}
2635	2697
2636	2698	const error_union_ty = self.air.getRefType(ty_op.ty);
...	...	@@ -2660,7 +2722,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2660	2722	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2661	2723	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2662	2724	if (self.liveness.isUnused(inst)) {
2663		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	2725	return self.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
2664	2726	}
2665	2727	const error_union_ty = self.air.getRefType(ty_op.ty);
2666	2728	const payload_ty = error_union_ty.errorUnionPayload();
...	...	@@ -2687,7 +2749,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2687	2749
2688	2750	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2689	2751	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2690		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	2752	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
2691	2753	const src_mcv = try self.resolveInst(ty_op.operand);
2692	2754	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
2693	2755
...	...	@@ -2701,7 +2763,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2701	2763
2702	2764	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2703	2765	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2704		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2766	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2705	2767	const operand = try self.resolveInst(ty_op.operand);
2706	2768	const dst_mcv: MCValue = blk: {
2707	2769	switch (operand) {
...	...	@@ -2720,7 +2782,7 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2720	2782	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2721	2783	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2722	2784	const result: MCValue = result: {
2723		if (self.liveness.isUnused(inst)) break :result .dead;
	2785	if (self.liveness.isUnused(inst)) break :result .unreach;
2724	2786
2725	2787	const src_ty = self.air.typeOf(ty_op.operand);
2726	2788	const src_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -2756,7 +2818,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2756	2818	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
2757	2819	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2758	2820	const result: MCValue = result: {
2759		if (self.liveness.isUnused(inst)) break :result .dead;
	2821	if (self.liveness.isUnused(inst)) break :result .unreach;
2760	2822
2761	2823	const dst_ty = self.air.typeOfIndex(inst);
2762	2824	const opt_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -2834,7 +2896,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2834	2896	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2835	2897	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2836	2898	const slice_ty = self.air.typeOf(bin_op.lhs);
2837		const result = if (!slice_ty.isVolatilePtr() and self.liveness.isUnused(inst)) .dead else result: {
	2899	const result = if (!slice_ty.isVolatilePtr() and self.liveness.isUnused(inst)) .unreach else result: {
2838	2900	var buf: Type.SlicePtrFieldTypeBuffer = undefined;
2839	2901	const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
2840	2902	const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
...	...	@@ -2849,7 +2911,7 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2849	2911	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2850	2912	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2851	2913	const result: MCValue = if (self.liveness.isUnused(inst))
2852		.dead
	2914	.unreach
2853	2915	else
2854	2916	try self.genSliceElemPtr(extra.lhs, extra.rhs);
2855	2917	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
...	...	@@ -2859,7 +2921,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2859	2921	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2860	2922
2861	2923	if (self.liveness.isUnused(inst)) {
2862		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
	2924	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
2863	2925	}
2864	2926
2865	2927	const array_ty = self.air.typeOf(bin_op.lhs);
...	...	@@ -2928,7 +2990,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2928	2990	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2929	2991	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2930	2992	const ptr_ty = self.air.typeOf(bin_op.lhs);
2931		const result = if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst)) .dead else result: {
	2993	const result = if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst)) .unreach else result: {
2932	2994	// this is identical to the `airPtrElemPtr` codegen expect here an
2933	2995	// additional `mov` is needed at the end to get the actual value
2934	2996
...	...	@@ -2971,7 +3033,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2971	3033	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2972	3034	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2973	3035
2974		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	3036	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
2975	3037	const ptr_ty = self.air.typeOf(extra.lhs);
2976	3038	const ptr = try self.resolveInst(extra.lhs);
2977	3039	const ptr_lock: ?RegisterLock = switch (ptr) {
...	...	@@ -3041,7 +3103,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3041	3103	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3042	3104	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3043	3105	if (self.liveness.isUnused(inst)) {
3044		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	3106	return self.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
3045	3107	}
3046	3108
3047	3109	const tag_ty = self.air.typeOfIndex(inst);
...	...	@@ -3094,7 +3156,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3094	3156	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3095	3157	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3096	3158	const result = result: {
3097		if (self.liveness.isUnused(inst)) break :result .dead;
	3159	if (self.liveness.isUnused(inst)) break :result .unreach;
3098	3160
3099	3161	const dst_ty = self.air.typeOfIndex(inst);
3100	3162	const src_ty = self.air.typeOf(ty_op.operand);
...	...	@@ -3163,7 +3225,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3163	3225	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3164	3226	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3165	3227	const result = result: {
3166		if (self.liveness.isUnused(inst)) break :result .dead;
	3228	if (self.liveness.isUnused(inst)) break :result .unreach;
3167	3229
3168	3230	const dst_ty = self.air.typeOfIndex(inst);
3169	3231	const src_ty = self.air.typeOf(ty_op.operand);
...	...	@@ -3221,7 +3283,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3221	3283	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
3222	3284	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3223	3285	const result: MCValue = result: {
3224		if (self.liveness.isUnused(inst)) break :result .dead;
	3286	if (self.liveness.isUnused(inst)) break :result .unreach;
3225	3287
3226	3288	const src_ty = self.air.typeOf(ty_op.operand);
3227	3289	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
...	...	@@ -3392,7 +3454,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
3392	3454	fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
3393	3455	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3394	3456	const result = result: {
3395		if (self.liveness.isUnused(inst)) break :result .dead;
	3457	if (self.liveness.isUnused(inst)) break :result .unreach;
3396	3458
3397	3459	const src_ty = self.air.typeOf(ty_op.operand);
3398	3460	const src_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -3416,7 +3478,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
3416	3478	fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3417	3479	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3418	3480	const result = result: {
3419		if (self.liveness.isUnused(inst)) break :result .dead;
	3481	if (self.liveness.isUnused(inst)) break :result .unreach;
3420	3482
3421	3483	const src_ty = self.air.typeOf(ty_op.operand);
3422	3484	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
...	...	@@ -3529,7 +3591,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3529	3591	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
3530	3592	const un_op = self.air.instructions.items(.data)[inst].un_op;
3531	3593	const result: MCValue = if (self.liveness.isUnused(inst))
3532		.dead
	3594	.unreach
3533	3595	else
3534	3596	return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch});
3535	3597	return self.finishAir(inst, result, .{ un_op, .none, .none });
...	...	@@ -3564,10 +3626,7 @@ fn reuseOperand(
3564	3626
3565	3627	// Prevent the operand deaths processing code from deallocating it.
3566	3628	self.liveness.clearOperandDeath(inst, op_index);
3567
3568		// That makes us responsible for doing the rest of the stuff that processDeath would have done.
3569		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
3570		branch.inst_table.putAssumeCapacity(Air.refToIndex(operand).?, .dead);
	3629	if (self.getResolvedInstValue(Air.refToIndex(operand).?)) \|tracking\| tracking.reuse(self);
3571	3630
3572	3631	return true;
3573	3632	}
...	...	@@ -3709,7 +3768,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
3709	3768
3710	3769	const ptr = try self.resolveInst(ty_op.operand);
3711	3770	const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
3712		if (self.liveness.isUnused(inst) and !is_volatile) break :result .dead;
	3771	if (self.liveness.isUnused(inst) and !is_volatile) break :result .unreach;
3713	3772
3714	3773	const dst_mcv: MCValue = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr))
3715	3774	// The MCValue that holds the pointer can be re-used as the value.
...	...	@@ -4008,7 +4067,7 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
4008	4067
4009	4068	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
4010	4069	if (self.liveness.isUnused(inst)) {
4011		return MCValue.dead;
	4070	return MCValue.unreach;
4012	4071	}
4013	4072
4014	4073	const mcv = try self.resolveInst(operand);
...	...	@@ -4077,7 +4136,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
4077	4136	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4078	4137	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
4079	4138	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
4080		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	4139	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
4081	4140	const operand = extra.struct_operand;
4082	4141	const index = extra.field_index;
4083	4142
...	...	@@ -4226,7 +4285,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4226	4285	fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
4227	4286	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
4228	4287	const result: MCValue = if (self.liveness.isUnused(inst))
4229		.dead
	4288	.unreach
4230	4289	else
4231	4290	return self.fail("TODO implement airFieldParentPtr for {}", .{self.target.cpu.arch});
4232	4291	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -5449,7 +5508,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
5449	5508	const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5450	5509
5451	5510	const result: MCValue = result: {
5452		if (self.liveness.isUnused(inst)) break :result .dead;
	5511	if (self.liveness.isUnused(inst)) break :result .unreach;
5453	5512
5454	5513	const dst_mcv: MCValue = switch (mcv) {
5455	5514	.register => \|reg\| blk: {
...	...	@@ -5541,7 +5600,7 @@ fn airBreakpoint(self: *Self) !void {
5541	5600	}
5542	5601
5543	5602	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
5544		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	5603	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
5545	5604	const dst_mcv = try self.allocRegOrMem(inst, true);
5546	5605	try self.setRegOrMem(Type.usize, dst_mcv, .{
5547	5606	.stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)),
...	...	@@ -5552,7 +5611,7 @@ fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
5552	5611	}
5553	5612
5554	5613	fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
5555		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	5614	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
5556	5615	const dst_mcv = try self.allocRegOrMem(inst, true);
5557	5616	try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp });
5558	5617	break :result dst_mcv;
...	...	@@ -5754,7 +5813,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5754	5813	}
5755	5814
5756	5815	const result: MCValue = result: {
5757		if (self.liveness.isUnused(inst)) break :result .dead;
	5816	if (self.liveness.isUnused(inst)) break :result .unreach;
5758	5817
5759	5818	switch (info.return_value) {
5760	5819	.register => {
...	...	@@ -5776,12 +5835,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5776	5835	std.mem.copy(Air.Inst.Ref, buf[1..], args);
5777	5836	return self.finishAir(inst, result, buf);
5778	5837	}
5779		var bt = try self.iterateBigTomb(inst, 1 + args.len);
5780		bt.feed(callee);
5781		for (args) \|arg\| {
5782		bt.feed(arg);
5783		}
5784		return bt.finishAir(result);
	5838	var bt = self.liveness.iterateBigTomb(inst);
	5839	self.feed(&bt, callee);
	5840	for (args) \|arg\| self.feed(&bt, arg);
	5841	return self.finishAirResult(inst, result);
5785	5842	}
5786	5843
5787	5844	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5804,7 +5861,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
5804	5861	// which is available if the jump is 127 bytes or less forward.
5805	5862	const jmp_reloc = try self.asmJmpReloc(undefined);
5806	5863	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
5807		return self.finishAir(inst, .dead, .{ un_op, .none, .none });
	5864	return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
5808	5865	}
5809	5866
5810	5867	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5834,12 +5891,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
5834	5891	// which is available if the jump is 127 bytes or less forward.
5835	5892	const jmp_reloc = try self.asmJmpReloc(undefined);
5836	5893	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
5837		return self.finishAir(inst, .dead, .{ un_op, .none, .none });
	5894	return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
5838	5895	}
5839	5896
5840	5897	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
5841	5898	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
5842		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	5899	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
5843	5900	const ty = self.air.typeOf(bin_op.lhs);
5844	5901	const ty_abi_size = ty.abiSize(self.target.*);
5845	5902	const can_reuse = ty_abi_size <= 8;
...	...	@@ -5904,7 +5961,7 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
5904	5961
5905	5962	fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5906	5963	const un_op = self.air.instructions.items(.data)[inst].un_op;
5907		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	5964	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
5908	5965	const addr_reg = try self.register_manager.allocReg(null, gp);
5909	5966	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
5910	5967	defer self.register_manager.unlockReg(addr_lock);
...	...	@@ -5993,7 +6050,7 @@ fn genTry(
5993	6050	try self.genBody(body);
5994	6051	try self.performReloc(reloc);
5995	6052	const result = if (self.liveness.isUnused(inst))
5996		.dead
	6053	.unreach
5997	6054	else
5998	6055	try self.genUnwrapErrorUnionPayloadMir(inst, err_union_ty, err_union);
5999	6056	return result;
...	...	@@ -6018,12 +6075,12 @@ fn airDbgInline(self: *Self, inst: Air.Inst.Index) !void {
6018	6075	const function = self.air.values[ty_pl.payload].castTag(.function).?.data;
6019	6076	// TODO emit debug info for function change
6020	6077	_ = function;
6021		return self.finishAir(inst, .dead, .{ .none, .none, .none });
	6078	return self.finishAir(inst, .unreach, .{ .none, .none, .none });
6022	6079	}
6023	6080
6024	6081	fn airDbgBlock(self: *Self, inst: Air.Inst.Index) !void {
6025	6082	// TODO emit debug info lexical block
6026		return self.finishAir(inst, .dead, .{ .none, .none, .none });
	6083	return self.finishAir(inst, .unreach, .{ .none, .none, .none });
6027	6084	}
6028	6085
6029	6086	fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -6039,7 +6096,7 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
6039	6096	const tag = self.air.instructions.items(.tag)[inst];
6040	6097	try self.genVarDbgInfo(tag, ty, mcv, name);
6041	6098
6042		return self.finishAir(inst, .dead, .{ operand, .none, .none });
	6099	return self.finishAir(inst, .unreach, .{ operand, .none, .none });
6043	6100	}
6044	6101
6045	6102	fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
...	...	@@ -6087,60 +6144,31 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
6087	6144	if (Air.refToIndex(pl_op.operand)) \|op_inst\| self.processDeath(op_inst);
6088	6145	}
6089	6146
6090		// Capture the state of register and stack allocation state so that we can revert to it.
6091		const saved_state = self.captureState();
6092
	6147	const outer_state = try self.saveState();
6093	6148	{
6094		try self.branch_stack.append(.{});
6095		errdefer _ = self.branch_stack.pop();
	6149	self.scope_index += 1;
	6150	const inner_state = try self.saveState();
6096	6151
6097		try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
6098		for (liveness_condbr.then_deaths) \|operand\| {
6099		self.processDeath(operand);
6100		}
	6152	for (liveness_condbr.then_deaths) \|operand\| self.processDeath(operand);
6101	6153	try self.genBody(then_body);
6102		}
6103
6104		// Revert to the previous register and stack allocation state.
6105
6106		var then_branch = self.branch_stack.pop();
6107		defer then_branch.deinit(self.gpa);
6108
6109		self.revertState(saved_state);
6110
6111		try self.performReloc(reloc);
	6154	try self.restoreState(inner_state, .{
	6155	.emit_instructions = false,
	6156	.update_tracking = true,
	6157	.resurrect = true,
	6158	.close_scope = true,
	6159	});
6112	6160
6113		{
6114		try self.branch_stack.append(.{});
6115		errdefer _ = self.branch_stack.pop();
	6161	try self.performReloc(reloc);
6116	6162
6117		try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
6118		for (liveness_condbr.else_deaths) \|operand\| {
6119		self.processDeath(operand);
6120		}
	6163	for (liveness_condbr.else_deaths) \|operand\| self.processDeath(operand);
6121	6164	try self.genBody(else_body);
6122	6165	}
6123
6124		var else_branch = self.branch_stack.pop();
6125		defer else_branch.deinit(self.gpa);
6126
6127		// At this point, each branch will possibly have conflicting values for where
6128		// each instruction is stored. They agree, however, on which instructions are alive/dead.
6129		// We use the first ("then") branch as canonical, and here emit
6130		// instructions into the second ("else") branch to make it conform.
6131		// We continue respect the data structure semantic guarantees of the else_branch so
6132		// that we can use all the code emitting abstractions. This is why at the bottom we
6133		// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
6134		// rather than assigning it.
6135		log.debug("airCondBr: %{d}", .{inst});
6136		log.debug("Upper branches:", .{});
6137		for (self.branch_stack.items) \|bs\| {
6138		log.debug("{}", .{bs.fmtDebug()});
6139		}
6140		log.debug("Then branch: {}", .{then_branch.fmtDebug()});
6141		log.debug("Else branch: {}", .{else_branch.fmtDebug()});
6142
6143		try self.canonicaliseBranches(true, &then_branch, &else_branch, true, true);
	6166	try self.restoreState(outer_state, .{
	6167	.emit_instructions = false,
	6168	.update_tracking = false,
	6169	.resurrect = false,
	6170	.close_scope = true,
	6171	});
6144	6172
6145	6173	// We already took care of pl_op.operand earlier, so we're going
6146	6174	// to pass .none here
...	...	@@ -6314,7 +6342,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCVa
6314	6342
6315	6343	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
6316	6344	const un_op = self.air.instructions.items(.data)[inst].un_op;
6317		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	6345	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
6318	6346	const operand = try self.resolveInst(un_op);
6319	6347	const ty = self.air.typeOf(un_op);
6320	6348	break :result try self.isNull(inst, ty, operand);
...	...	@@ -6324,7 +6352,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
6324	6352
6325	6353	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
6326	6354	const un_op = self.air.instructions.items(.data)[inst].un_op;
6327		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	6355	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
6328	6356	const operand = try self.resolveInst(un_op);
6329	6357	const ty = self.air.typeOf(un_op);
6330	6358	break :result try self.isNullPtr(inst, ty, operand);
...	...	@@ -6334,7 +6362,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
6334	6362
6335	6363	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
6336	6364	const un_op = self.air.instructions.items(.data)[inst].un_op;
6337		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	6365	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
6338	6366	const operand = try self.resolveInst(un_op);
6339	6367	const ty = self.air.typeOf(un_op);
6340	6368	break :result switch (try self.isNull(inst, ty, operand)) {
...	...	@@ -6347,7 +6375,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
6347	6375
6348	6376	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
6349	6377	const un_op = self.air.instructions.items(.data)[inst].un_op;
6350		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	6378	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
6351	6379	const operand = try self.resolveInst(un_op);
6352	6380	const ty = self.air.typeOf(un_op);
6353	6381	break :result switch (try self.isNullPtr(inst, ty, operand)) {
...	...	@@ -6360,7 +6388,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
6360	6388
6361	6389	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
6362	6390	const un_op = self.air.instructions.items(.data)[inst].un_op;
6363		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	6391	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
6364	6392	const operand = try self.resolveInst(un_op);
6365	6393	const ty = self.air.typeOf(un_op);
6366	6394	break :result try self.isErr(inst, ty, operand);
...	...	@@ -6372,7 +6400,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6372	6400	const un_op = self.air.instructions.items(.data)[inst].un_op;
6373	6401
6374	6402	if (self.liveness.isUnused(inst)) {
6375		return self.finishAir(inst, .dead, .{ un_op, .none, .none });
	6403	return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
6376	6404	}
6377	6405
6378	6406	const operand_ptr = try self.resolveInst(un_op);
...	...	@@ -6400,7 +6428,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6400	6428
6401	6429	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
6402	6430	const un_op = self.air.instructions.items(.data)[inst].un_op;
6403		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	6431	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
6404	6432	const operand = try self.resolveInst(un_op);
6405	6433	const ty = self.air.typeOf(un_op);
6406	6434	break :result try self.isNonErr(inst, ty, operand);
...	...	@@ -6412,7 +6440,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6412	6440	const un_op = self.air.instructions.items(.data)[inst].un_op;
6413	6441
6414	6442	if (self.liveness.isUnused(inst)) {
6415		return self.finishAir(inst, .dead, .{ un_op, .none, .none });
	6443	return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
6416	6444	}
6417	6445
6418	6446	const operand_ptr = try self.resolveInst(un_op);
...	...	@@ -6445,27 +6473,16 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
6445	6473	const body = self.air.extra[loop.end..][0..loop.data.body_len];
6446	6474	const jmp_target = @intCast(u32, self.mir_instructions.len);
6447	6475
6448		{
6449		try self.branch_stack.append(.{});
6450		errdefer _ = self.branch_stack.pop();
6451
6452		try self.genBody(body);
6453		}
6454
6455		var branch = self.branch_stack.pop();
6456		defer branch.deinit(self.gpa);
6457
6458		log.debug("airLoop: %{d}", .{inst});
6459		log.debug("Upper branches:", .{});
6460		for (self.branch_stack.items) \|bs\| {
6461		log.debug("{}", .{bs.fmtDebug()});
6462		}
6463		log.debug("Loop branch: {}", .{branch.fmtDebug()});
6464
6465		var dummy_branch = Branch{};
6466		defer dummy_branch.deinit(self.gpa);
6467		try self.canonicaliseBranches(true, &dummy_branch, &branch, true, false);
	6476	self.scope_index += 1;
	6477	const state = try self.saveState();
6468	6478
	6479	try self.genBody(body);
	6480	try self.restoreState(state, .{
	6481	.emit_instructions = true,
	6482	.update_tracking = false,
	6483	.resurrect = false,
	6484	.close_scope = true,
	6485	});
6469	6486	_ = try self.asmJmpReloc(jmp_target);
6470	6487
6471	6488	return self.finishAirBookkeeping();
...	...	@@ -6473,68 +6490,52 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
6473	6490
6474	6491	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6475	6492	// A block is a setup to be able to jump to the end.
6476		const branch_depth = @intCast(u32, self.branch_stack.items.len);
6477		try self.blocks.putNoClobber(self.gpa, inst, .{ .branch_depth = branch_depth });
	6493	const ty = self.air.typeOfIndex(inst);
	6494
	6495	// Here we use .{ .long = .unreach } to represent a null value so that the
	6496	// first break instruction will choose a MCValue for the block result and
	6497	// overwrite this field. Following break instructions will use that MCValue
	6498	// to put their block results.
	6499	self.inst_tracking.putAssumeCapacityNoClobber(inst, .{
	6500	.long = .unreach,
	6501	.short = if (ty.isNoReturn()) .unreach else .none,
	6502	});
	6503
	6504	self.scope_index += 1;
	6505	try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
6478	6506	defer {
6479	6507	var block_data = self.blocks.fetchRemove(inst).?.value;
6480	6508	block_data.deinit(self.gpa);
6481	6509	}
6482	6510
6483		const ty = self.air.typeOfIndex(inst);
6484		const unused = !ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(inst);
6485
6486		{
6487		// Here we use `.none` to represent a null value so that the first break
6488		// instruction will choose a MCValue for the block result and overwrite
6489		// this field. Following break instructions will use that MCValue to put
6490		// their block results.
6491		const result: MCValue = if (unused) .dead else .none;
6492		const branch = &self.branch_stack.items[branch_depth - 1];
6493		try branch.inst_table.putNoClobber(self.gpa, inst, result);
6494		}
6495
6496		{
6497		try self.branch_stack.append(.{});
6498		errdefer _ = self.branch_stack.pop();
6499
6500		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
6501		const extra = self.air.extraData(Air.Block, ty_pl.payload);
6502		const body = self.air.extra[extra.end..][0..extra.data.body_len];
6503		try self.genBody(body);
6504		}
	6511	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	6512	const extra = self.air.extraData(Air.Block, ty_pl.payload);
	6513	const body = self.air.extra[extra.end..][0..extra.data.body_len];
	6514	try self.genBody(body);
6505	6515
	6516	const tracking = self.inst_tracking.getPtr(inst).?;
6506	6517	const block_data = self.blocks.getPtr(inst).?;
6507		const target_branch = self.branch_stack.pop();
6508
6509		log.debug("airBlock: %{d}", .{inst});
6510		log.debug("Upper branches:", .{});
6511		for (self.branch_stack.items) \|bs\| {
6512		log.debug("{}", .{bs.fmtDebug()});
6513		}
6514		log.debug("Block branch: {}", .{block_data.branch.fmtDebug()});
6515		log.debug("Target branch: {}", .{target_branch.fmtDebug()});
6516
6517		try self.canonicaliseBranches(true, &block_data.branch, &target_branch, false, false);
6518
	6518	if (tracking.short != .unreach) try self.restoreState(block_data.state, .{
	6519	.emit_instructions = false,
	6520	.update_tracking = true,
	6521	.resurrect = false,
	6522	.close_scope = true,
	6523	});
6519	6524	for (block_data.relocs.items) \|reloc\| try self.performReloc(reloc);
6520	6525
6521		const result = if (unused) .dead else self.getResolvedInstValue(inst).?.*;
6522		self.getValue(result, inst);
	6526	if (self.liveness.isUnused(inst)) tracking.die(self);
	6527	self.getValue(tracking.short, inst);
6523	6528	self.finishAirBookkeeping();
6524	6529	}
6525	6530
6526		fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
	6531	fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
6527	6532	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
6528	6533	const condition = try self.resolveInst(pl_op.operand);
6529	6534	const condition_ty = self.air.typeOf(pl_op.operand);
6530	6535	const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
6531	6536	var extra_index: usize = switch_br.end;
6532	6537	var case_i: u32 = 0;
6533		const liveness = try self.liveness.getSwitchBr(
6534		self.gpa,
6535		inst,
6536		switch_br.data.cases_len + 1,
6537		);
	6538	const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.data.cases_len + 1);
6538	6539	defer self.gpa.free(liveness.deaths);
6539	6540
6540	6541	// If the condition dies here in this switch instruction, process
...	...	@@ -6544,186 +6545,64 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
6544	6545	if (Air.refToIndex(pl_op.operand)) \|op_inst\| self.processDeath(op_inst);
6545	6546	}
6546	6547
6547		log.debug("airSwitch: %{d}", .{inst});
6548		log.debug("Upper branches:", .{});
6549		for (self.branch_stack.items) \|bs\| {
6550		log.debug("{}", .{bs.fmtDebug()});
6551		}
6552
6553		var prev_branch: ?Branch = null;
6554		defer if (prev_branch) \|*branch\| branch.deinit(self.gpa);
6555
6556		// Capture the state of register and stack allocation state so that we can revert to it.
6557		const saved_state = self.captureState();
6558
6559		const cases_len = switch_br.data.cases_len + @boolToInt(switch_br.data.else_body_len > 0);
6560		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
6561		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
6562		const items = @ptrCast([]const Air.Inst.Ref, self.air.extra[case.end..][0..case.data.items_len]);
6563		const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
6564		extra_index = case.end + items.len + case_body.len;
6565
6566		// Revert to the previous register and stack allocation state.
6567		if (prev_branch) \|_\| self.revertState(saved_state);
6568
6569		var relocs = try self.gpa.alloc(u32, items.len);
6570		defer self.gpa.free(relocs);
6571
6572		for (items, relocs) \|item, *reloc\| {
6573		try self.spillEflagsIfOccupied();
6574		const item_mcv = try self.resolveInst(item);
6575		try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
6576		reloc.* = try self.asmJccReloc(undefined, .ne);
6577		}
	6548	const outer_state = try self.saveState();
	6549	{
	6550	self.scope_index += 1;
	6551	const inner_state = try self.saveState();
	6552
	6553	while (case_i < switch_br.data.cases_len) : (case_i += 1) {
	6554	const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
	6555	const items = @ptrCast(
	6556	[]const Air.Inst.Ref,
	6557	self.air.extra[case.end..][0..case.data.items_len],
	6558	);
	6559	const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
	6560	extra_index = case.end + items.len + case_body.len;
6578	6561
6579		{
6580		if (cases_len > 1) try self.branch_stack.append(.{});
6581		errdefer _ = if (cases_len > 1) self.branch_stack.pop();
	6562	var relocs = try self.gpa.alloc(u32, items.len);
	6563	defer self.gpa.free(relocs);
6582	6564
6583		try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len);
6584		for (liveness.deaths[case_i]) \|operand\| {
6585		self.processDeath(operand);
	6565	for (items, relocs) \|item, *reloc\| {
	6566	try self.spillEflagsIfOccupied();
	6567	const item_mcv = try self.resolveInst(item);
	6568	try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
	6569	reloc.* = try self.asmJccReloc(undefined, .ne);
6586	6570	}
6587	6571
6588		try self.genBody(case_body);
6589		}
	6572	for (liveness.deaths[case_i]) \|operand\| self.processDeath(operand);
6590	6573
6591		// Consolidate returned MCValues between prongs like we do in airCondBr.
6592		if (cases_len > 1) {
6593		var case_branch = self.branch_stack.pop();
6594		errdefer case_branch.deinit(self.gpa);
	6574	try self.genBody(case_body);
	6575	if (case_i < switch_br.data.cases_len - 1 or switch_br.data.else_body_len > 0)
	6576	try self.restoreState(inner_state, .{
	6577	.emit_instructions = false,
	6578	.update_tracking = true,
	6579	.resurrect = true,
	6580	.close_scope = true,
	6581	});
6595	6582
6596		log.debug("Case-{d} branch: {}", .{ case_i, case_branch.fmtDebug() });
6597		const final = case_i == cases_len - 1;
6598		if (prev_branch) \|*canon_branch\| {
6599		try self.canonicaliseBranches(final, canon_branch, &case_branch, true, true);
6600		canon_branch.deinit(self.gpa);
6601		}
6602		prev_branch = case_branch;
	6583	for (relocs) \|reloc\| try self.performReloc(reloc);
6603	6584	}
6604	6585
6605		for (relocs) \|reloc\| try self.performReloc(reloc);
6606		}
6607
6608		if (switch_br.data.else_body_len > 0) {
6609		const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
6610
6611		// Revert to the previous register and stack allocation state.
6612		if (prev_branch) \|_\| self.revertState(saved_state);
6613
6614		{
6615		if (cases_len > 1) try self.branch_stack.append(.{});
6616		errdefer _ = if (cases_len > 1) self.branch_stack.pop();
	6586	if (switch_br.data.else_body_len > 0) {
	6587	const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
6617	6588
6618	6589	const else_deaths = liveness.deaths.len - 1;
6619		try self.ensureProcessDeathCapacity(liveness.deaths[else_deaths].len);
6620		for (liveness.deaths[else_deaths]) \|operand\| {
6621		self.processDeath(operand);
6622		}
	6590	for (liveness.deaths[else_deaths]) \|operand\| self.processDeath(operand);
6623	6591
6624	6592	try self.genBody(else_body);
6625	6593	}
6626
6627		// Consolidate returned MCValues between a prong and the else branch like we do in airCondBr.
6628		if (cases_len > 1) {
6629		var else_branch = self.branch_stack.pop();
6630		errdefer else_branch.deinit(self.gpa);
6631
6632		log.debug("Else branch: {}", .{else_branch.fmtDebug()});
6633		if (prev_branch) \|*canon_branch\| {
6634		try self.canonicaliseBranches(true, canon_branch, &else_branch, true, true);
6635		canon_branch.deinit(self.gpa);
6636		}
6637		prev_branch = else_branch;
6638		}
6639	6594	}
	6595	try self.restoreState(outer_state, .{
	6596	.emit_instructions = false,
	6597	.update_tracking = false,
	6598	.resurrect = false,
	6599	.close_scope = true,
	6600	});
6640	6601
6641	6602	// We already took care of pl_op.operand earlier, so we're going to pass .none here
6642	6603	return self.finishAir(inst, .unreach, .{ .none, .none, .none });
6643	6604	}
6644	6605
6645		fn canonicaliseBranches(
6646		self: *Self,
6647		update_parent: bool,
6648		canon_branch: *Branch,
6649		target_branch: *const Branch,
6650		comptime set_values: bool,
6651		comptime assert_same_deaths: bool,
6652		) !void {
6653		var hazard_map = std.AutoHashMap(MCValue, void).init(self.gpa);
6654		defer hazard_map.deinit();
6655
6656		const parent_branch =
6657		if (update_parent) &self.branch_stack.items[self.branch_stack.items.len - 1] else undefined;
6658
6659		if (update_parent) try self.ensureProcessDeathCapacity(target_branch.inst_table.count());
6660		var target_it = target_branch.inst_table.iterator();
6661		while (target_it.next()) \|target_entry\| {
6662		const target_key = target_entry.key_ptr.*;
6663		const target_value = target_entry.value_ptr.*;
6664		const canon_mcv = if (canon_branch.inst_table.fetchSwapRemove(target_key)) \|canon_entry\| blk: {
6665		// The instruction's MCValue is overridden in both branches.
6666		if (target_value == .dead) {
6667		if (update_parent) {
6668		parent_branch.inst_table.putAssumeCapacity(target_key, .dead);
6669		}
6670		if (assert_same_deaths) assert(canon_entry.value == .dead);
6671		continue;
6672		}
6673		if (update_parent) {
6674		parent_branch.inst_table.putAssumeCapacity(target_key, canon_entry.value);
6675		}
6676		break :blk canon_entry.value;
6677		} else blk: {
6678		if (target_value == .dead) {
6679		if (update_parent) {
6680		parent_branch.inst_table.putAssumeCapacity(target_key, .dead);
6681		}
6682		continue;
6683		}
6684		// The instruction is only overridden in the else branch.
6685		// If integer overflow occurs, the question is: why wasn't the instruction marked dead?
6686		break :blk self.getResolvedInstValue(target_key).?.*;
6687		};
6688		log.debug("consolidating target_entry %{d} {}=>{}", .{ target_key, target_value, canon_mcv });
6689		// TODO handle the case where the destination stack offset / register has something
6690		// going on there.
6691		assert(!hazard_map.contains(target_value));
6692		try hazard_map.putNoClobber(canon_mcv, {});
6693		if (set_values) {
6694		try self.setRegOrMem(self.air.typeOfIndex(target_key), canon_mcv, target_value);
6695		} else self.getValue(canon_mcv, target_key);
6696		self.freeValue(target_value);
6697		// TODO track the new register / stack allocation
6698		}
6699
6700		if (update_parent) try self.ensureProcessDeathCapacity(canon_branch.inst_table.count());
6701		var canon_it = canon_branch.inst_table.iterator();
6702		while (canon_it.next()) \|canon_entry\| {
6703		const canon_key = canon_entry.key_ptr.*;
6704		const canon_value = canon_entry.value_ptr.*;
6705		// We already deleted the items from this table that matched the target_branch.
6706		// So these are all instructions that are only overridden in the canon branch.
6707		const parent_mcv =
6708		if (canon_value != .dead) self.getResolvedInstValue(canon_key).?.* else undefined;
6709		if (canon_value != .dead) {
6710		log.debug("consolidating canon_entry %{d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
6711		// TODO handle the case where the destination stack offset / register has something
6712		// going on there.
6713		assert(!hazard_map.contains(parent_mcv));
6714		try hazard_map.putNoClobber(canon_value, {});
6715		if (set_values) {
6716		try self.setRegOrMem(self.air.typeOfIndex(canon_key), canon_value, parent_mcv);
6717		} else self.getValue(canon_value, canon_key);
6718		self.freeValue(parent_mcv);
6719		// TODO track the new register / stack allocation
6720		}
6721		if (update_parent) {
6722		parent_branch.inst_table.putAssumeCapacity(canon_key, canon_value);
6723		}
6724		}
6725		}
6726
6727	6606	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
6728	6607	const next_inst = @intCast(u32, self.mir_instructions.len);
6729	6608	switch (self.mir_instructions.items(.tag)[reloc]) {
...	...	@@ -6739,69 +6618,38 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
6739	6618
6740	6619	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6741	6620	const br = self.air.instructions.items(.data)[inst].br;
6742		const block = br.block_inst;
6743
6744		// The first break instruction encounters `.none` here and chooses a
6745		// machine code value for the block result, populating this field.
6746		// Following break instructions encounter that value and use it for
6747		// the location to store their block results.
6748		if (self.getResolvedInstValue(block)) \|dst_mcv\| {
6749		const src_mcv = try self.resolveInst(br.operand);
6750		switch (dst_mcv.*) {
6751		.none => {
6752		const result = result: {
6753		if (self.reuseOperand(inst, br.operand, 0, src_mcv)) break :result src_mcv;
6754
6755		const new_mcv = try self.allocRegOrMem(block, true);
6756		try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, src_mcv);
6757		break :result new_mcv;
6758		};
6759		dst_mcv.* = result;
6760		self.freeValue(result);
6761		},
6762		else => try self.setRegOrMem(self.air.typeOfIndex(block), dst_mcv.*, src_mcv),
6763		}
6764		}
	6621	const block_ty = self.air.typeOfIndex(br.block_inst);
	6622	const block_unused =
	6623	!block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);
6765	6624
6766		// Process operand death early so that it is properly accounted for in the Branch below.
	6625	// Process operand death early so that it is properly accounted for in the State below.
	6626	const src_mcv = try self.resolveInst(br.operand);
6767	6627	if (self.liveness.operandDies(inst, 0)) {
6768	6628	if (Air.refToIndex(br.operand)) \|op_inst\| self.processDeath(op_inst);
6769	6629	}
6770	6630
6771		const block_data = self.blocks.getPtr(block).?;
6772		{
6773		var branch = Branch{};
6774		errdefer branch.deinit(self.gpa);
6775
6776		var branch_i = self.branch_stack.items.len - 1;
6777		while (branch_i >= block_data.branch_depth) : (branch_i -= 1) {
6778		const table = &self.branch_stack.items[branch_i].inst_table;
6779		try branch.inst_table.ensureUnusedCapacity(self.gpa, table.count());
6780		var it = table.iterator();
6781		while (it.next()) \|entry\| {
6782		// This loop could be avoided by tracking inst depth, which
6783		// will be needed later anyway for reusing loop deaths.
6784		var parent_branch_i = block_data.branch_depth - 1;
6785		while (parent_branch_i > 0) : (parent_branch_i -= 1) {
6786		const parent_table = &self.branch_stack.items[parent_branch_i].inst_table;
6787		if (parent_table.contains(entry.key_ptr.*)) break;
6788		} else continue;
6789		const gop = branch.inst_table.getOrPutAssumeCapacity(entry.key_ptr.*);
6790		if (!gop.found_existing) gop.value_ptr.* = entry.value_ptr.*;
6791		}
6792		}
	6631	const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
	6632	const block_data = self.blocks.getPtr(br.block_inst).?;
	6633	if (block_tracking.long == .unreach) {
	6634	const result = result: {
	6635	if (block_unused) break :result .none;
	6636	if (self.reuseOperand(inst, br.operand, 0, src_mcv)) break :result src_mcv;
6793	6637
6794		log.debug("airBr: %{d}", .{inst});
6795		log.debug("Upper branches:", .{});
6796		for (self.branch_stack.items) \|bs\| {
6797		log.debug("{}", .{bs.fmtDebug()});
6798		}
6799		log.debug("Prev branch: {}", .{block_data.branch.fmtDebug()});
6800		log.debug("Cur branch: {}", .{branch.fmtDebug()});
6801
6802		try self.canonicaliseBranches(false, &block_data.branch, &branch, true, false);
6803		block_data.branch.deinit(self.gpa);
6804		block_data.branch = branch;
	6638	const new_mcv = try self.allocRegOrMem(br.block_inst, true);
	6639	try self.setRegOrMem(block_ty, new_mcv, src_mcv);
	6640	break :result new_mcv;
	6641	};
	6642	block_tracking.* = InstTracking.init(result);
	6643	try self.saveRetroactiveState(&block_data.state, true);
	6644	self.freeValue(result);
	6645	} else {
	6646	if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);
	6647	try self.restoreState(block_data.state, .{
	6648	.emit_instructions = true,
	6649	.update_tracking = false,
	6650	.resurrect = false,
	6651	.close_scope = false,
	6652	});
6805	6653	}
6806	6654
6807	6655	// Emit a jump with a relocation. It will be patched up after the block ends.
...	...	@@ -6825,7 +6673,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6825	6673	extra_i += inputs.len;
6826	6674
6827	6675	var result: MCValue = .none;
6828		if (!is_volatile and self.liveness.isUnused(inst)) result = .dead else {
	6676	if (!is_volatile and self.liveness.isUnused(inst)) result = .unreach else {
6829	6677	var args = std.StringArrayHashMap(MCValue).init(self.gpa);
6830	6678	try args.ensureTotalCapacity(outputs.len + inputs.len + clobbers_len);
6831	6679	defer {
...	...	@@ -7051,25 +6899,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7051	6899	std.mem.copy(Air.Inst.Ref, buf[buf_index..], inputs);
7052	6900	return self.finishAir(inst, result, buf);
7053	6901	}
7054		var bt = try self.iterateBigTomb(inst, outputs.len + inputs.len);
7055		for (outputs) \|output\| {
7056		if (output == .none) continue;
7057
7058		bt.feed(output);
7059		}
7060		for (inputs) \|input\| {
7061		bt.feed(input);
7062		}
7063		return bt.finishAir(result);
7064		}
7065
7066		fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
7067		try self.ensureProcessDeathCapacity(operand_count + 1);
7068		return BigTomb{
7069		.function = self,
7070		.inst = inst,
7071		.lbt = self.liveness.iterateBigTomb(inst),
7072		};
	6902	var bt = self.liveness.iterateBigTomb(inst);
	6903	for (outputs) \|output\| if (output != .none) self.feed(&bt, output);
	6904	for (inputs) \|input\| self.feed(&bt, input);
	6905	return self.finishAirResult(inst, result);
7073	6906	}
7074	6907
7075	6908	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
...	...	@@ -7951,7 +7784,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
7951	7784
7952	7785	fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7953	7786	const un_op = self.air.instructions.items(.data)[inst].un_op;
7954		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	7787	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
7955	7788	const src_mcv = try self.resolveInst(un_op);
7956	7789	if (self.reuseOperand(inst, un_op, 0, src_mcv)) break :result src_mcv;
7957	7790
...	...	@@ -7965,7 +7798,7 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7965	7798
7966	7799	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7967	7800	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
7968		const result = if (self.liveness.isUnused(inst)) .dead else result: {
	7801	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
7969	7802	const operand = try self.resolveInst(ty_op.operand);
7970	7803	if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
7971	7804
...	...	@@ -7990,7 +7823,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
7990	7823	const ptr = try self.resolveInst(ty_op.operand);
7991	7824	const array_ty = ptr_ty.childType();
7992	7825	const array_len = array_ty.arrayLen();
7993		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
	7826	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else blk: {
7994	7827	const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
7995	7828	try self.genSetStack(ptr_ty, stack_offset, ptr, .{});
7996	7829	try self.genSetStack(Type.u64, stack_offset - 8, .{ .immediate = array_len }, .{});
...	...	@@ -8002,7 +7835,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
8002	7835	fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
8003	7836	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
8004	7837	const result: MCValue = if (self.liveness.isUnused(inst))
8005		.dead
	7838	.unreach
8006	7839	else
8007	7840	return self.fail("TODO implement airIntToFloat for {}", .{self.target.cpu.arch});
8008	7841	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -8011,7 +7844,7 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
8011	7844	fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
8012	7845	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
8013	7846	if (self.liveness.isUnused(inst))
8014		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	7847	return self.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
8015	7848
8016	7849	const src_ty = self.air.typeOf(ty_op.operand);
8017	7850	const dst_ty = self.air.typeOfIndex(inst);
...	...	@@ -8113,7 +7946,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
8113	7946	}
8114	7947
8115	7948	const result: MCValue = result: {
8116		if (self.liveness.isUnused(inst)) break :result .dead;
	7949	if (self.liveness.isUnused(inst)) break :result .unreach;
8117	7950
8118	7951	if (val_abi_size <= 8) {
8119	7952	self.eflags_inst = inst;
...	...	@@ -8211,7 +8044,7 @@ fn atomicOp(
8211	8044	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8212	8045	} } });
8213	8046
8214		return if (unused) .none else dst_mcv;
	8047	return if (unused) .unreach else dst_mcv;
8215	8048	},
8216	8049	.loop => _ = if (val_abi_size <= 8) {
8217	8050	const tmp_reg = try self.register_manager.allocReg(null, gp);
...	...	@@ -8284,7 +8117,7 @@ fn atomicOp(
8284	8117	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8285	8118	} } });
8286	8119	_ = try self.asmJccReloc(loop, .ne);
8287		return if (unused) .none else .{ .register = .rax };
	8120	return if (unused) .unreach else .{ .register = .rax };
8288	8121	} else {
8289	8122	try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
8290	8123	.base = ptr_mem.sib.base,
...	...	@@ -8353,7 +8186,7 @@ fn atomicOp(
8353	8186	} });
8354	8187	_ = try self.asmJccReloc(loop, .ne);
8355	8188
8356		if (unused) return .none;
	8189	if (unused) return .unreach;
8357	8190	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
8358	8191	try self.asmMemoryRegister(
8359	8192	.mov,
...	...	@@ -8396,7 +8229,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
8396	8229	const atomic_load = self.air.instructions.items(.data)[inst].atomic_load;
8397	8230
8398	8231	const result: MCValue = result: {
8399		if (self.liveness.isUnused(inst)) break :result .dead;
	8232	if (self.liveness.isUnused(inst)) break :result .unreach;
8400	8233
8401	8234	const ptr_ty = self.air.typeOf(atomic_load.ptr);
8402	8235	const ptr_mcv = try self.resolveInst(atomic_load.ptr);
...	...	@@ -8458,7 +8291,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
8458	8291
8459	8292	try self.genInlineMemset(dst_ptr, src_val, len, .{});
8460	8293
8461		return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
	8294	return self.finishAir(inst, .unreach, .{ pl_op.operand, extra.lhs, extra.rhs });
8462	8295	}
8463	8296
8464	8297	fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -8488,13 +8321,13 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8488	8321
8489	8322	try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});
8490	8323
8491		return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
	8324	return self.finishAir(inst, .unreach, .{ pl_op.operand, extra.lhs, extra.rhs });
8492	8325	}
8493	8326
8494	8327	fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
8495	8328	const un_op = self.air.instructions.items(.data)[inst].un_op;
8496	8329	const operand = try self.resolveInst(un_op);
8497		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
	8330	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else {
8498	8331	_ = operand;
8499	8332	return self.fail("TODO implement airTagName for x86_64", .{});
8500	8333	};
...	...	@@ -8503,7 +8336,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
8503	8336
8504	8337	fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8505	8338	const un_op = self.air.instructions.items(.data)[inst].un_op;
8506		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	8339	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
8507	8340	const err_ty = self.air.typeOf(un_op);
8508	8341	const err_mcv = try self.resolveInst(un_op);
8509	8342	const err_reg = try self.copyToTmpRegister(err_ty, err_mcv);
...	...	@@ -8589,26 +8422,26 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8589	8422
8590	8423	fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
8591	8424	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
8592		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSplat for x86_64", .{});
	8425	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airSplat for x86_64", .{});
8593	8426	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8594	8427	}
8595	8428
8596	8429	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
8597	8430	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
8598	8431	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
8599		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSelect for x86_64", .{});
	8432	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airSelect for x86_64", .{});
8600	8433	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
8601	8434	}
8602	8435
8603	8436	fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
8604	8437	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
8605		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airShuffle for x86_64", .{});
	8438	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airShuffle for x86_64", .{});
8606	8439	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8607	8440	}
8608	8441
8609	8442	fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
8610	8443	const reduce = self.air.instructions.items(.data)[inst].reduce;
8611		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airReduce for x86_64", .{});
	8444	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airReduce for x86_64", .{});
8612	8445	return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
8613	8446	}
8614	8447
...	...	@@ -8620,7 +8453,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8620	8453	const abi_size = @intCast(u32, result_ty.abiSize(self.target.*));
8621	8454	const abi_align = result_ty.abiAlignment(self.target.*);
8622	8455	const result: MCValue = res: {
8623		if (self.liveness.isUnused(inst)) break :res MCValue.dead;
	8456	if (self.liveness.isUnused(inst)) break :res MCValue.unreach;
8624	8457	switch (result_ty.zigTypeTag()) {
8625	8458	.Struct => {
8626	8459	const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
...	...	@@ -8739,18 +8572,16 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8739	8572	std.mem.copy(Air.Inst.Ref, &buf, elements);
8740	8573	return self.finishAir(inst, result, buf);
8741	8574	}
8742		var bt = try self.iterateBigTomb(inst, elements.len);
8743		for (elements) \|elem\| {
8744		bt.feed(elem);
8745		}
8746		return bt.finishAir(result);
	8575	var bt = self.liveness.iterateBigTomb(inst);
	8576	for (elements) \|elem\| self.feed(&bt, elem);
	8577	return self.finishAirResult(inst, result);
8747	8578	}
8748	8579
8749	8580	fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
8750	8581	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
8751	8582	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
8752	8583	const result: MCValue = res: {
8753		if (self.liveness.isUnused(inst)) break :res MCValue.dead;
	8584	if (self.liveness.isUnused(inst)) break :res MCValue.unreach;
8754	8585	return self.fail("TODO implement airAggregateInit for x86_64", .{});
8755	8586	};
8756	8587	return self.finishAir(inst, result, .{ extra.init, .none, .none });
...	...	@@ -8758,63 +8589,57 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
8758	8589
8759	8590	fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
8760	8591	const prefetch = self.air.instructions.items(.data)[inst].prefetch;
8761		return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
	8592	return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
8762	8593	}
8763	8594
8764	8595	fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
8765	8596	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
8766	8597	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
8767		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
	8598	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else {
8768	8599	return self.fail("TODO implement airMulAdd for x86_64", .{});
8769	8600	};
8770	8601	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
8771	8602	}
8772	8603
8773		fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
8774		// First section of indexes correspond to a set number of constant values.
8775		const ref_int = @enumToInt(inst);
8776		if (ref_int < Air.Inst.Ref.typed_value_map.len) {
8777		const tv = Air.Inst.Ref.typed_value_map[ref_int];
8778		if (!tv.ty.hasRuntimeBitsIgnoreComptime() and !tv.ty.isError()) {
8779		return .none;
8780		}
8781		return self.genTypedValue(tv);
8782		}
	8604	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
	8605	const ty = self.air.typeOf(ref);
8783	8606
8784	8607	// If the type has no codegen bits, no need to store it.
8785		const inst_ty = self.air.typeOf(inst);
8786		if (!inst_ty.hasRuntimeBitsIgnoreComptime() and !inst_ty.isError())
8787		return .none;
8788
8789		const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len);
8790		switch (self.air.instructions.items(.tag)[inst_index]) {
8791		.constant => {
8792		// Constants have static lifetimes, so they are always memoized in the outer most table.
8793		const branch = &self.branch_stack.items[0];
8794		const gop = try branch.inst_table.getOrPut(self.gpa, inst_index);
8795		if (!gop.found_existing) {
8796		const ty_pl = self.air.instructions.items(.data)[inst_index].ty_pl;
8797		gop.value_ptr.* = try self.genTypedValue(.{
8798		.ty = inst_ty,
8799		.val = self.air.values[ty_pl.payload],
8800		});
8801		}
8802		return gop.value_ptr.*;
8803		},
8804		.const_ty => unreachable,
8805		else => return self.getResolvedInstValue(inst_index).?.*,
	8608	if (!ty.hasRuntimeBitsIgnoreComptime() and !ty.isError()) return .none;
	8609
	8610	if (Air.refToIndex(ref)) \|inst\| {
	8611	const mcv = switch (self.air.instructions.items(.tag)[inst]) {
	8612	.constant => tracking: {
	8613	const gop = try self.const_tracking.getOrPut(self.gpa, inst);
	8614	if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(try self.genTypedValue(.{
	8615	.ty = ty,
	8616	.val = self.air.value(ref).?,
	8617	}));
	8618	break :tracking gop.value_ptr;
	8619	},
	8620	.const_ty => unreachable,
	8621	else => self.inst_tracking.getPtr(inst).?,
	8622	}.short;
	8623	switch (mcv) {
	8624	.none, .unreach => unreachable,
	8625	else => return mcv,
	8626	}
8806	8627	}
	8628
	8629	return self.genTypedValue(.{ .ty = ty, .val = self.air.value(ref).? });
8807	8630	}
8808	8631
8809		fn getResolvedInstValue(self: Self, inst: Air.Inst.Index) ?MCValue {
8810		// Treat each stack item as a "layer" on top of the previous one.
8811		var i: usize = self.branch_stack.items.len;
8812		while (true) {
8813		i -= 1;
8814		if (self.branch_stack.items[i].inst_table.getPtr(inst)) \|mcv\| {
8815		return if (mcv.* != .dead) mcv else null;
8816		}
8817		}
	8632	fn getResolvedInstValue(self: Self, inst: Air.Inst.Index) ?InstTracking {
	8633	const tracking = switch (self.air.instructions.items(.tag)[inst]) {
	8634	.constant => self.const_tracking.getPtr(inst) orelse return null,
	8635	.const_ty => unreachable,
	8636	else => self.inst_tracking.getPtr(inst).?,
	8637	};
	8638	return switch (tracking.short) {
	8639	.unreach => unreachable,
	8640	.dead => null,
	8641	else => tracking,
	8642	};
8818	8643	}
8819	8644
8820	8645	/// If the MCValue is an immediate, and it does not fit within this type,

src/register_manager.zig+4

...	...	@@ -95,6 +95,10 @@ pub fn RegisterManager(
95	95	return indexOfReg(tracked_registers, reg);
96	96	}
97	97
	98	pub fn regAtTrackedIndex(index: RegisterBitSet.ShiftInt) Register {
	99	return tracked_registers[index];
	100	}
	101
98	102	/// Returns true when this register is not tracked
99	103	pub fn isRegFree(self: Self, reg: Register) bool {
100	104	const index = indexOfRegIntoTracked(reg) orelse return true;

test/behavior/for.zig-1

...	...	@@ -274,7 +274,6 @@ test "two counters" {
274	274	test "1-based counter and ptr to array" {
275	275	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
276	276	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
277		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
278	277
279	278	var ok: usize = 0;
280	279