Commit 011bc59e8aa4b62a286e4dd6e92371af5dac8b9f | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-04-27 15:45:01-07:00
committer	GitHub <noreply@github.com> 2023-04-27 15:45:01-07:00
log	011bc59e8aa4b62a286e4dd6e92371af5dac8b9f
tree	9205ef1625f18435be9dfbffcc709dee73b27724
parent	aa3405aabc539c66a9f110bc044924db07e892b6
parent	f56054d129d19a006ff19c56b48bc9c7d9c04593
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15435 from jacobly0/x86_64-frame

x86_64: add frame indices

27 files changed, 2751 insertions(+), 2647 deletions(-)

lib
- test_runner.zig
src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    bits.zig
    encoder.zig
    encodings.zig
- link
  - Coff.zig
  - Coff
    Relocation.zig
  - MachO.zig
  - MachO
    Relocation.zig
test
- behavior
- src
  - Cases.zig

lib/test_runner.zig+20-4

...	...	@@ -233,14 +233,30 @@ pub fn log(
233	233	/// Simpler main(), exercising fewer language features, so that
234	234	/// work-in-progress backends can handle it.
235	235	pub fn mainSimple() anyerror!void {
236		//const stderr = std.io.getStdErr();
	236	const enable_print = false;
	237
	238	var passed: u64 = 0;
	239	var skipped: u64 = 0;
	240	var failed: u64 = 0;
	241	const stderr = if (enable_print) std.io.getStdErr() else {};
237	242	for (builtin.test_functions) \|test_fn\| {
238	243	test_fn.func() catch \|err\| {
	244	if (enable_print) stderr.writeAll(test_fn.name) catch {};
239	245	if (err != error.SkipZigTest) {
240		//stderr.writeAll(test_fn.name) catch {};
241		//stderr.writeAll("\n") catch {};
242		return err;
	246	if (enable_print) stderr.writeAll("... FAIL\n") catch {};
	247	failed += 1;
	248	if (!enable_print) return err;
	249	continue;
243	250	}
	251	if (enable_print) stderr.writeAll("... SKIP\n") catch {};
	252	skipped += 1;
	253	continue;
244	254	};
	255	//if (enable_print) stderr.writeAll("... PASS\n") catch {};
	256	passed += 1;
	257	}
	258	if (enable_print) {
	259	stderr.writer().print("{} passed, {} skipped, {} failed\n", .{ passed, skipped, failed }) catch {};
	260	if (failed != 0) std.process.exit(1);
245	261	}
246	262	}

src/arch/x86_64/CodeGen.zig+2310-2389

...	...	@@ -6,6 +6,7 @@ const codegen = @import("../../codegen.zig");
6	6	const leb128 = std.leb;
7	7	const link = @import("../../link.zig");
8	8	const log = std.log.scoped(.codegen);
	9	const tracking_log = std.log.scoped(.tracking);
9	10	const math = std.math;
10	11	const mem = std.mem;
11	12	const trace = @import("../../tracy.zig").trace;
...	...	@@ -40,6 +41,7 @@ const Memory = bits.Memory;
40	41	const Register = bits.Register;
41	42	const RegisterManager = abi.RegisterManager;
42	43	const RegisterLock = RegisterManager.RegisterLock;
	44	const FrameIndex = bits.FrameIndex;
43	45
44	46	const gp = abi.RegisterClass.gp;
45	47	const sse = abi.RegisterClass.sse;
...	...	@@ -47,6 +49,7 @@ const sse = abi.RegisterClass.sse;
47	49	const InnerError = CodeGenError \|\| error{OutOfRegisters};
48	50
49	51	const debug_wip_mir = false;
	52	const debug_tracking = false;
50	53
51	54	gpa: Allocator,
52	55	air: Air,
...	...	@@ -57,11 +60,10 @@ target: *const std.Target,
57	60	mod_fn: *const Module.Fn,
58	61	err_msg: ?*ErrorMsg,
59	62	args: []MCValue,
60		ret_mcv: MCValue,
	63	ret_mcv: InstTracking,
61	64	fn_type: Type,
62	65	arg_index: u32,
63	66	src_loc: Module.SrcLoc,
64		stack_align: u32,
65	67
66	68	eflags_inst: ?Air.Inst.Index = null,
67	69
...	...	@@ -86,17 +88,13 @@ inst_tracking: InstTrackingMap = .{},
86	88	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
87	89
88	90	register_manager: RegisterManager = .{},
89		/// Maps offset to what is stored there.
90		stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
91	91
92	92	/// Generation of the current scope, increments by 1 for every entered scope.
93	93	scope_generation: u32 = 0,
94	94
95		/// Offset from the stack base, representing the end of the stack frame.
96		max_end_stack: u32 = 0,
97		/// Represents the current end stack offset. If there is no existing slot
98		/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
99		next_stack_offset: u32 = 0,
	95	frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
	96	free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .{},
	97	frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
100	98
101	99	/// Debug field, used to find bugs in the compiler.
102	100	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
...	...	@@ -108,6 +106,9 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}
108	106
109	107	const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged(Mir.Inst.Index, Air.Inst.Index){} else {};
110	108
	109	const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
	110	const RegisterOffset = struct { reg: Register, off: i32 = 0 };
	111
111	112	pub const MCValue = union(enum) {
112	113	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
113	114	/// TODO Look into deleting this tag and using `dead` instead, since every use
...	...	@@ -123,46 +124,59 @@ pub const MCValue = union(enum) {
123	124	/// A pointer-sized integer that fits in a register.
124	125	/// If the type is a pointer, this is the pointer address in virtual address space.
125	126	immediate: u64,
126		/// The value is in a GP register.
	127	/// The value resides in the EFLAGS register.
	128	eflags: Condition,
	129	/// The value is in a register.
127	130	register: Register,
	131	/// The value is a constant offset from the value in a register.
	132	register_offset: RegisterOffset,
128	133	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
129	134	register_overflow: struct { reg: Register, eflags: Condition },
130	135	/// The value is in memory at a hard-coded address.
131	136	/// If the type is a pointer, it means the pointer address is at this memory location.
132	137	memory: u64,
	138	/// The value is in memory at a constant offset from the address in a register.
	139	indirect: RegisterOffset,
133	140	/// The value is in memory.
134	141	/// Payload is a symbol index.
135	142	load_direct: u32,
136		/// The value is a pointer to value in memory.
	143	/// The value is a pointer to a value in memory.
137	144	/// Payload is a symbol index.
138	145	lea_direct: u32,
139	146	/// The value is in memory referenced indirectly via GOT.
140	147	/// Payload is a symbol index.
141	148	load_got: u32,
	149	/// The value is a pointer to a value referenced indirectly via GOT.
	150	/// Payload is a symbol index.
	151	lea_got: u32,
142	152	/// The value is a threadlocal variable.
143	153	/// Payload is a symbol index.
144	154	load_tlv: u32,
145		/// The value is a pointer to threadlocal variable.
	155	/// The value is a pointer to a threadlocal variable.
146	156	/// Payload is a symbol index.
147	157	lea_tlv: u32,
148		/// The value is one of the stack variables.
149		/// If the type is a pointer, it means the pointer address is in the stack at this offset.
150		stack_offset: i32,
151		/// The value is a pointer to one of the stack variables (payload is stack offset).
152		ptr_stack_offset: i32,
153		/// The value resides in the EFLAGS register.
154		eflags: Condition,
	158	/// The value stored at an offset from a frame index
	159	/// Payload is a frame address.
	160	load_frame: FrameAddr,
	161	/// The address of an offset from a frame index
	162	/// Payload is a frame address.
	163	lea_frame: FrameAddr,
	164	/// This indicates that we have already allocated a frame index for this instruction,
	165	/// but it has not been spilled there yet in the current control flow.
	166	/// Payload is a frame index.
	167	reserved_frame: FrameIndex,
155	168
156	169	fn isMemory(mcv: MCValue) bool {
157	170	return switch (mcv) {
158	171	.memory,
159		.stack_offset,
160		.ptr_stack_offset,
161	172	.load_direct,
162	173	.lea_direct,
163	174	.load_got,
	175	.lea_got,
164	176	.load_tlv,
165	177	.lea_tlv,
	178	.load_frame,
	179	.lea_frame,
166	180	=> true,
167	181	else => false,
168	182	};
...	...	@@ -181,6 +195,134 @@ pub const MCValue = union(enum) {
181	195	else => false,
182	196	};
183	197	}
	198
	199	fn getReg(mcv: MCValue) ?Register {
	200	return switch (mcv) {
	201	.register => \|reg\| reg,
	202	.register_offset, .indirect => \|ro\| ro.reg,
	203	.register_overflow => \|ro\| ro.reg,
	204	else => null,
	205	};
	206	}
	207
	208	fn getCondition(mcv: MCValue) ?Condition {
	209	return switch (mcv) {
	210	.eflags => \|cc\| cc,
	211	.register_overflow => \|reg_ov\| reg_ov.eflags,
	212	else => null,
	213	};
	214	}
	215
	216	fn address(mcv: MCValue) MCValue {
	217	return switch (mcv) {
	218	.none,
	219	.unreach,
	220	.dead,
	221	.undef,
	222	.immediate,
	223	.register,
	224	.register_offset,
	225	.eflags,
	226	.register_overflow,
	227	.lea_direct,
	228	.lea_got,
	229	.lea_tlv,
	230	.lea_frame,
	231	.reserved_frame,
	232	=> unreachable, // not in memory
	233	.memory => \|addr\| .{ .immediate = addr },
	234	.indirect => \|reg_off\| switch (reg_off.off) {
	235	0 => .{ .register = reg_off.reg },
	236	else => .{ .register_offset = reg_off },
	237	},
	238	.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
	239	.load_got => \|sym_index\| .{ .lea_got = sym_index },
	240	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
	241	.load_frame => \|frame_addr\| .{ .lea_frame = frame_addr },
	242	};
	243	}
	244
	245	fn deref(mcv: MCValue) MCValue {
	246	return switch (mcv) {
	247	.none,
	248	.unreach,
	249	.dead,
	250	.undef,
	251	.eflags,
	252	.register_overflow,
	253	.memory,
	254	.indirect,
	255	.load_direct,
	256	.load_got,
	257	.load_tlv,
	258	.load_frame,
	259	.reserved_frame,
	260	=> unreachable, // not a dereferenceable
	261	.immediate => \|addr\| .{ .memory = addr },
	262	.register => \|reg\| .{ .indirect = .{ .reg = reg } },
	263	.register_offset => \|reg_off\| .{ .indirect = reg_off },
	264	.lea_direct => \|sym_index\| .{ .load_direct = sym_index },
	265	.lea_got => \|sym_index\| .{ .load_got = sym_index },
	266	.lea_tlv => \|sym_index\| .{ .load_tlv = sym_index },
	267	.lea_frame => \|frame_addr\| .{ .load_frame = frame_addr },
	268	};
	269	}
	270
	271	fn offset(mcv: MCValue, off: i32) MCValue {
	272	return switch (mcv) {
	273	.none,
	274	.unreach,
	275	.dead,
	276	.undef,
	277	.eflags,
	278	.register_overflow,
	279	.memory,
	280	.indirect,
	281	.load_direct,
	282	.lea_direct,
	283	.load_got,
	284	.lea_got,
	285	.load_tlv,
	286	.lea_tlv,
	287	.load_frame,
	288	.reserved_frame,
	289	=> unreachable, // not offsettable
	290	.immediate => \|imm\| .{ .immediate = @bitCast(u64, @bitCast(i64, imm) +% off) },
	291	.register => \|reg\| .{ .register_offset = .{ .reg = reg, .off = off } },
	292	.register_offset => \|reg_off\| .{
	293	.register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off },
	294	},
	295	.lea_frame => \|frame_addr\| .{
	296	.lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
	297	},
	298	};
	299	}
	300
	301	pub fn format(
	302	mcv: MCValue,
	303	comptime _: []const u8,
	304	_: std.fmt.FormatOptions,
	305	writer: anytype,
	306	) @TypeOf(writer).Error!void {
	307	switch (mcv) {
	308	.none, .unreach, .dead, .undef => try writer.print("({s})", .{@tagName(mcv)}),
	309	.immediate => \|pl\| try writer.print("0x{x}", .{pl}),
	310	.memory => \|pl\| try writer.print("[ds:0x{x}]", .{pl}),
	311	inline .eflags, .register => \|pl\| try writer.print("{s}", .{@tagName(pl)}),
	312	.register_offset => \|pl\| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }),
	313	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{ @tagName(pl.eflags), @tagName(pl.reg) }),
	314	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
	315	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),
	316	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),
	317	.load_got => \|pl\| try writer.print("[got:{d}]", .{pl}),
	318	.lea_got => \|pl\| try writer.print("got:{d}", .{pl}),
	319	.load_tlv => \|pl\| try writer.print("[tlv:{d}]", .{pl}),
	320	.lea_tlv => \|pl\| try writer.print("tlv:{d}", .{pl}),
	321	.load_frame => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }),
	322	.lea_frame => \|pl\| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
	323	.reserved_frame => \|pl\| try writer.print("(dead:{})", .{pl}),
	324	}
	325	}
184	326	};
185	327
186	328	const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);
...	...	@@ -189,89 +331,211 @@ const InstTracking = struct {
189	331	short: MCValue,
190	332
191	333	fn init(result: MCValue) InstTracking {
192		return .{ .long = result, .short = result };
	334	return .{ .long = switch (result) {
	335	.none,
	336	.unreach,
	337	.undef,
	338	.immediate,
	339	.memory,
	340	.load_direct,
	341	.lea_direct,
	342	.load_got,
	343	.lea_got,
	344	.load_tlv,
	345	.lea_tlv,
	346	.load_frame,
	347	.lea_frame,
	348	=> result,
	349	.dead,
	350	.reserved_frame,
	351	=> unreachable,
	352	.eflags,
	353	.register,
	354	.register_offset,
	355	.register_overflow,
	356	.indirect,
	357	=> .none,
	358	}, .short = result };
193	359	}
194	360
195	361	fn getReg(self: InstTracking) ?Register {
196		return switch (self.short) {
197		.register => \|reg\| reg,
198		.register_overflow => \|ro\| ro.reg,
199		else => null,
200		};
	362	return self.short.getReg();
201	363	}
202	364
203	365	fn getCondition(self: InstTracking) ?Condition {
204		return switch (self.short) {
205		.eflags => \|eflags\| eflags,
206		.register_overflow => \|ro\| ro.eflags,
207		else => null,
208		};
	366	return self.short.getCondition();
209	367	}
210	368
211	369	fn spill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
	370	if (std.meta.eql(self.long, self.short)) return; // Already spilled
	371	// Allocate or reuse frame index
	372	switch (self.long) {
	373	.none => self.long = try function.allocRegOrMem(inst, false),
	374	.load_frame => {},
	375	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
	376	else => unreachable,
	377	}
	378	tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
	379	try function.genCopy(function.air.typeOfIndex(inst), self.long, self.short);
	380	}
	381
	382	fn reuseFrame(self: *InstTracking) void {
212	383	switch (self.long) {
	384	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
	385	else => {},
	386	}
	387	self.short = switch (self.long) {
213	388	.none,
214		.dead,
215	389	.unreach,
216		=> unreachable,
	390	.undef,
	391	.immediate,
	392	.memory,
	393	.load_direct,
	394	.lea_direct,
	395	.load_got,
	396	.lea_got,
	397	.load_tlv,
	398	.lea_tlv,
	399	.load_frame,
	400	.lea_frame,
	401	=> self.long,
	402	.dead,
	403	.eflags,
217	404	.register,
	405	.register_offset,
218	406	.register_overflow,
219		.eflags,
220		=> self.long = try function.allocRegOrMem(inst, self.short == .eflags),
221		.stack_offset => {},
	407	.indirect,
	408	.reserved_frame,
	409	=> unreachable,
	410	};
	411	}
	412
	413	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
	414	function.freeValue(self.short);
	415	self.reuseFrame();
	416	tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
	417	}
	418
	419	fn verifyMaterialize(self: *InstTracking, target: InstTracking) void {
	420	switch (self.long) {
	421	.none,
	422	.unreach,
222	423	.undef,
223	424	.immediate,
224	425	.memory,
225	426	.load_direct,
226	427	.lea_direct,
227	428	.load_got,
	429	.lea_got,
228	430	.load_tlv,
229	431	.lea_tlv,
230		.ptr_stack_offset,
231		=> return, // these can be rematerialized without using a stack slot
	432	.lea_frame,
	433	=> assert(std.meta.eql(self.long, target.long)),
	434	.load_frame,
	435	.reserved_frame,
	436	=> switch (target.long) {
	437	.none,
	438	.load_frame,
	439	.reserved_frame,
	440	=> {},
	441	else => unreachable,
	442	},
	443	.dead,
	444	.eflags,
	445	.register,
	446	.register_offset,
	447	.register_overflow,
	448	.indirect,
	449	=> unreachable,
232	450	}
233		log.debug("spilling %{d} from {} to {}", .{ inst, self.short, self.long });
234		const ty = function.air.typeOfIndex(inst);
235		try function.setRegOrMem(ty, self.long, self.short);
236	451	}
237	452
238		fn trackSpill(self: InstTracking, function: Self) void {
239		if (self.getReg()) \|reg\| function.register_manager.freeReg(reg);
240		switch (self.short) {
241		.none, .dead, .unreach => unreachable,
242		else => {},
243		}
244		self.short = self.long;
	453	fn materialize(
	454	self: *InstTracking,
	455	function: *Self,
	456	inst: Air.Inst.Index,
	457	target: InstTracking,
	458	) !void {
	459	self.verifyMaterialize(target);
	460	try self.materializeUnsafe(function, inst, target);
245	461	}
246	462
247		fn materialize(self: InstTracking, function: Self, inst: Air.Inst.Index, reg: Register) !void {
	463	fn materializeUnsafe(
	464	self: *InstTracking,
	465	function: *Self,
	466	inst: Air.Inst.Index,
	467	target: InstTracking,
	468	) !void {
248	469	const ty = function.air.typeOfIndex(inst);
249		try function.genSetReg(ty, reg, self.long);
	470	if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame)
	471	try function.genCopy(ty, target.long, self.short);
	472	try function.genCopy(ty, target.short, self.short);
250	473	}
251	474
252		fn trackMaterialize(self: InstTracking, function: Self, inst: Air.Inst.Index, reg: Register) void {
253		assert(inst == function.register_manager.registers[
254		RegisterManager.indexOfRegIntoTracked(reg).?
255		]);
256		self.short = .{ .register = reg };
	475	fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
	476	self.verifyMaterialize(target);
	477	// Don't clobber reserved frame indices
	478	self.long = if (target.long == .none) switch (self.long) {
	479	.load_frame => \|addr\| .{ .reserved_frame = addr.index },
	480	.reserved_frame => self.long,
	481	else => target.long,
	482	} else target.long;
	483	self.short = target.short;
	484	tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });
257	485	}
258	486
259		fn resurrect(self: *InstTracking, scope_generation: u32) void {
	487	fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
260	488	switch (self.short) {
261	489	.dead => \|die_generation\| if (die_generation >= scope_generation) {
262		self.short = self.long;
	490	self.reuseFrame();
	491	tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });
263	492	},
264	493	else => {},
265	494	}
266	495	}
267	496
268		fn die(self: InstTracking, function: Self) void {
	497	fn die(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
269	498	function.freeValue(self.short);
270		self.reuse(function);
	499	self.short = .{ .dead = function.scope_generation };
	500	tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
271	501	}
272	502
273		fn reuse(self: InstTracking, function: Self) void {
	503	fn reuse(
	504	self: *InstTracking,
	505	function: *Self,
	506	new_inst: Air.Inst.Index,
	507	old_inst: Air.Inst.Index,
	508	) void {
274	509	self.short = .{ .dead = function.scope_generation };
	510	tracking_log.debug("%{d} => {} (reuse %{d})", .{ new_inst, self.*, old_inst });
	511	}
	512
	513	pub fn format(
	514	self: InstTracking,
	515	comptime _: []const u8,
	516	_: std.fmt.FormatOptions,
	517	writer: anytype,
	518	) @TypeOf(writer).Error!void {
	519	if (!std.meta.eql(self.long, self.short)) try writer.print("\|{}\| ", .{self.long});
	520	try writer.print("{}", .{self.short});
	521	}
	522	};
	523
	524	const FrameAlloc = struct {
	525	abi_size: u31,
	526	abi_align: u5,
	527	ref_count: u16,
	528
	529	fn init(alloc_abi: struct { size: u64, alignment: u32 }) FrameAlloc {
	530	assert(math.isPowerOfTwo(alloc_abi.alignment));
	531	return .{
	532	.abi_size = @intCast(u31, alloc_abi.size),
	533	.abi_align = math.log2_int(u32, alloc_abi.alignment),
	534	.ref_count = 0,
	535	};
	536	}
	537	fn initType(ty: Type, target: Target) FrameAlloc {
	538	return init(.{ .size = ty.abiSize(target), .alignment = ty.abiAlignment(target) });
275	539	}
276	540	};
277	541
...	...	@@ -332,12 +596,13 @@ pub fn generate(
332	596	.fn_type = fn_type,
333	597	.arg_index = 0,
334	598	.src_loc = src_loc,
335		.stack_align = undefined,
336	599	.end_di_line = module_fn.rbrace_line,
337	600	.end_di_column = module_fn.rbrace_column,
338	601	};
339	602	defer {
340		function.stack.deinit(gpa);
	603	function.frame_allocs.deinit(gpa);
	604	function.free_frame_indices.deinit(gpa);
	605	function.frame_locs.deinit(gpa);
341	606	var block_it = function.blocks.valueIterator();
342	607	while (block_it.next()) \|block\| block.deinit(gpa);
343	608	function.blocks.deinit(gpa);
...	...	@@ -349,7 +614,23 @@ pub fn generate(
349	614	if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);
350	615	}
351	616
352		var call_info = function.resolveCallingConventionValues(fn_type, &.{}) catch \|err\| switch (err) {
	617	try function.frame_allocs.resize(gpa, FrameIndex.named_count);
	618	function.frame_allocs.set(
	619	@enumToInt(FrameIndex.stack_frame),
	620	FrameAlloc.init(.{
	621	.size = 0,
	622	.alignment = if (mod.align_stack_fns.get(module_fn)) \|set_align_stack\|
	623	set_align_stack.alignment
	624	else
	625	1,
	626	}),
	627	);
	628	function.frame_allocs.set(
	629	@enumToInt(FrameIndex.call_frame),
	630	FrameAlloc.init(.{ .size = 0, .alignment = 1 }),
	631	);
	632
	633	var call_info = function.resolveCallingConventionValues(fn_type, &.{}, .args_frame) catch \|err\| switch (err) {
353	634	error.CodegenFail => return Result{ .fail = function.err_msg.? },
354	635	error.OutOfRegisters => return Result{
355	636	.fail = try ErrorMsg.create(
...	...	@@ -365,8 +646,18 @@ pub fn generate(
365	646
366	647	function.args = call_info.args;
367	648	function.ret_mcv = call_info.return_value;
368		function.stack_align = call_info.stack_align;
369		function.max_end_stack = call_info.stack_byte_count;
	649	function.frame_allocs.set(@enumToInt(FrameIndex.ret_addr), FrameAlloc.init(.{
	650	.size = Type.usize.abiSize(function.target.*),
	651	.alignment = @min(Type.usize.abiAlignment(function.target.*), call_info.stack_align),
	652	}));
	653	function.frame_allocs.set(@enumToInt(FrameIndex.base_ptr), FrameAlloc.init(.{
	654	.size = Type.usize.abiSize(function.target.*),
	655	.alignment = @min(Type.usize.abiAlignment(function.target.) 2, call_info.stack_align),
	656	}));
	657	function.frame_allocs.set(
	658	@enumToInt(FrameIndex.args_frame),
	659	FrameAlloc.init(.{ .size = call_info.stack_byte_count, .alignment = call_info.stack_align }),
	660	);
370	661
371	662	function.gen() catch \|err\| switch (err) {
372	663	error.CodegenFail => return Result{ .fail = function.err_msg.? },
...	...	@@ -379,6 +670,7 @@ pub fn generate(
379	670	var mir = Mir{
380	671	.instructions = function.mir_instructions.toOwnedSlice(),
381	672	.extra = try function.mir_extra.toOwnedSlice(bin_file.allocator),
	673	.frame_locs = function.frame_locs.toOwnedSlice(),
382	674	};
383	675	defer mir.deinit(bin_file.allocator);
384	676
...	...	@@ -432,6 +724,7 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
432	724	.mir = .{
433	725	.instructions = self.mir_instructions.slice(),
434	726	.extra = self.mir_extra.items,
	727	.frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
435	728	},
436	729	.target = self.target,
437	730	.src_loc = self.src_loc,
...	...	@@ -458,6 +751,14 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
458	751	}
459	752	}
460	753
	754	fn dumpTracking(self: *Self) !void {
	755	if (!debug_tracking) return;
	756	const stderr = std.io.getStdErr().writer();
	757
	758	var it = self.inst_tracking.iterator();
	759	while (it.next()) \|entry\| try stderr.print("%{d} = {}\n", .{ entry.key_ptr., entry.value_ptr. });
	760	}
	761
461	762	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
462	763	const gpa = self.gpa;
463	764	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
...	...	@@ -550,6 +851,14 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
550	851	});
551	852	}
552	853
	854	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
	855	return self.addInst(.{
	856	.tag = .dead,
	857	.ops = undefined,
	858	.data = undefined,
	859	});
	860	}
	861
553	862	fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
554	863	_ = try self.addInst(.{
555	864	.tag = tag,
...	...	@@ -765,44 +1074,32 @@ fn gen(self: *Self) InnerError!void {
765	1074	const cc = self.fn_type.fnCallingConvention();
766	1075	if (cc != .Naked) {
767	1076	try self.asmRegister(.push, .rbp);
	1077	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
768	1078	try self.asmRegisterRegister(.mov, .rbp, .rsp);
769
770		// We want to subtract the aligned stack frame size from rsp here, but we don't
771		// yet know how big it will be, so we leave room for a 4-byte stack size.
772		// TODO During semantic analysis, check if there are no function calls. If there
773		// are none, here we can omit the part where we subtract and then add rsp.
774		const backpatch_stack_sub = try self.addInst(.{
775		.tag = .dead,
776		.ops = undefined,
777		.data = undefined,
778		});
779
780		if (self.ret_mcv == .stack_offset) {
781		// The address where to store the return value for the caller is in a
782		// register which the callee is free to clobber. Therefore, we purposely
783		// spill it to stack immediately.
784		const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + 8, 8);
785		self.next_stack_offset = stack_offset;
786		self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);
787
788		const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
789		try self.genSetStack(Type.usize, @intCast(i32, stack_offset), MCValue{ .register = ret_reg }, .{});
790		self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };
791		log.debug("gen: spilling {s} to stack at offset {}", .{ @tagName(ret_reg), stack_offset });
	1079	const backpatch_frame_align = try self.asmPlaceholder();
	1080	const backpatch_stack_alloc = try self.asmPlaceholder();
	1081
	1082	switch (self.ret_mcv.long) {
	1083	.none, .unreach => {},
	1084	.indirect => {
	1085	// The address where to store the return value for the caller is in a
	1086	// register which the callee is free to clobber. Therefore, we purposely
	1087	// spill it to stack immediately.
	1088	const frame_index =
	1089	try self.allocFrameIndex(FrameAlloc.initType(Type.usize, self.target.*));
	1090	try self.genSetMem(
	1091	.{ .frame = frame_index },
	1092	0,
	1093	Type.usize,
	1094	self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
	1095	);
	1096	self.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } };
	1097	tracking_log.debug("spill {} to {}", .{ self.ret_mcv.long, frame_index });
	1098	},
	1099	else => unreachable,
792	1100	}
793	1101
794		_ = try self.addInst(.{
795		.tag = .dbg_prologue_end,
796		.ops = undefined,
797		.data = undefined,
798		});
799
800		// Push callee-preserved regs that were used actually in use.
801		const backpatch_push_callee_preserved_regs = try self.addInst(.{
802		.tag = .dead,
803		.ops = undefined,
804		.data = undefined,
805		});
	1102	try self.asmOpOnly(.dbg_prologue_end);
806	1103
807	1104	try self.genBody(self.air.getMainBody());
808	1105
...	...	@@ -814,69 +1111,43 @@ fn gen(self: *Self) InnerError!void {
814	1111	// }
815	1112	// Eliding the reloc will cause a miscompilation in this case.
816	1113	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
817		self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);
	1114	self.mir_instructions.items(.data)[jmp_reloc].inst =
	1115	@intCast(u32, self.mir_instructions.len);
818	1116	}
819	1117
820		// Create list of registers to save in the prologue.
821		// TODO handle register classes
822		var reg_list = Mir.RegisterList{};
823		const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
824		for (callee_preserved_regs) \|reg\| {
825		if (self.register_manager.isRegAllocated(reg)) {
826		reg_list.push(callee_preserved_regs, reg);
827		}
828		}
829		const saved_regs_stack_space: u32 = reg_list.count() * 8;
830
831		// Pop saved callee-preserved regs.
832		const backpatch_pop_callee_preserved_regs = try self.addInst(.{
833		.tag = .dead,
834		.ops = undefined,
835		.data = undefined,
836		});
837
838		_ = try self.addInst(.{
839		.tag = .dbg_epilogue_begin,
840		.ops = undefined,
841		.data = undefined,
842		});
843
844		// Maybe add rsp, x if required. This is backpatched later.
845		const backpatch_stack_add = try self.addInst(.{
846		.tag = .dead,
847		.ops = undefined,
848		.data = undefined,
849		});
850
	1118	try self.asmOpOnly(.dbg_epilogue_begin);
	1119	const backpatch_stack_dealloc = try self.asmPlaceholder();
	1120	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
851	1121	try self.asmRegister(.pop, .rbp);
852	1122	try self.asmOpOnly(.ret);
853	1123
854		// Adjust the stack
855		if (self.max_end_stack > math.maxInt(i32)) {
856		return self.failSymbol("too much stack used in call parameters", .{});
	1124	const frame_layout = try self.computeFrameLayout();
	1125	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
	1126	const need_stack_adjust = frame_layout.stack_adjust > 0;
	1127	const need_save_reg = frame_layout.save_reg_list.count() > 0;
	1128	if (need_frame_align) {
	1129	self.mir_instructions.set(backpatch_frame_align, .{
	1130	.tag = .@"and",
	1131	.ops = .ri_s,
	1132	.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },
	1133	});
857	1134	}
858
859		const aligned_stack_end = @intCast(
860		u32,
861		mem.alignForward(self.max_end_stack + saved_regs_stack_space, self.stack_align),
862		);
863		if (aligned_stack_end > 0) {
864		self.mir_instructions.set(backpatch_stack_sub, .{
	1135	if (need_stack_adjust) {
	1136	self.mir_instructions.set(backpatch_stack_alloc, .{
865	1137	.tag = .sub,
866		.ops = .ri_u,
867		.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
	1138	.ops = .ri_s,
	1139	.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },
868	1140	});
869		self.mir_instructions.set(backpatch_stack_add, .{
870		.tag = .add,
871		.ops = .ri_u,
872		.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
873		});
874
875		const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
876		.base_reg = @enumToInt(Register.rbp),
877		.register_list = reg_list.asInt(),
878		.stack_end = aligned_stack_end,
	1141	}
	1142	if (need_frame_align or need_stack_adjust) {
	1143	self.mir_instructions.set(backpatch_stack_dealloc, .{
	1144	.tag = .mov,
	1145	.ops = .rr,
	1146	.data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },
879	1147	});
	1148	}
	1149	if (need_save_reg) {
	1150	const save_reg_list = frame_layout.save_reg_list.asInt();
880	1151	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
881	1152	.tag = .push_regs,
882	1153	.ops = undefined,
...	...	@@ -889,30 +1160,19 @@ fn gen(self: *Self) InnerError!void {
889	1160	});
890	1161	}
891	1162	} else {
892		_ = try self.addInst(.{
893		.tag = .dbg_prologue_end,
894		.ops = undefined,
895		.data = undefined,
896		});
897
	1163	try self.asmOpOnly(.dbg_prologue_end);
898	1164	try self.genBody(self.air.getMainBody());
899
900		_ = try self.addInst(.{
901		.tag = .dbg_epilogue_begin,
902		.ops = undefined,
903		.data = undefined,
904		});
	1165	try self.asmOpOnly(.dbg_epilogue_begin);
905	1166	}
906	1167
907	1168	// Drop them off at the rbrace.
908		const payload = try self.addExtra(Mir.DbgLineColumn{
909		.line = self.end_di_line,
910		.column = self.end_di_column,
911		});
912	1169	_ = try self.addInst(.{
913	1170	.tag = .dbg_line,
914	1171	.ops = undefined,
915		.data = .{ .payload = payload },
	1172	.data = .{ .line_column = .{
	1173	.line = self.end_di_line,
	1174	.column = self.end_di_column,
	1175	} },
916	1176	});
917	1177	}
918	1178
...	...	@@ -932,6 +1192,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
932	1192	self.air,
933	1193	self.liveness,
934	1194	);
	1195	self.dumpTracking() catch {};
935	1196
936	1197	const old_air_bookkeeping = self.air_bookkeeping;
937	1198	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
...	...	@@ -1187,23 +1448,16 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1187	1448	while (it.next()) \|index\| {
1188	1449	const tracked_inst = self.register_manager.registers[index];
1189	1450	const tracking = self.getResolvedInstValue(tracked_inst);
1190		assert(RegisterManager.indexOfRegIntoTracked(switch (tracking.short) {
1191		.register => \|reg\| reg,
1192		.register_overflow => \|ro\| ro.reg,
1193		else => unreachable,
1194		}).? == index);
	1451	assert(RegisterManager.indexOfRegIntoTracked(tracking.getReg().?).? == index);
1195	1452	}
1196	1453	}
1197	1454	}
1198	1455	}
	1456	self.dumpTracking() catch {};
1199	1457	}
1200	1458
1201	1459	fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
1202		const reg = switch (value) {
1203		.register => \|reg\| reg,
1204		.register_overflow => \|ro\| ro.reg,
1205		else => return,
1206		};
	1460	const reg = value.getReg() orelse return;
1207	1461	if (self.register_manager.isRegFree(reg)) {
1208	1462	self.register_manager.getRegAssumeFree(reg, inst);
1209	1463	}
...	...	@@ -1214,8 +1468,11 @@ fn freeValue(self: *Self, value: MCValue) void {
1214	1468	.register => \|reg\| {
1215	1469	self.register_manager.freeReg(reg);
1216	1470	},
1217		.register_overflow => \|ro\| {
1218		self.register_manager.freeReg(ro.reg);
	1471	.register_offset => \|reg_off\| {
	1472	self.register_manager.freeReg(reg_off.reg);
	1473	},
	1474	.register_overflow => \|reg_ov\| {
	1475	self.register_manager.freeReg(reg_ov.reg);
1219	1476	self.eflags_inst = null;
1220	1477	},
1221	1478	.eflags => {
...	...	@@ -1231,10 +1488,10 @@ fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) void {
1231	1488
1232	1489	/// Asserts there is already capacity to insert into top branch inst_table.
1233	1490	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
1234		const air_tags = self.air.instructions.items(.tag);
1235		if (air_tags[inst] == .constant) return;
1236		log.debug("%{d} => {}", .{ inst, MCValue.dead });
1237		self.inst_tracking.getPtr(inst).?.die(self);
	1491	switch (self.air.instructions.items(.tag)[inst]) {
	1492	.constant, .const_ty => unreachable,
	1493	else => self.inst_tracking.getPtr(inst).?.die(self, inst),
	1494	}
1238	1495	}
1239	1496
1240	1497	/// Called when there are no operands, and the instruction is always unreferenced.
...	...	@@ -1249,7 +1506,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
1249	1506	.none, .dead, .unreach => {},
1250	1507	else => unreachable, // Why didn't the result die?
1251	1508	} else {
1252		log.debug("%{d} => {}", .{ inst, result });
	1509	tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
1253	1510	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
1254	1511	// In some cases, an operand may be reused as the result.
1255	1512	// If that operand died and was a register, it was freed by
...	...	@@ -1273,36 +1530,123 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
1273	1530	self.finishAirResult(inst, result);
1274	1531	}
1275	1532
1276		fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
1277		if (abi_align > self.stack_align)
1278		self.stack_align = abi_align;
1279		// TODO find a free slot instead of always appending
1280		const offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);
1281		self.next_stack_offset = offset;
1282		self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);
1283		try self.stack.putNoClobber(self.gpa, offset, .{
1284		.inst = inst,
1285		.size = abi_size,
1286		});
1287		return offset;
	1533	const FrameLayout = struct {
	1534	stack_mask: u32,
	1535	stack_adjust: u32,
	1536	save_reg_list: Mir.RegisterList,
	1537	};
	1538
	1539	fn setFrameLoc(
	1540	self: *Self,
	1541	frame_index: FrameIndex,
	1542	base: Register,
	1543	offset: *i32,
	1544	comptime aligned: bool,
	1545	) void {
	1546	const frame_i = @enumToInt(frame_index);
	1547	if (aligned) {
	1548	const alignment = @as(i32, 1) << self.frame_allocs.items(.abi_align)[frame_i];
	1549	offset.* = mem.alignForwardGeneric(i32, offset.*, alignment);
	1550	}
	1551	self.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* });
	1552	offset.* += self.frame_allocs.items(.abi_size)[frame_i];
	1553	}
	1554
	1555	fn computeFrameLayout(self: *Self) !FrameLayout {
	1556	const frame_allocs_len = self.frame_allocs.len;
	1557	try self.frame_locs.resize(self.gpa, frame_allocs_len);
	1558	const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
	1559	defer self.gpa.free(stack_frame_order);
	1560
	1561	const frame_size = self.frame_allocs.items(.abi_size);
	1562	const frame_align = self.frame_allocs.items(.abi_align);
	1563	const frame_offset = self.frame_locs.items(.disp);
	1564
	1565	for (stack_frame_order, FrameIndex.named_count..) \|*frame_order, frame_index\|
	1566	frame_order.* = @intToEnum(FrameIndex, frame_index);
	1567	{
	1568	const SortContext = struct {
	1569	frame_align: @TypeOf(frame_align),
	1570	pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool {
	1571	return context.frame_align[@enumToInt(lhs)] > context.frame_align[@enumToInt(rhs)];
	1572	}
	1573	};
	1574	const sort_context = SortContext{ .frame_align = frame_align };
	1575	std.sort.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
	1576	}
	1577
	1578	const call_frame_align = frame_align[@enumToInt(FrameIndex.call_frame)];
	1579	const stack_frame_align = frame_align[@enumToInt(FrameIndex.stack_frame)];
	1580	const args_frame_align = frame_align[@enumToInt(FrameIndex.args_frame)];
	1581	const needed_align = @max(call_frame_align, stack_frame_align);
	1582	const need_align_stack = needed_align > args_frame_align;
	1583
	1584	// Create list of registers to save in the prologue.
	1585	// TODO handle register classes
	1586	var save_reg_list = Mir.RegisterList{};
	1587	const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
	1588	for (callee_preserved_regs) \|reg\| {
	1589	if (self.register_manager.isRegAllocated(reg)) {
	1590	save_reg_list.push(callee_preserved_regs, reg);
	1591	}
	1592	}
	1593
	1594	var rbp_offset = @intCast(i32, save_reg_list.count() * 8);
	1595	self.setFrameLoc(.base_ptr, .rbp, &rbp_offset, false);
	1596	self.setFrameLoc(.ret_addr, .rbp, &rbp_offset, false);
	1597	self.setFrameLoc(.args_frame, .rbp, &rbp_offset, false);
	1598	const stack_frame_align_offset =
	1599	if (need_align_stack) 0 else frame_offset[@enumToInt(FrameIndex.args_frame)];
	1600
	1601	var rsp_offset: i32 = 0;
	1602	self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true);
	1603	self.setFrameLoc(.stack_frame, .rsp, &rsp_offset, true);
	1604	for (stack_frame_order) \|frame_index\| self.setFrameLoc(frame_index, .rsp, &rsp_offset, true);
	1605	rsp_offset += stack_frame_align_offset;
	1606	rsp_offset = mem.alignForwardGeneric(i32, rsp_offset, @as(i32, 1) << needed_align);
	1607	rsp_offset -= stack_frame_align_offset;
	1608	frame_size[@enumToInt(FrameIndex.call_frame)] =
	1609	@intCast(u31, rsp_offset - frame_offset[@enumToInt(FrameIndex.stack_frame)]);
	1610
	1611	return .{
	1612	.stack_mask = @as(u32, math.maxInt(u32)) << (if (need_align_stack) needed_align else 0),
	1613	.stack_adjust = @intCast(u32, rsp_offset - frame_offset[@enumToInt(FrameIndex.call_frame)]),
	1614	.save_reg_list = save_reg_list,
	1615	};
1288	1616	}
1289	1617
1290		/// Use a pointer instruction as the basis for allocating stack memory.
1291		fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
1292		const ptr_ty = self.air.typeOfIndex(inst);
1293		const elem_ty = ptr_ty.elemType();
	1618	fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
	1619	const frame_allocs_slice = self.frame_allocs.slice();
	1620	const frame_size = frame_allocs_slice.items(.abi_size);
	1621	const frame_align = frame_allocs_slice.items(.abi_align);
	1622	for (self.free_frame_indices.keys(), 0..) \|frame_index, free_i\| {
	1623	const abi_size = frame_size[@enumToInt(frame_index)];
	1624	if (abi_size != alloc.abi_size) continue;
	1625	const abi_align = &frame_align[@enumToInt(frame_index)];
	1626	abi_align.* = @max(abi_align.*, alloc.abi_align);
1294	1627
1295		if (!elem_ty.hasRuntimeBitsIgnoreComptime()) {
1296		return self.allocMem(inst, @sizeOf(usize), @alignOf(usize));
	1628	const stack_frame_align = &frame_align[@enumToInt(FrameIndex.stack_frame)];
	1629	stack_frame_align.* = @max(stack_frame_align.*, alloc.abi_align);
	1630
	1631	_ = self.free_frame_indices.swapRemoveAt(free_i);
	1632	return frame_index;
1297	1633	}
	1634	const frame_index = @intToEnum(FrameIndex, self.frame_allocs.len);
	1635	try self.frame_allocs.append(self.gpa, alloc);
	1636	return frame_index;
	1637	}
1298	1638
1299		const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) orelse {
1300		const mod = self.bin_file.options.module.?;
1301		return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});
1302		};
1303		// TODO swap this for inst.ty.ptrAlign
1304		const abi_align = ptr_ty.ptrAlignment(self.target.*);
1305		return self.allocMem(inst, abi_size, abi_align);
	1639	/// Use a pointer instruction as the basis for allocating stack memory.
	1640	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
	1641	const ptr_ty = self.air.typeOfIndex(inst);
	1642	const val_ty = ptr_ty.childType();
	1643	return self.allocFrameIndex(FrameAlloc.init(.{
	1644	.size = math.cast(u32, val_ty.abiSize(self.target.*)) orelse {
	1645	const mod = self.bin_file.options.module.?;
	1646	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(mod)});
	1647	},
	1648	.alignment = @max(ptr_ty.ptrAlignment(self.target.*), 1),
	1649	}));
1306	1650	}
1307	1651
1308	1652	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
...	...	@@ -1320,42 +1664,35 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
1320	1664	};
1321	1665
1322	1666	if (reg_ok) {
1323		switch (elem_ty.zigTypeTag()) {
1324		.Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}),
1325		.Float => {
1326		if (intrinsicsAllowed(self.target.*, elem_ty)) {
1327		const ptr_bytes: u64 = 32;
1328		if (abi_size <= ptr_bytes) {
1329		if (self.register_manager.tryAllocReg(inst, sse)) \|reg\| {
1330		return MCValue{ .register = registerAlias(reg, abi_size) };
1331		}
1332		}
1333		}
1334
1335		return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{});
1336		},
1337		else => {
1338		// Make sure the type can fit in a register before we try to allocate one.
1339		const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1340		const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1341		if (abi_size <= ptr_bytes) {
1342		if (self.register_manager.tryAllocReg(inst, gp)) \|reg\| {
1343		return MCValue{ .register = registerAlias(reg, abi_size) };
1344		}
1345		}
1346		},
	1667	// Make sure the type can fit in a register before we try to allocate one.
	1668	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
	1669	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
	1670	if (abi_size <= ptr_bytes) {
	1671	if (self.register_manager.tryAllocReg(inst, try self.regClassForType(elem_ty))) \|reg\| {
	1672	return MCValue{ .register = registerAlias(reg, abi_size) };
	1673	}
1347	1674	}
1348	1675	}
1349	1676
1350		const abi_align = elem_ty.abiAlignment(self.target.*);
1351		if (abi_align > self.stack_align)
1352		self.stack_align = abi_align;
1353		const stack_offset = try self.allocMem(inst, abi_size, abi_align);
1354		return MCValue{ .stack_offset = @intCast(i32, stack_offset) };
	1677	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(elem_ty, self.target.*));
	1678	return .{ .load_frame = .{ .index = frame_index } };
	1679	}
	1680
	1681	fn regClassForType(self: *Self, ty: Type) !RegisterManager.RegisterBitSet {
	1682	return switch (ty.zigTypeTag()) {
	1683	.Vector => self.fail("TODO regClassForType for {}", .{ty.fmt(self.bin_file.options.module.?)}),
	1684	.Float => switch (ty.floatBits(self.target.*)) {
	1685	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) sse else gp,
	1686	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) sse else gp,
	1687	else => gp,
	1688	},
	1689	else => gp,
	1690	};
1355	1691	}
1356	1692
1357	1693	const State = struct {
1358	1694	registers: RegisterManager.TrackedRegisters,
	1695	reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
1359	1696	free_registers: RegisterManager.RegisterBitSet,
1360	1697	inst_tracking_len: u32,
1361	1698	scope_generation: u32,
...	...	@@ -1370,8 +1707,14 @@ fn initRetroactiveState(self: *Self) State {
1370	1707
1371	1708	fn saveRetroactiveState(self: Self, state: State) !void {
1372	1709	try self.spillEflagsIfOccupied();
1373		state.registers = self.register_manager.registers;
1374		state.free_registers = self.register_manager.free_registers;
	1710	const free_registers = self.register_manager.free_registers;
	1711	var it = free_registers.iterator(.{ .kind = .unset });
	1712	while (it.next()) \|index\| {
	1713	const tracked_inst = self.register_manager.registers[index];
	1714	state.registers[index] = tracked_inst;
	1715	state.reg_tracking[index] = self.inst_tracking.get(tracked_inst).?;
	1716	}
	1717	state.free_registers = free_registers;
1375	1718	}
1376	1719
1377	1720	fn saveState(self: *Self) !State {
...	...	@@ -1387,18 +1730,18 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1387	1730	close_scope: bool,
1388	1731	}) !void {
1389	1732	if (opts.close_scope) {
1390		for (self.inst_tracking.values()[state.inst_tracking_len..]) \|*tracking\| tracking.die(self);
	1733	for (
	1734	self.inst_tracking.keys()[state.inst_tracking_len..],
	1735	self.inst_tracking.values()[state.inst_tracking_len..],
	1736	) \|inst, *tracking\| tracking.die(self, inst);
1391	1737	self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
1392	1738	}
1393	1739
1394		if (opts.resurrect) for (self.inst_tracking.values()[0..state.inst_tracking_len]) \|*tracking\|
1395		tracking.resurrect(state.scope_generation);
1396		const air_tags = self.air.instructions.items(.tag);
1397		for (deaths) \|death\| switch (air_tags[death]) {
1398		.constant => {},
1399		.const_ty => unreachable,
1400		else => self.inst_tracking.getPtr(death).?.die(self),
1401		};
	1740	if (opts.resurrect) for (
	1741	self.inst_tracking.keys()[0..state.inst_tracking_len],
	1742	self.inst_tracking.values()[0..state.inst_tracking_len],
	1743	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);
	1744	for (deaths) \|death\| self.processDeath(death);
1402	1745
1403	1746	for (0..state.registers.len) \|index\| {
1404	1747	const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
...	...	@@ -1411,26 +1754,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1411	1754	state.registers[index];
1412	1755	if (std.debug.runtime_safety) if (target_maybe_inst) \|target_inst\|
1413	1756	assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
1414		if (current_maybe_inst == target_maybe_inst) continue;
1415		const reg = RegisterManager.regAtTrackedIndex(
1416		@intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
1417		);
1418	1757	if (opts.emit_instructions) {
1419	1758	if (current_maybe_inst) \|current_inst\| {
1420	1759	try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
1421	1760	}
1422	1761	if (target_maybe_inst) \|target_inst\| {
1423		try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg);
	1762	try self.inst_tracking.getPtr(target_inst).?.materialize(
	1763	self,
	1764	target_inst,
	1765	state.reg_tracking[index],
	1766	);
1424	1767	}
1425	1768	}
1426	1769	if (opts.update_tracking) {
1427	1770	if (current_maybe_inst) \|current_inst\| {
1428		self.inst_tracking.getPtr(current_inst).?.trackSpill(self);
	1771	self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
	1772	}
	1773	{
	1774	const reg = RegisterManager.regAtTrackedIndex(
	1775	@intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
	1776	);
	1777	self.register_manager.freeReg(reg);
	1778	self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1429	1779	}
1430		self.register_manager.freeReg(reg);
1431		self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1432	1780	if (target_maybe_inst) \|target_inst\| {
1433		self.inst_tracking.getPtr(target_inst).?.trackMaterialize(self, target_inst, reg);
	1781	self.inst_tracking.getPtr(target_inst).?.trackMaterialize(
	1782	target_inst,
	1783	state.reg_tracking[index],
	1784	);
1434	1785	}
1435	1786	}
1436	1787	}
...	...	@@ -1438,7 +1789,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1438	1789	try self.inst_tracking.getPtr(inst).?.spill(self, inst);
1439	1790	if (opts.update_tracking) if (self.eflags_inst) \|inst\| {
1440	1791	self.eflags_inst = null;
1441		self.inst_tracking.getPtr(inst).?.trackSpill(self);
	1792	self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
1442	1793	};
1443	1794
1444	1795	if (opts.update_tracking and std.debug.runtime_safety) {
...	...	@@ -1452,9 +1803,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1452	1803
1453	1804	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1454	1805	const tracking = self.inst_tracking.getPtr(inst).?;
1455		assert(tracking.getReg().?.to64() == reg.to64());
	1806	assert(tracking.getReg().?.id() == reg.id());
1456	1807	try tracking.spill(self, inst);
1457		tracking.trackSpill(self);
	1808	tracking.trackSpill(self, inst);
1458	1809	}
1459	1810
1460	1811	pub fn spillEflagsIfOccupied(self: *Self) !void {
...	...	@@ -1463,7 +1814,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
1463	1814	const tracking = self.inst_tracking.getPtr(inst).?;
1464	1815	assert(tracking.getCondition() != null);
1465	1816	try tracking.spill(self, inst);
1466		tracking.trackSpill(self);
	1817	tracking.trackSpill(self, inst);
1467	1818	}
1468	1819	}
1469	1820
...	...	@@ -1477,15 +1828,8 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {
1477	1828	/// allocated. A second call to `copyToTmpRegister` may return the same register.
1478	1829	/// This can have a side effect of spilling instructions to the stack to free up a register.
1479	1830	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1480		const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
1481		.Float => blk: {
1482		if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
1483		return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
1484		},
1485		else => gp,
1486		};
1487		const reg: Register = try self.register_manager.allocReg(null, reg_class);
1488		try self.genSetReg(ty, reg, mcv);
	1831	const reg: Register = try self.register_manager.allocReg(null, try self.regClassForType(ty));
	1832	try self.genSetReg(reg, ty, mcv);
1489	1833	return reg;
1490	1834	}
1491	1835
...	...	@@ -1493,28 +1837,35 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1493	1837	/// `reg_owner` is the instruction that gets associated with the register in the register table.
1494	1838	/// This can have a side effect of spilling instructions to the stack to free up a register.
1495	1839	/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
1496		fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {
1497		const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
1498		.Float => blk: {
1499		if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
1500		return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
1501		},
1502		else => gp,
1503		};
1504		const reg: Register = try self.register_manager.allocReg(reg_owner, reg_class);
1505		try self.genSetReg(ty, reg, mcv);
	1840	fn copyToRegisterWithInstTracking(
	1841	self: *Self,
	1842	reg_owner: Air.Inst.Index,
	1843	ty: Type,
	1844	mcv: MCValue,
	1845	) !MCValue {
	1846	const reg: Register = try self.register_manager.allocReg(reg_owner, try self.regClassForType(ty));
	1847	try self.genSetReg(reg, ty, mcv);
1506	1848	return MCValue{ .register = reg };
1507	1849	}
1508	1850
1509	1851	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1510		const stack_offset = try self.allocMemPtr(inst);
1511		const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };
	1852	const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
1512	1853	return self.finishAir(inst, result, .{ .none, .none, .none });
1513	1854	}
1514	1855
1515	1856	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1516		const stack_offset = try self.allocMemPtr(inst);
1517		const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };
	1857	const result: MCValue = switch (self.ret_mcv.long) {
	1858	else => unreachable,
	1859	.none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
	1860	.load_frame => .{ .register_offset = .{
	1861	.reg = (try self.copyToRegisterWithInstTracking(
	1862	inst,
	1863	self.air.typeOfIndex(inst),
	1864	self.ret_mcv.long,
	1865	)).register,
	1866	.off = self.ret_mcv.short.indirect.off,
	1867	} },
	1868	};
1518	1869	return self.finishAir(inst, result, .{ .none, .none, .none });
1519	1870	}
1520	1871
...	...	@@ -1577,12 +1928,14 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1577	1928	registerAlias(src_reg, min_abi_size),
1578	1929	);
1579	1930	},
1580		.stack_offset => \|src_off\| {
1581		try self.asmRegisterMemory(tag, dst_alias, Memory.sib(
1582		Memory.PtrSize.fromSize(min_abi_size),
1583		.{ .base = .rbp, .disp = -src_off },
1584		));
1585		},
	1931	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
	1932	tag,
	1933	dst_alias,
	1934	Memory.sib(Memory.PtrSize.fromSize(min_abi_size), .{
	1935	.base = .{ .frame = frame_addr.index },
	1936	.disp = frame_addr.off,
	1937	}),
	1938	),
1586	1939	else => return self.fail("TODO airIntCast from {s} to {s}", .{
1587	1940	@tagName(src_mcv),
1588	1941	@tagName(dst_mcv),
...	...	@@ -1591,7 +1944,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1591	1944	if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg);
1592	1945	},
1593	1946	else => {
1594		try self.setRegOrMem(min_ty, dst_mcv, src_mcv);
	1947	try self.genCopy(min_ty, dst_mcv, src_mcv);
1595	1948	const extra = dst_abi_size * 8 - dst_int_info.bits;
1596	1949	if (extra > 0) {
1597	1950	try self.genShiftBinOpMir(switch (signedness) {
...	...	@@ -1653,16 +2006,22 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1653	2006	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1654	2007	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1655	2008
	2009	const slice_ty = self.air.typeOfIndex(inst);
1656	2010	const ptr = try self.resolveInst(bin_op.lhs);
1657	2011	const ptr_ty = self.air.typeOf(bin_op.lhs);
1658	2012	const len = try self.resolveInst(bin_op.rhs);
1659	2013	const len_ty = self.air.typeOf(bin_op.rhs);
1660	2014
1661		const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
1662		try self.genSetStack(ptr_ty, stack_offset, ptr, .{});
1663		try self.genSetStack(len_ty, stack_offset - 8, len, .{});
1664		const result = MCValue{ .stack_offset = stack_offset };
	2015	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
	2016	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
	2017	try self.genSetMem(
	2018	.{ .frame = frame_index },
	2019	@intCast(i32, ptr_ty.abiSize(self.target.*)),
	2020	len_ty,
	2021	len,
	2022	);
1665	2023
	2024	const result = MCValue{ .load_frame = .{ .index = frame_index } };
1666	2025	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1667	2026	}
1668	2027
...	...	@@ -1778,14 +2137,14 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1778	2137
1779	2138	const reg_bits = self.regBitSize(ty);
1780	2139	const cc: Condition = if (ty.isSignedInt()) cc: {
1781		try self.genSetReg(ty, limit_reg, dst_mcv);
	2140	try self.genSetReg(limit_reg, ty, dst_mcv);
1782	2141	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1783	2142	try self.genBinOpMir(.xor, ty, limit_mcv, .{
1784	2143	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1785	2144	});
1786	2145	break :cc .o;
1787	2146	} else cc: {
1788		try self.genSetReg(ty, limit_reg, .{
	2147	try self.genSetReg(limit_reg, ty, .{
1789	2148	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1790	2149	});
1791	2150	break :cc .c;
...	...	@@ -1829,14 +2188,14 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1829	2188
1830	2189	const reg_bits = self.regBitSize(ty);
1831	2190	const cc: Condition = if (ty.isSignedInt()) cc: {
1832		try self.genSetReg(ty, limit_reg, dst_mcv);
	2191	try self.genSetReg(limit_reg, ty, dst_mcv);
1833	2192	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1834	2193	try self.genBinOpMir(.xor, ty, limit_mcv, .{
1835	2194	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1836	2195	});
1837	2196	break :cc .o;
1838	2197	} else cc: {
1839		try self.genSetReg(ty, limit_reg, .{ .immediate = 0 });
	2198	try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
1840	2199	break :cc .c;
1841	2200	};
1842	2201	try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
...	...	@@ -1880,7 +2239,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1880	2239
1881	2240	const reg_bits = self.regBitSize(ty);
1882	2241	const cc: Condition = if (ty.isSignedInt()) cc: {
1883		try self.genSetReg(ty, limit_reg, lhs_mcv);
	2242	try self.genSetReg(limit_reg, ty, lhs_mcv);
1884	2243	try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
1885	2244	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1886	2245	try self.genBinOpMir(.xor, ty, limit_mcv, .{
...	...	@@ -1888,7 +2247,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1888	2247	});
1889	2248	break :cc .o;
1890	2249	} else cc: {
1891		try self.genSetReg(ty, limit_reg, .{
	2250	try self.genSetReg(limit_reg, ty, .{
1892	2251	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1893	2252	});
1894	2253	break :cc .c;
...	...	@@ -1916,17 +2275,18 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1916	2275	.Int => {
1917	2276	try self.spillEflagsIfOccupied();
1918	2277
1919		const partial_mcv = switch (tag) {
1920		.add_with_overflow => try self.genBinOp(null, .add, bin_op.lhs, bin_op.rhs),
1921		.sub_with_overflow => try self.genBinOp(null, .sub, bin_op.lhs, bin_op.rhs),
	2278	const partial_mcv = try self.genBinOp(null, switch (tag) {
	2279	.add_with_overflow => .add,
	2280	.sub_with_overflow => .sub,
1922	2281	else => unreachable,
1923		};
	2282	}, bin_op.lhs, bin_op.rhs);
1924	2283	const int_info = ty.intInfo(self.target.*);
1925	2284	const cc: Condition = switch (int_info.signedness) {
1926	2285	.unsigned => .c,
1927	2286	.signed => .o,
1928	2287	};
1929	2288
	2289	const tuple_ty = self.air.typeOfIndex(inst);
1930	2290	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
1931	2291	switch (partial_mcv) {
1932	2292	.register => \|reg\| {
...	...	@@ -1936,27 +2296,32 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1936	2296	else => {},
1937	2297	}
1938	2298
1939		const abi_size = @intCast(i32, ty.abiSize(self.target.*));
1940		const dst_mcv = try self.allocRegOrMem(inst, false);
1941		try self.genSetStack(
	2299	const frame_index =
	2300	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2301	try self.genSetMem(
	2302	.{ .frame = frame_index },
	2303	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
1942	2304	Type.u1,
1943		dst_mcv.stack_offset - abi_size,
1944	2305	.{ .eflags = cc },
1945		.{},
1946	2306	);
1947		try self.genSetStack(ty, dst_mcv.stack_offset, partial_mcv, .{});
1948		break :result dst_mcv;
	2307	try self.genSetMem(
	2308	.{ .frame = frame_index },
	2309	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2310	ty,
	2311	partial_mcv,
	2312	);
	2313	break :result .{ .load_frame = .{ .index = frame_index } };
1949	2314	}
1950	2315
1951		const tuple_ty = self.air.typeOfIndex(inst);
1952		const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
1953		const tuple_align = tuple_ty.abiAlignment(self.target.*);
1954		const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));
1955		const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
1956
1957		try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);
1958
1959		break :result .{ .stack_offset = stack_offset };
	2316	const frame_index =
	2317	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2318	try self.genSetFrameTruncatedOverflowCompare(
	2319	tuple_ty,
	2320	frame_index,
	2321	partial_mcv.register,
	2322	cc,
	2323	);
	2324	break :result .{ .load_frame = .{ .index = frame_index } };
1960	2325	},
1961	2326	else => unreachable,
1962	2327	}
...	...	@@ -1998,6 +2363,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1998	2363	try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);
1999	2364	const cc = Condition.ne;
2000	2365
	2366	const tuple_ty = self.air.typeOfIndex(inst);
2001	2367	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
2002	2368	switch (partial_mcv) {
2003	2369	.register => \|reg\| {
...	...	@@ -2007,27 +2373,32 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2007	2373	else => {},
2008	2374	}
2009	2375
2010		const abi_size = @intCast(i32, lhs_ty.abiSize(self.target.*));
2011		const dst_mcv = try self.allocRegOrMem(inst, false);
2012		try self.genSetStack(
2013		Type.u1,
2014		dst_mcv.stack_offset - abi_size,
	2376	const frame_index =
	2377	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2378	try self.genSetMem(
	2379	.{ .frame = frame_index },
	2380	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
	2381	tuple_ty.structFieldType(1),
2015	2382	.{ .eflags = cc },
2016		.{},
2017	2383	);
2018		try self.genSetStack(lhs_ty, dst_mcv.stack_offset, partial_mcv, .{});
2019		break :result dst_mcv;
	2384	try self.genSetMem(
	2385	.{ .frame = frame_index },
	2386	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2387	tuple_ty.structFieldType(0),
	2388	partial_mcv,
	2389	);
	2390	break :result .{ .load_frame = .{ .index = frame_index } };
2020	2391	}
2021	2392
2022		const tuple_ty = self.air.typeOfIndex(inst);
2023		const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
2024		const tuple_align = tuple_ty.abiAlignment(self.target.*);
2025		const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));
2026		const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
2027
2028		try self.genSetStackTruncatedOverflowCompare(lhs_ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);
2029
2030		break :result .{ .stack_offset = stack_offset };
	2393	const frame_index =
	2394	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2395	try self.genSetFrameTruncatedOverflowCompare(
	2396	tuple_ty,
	2397	frame_index,
	2398	partial_mcv.register,
	2399	cc,
	2400	);
	2401	break :result .{ .load_frame = .{ .index = frame_index } };
2031	2402	},
2032	2403	else => unreachable,
2033	2404	}
...	...	@@ -2035,17 +2406,17 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2035	2406	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2036	2407	}
2037	2408
2038		fn genSetStackTruncatedOverflowCompare(
	2409	fn genSetFrameTruncatedOverflowCompare(
2039	2410	self: *Self,
2040		ty: Type,
2041		stack_offset: i32,
2042		overflow_bit_offset: i32,
	2411	tuple_ty: Type,
	2412	frame_index: FrameIndex,
2043	2413	reg: Register,
2044	2414	cc: Condition,
2045	2415	) !void {
2046	2416	const reg_lock = self.register_manager.lockReg(reg);
2047	2417	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
2048	2418
	2419	const ty = tuple_ty.structFieldType(0);
2049	2420	const int_info = ty.intInfo(self.target.*);
2050	2421	const extended_ty = switch (int_info.signedness) {
2051	2422	.signed => Type.isize,
...	...	@@ -2062,7 +2433,7 @@ fn genSetStackTruncatedOverflowCompare(
2062	2433	try self.asmSetccRegister(overflow_reg.to8(), cc);
2063	2434
2064	2435	const scratch_reg = temp_regs[1];
2065		try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });
	2436	try self.genSetReg(scratch_reg, extended_ty, .{ .register = reg });
2066	2437	try self.truncateRegister(ty, scratch_reg);
2067	2438	try self.genBinOpMir(
2068	2439	.cmp,
...	...	@@ -2080,10 +2451,18 @@ fn genSetStackTruncatedOverflowCompare(
2080	2451	.{ .register = eq_reg },
2081	2452	);
2082	2453
2083		try self.genSetStack(ty, stack_offset, .{ .register = scratch_reg }, .{});
2084		try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{
2085		.register = overflow_reg.to8(),
2086		}, .{});
	2454	try self.genSetMem(
	2455	.{ .frame = frame_index },
	2456	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
	2457	tuple_ty.structFieldType(1),
	2458	.{ .register = overflow_reg.to8() },
	2459	);
	2460	try self.genSetMem(
	2461	.{ .frame = frame_index },
	2462	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2463	ty,
	2464	.{ .register = scratch_reg },
	2465	);
2087	2466	}
2088	2467
2089	2468	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -2101,6 +2480,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2101	2480	.unsigned => .c,
2102	2481	.signed => .o,
2103	2482	};
	2483
	2484	const tuple_ty = self.air.typeOfIndex(inst);
2104	2485	if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {
2105	2486	var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
2106	2487	.signed => .int_signed,
...	...	@@ -2127,16 +2508,21 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2127	2508
2128	2509	// For now, this is the only supported multiply that doesn't fit in a register.
2129	2510	assert(dst_info.bits == 128 and src_pl.data == 64);
2130		const dst_abi_size = @intCast(i32, dst_ty.abiSize(self.target.*));
2131		const dst_mcv = try self.allocRegOrMem(inst, false);
2132		try self.genSetStack(
2133		Type.u1,
2134		dst_mcv.stack_offset - dst_abi_size,
2135		.{ .immediate = 0 }, // 64x64 -> 128 never overflows
2136		.{},
	2511	const frame_index =
	2512	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2513	try self.genSetMem(
	2514	.{ .frame = frame_index },
	2515	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
	2516	tuple_ty.structFieldType(1),
	2517	.{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
2137	2518	);
2138		try self.genSetStack(dst_ty, dst_mcv.stack_offset, partial_mcv, .{});
2139		break :result dst_mcv;
	2519	try self.genSetMem(
	2520	.{ .frame = frame_index },
	2521	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2522	tuple_ty.structFieldType(0),
	2523	partial_mcv,
	2524	);
	2525	break :result .{ .load_frame = .{ .index = frame_index } };
2140	2526	}
2141	2527
2142	2528	const dst_reg: Register = dst_reg: {
...	...	@@ -2184,15 +2570,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2184	2570	}
2185	2571	};
2186	2572
2187		const tuple_ty = self.air.typeOfIndex(inst);
2188		const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
2189		const tuple_align = tuple_ty.abiAlignment(self.target.*);
2190		const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));
2191		const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
2192
2193		try self.genSetStackTruncatedOverflowCompare(dst_ty, stack_offset, overflow_bit_offset, dst_reg, cc);
2194
2195		break :result .{ .stack_offset = stack_offset };
	2573	const frame_index =
	2574	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2575	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, dst_reg, cc);
	2576	break :result .{ .load_frame = .{ .index = frame_index } };
2196	2577	},
2197	2578	else => unreachable,
2198	2579	}
...	...	@@ -2215,31 +2596,40 @@ fn genIntMulDivOpMir(
2215	2596	return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
2216	2597	}
2217	2598
2218		lhs: {
2219		switch (lhs) {
2220		.register => \|reg\| if (reg.to64() == .rax) break :lhs,
2221		else => {},
2222		}
2223		try self.genSetReg(ty, .rax, lhs);
2224		}
2225
	2599	try self.genSetReg(.rax, ty, lhs);
2226	2600	switch (tag) {
2227	2601	else => unreachable,
2228	2602	.mul, .imul => {},
2229	2603	.div => try self.asmRegisterRegister(.xor, .edx, .edx),
2230		.idiv => try self.asmOpOnly(.cqo),
	2604	.idiv => switch (self.regBitSize(ty)) {
	2605	8 => try self.asmOpOnly(.cbw),
	2606	16 => try self.asmOpOnly(.cwd),
	2607	32 => try self.asmOpOnly(.cdq),
	2608	64 => try self.asmOpOnly(.cqo),
	2609	else => unreachable,
	2610	},
2231	2611	}
2232	2612
2233		const factor: MCValue = switch (rhs) {
2234		.register, .stack_offset => rhs,
	2613	const mat_rhs: MCValue = switch (rhs) {
	2614	.register, .indirect, .load_frame => rhs,
2235	2615	else => .{ .register = try self.copyToTmpRegister(ty, rhs) },
2236	2616	};
2237		switch (factor) {
2238		.register => \|reg\| try self.asmRegister(tag, reg),
2239		.stack_offset => \|off\| try self.asmMemory(tag, Memory.sib(
2240		Memory.PtrSize.fromSize(abi_size),
2241		.{ .base = .rbp, .disp = -off },
2242		)),
	2617	switch (mat_rhs) {
	2618	.register => \|reg\| try self.asmRegister(tag, registerAlias(reg, abi_size)),
	2619	.indirect, .load_frame => try self.asmMemory(
	2620	tag,
	2621	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (mat_rhs) {
	2622	.indirect => \|reg_off\| .{
	2623	.base = .{ .reg = reg_off.reg },
	2624	.disp = reg_off.off,
	2625	},
	2626	.load_frame => \|frame_addr\| .{
	2627	.base = .{ .frame = frame_addr.index },
	2628	.disp = frame_addr.off,
	2629	},
	2630	else => unreachable,
	2631	}),
	2632	),
2243	2633	else => unreachable,
2244	2634	}
2245	2635	}
...	...	@@ -2247,7 +2637,8 @@ fn genIntMulDivOpMir(
2247	2637	/// Always returns a register.
2248	2638	/// Clobbers .rax and .rdx registers.
2249	2639	fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
2250		const signedness = ty.intInfo(self.target.*).signedness;
	2640	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
	2641	const int_info = ty.intInfo(self.target.*);
2251	2642	const dividend: Register = switch (lhs) {
2252	2643	.register => \|reg\| reg,
2253	2644	else => try self.copyToTmpRegister(ty, lhs),
...	...	@@ -2262,16 +2653,32 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
2262	2653	const divisor_lock = self.register_manager.lockReg(divisor);
2263	2654	defer if (divisor_lock) \|lock\| self.register_manager.unlockReg(lock);
2264	2655
2265		try self.genIntMulDivOpMir(switch (signedness) {
	2656	try self.genIntMulDivOpMir(switch (int_info.signedness) {
2266	2657	.signed => .idiv,
2267	2658	.unsigned => .div,
2268		}, Type.isize, .{ .register = dividend }, .{ .register = divisor });
	2659	}, ty, .{ .register = dividend }, .{ .register = divisor });
2269	2660
2270		try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64());
2271		try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63));
2272		try self.asmRegisterRegister(.@"test", .rdx, .rdx);
2273		try self.asmCmovccRegisterRegister(divisor.to64(), .rdx, .e);
2274		try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
	2661	try self.asmRegisterRegister(
	2662	.xor,
	2663	registerAlias(divisor, abi_size),
	2664	registerAlias(dividend, abi_size),
	2665	);
	2666	try self.asmRegisterImmediate(
	2667	.sar,
	2668	registerAlias(divisor, abi_size),
	2669	Immediate.u(int_info.bits - 1),
	2670	);
	2671	try self.asmRegisterRegister(
	2672	.@"test",
	2673	registerAlias(.rdx, abi_size),
	2674	registerAlias(.rdx, abi_size),
	2675	);
	2676	try self.asmCmovccRegisterRegister(
	2677	registerAlias(divisor, abi_size),
	2678	registerAlias(.rdx, abi_size),
	2679	.z,
	2680	);
	2681	try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });
2275	2682	return MCValue{ .register = divisor };
2276	2683	}
2277	2684
...	...	@@ -2315,7 +2722,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
2315	2722	}
2316	2723
2317	2724	const pl_mcv = try self.allocRegOrMem(inst, true);
2318		try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) {
	2725	try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) {
2319	2726	else => opt_mcv,
2320	2727	.register_overflow => \|ro\| .{ .register = ro.reg },
2321	2728	});
...	...	@@ -2361,11 +2768,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2361	2768
2362	2769	const pl_ty = dst_ty.childType();
2363	2770	const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
2364		try self.asmMemoryImmediate(
2365		.mov,
2366		Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }),
2367		Immediate.u(1),
2368		);
	2771	try self.genSetMem(.{ .reg = dst_mcv.register }, pl_abi_size, Type.bool, .{ .immediate = 1 });
2369	2772	break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
2370	2773	};
2371	2774	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -2389,10 +2792,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2389	2792
2390	2793	const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2391	2794	switch (operand) {
2392		.stack_offset => \|off\| {
2393		const offset = off - @intCast(i32, err_off);
2394		break :result MCValue{ .stack_offset = offset };
2395		},
2396	2795	.register => \|reg\| {
2397	2796	// TODO reuse operand
2398	2797	const eu_lock = self.register_manager.lockReg(reg);
...	...	@@ -2407,6 +2806,10 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2407	2806	}
2408	2807	break :result result;
2409	2808	},
	2809	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
	2810	.index = frame_addr.index,
	2811	.off = frame_addr.off + @intCast(i32, err_off),
	2812	} },
2410	2813	else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),
2411	2814	}
2412	2815	};
...	...	@@ -2434,10 +2837,10 @@ fn genUnwrapErrorUnionPayloadMir(
2434	2837
2435	2838	const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2436	2839	switch (err_union) {
2437		.stack_offset => \|off\| {
2438		const offset = off - @intCast(i32, payload_off);
2439		break :result MCValue{ .stack_offset = offset };
2440		},
	2840	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
	2841	.index = frame_addr.index,
	2842	.off = frame_addr.off + @intCast(i32, payload_off),
	2843	} },
2441	2844	.register => \|reg\| {
2442	2845	// TODO reuse operand
2443	2846	const eu_lock = self.register_manager.lockReg(reg);
...	...	@@ -2488,7 +2891,10 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2488	2891	try self.asmRegisterMemory(
2489	2892	.mov,
2490	2893	registerAlias(dst_reg, err_abi_size),
2491		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
	2894	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	2895	.base = .{ .reg = src_reg },
	2896	.disp = err_off,
	2897	}),
2492	2898	);
2493	2899
2494	2900	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -2523,7 +2929,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2523	2929	try self.asmRegisterMemory(
2524	2930	.lea,
2525	2931	registerAlias(dst_reg, dst_abi_size),
2526		Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
	2932	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2527	2933	);
2528	2934
2529	2935	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -2548,7 +2954,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2548	2954	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
2549	2955	try self.asmMemoryImmediate(
2550	2956	.mov,
2551		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
	2957	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	2958	.base = .{ .reg = src_reg },
	2959	.disp = err_off,
	2960	}),
2552	2961	Immediate.u(0),
2553	2962	);
2554	2963
...	...	@@ -2567,7 +2976,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2567	2976	try self.asmRegisterMemory(
2568	2977	.lea,
2569	2978	registerAlias(dst_reg, dst_abi_size),
2570		Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
	2979	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2571	2980	);
2572	2981	break :result .{ .register = dst_reg };
2573	2982	};
...	...	@@ -2608,7 +3017,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2608	3017	defer if (pl_lock) \|lock\| self.register_manager.unlockReg(lock);
2609	3018
2610	3019	const opt_mcv = try self.allocRegOrMem(inst, true);
2611		try self.setRegOrMem(pl_ty, opt_mcv, pl_mcv);
	3020	try self.genCopy(pl_ty, opt_mcv, pl_mcv);
2612	3021
2613	3022	if (!same_repr) {
2614	3023	const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
...	...	@@ -2621,9 +3030,12 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2621	3030	Immediate.u(@intCast(u6, pl_abi_size * 8)),
2622	3031	),
2623	3032
2624		.stack_offset => \|off\| try self.asmMemoryImmediate(
	3033	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(
2625	3034	.mov,
2626		Memory.sib(.byte, .{ .base = .rbp, .disp = pl_abi_size - off }),
	3035	Memory.sib(.byte, .{
	3036	.base = .{ .frame = frame_addr.index },
	3037	.disp = frame_addr.off + pl_abi_size,
	3038	}),
2627	3039	Immediate.u(1),
2628	3040	),
2629	3041	}
...	...	@@ -2637,26 +3049,21 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2637	3049	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2638	3050	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2639	3051
2640		const error_union_ty = self.air.getRefType(ty_op.ty);
2641		const payload_ty = error_union_ty.errorUnionPayload();
	3052	const eu_ty = self.air.getRefType(ty_op.ty);
	3053	const pl_ty = eu_ty.errorUnionPayload();
	3054	const err_ty = eu_ty.errorUnionSet();
2642	3055	const operand = try self.resolveInst(ty_op.operand);
2643	3056
2644	3057	const result: MCValue = result: {
2645		if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
2646		break :result operand;
2647		}
2648
2649		const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2650		const abi_align = error_union_ty.abiAlignment(self.target.*);
2651		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2652		const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2653		const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2654		try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), operand, .{});
2655		try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), .{ .immediate = 0 }, .{});
	3058	if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result .{ .immediate = 0 };
2656	3059
2657		break :result MCValue{ .stack_offset = stack_offset };
	3060	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
	3061	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
	3062	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
	3063	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
	3064	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
	3065	break :result .{ .load_frame = .{ .index = frame_index } };
2658	3066	};
2659
2660	3067	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2661	3068	}
2662	3069
...	...	@@ -2664,26 +3071,21 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2664	3071	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2665	3072	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2666	3073
2667		const error_union_ty = self.air.getRefType(ty_op.ty);
2668		const payload_ty = error_union_ty.errorUnionPayload();
2669		const operand = try self.resolveInst(ty_op.operand);
	3074	const eu_ty = self.air.getRefType(ty_op.ty);
	3075	const pl_ty = eu_ty.errorUnionPayload();
	3076	const err_ty = eu_ty.errorUnionSet();
2670	3077
2671	3078	const result: MCValue = result: {
2672		if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
2673		break :result operand;
2674		}
2675
2676		const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2677		const abi_align = error_union_ty.abiAlignment(self.target.*);
2678		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2679		const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2680		const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2681		try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), operand, .{});
2682		try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), .undef, .{});
	3079	if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result try self.resolveInst(ty_op.operand);
2683	3080
2684		break :result MCValue{ .stack_offset = stack_offset };
	3081	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
	3082	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
	3083	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
	3084	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
	3085	const operand = try self.resolveInst(ty_op.operand);
	3086	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
	3087	break :result .{ .load_frame = .{ .index = frame_index } };
2685	3088	};
2686
2687	3089	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2688	3090	}
2689	3091
...	...	@@ -2695,7 +3097,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2695	3097
2696	3098	const dst_mcv = try self.allocRegOrMem(inst, true);
2697	3099	const dst_ty = self.air.typeOfIndex(inst);
2698		try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);
	3100	try self.genCopy(dst_ty, dst_mcv, src_mcv);
2699	3101	break :result dst_mcv;
2700	3102	};
2701	3103	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -2707,9 +3109,10 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2707	3109	const operand = try self.resolveInst(ty_op.operand);
2708	3110	const dst_mcv: MCValue = blk: {
2709	3111	switch (operand) {
2710		.stack_offset => \|off\| {
2711		break :blk MCValue{ .stack_offset = off - 8 };
2712		},
	3112	.load_frame => \|frame_addr\| break :blk .{ .load_frame = .{
	3113	.index = frame_addr.index,
	3114	.off = frame_addr.off + 8,
	3115	} },
2713	3116	else => return self.fail("TODO implement slice_len for {}", .{operand}),
2714	3117	}
2715	3118	};
...	...	@@ -2743,7 +3146,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2743	3146	.lea,
2744	3147	registerAlias(dst_reg, dst_abi_size),
2745	3148	Memory.sib(.qword, .{
2746		.base = src_reg,
	3149	.base = .{ .reg = src_reg },
2747	3150	.disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),
2748	3151	}),
2749	3152	);
...	...	@@ -2771,7 +3174,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
2771	3174	// Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
2772	3175	// and set the register directly to the scaled offset as an immediate.
2773	3176	const reg = try self.register_manager.allocReg(null, gp);
2774		try self.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });
	3177	try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size });
2775	3178	break :blk reg;
2776	3179	},
2777	3180	else => {
...	...	@@ -2812,10 +3215,14 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2812	3215
2813	3216	const addr_reg = try self.register_manager.allocReg(null, gp);
2814	3217	switch (slice_mcv) {
2815		.stack_offset => \|off\| try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
2816		.base = .rbp,
2817		.disp = -off,
2818		})),
	3218	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
	3219	.mov,
	3220	addr_reg.to64(),
	3221	Memory.sib(.qword, .{
	3222	.base = .{ .frame = frame_addr.index },
	3223	.disp = frame_addr.off,
	3224	}),
	3225	),
2819	3226	else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
2820	3227	}
2821	3228	// TODO we could allocate register here, but need to expect addr register and potentially
...	...	@@ -2834,7 +3241,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2834	3241	const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
2835	3242	const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
2836	3243	const dst_mcv = try self.allocRegOrMem(inst, false);
2837		try self.load(dst_mcv, elem_ptr, slice_ptr_field_type);
	3244	try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
2838	3245
2839	3246	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2840	3247	}
...	...	@@ -2872,33 +3279,27 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2872	3279	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2873	3280	defer self.register_manager.unlockReg(offset_reg_lock);
2874	3281
2875		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	3282	const addr_reg = try self.register_manager.allocReg(null, gp);
2876	3283	switch (array) {
2877	3284	.register => {
2878		const off = @intCast(i32, try self.allocMem(
2879		inst,
2880		@intCast(u32, array_ty.abiSize(self.target.*)),
2881		array_ty.abiAlignment(self.target.*),
2882		));
2883		try self.genSetStack(array_ty, off, array, .{});
2884		try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{
2885		.base = .rbp,
2886		.disp = -off,
2887		}));
2888		},
2889		.stack_offset => \|off\| {
2890		try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{
2891		.base = .rbp,
2892		.disp = -off,
2893		}));
	3285	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
	3286	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
	3287	try self.asmRegisterMemory(
	3288	.lea,
	3289	addr_reg,
	3290	Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
	3291	);
2894	3292	},
2895		.load_got => try self.genSetReg(array_ty, addr_reg, array),
2896		.memory, .load_direct, .load_tlv => try self.genSetReg(Type.usize, addr_reg, switch (array) {
2897		.memory => \|addr\| .{ .immediate = addr },
2898		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
2899		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
2900		else => unreachable,
2901		}),
	3293	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
	3294	.lea,
	3295	addr_reg,
	3296	Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
	3297	),
	3298	.memory,
	3299	.load_direct,
	3300	.load_got,
	3301	.load_tlv,
	3302	=> try self.genSetReg(addr_reg, Type.usize, array.address()),
2902	3303	.lea_direct, .lea_tlv => unreachable,
2903	3304	else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2904	3305	}
...	...	@@ -2907,7 +3308,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2907	3308	// offset register.
2908	3309	const dst_mcv = try self.allocRegOrMem(inst, false);
2909	3310	try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2910		try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
	3311	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
2911	3312
2912	3313	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2913	3314	}
...	...	@@ -2948,7 +3349,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2948	3349	else => null,
2949	3350	};
2950	3351	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2951		try self.load(dst_mcv, .{ .register = elem_ptr_reg }, ptr_ty);
	3352	try self.load(dst_mcv, ptr_ty, .{ .register = elem_ptr_reg });
2952	3353
2953	3354	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2954	3355	}
...	...	@@ -2987,8 +3388,8 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2987	3388
2988	3389	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2989	3390	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2990		const ptr_ty = self.air.typeOf(bin_op.lhs);
2991		const union_ty = ptr_ty.childType();
	3391	const ptr_union_ty = self.air.typeOf(bin_op.lhs);
	3392	const union_ty = ptr_union_ty.childType();
2992	3393	const tag_ty = self.air.typeOf(bin_op.rhs);
2993	3394	const layout = union_ty.unionGetLayout(self.target.*);
2994	3395
...	...	@@ -3012,12 +3413,15 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3012	3413
3013	3414	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
3014	3415	// TODO reusing the operand
3015		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3016		try self.genBinOpMir(.add, ptr_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
	3416	const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
	3417	try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
3017	3418	break :blk MCValue{ .register = reg };
3018	3419	} else ptr;
3019	3420
3020		try self.store(adjusted_ptr, tag, ptr_ty, tag_ty);
	3421	var ptr_tag_pl = ptr_union_ty.ptrInfo();
	3422	ptr_tag_pl.data.pointee_type = tag_ty;
	3423	const ptr_tag_ty = Type.initPayload(&ptr_tag_pl.base);
	3424	try self.store(ptr_tag_ty, adjusted_ptr, tag);
3021	3425
3022	3426	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3023	3427	}
...	...	@@ -3044,11 +3448,14 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3044	3448	const tag_abi_size = tag_ty.abiSize(self.target.*);
3045	3449	const dst_mcv: MCValue = blk: {
3046	3450	switch (operand) {
3047		.stack_offset => \|off\| {
	3451	.load_frame => \|frame_addr\| {
3048	3452	if (tag_abi_size <= 8) {
3049		const offset: i32 = if (layout.tag_align < layout.payload_align) @intCast(i32, layout.payload_size) else 0;
	3453	const off: i32 = if (layout.tag_align < layout.payload_align)
	3454	@intCast(i32, layout.payload_size)
	3455	else
	3456	0;
3050	3457	break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{
3051		.stack_offset = off - offset,
	3458	.load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
3052	3459	});
3053	3460	}
3054	3461
...	...	@@ -3131,7 +3538,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3131	3538	.nz,
3132	3539	);
3133	3540
3134		try self.genSetReg(dst_ty, dst_reg, .{ .immediate = src_bits - 1 });
	3541	try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
3135	3542	try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
3136	3543	}
3137	3544	break :result dst_mcv;
...	...	@@ -3171,7 +3578,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3171	3578	};
3172	3579	const tmp_mcv = tmp: {
3173	3580	if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;
3174		try self.genSetReg(src_ty, dst_reg, src_mcv);
	3581	try self.genSetReg(dst_reg, src_ty, src_mcv);
3175	3582	break :tmp dst_mcv;
3176	3583	};
3177	3584	try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
...	...	@@ -3345,7 +3752,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
3345	3752	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
3346	3753	defer self.register_manager.unlockReg(dst_lock);
3347	3754
3348		try self.genSetReg(src_ty, dst_mcv.register, src_mcv);
	3755	try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
3349	3756	switch (src_bits) {
3350	3757	else => unreachable,
3351	3758	16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
...	...	@@ -3450,7 +3857,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3450	3857	.lea,
3451	3858	if (src_abi_size > 4) tmp.to64() else tmp.to32(),
3452	3859	Memory.sib(.qword, .{
3453		.base = dst.to64(),
	3860	.base = .{ .reg = dst.to64() },
3454	3861	.scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
3455	3862	}),
3456	3863	);
...	...	@@ -3473,7 +3880,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3473	3880	.lea,
3474	3881	if (src_abi_size > 4) dst.to64() else dst.to32(),
3475	3882	Memory.sib(.qword, .{
3476		.base = tmp.to64(),
	3883	.base = .{ .reg = tmp.to64() },
3477	3884	.scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
3478	3885	}),
3479	3886	);
...	...	@@ -3506,6 +3913,17 @@ fn reuseOperand(
3506	3913	operand: Air.Inst.Ref,
3507	3914	op_index: Liveness.OperandInt,
3508	3915	mcv: MCValue,
	3916	) bool {
	3917	return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
	3918	}
	3919
	3920	fn reuseOperandAdvanced(
	3921	self: *Self,
	3922	inst: Air.Inst.Index,
	3923	operand: Air.Inst.Ref,
	3924	op_index: Liveness.OperandInt,
	3925	mcv: MCValue,
	3926	tracked_inst: Air.Inst.Index,
3509	3927	) bool {
3510	3928	if (!self.liveness.operandDies(inst, op_index))
3511	3929	return false;
...	...	@@ -3516,25 +3934,23 @@ fn reuseOperand(
3516	3934	// new instruction.
3517	3935	if (!self.register_manager.isRegFree(reg)) {
3518	3936	if (RegisterManager.indexOfRegIntoTracked(reg)) \|index\| {
3519		self.register_manager.registers[index] = inst;
	3937	self.register_manager.registers[index] = tracked_inst;
3520	3938	}
3521	3939	}
3522		log.debug("%{d} => {} (reused)", .{ inst, reg });
3523		},
3524		.stack_offset => \|off\| {
3525		log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
3526	3940	},
	3941	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
3527	3942	else => return false,
3528	3943	}
3529	3944
3530	3945	// Prevent the operand deaths processing code from deallocating it.
3531	3946	self.liveness.clearOperandDeath(inst, op_index);
3532		self.getResolvedInstValue(Air.refToIndex(operand).?).reuse(self);
	3947	const op_inst = Air.refToIndex(operand).?;
	3948	self.getResolvedInstValue(op_inst).reuse(self, tracked_inst, op_inst);
3533	3949
3534	3950	return true;
3535	3951	}
3536	3952
3537		fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {
	3953	fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
3538	3954	const ptr_info = ptr_ty.ptrInfo().data;
3539	3955
3540	3956	const val_ty = ptr_info.pointee_type;
...	...	@@ -3564,10 +3980,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3564	3980	if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;
3565	3981	if (load_abi_size <= 8) {
3566	3982	const load_reg = registerAlias(dst_reg, load_abi_size);
3567		try self.asmRegisterMemory(.mov, load_reg, Memory.sib(
3568		Memory.PtrSize.fromSize(load_abi_size),
3569		.{ .base = ptr_reg, .disp = val_byte_off },
3570		));
	3983	try self.asmRegisterMemory(
	3984	.mov,
	3985	load_reg,
	3986	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
	3987	.base = .{ .reg = ptr_reg },
	3988	.disp = val_byte_off,
	3989	}),
	3990	);
3571	3991	try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
3572	3992	} else {
3573	3993	const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
...	...	@@ -3575,85 +3995,60 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3575	3995	defer self.register_manager.unlockReg(tmp_lock);
3576	3996
3577	3997	const dst_alias = registerAlias(dst_reg, val_abi_size);
3578		try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(
3579		Memory.PtrSize.fromSize(val_abi_size),
3580		.{ .base = ptr_reg, .disp = val_byte_off },
3581		));
3582		try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(
3583		Memory.PtrSize.fromSize(val_abi_size),
3584		.{ .base = ptr_reg, .disp = val_byte_off + 1 },
3585		));
	3998	try self.asmRegisterMemory(
	3999	.mov,
	4000	dst_alias,
	4001	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
	4002	.base = .{ .reg = ptr_reg },
	4003	.disp = val_byte_off,
	4004	}),
	4005	);
	4006	try self.asmRegisterMemory(
	4007	.mov,
	4008	tmp_reg,
	4009	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
	4010	.base = .{ .reg = ptr_reg },
	4011	.disp = val_byte_off + 1,
	4012	}),
	4013	);
3586	4014	try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
3587	4015	}
3588	4016
3589	4017	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
3590		try self.setRegOrMem(val_ty, dst_mcv, .{ .register = dst_reg });
3591		}
3592
3593		fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
3594		const elem_ty = ptr_ty.elemType();
3595		const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
3596		switch (ptr) {
3597		.none => unreachable,
3598		.undef => unreachable,
3599		.unreach => unreachable,
3600		.dead => unreachable,
3601		.eflags => unreachable,
3602		.register_overflow => unreachable,
3603		.immediate => \|imm\| {
3604		try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm });
3605		},
3606		.stack_offset => {
3607		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3608		try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
3609		},
3610		.ptr_stack_offset => \|off\| {
3611		try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
3612		},
3613		.register => \|reg\| {
3614		const reg_lock = self.register_manager.lockReg(reg);
3615		defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	4018	try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg });
	4019	}
3616	4020
3617		switch (dst_mcv) {
3618		.dead => unreachable,
3619		.undef => unreachable,
3620		.eflags => unreachable,
3621		.register => \|dst_reg\| {
3622		try self.asmRegisterMemory(
3623		.mov,
3624		registerAlias(dst_reg, abi_size),
3625		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }),
3626		);
3627		},
3628		.stack_offset => \|off\| {
3629		if (abi_size <= 8) {
3630		const tmp_reg = try self.register_manager.allocReg(null, gp);
3631		try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
3632		return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
3633		}
	4021	fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
	4022	const dst_ty = ptr_ty.childType();
	4023	switch (ptr_mcv) {
	4024	.none,
	4025	.unreach,
	4026	.dead,
	4027	.undef,
	4028	.eflags,
	4029	.register_overflow,
	4030	.reserved_frame,
	4031	=> unreachable, // not a valid pointer
	4032	.immediate,
	4033	.register,
	4034	.register_offset,
	4035	.lea_direct,
	4036	.lea_got,
	4037	.lea_tlv,
	4038	.lea_frame,
	4039	=> try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
	4040	.memory,
	4041	.indirect,
	4042	.load_direct,
	4043	.load_got,
	4044	.load_tlv,
	4045	.load_frame,
	4046	=> {
	4047	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
	4048	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	4049	defer self.register_manager.unlockReg(addr_lock);
3634	4050
3635		try self.genInlineMemcpy(
3636		.{ .ptr_stack_offset = off },
3637		ptr,
3638		.{ .immediate = abi_size },
3639		.{},
3640		);
3641		},
3642		else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
3643		}
3644		},
3645		.load_direct => \|sym_index\| {
3646		const addr_reg = try self.copyToTmpRegister(Type.usize, .{ .lea_direct = sym_index });
3647		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3648		defer self.register_manager.unlockReg(addr_reg_lock);
3649		// Load the pointer, which is stored in memory
3650		try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3651		try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
3652		},
3653		.load_tlv => \|sym_index\| try self.load(dst_mcv, .{ .lea_tlv = sym_index }, ptr_ty),
3654		.memory, .load_got, .lea_direct, .lea_tlv => {
3655		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3656		try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
	4051	try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3657	4052	},
3658	4053	}
3659	4054	}
...	...	@@ -3669,53 +4064,48 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
3669	4064	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
3670	4065	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
3671	4066
3672		const ptr = try self.resolveInst(ty_op.operand);
3673		const dst_mcv: MCValue = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr))
	4067	const ptr_mcv = try self.resolveInst(ty_op.operand);
	4068	const dst_mcv = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
3674	4069	// The MCValue that holds the pointer can be re-used as the value.
3675		ptr
	4070	ptr_mcv
3676	4071	else
3677	4072	try self.allocRegOrMem(inst, true);
3678	4073
3679	4074	const ptr_ty = self.air.typeOf(ty_op.operand);
3680	4075	if (ptr_ty.ptrInfo().data.host_size > 0) {
3681		try self.packedLoad(dst_mcv, ptr, ptr_ty);
	4076	try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv);
3682	4077	} else {
3683		try self.load(dst_mcv, ptr, ptr_ty);
	4078	try self.load(dst_mcv, ptr_ty, ptr_mcv);
3684	4079	}
3685	4080	break :result dst_mcv;
3686	4081	};
3687	4082	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3688	4083	}
3689	4084
3690		fn packedStore(
3691		self: *Self,
3692		ptr_mcv: MCValue,
3693		val_mcv: MCValue,
3694		ptr_ty: Type,
3695		val_ty: Type,
3696		) InnerError!void {
	4085	fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
3697	4086	const ptr_info = ptr_ty.ptrInfo().data;
	4087	const src_ty = ptr_ty.childType();
3698	4088
3699	4089	const limb_abi_size = @min(ptr_info.host_size, 8);
3700	4090	const limb_abi_bits = limb_abi_size * 8;
3701	4091
3702		const val_bit_size = val_ty.bitSize(self.target.*);
3703		const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
3704		const val_bit_off = ptr_info.bit_offset % limb_abi_bits;
	4092	const src_bit_size = src_ty.bitSize(self.target.*);
	4093	const src_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
	4094	const src_bit_off = ptr_info.bit_offset % limb_abi_bits;
3705	4095
3706	4096	const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3707	4097	const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
3708	4098	defer self.register_manager.unlockReg(ptr_lock);
3709	4099
3710	4100	var limb_i: u16 = 0;
3711		while (limb_i * limb_abi_bits < val_bit_off + val_bit_size) : (limb_i += 1) {
3712		const part_bit_off = if (limb_i == 0) val_bit_off else 0;
	4101	while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) {
	4102	const part_bit_off = if (limb_i == 0) src_bit_off else 0;
3713	4103	const part_bit_size =
3714		@min(val_bit_off + val_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
3715		const limb_mem = Memory.sib(
3716		Memory.PtrSize.fromSize(limb_abi_size),
3717		.{ .base = ptr_reg, .disp = val_byte_off + limb_i * limb_abi_bits },
3718		);
	4104	@min(src_bit_off + src_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
	4105	const limb_mem = Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), .{
	4106	.base = .{ .reg = ptr_reg },
	4107	.disp = src_byte_off + limb_i * limb_abi_bits,
	4108	});
3719	4109
3720	4110	const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) <<
3721	4111	@intCast(u6, part_bit_off);
...	...	@@ -3731,211 +4121,62 @@ fn packedStore(
3731	4121	try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
3732	4122	}
3733	4123
3734		if (val_bit_size <= 64) {
	4124	if (src_bit_size <= 64) {
3735	4125	const tmp_reg = try self.register_manager.allocReg(null, gp);
3736	4126	const tmp_mcv = MCValue{ .register = tmp_reg };
3737	4127	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3738	4128	defer self.register_manager.unlockReg(tmp_lock);
3739	4129
3740		try self.genSetReg(val_ty, tmp_reg, val_mcv);
	4130	try self.genSetReg(tmp_reg, src_ty, src_mcv);
3741	4131	switch (limb_i) {
3742		0 => try self.genShiftBinOpMir(.shl, val_ty, tmp_mcv, .{ .immediate = val_bit_off }),
3743		1 => try self.genShiftBinOpMir(.shr, val_ty, tmp_mcv, .{
3744		.immediate = limb_abi_bits - val_bit_off,
	4132	0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),
	4133	1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{
	4134	.immediate = limb_abi_bits - src_bit_off,
3745	4135	}),
3746	4136	else => unreachable,
3747	4137	}
3748		try self.genBinOpMir(.@"and", val_ty, tmp_mcv, .{ .immediate = part_mask });
	4138	try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });
3749	4139	try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
3750	4140	} else return self.fail("TODO: implement packed store of {}", .{
3751		val_ty.fmt(self.bin_file.options.module.?),
	4141	src_ty.fmt(self.bin_file.options.module.?),
3752	4142	});
3753	4143	}
3754	4144	}
3755	4145
3756		fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
3757		const abi_size = @intCast(u32, value_ty.abiSize(self.target.*));
3758		switch (ptr) {
3759		.none => unreachable,
3760		.undef => unreachable,
3761		.unreach => unreachable,
3762		.dead => unreachable,
3763		.eflags => unreachable,
3764		.register_overflow => unreachable,
3765		.immediate, .stack_offset => {
3766		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3767		try self.store(.{ .register = reg }, value, ptr_ty, value_ty);
3768		},
3769		.ptr_stack_offset => \|off\| {
3770		try self.genSetStack(value_ty, off, value, .{});
	4146	fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
	4147	const src_ty = ptr_ty.childType();
	4148	switch (ptr_mcv) {
	4149	.none,
	4150	.unreach,
	4151	.dead,
	4152	.undef,
	4153	.eflags,
	4154	.register_overflow,
	4155	.reserved_frame,
	4156	=> unreachable, // not a valid pointer
	4157	.immediate,
	4158	.register,
	4159	.register_offset,
	4160	.lea_direct,
	4161	.lea_got,
	4162	.lea_tlv,
	4163	.lea_frame,
	4164	=> try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
	4165	.memory,
	4166	.indirect,
	4167	.load_direct,
	4168	.load_got,
	4169	.load_tlv,
	4170	.load_frame,
	4171	=> {
	4172	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
	4173	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	4174	defer self.register_manager.unlockReg(addr_lock);
	4175
	4176	try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
3771	4177	},
3772		.register => \|reg\| {
3773		const reg_lock = self.register_manager.lockReg(reg);
3774		defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
3775
3776		switch (value) {
3777		.none => unreachable,
3778		.dead => unreachable,
3779		.unreach => unreachable,
3780		.eflags => \|cc\| try self.asmSetccMemory(
3781		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3782		cc,
3783		),
3784		.undef => if (self.wantSafety()) switch (abi_size) {
3785		1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
3786		2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
3787		4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
3788		8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
3789		else => try self.genInlineMemset(
3790		ptr,
3791		.{ .immediate = 0xaa },
3792		.{ .immediate = abi_size },
3793		.{},
3794		),
3795		},
3796		.immediate => \|imm\| switch (self.regBitSize(value_ty)) {
3797		8 => try self.asmMemoryImmediate(
3798		.mov,
3799		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3800		if (math.cast(i8, @bitCast(i64, imm))) \|small\|
3801		Immediate.s(small)
3802		else
3803		Immediate.u(@intCast(u8, imm)),
3804		),
3805		16 => try self.asmMemoryImmediate(
3806		.mov,
3807		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3808		if (math.cast(i16, @bitCast(i64, imm))) \|small\|
3809		Immediate.s(small)
3810		else
3811		Immediate.u(@intCast(u16, imm)),
3812		),
3813		32 => try self.asmMemoryImmediate(
3814		.mov,
3815		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3816		if (math.cast(i32, @bitCast(i64, imm))) \|small\|
3817		Immediate.s(small)
3818		else
3819		Immediate.u(@intCast(u32, imm)),
3820		),
3821		64 => if (math.cast(i32, @bitCast(i64, imm))) \|small\|
3822		try self.asmMemoryImmediate(
3823		.mov,
3824		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3825		Immediate.s(small),
3826		)
3827		else
3828		try self.asmMemoryRegister(
3829		.mov,
3830		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3831		registerAlias(try self.copyToTmpRegister(value_ty, value), abi_size),
3832		),
3833		else => unreachable,
3834		},
3835		.register => \|src_reg\| try self.genInlineMemcpyRegisterRegister(
3836		value_ty,
3837		reg,
3838		src_reg,
3839		0,
3840		),
3841		.register_overflow => \|ro\| {
3842		const ro_reg_lock = self.register_manager.lockReg(ro.reg);
3843		defer if (ro_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
3844
3845		const wrapped_ty = value_ty.structFieldType(0);
3846		try self.genInlineMemcpyRegisterRegister(wrapped_ty, reg, ro.reg, 0);
3847
3848		const overflow_bit_ty = value_ty.structFieldType(1);
3849		const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);
3850		const tmp_reg = try self.register_manager.allocReg(null, gp);
3851		try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
3852		try self.genInlineMemcpyRegisterRegister(
3853		overflow_bit_ty,
3854		reg,
3855		tmp_reg,
3856		-@intCast(i32, overflow_bit_offset),
3857		);
3858		},
3859		.memory, .load_tlv, .load_direct => if (abi_size <= 8) {
3860		const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3861		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3862		defer self.register_manager.unlockReg(tmp_lock);
3863
3864		try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3865		} else {
3866		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3867		const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3868		defer self.register_manager.unlockReg(addr_lock);
3869
3870		try self.genSetReg(Type.usize, addr_reg, switch (value) {
3871		.memory => \|addr\| .{ .immediate = addr },
3872		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
3873		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
3874		else => unreachable,
3875		});
3876		try self.genInlineMemcpy(
3877		ptr,
3878		.{ .register = addr_reg },
3879		.{ .immediate = abi_size },
3880		.{},
3881		);
3882		},
3883		.stack_offset => \|off\| if (abi_size <= 8) {
3884		const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3885		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3886		defer self.register_manager.unlockReg(tmp_lock);
3887
3888		try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3889		} else try self.genInlineMemcpy(
3890		ptr,
3891		.{ .ptr_stack_offset = off },
3892		.{ .immediate = abi_size },
3893		.{},
3894		),
3895		.ptr_stack_offset, .load_got, .lea_direct, .lea_tlv => {
3896		const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3897		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3898		defer self.register_manager.unlockReg(tmp_lock);
3899
3900		try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3901		},
3902		}
3903		},
3904		.memory, .load_direct, .load_tlv => {
3905		const value_lock: ?RegisterLock = switch (value) {
3906		.register => \|reg\| self.register_manager.lockReg(reg),
3907		else => null,
3908		};
3909		defer if (value_lock) \|lock\| self.register_manager.unlockReg(lock);
3910
3911		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3912		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3913		defer self.register_manager.unlockReg(addr_reg_lock);
3914
3915		switch (ptr) {
3916		.memory => \|addr\| try self.genSetReg(ptr_ty, addr_reg, .{ .immediate = addr }),
3917		.load_direct => \|sym_index\| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_direct = sym_index }),
3918		.load_tlv => \|sym_index\| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_tlv = sym_index }),
3919		else => unreachable,
3920		}
3921		if (ptr != .load_tlv) {
3922		// Load the pointer, which is stored in memory
3923		try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3924		}
3925
3926		const new_ptr = MCValue{ .register = addr_reg };
3927		try self.store(new_ptr, value, ptr_ty, value_ty);
3928		},
3929		.load_got, .lea_direct, .lea_tlv => {
3930		const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
3931		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3932		defer self.register_manager.unlockReg(addr_reg_lock);
3933
3934		const new_ptr = MCValue{ .register = addr_reg };
3935		try self.store(new_ptr, value, ptr_ty, value_ty);
3936		},
3937		}
3938		}
	4178	}
	4179	}
3939	4180
3940	4181	fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3941	4182	if (safety) {
...	...	@@ -3944,15 +4185,13 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3944	4185	// TODO if the value is undef, don't lower this instruction
3945	4186	}
3946	4187	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3947		const ptr = try self.resolveInst(bin_op.lhs);
	4188	const ptr_mcv = try self.resolveInst(bin_op.lhs);
3948	4189	const ptr_ty = self.air.typeOf(bin_op.lhs);
3949		const value = try self.resolveInst(bin_op.rhs);
3950		const value_ty = self.air.typeOf(bin_op.rhs);
3951		log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value });
	4190	const src_mcv = try self.resolveInst(bin_op.rhs);
3952	4191	if (ptr_ty.ptrInfo().data.host_size > 0) {
3953		try self.packedStore(ptr, value, ptr_ty, value_ty);
	4192	try self.packedStore(ptr_ty, ptr_mcv, src_mcv);
3954	4193	} else {
3955		try self.store(ptr, value, ptr_ty, value_ty);
	4194	try self.store(ptr_ty, ptr_mcv, src_mcv);
3956	4195	}
3957	4196	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3958	4197	}
...	...	@@ -3971,67 +4210,65 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
3971	4210	}
3972	4211
3973	4212	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
	4213	const ptr_field_ty = self.air.typeOfIndex(inst);
3974	4214	const mcv = try self.resolveInst(operand);
3975		const ptr_ty = self.air.typeOf(operand);
3976		const container_ty = ptr_ty.childType();
	4215	const ptr_container_ty = self.air.typeOf(operand);
	4216	const container_ty = ptr_container_ty.childType();
3977	4217	const field_offset = switch (container_ty.containerLayout()) {
3978	4218	.Auto, .Extern => @intCast(u32, container_ty.structFieldOffset(index, self.target.*)),
3979		.Packed => if (container_ty.zigTypeTag() == .Struct and ptr_ty.ptrInfo().data.host_size == 0)
	4219	.Packed => if (container_ty.zigTypeTag() == .Struct and
	4220	ptr_field_ty.ptrInfo().data.host_size == 0)
3980	4221	container_ty.packedStructFieldByteOffset(index, self.target.*)
3981	4222	else
3982	4223	0,
3983	4224	};
3984	4225
3985		const dst_mcv: MCValue = result: {
	4226	const result: MCValue = result: {
3986	4227	switch (mcv) {
3987		.stack_offset, .lea_tlv, .load_tlv => {
3988		const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
	4228	.load_frame, .lea_tlv, .load_tlv => {
	4229	const offset_reg = try self.copyToTmpRegister(Type.usize, .{
3989	4230	.immediate = field_offset,
3990	4231	});
3991	4232	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3992	4233	defer self.register_manager.unlockReg(offset_reg_lock);
3993	4234
3994		const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, switch (mcv) {
	4235	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, Type.usize, switch (mcv) {
3995	4236	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
3996	4237	else => mcv,
3997	4238	});
3998		try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
	4239	try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });
3999	4240	break :result dst_mcv;
4000	4241	},
4001		.ptr_stack_offset => \|off\| {
4002		const ptr_stack_offset = off - @intCast(i32, field_offset);
4003		break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };
4004		},
4005		.register => \|reg\| {
4006		const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
4007		defer self.register_manager.unlockReg(reg_lock);
4008
4009		const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
4010		.immediate = field_offset,
4011		});
4012		const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4013		defer self.register_manager.unlockReg(offset_reg_lock);
4014
4015		const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
4016		const result_reg: Register = blk: {
4017		if (can_reuse_operand) {
4018		break :blk reg;
4019		} else {
4020		const result_reg = try self.register_manager.allocReg(inst, gp);
4021		try self.genSetReg(ptr_ty, result_reg, mcv);
4022		break :blk result_reg;
4023		}
4024		};
4025		const result_reg_lock = self.register_manager.lockReg(result_reg);
4026		defer if (result_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	4242	.indirect => \|reg_off\| break :result .{ .indirect = .{
	4243	.reg = reg_off.reg,
	4244	.off = reg_off.off + @intCast(i32, field_offset),
	4245	} },
	4246	.lea_frame => \|frame_addr\| break :result .{ .lea_frame = .{
	4247	.index = frame_addr.index,
	4248	.off = frame_addr.off + @intCast(i32, field_offset),
	4249	} },
	4250	.register, .register_offset => {
	4251	const src_reg = mcv.getReg().?;
	4252	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	4253	defer self.register_manager.unlockReg(src_lock);
4027	4254
4028		try self.genBinOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });
4029		break :result MCValue{ .register = result_reg };
	4255	const dst_mcv: MCValue = if (self.reuseOperand(inst, operand, 0, mcv))
	4256	mcv
	4257	else
	4258	.{ .register = try self.copyToTmpRegister(ptr_field_ty, mcv) };
	4259	break :result .{ .register_offset = .{
	4260	.reg = dst_mcv.getReg().?,
	4261	.off = switch (dst_mcv) {
	4262	.register => 0,
	4263	.register_offset => \|reg_off\| reg_off.off,
	4264	else => unreachable,
	4265	} + @intCast(i32, field_offset),
	4266	} };
4030	4267	},
4031	4268	else => return self.fail("TODO implement fieldPtr for {}", .{mcv}),
4032	4269	}
4033	4270	};
4034		return dst_mcv;
	4271	return result;
4035	4272	}
4036	4273
4037	4274	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4055,7 +4292,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4055	4292	};
4056	4293
4057	4294	switch (src_mcv) {
4058		.stack_offset => \|src_off\| {
	4295	.load_frame => \|frame_addr\| {
4059	4296	const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));
4060	4297	const limb_abi_size = @min(field_abi_size, 8);
4061	4298	const limb_abi_bits = limb_abi_size * 8;
...	...	@@ -4063,11 +4300,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4063	4300	const field_bit_off = field_off % limb_abi_bits;
4064	4301
4065	4302	if (field_bit_off == 0) {
4066		const off_mcv = MCValue{ .stack_offset = src_off - field_byte_off };
	4303	const off_mcv = MCValue{ .load_frame = .{
	4304	.index = frame_addr.index,
	4305	.off = frame_addr.off + field_byte_off,
	4306	} };
4067	4307	if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;
4068	4308
4069	4309	const dst_mcv = try self.allocRegOrMem(inst, true);
4070		try self.setRegOrMem(field_ty, dst_mcv, off_mcv);
	4310	try self.genCopy(field_ty, dst_mcv, off_mcv);
4071	4311	break :result dst_mcv;
4072	4312	}
4073	4313
...	...	@@ -4081,10 +4321,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4081	4321	if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
4082	4322	if (load_abi_size <= 8) {
4083	4323	const load_reg = registerAlias(dst_reg, load_abi_size);
4084		try self.asmRegisterMemory(.mov, load_reg, Memory.sib(
4085		Memory.PtrSize.fromSize(load_abi_size),
4086		.{ .base = .rbp, .disp = field_byte_off - src_off },
4087		));
	4324	try self.asmRegisterMemory(
	4325	.mov,
	4326	load_reg,
	4327	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
	4328	.base = .{ .frame = frame_addr.index },
	4329	.disp = frame_addr.off + field_byte_off,
	4330	}),
	4331	);
4088	4332	try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));
4089	4333	} else {
4090	4334	const tmp_reg = registerAlias(
...	...	@@ -4095,14 +4339,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4095	4339	defer self.register_manager.unlockReg(tmp_lock);
4096	4340
4097	4341	const dst_alias = registerAlias(dst_reg, field_abi_size);
4098		try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(
4099		Memory.PtrSize.fromSize(field_abi_size),
4100		.{ .base = .rbp, .disp = field_byte_off - src_off },
4101		));
4102		try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(
4103		Memory.PtrSize.fromSize(field_abi_size),
4104		.{ .base = .rbp, .disp = field_byte_off + 1 - src_off },
4105		));
	4342	try self.asmRegisterMemory(
	4343	.mov,
	4344	dst_alias,
	4345	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
	4346	.base = .{ .frame = frame_addr.index },
	4347	.disp = frame_addr.off + field_byte_off,
	4348	}),
	4349	);
	4350	try self.asmRegisterMemory(
	4351	.mov,
	4352	tmp_reg,
	4353	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
	4354	.base = .{ .frame = frame_addr.index },
	4355	.disp = frame_addr.off + field_byte_off + @intCast(i32, limb_abi_size),
	4356	}),
	4357	);
4106	4358	try self.asmRegisterRegisterImmediate(
4107	4359	.shrd,
4108	4360	dst_alias,
...	...	@@ -4249,41 +4501,51 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
4249	4501
4250	4502	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
4251	4503	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	4504	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
	4505	mir_tag,
	4506	dst_ty.fmt(self.bin_file.options.module.?),
	4507	});
4252	4508	switch (dst_mcv) {
4253		.none => unreachable,
4254		.undef => unreachable,
4255		.dead, .unreach, .immediate => unreachable,
4256		.eflags => unreachable,
4257		.register_overflow => unreachable,
	4509	.none,
	4510	.unreach,
	4511	.dead,
	4512	.undef,
	4513	.immediate,
	4514	.register_offset,
	4515	.eflags,
	4516	.register_overflow,
	4517	.lea_direct,
	4518	.lea_got,
	4519	.lea_tlv,
	4520	.lea_frame,
	4521	.reserved_frame,
	4522	=> unreachable, // unmodifiable destination
4258	4523	.register => \|dst_reg\| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
4259		.stack_offset => \|off\| {
4260		if (abi_size > 8) {
4261		return self.fail("TODO implement {} for stack dst with large ABI", .{mir_tag});
4262		}
4263
4264		try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4265		.base = .rbp,
4266		.disp = -off,
4267		}));
4268		},
4269		.ptr_stack_offset => unreachable,
4270		.load_got, .lea_direct, .lea_tlv => unreachable,
4271		.memory, .load_direct, .load_tlv => {
4272		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	4524	.memory, .load_got, .load_direct, .load_tlv => {
	4525	const addr_reg = try self.register_manager.allocReg(null, gp);
4273	4526	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4274	4527	defer self.register_manager.unlockReg(addr_reg_lock);
4275	4528
4276		try self.genSetReg(Type.usize, addr_reg, switch (dst_mcv) {
4277		.memory => \|addr\| .{ .immediate = addr },
4278		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
4279		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4280		else => unreachable,
4281		});
	4529	try self.genSetReg(addr_reg, Type.usize, dst_mcv.address());
4282	4530	try self.asmMemory(
4283	4531	mir_tag,
4284		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
	4532	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
4285	4533	);
4286	4534	},
	4535	.indirect, .load_frame => try self.asmMemory(
	4536	mir_tag,
	4537	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (dst_mcv) {
	4538	.indirect => \|reg_off\| .{
	4539	.base = .{ .reg = reg_off.reg },
	4540	.disp = reg_off.off,
	4541	},
	4542	.load_frame => \|frame_addr\| .{
	4543	.base = .{ .frame = frame_addr.index },
	4544	.disp = frame_addr.off,
	4545	},
	4546	else => unreachable,
	4547	}),
	4548	),
4287	4549	}
4288	4550	}
4289	4551
...	...	@@ -4305,7 +4567,7 @@ fn genShiftBinOpMir(
4305	4567	else => {},
4306	4568	}
4307	4569	self.register_manager.getRegAssumeFree(.rcx, null);
4308		try self.genSetReg(Type.u8, .rcx, shift_mcv);
	4570	try self.genSetReg(.cl, Type.u8, shift_mcv);
4309	4571	break :rhs .{ .register = .rcx };
4310	4572	};
4311	4573
...	...	@@ -4328,21 +4590,42 @@ fn genShiftBinOpMir(
4328	4590	@tagName(rhs_mcv),
4329	4591	}),
4330	4592	},
4331		.stack_offset => \|lhs_off\| switch (rhs_mcv) {
4332		.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
4333		tag,
4334		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
4335		Immediate.u(rhs_imm),
4336		),
4337		.register => \|rhs_reg\| try self.asmMemoryRegister(
4338		tag,
4339		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
4340		registerAlias(rhs_reg, 1),
4341		),
4342		else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4343		@tagName(lhs_mcv),
4344		@tagName(rhs_mcv),
4345		}),
	4593	.memory, .indirect, .load_frame => {
	4594	const lhs_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (lhs_mcv) {
	4595	.memory => \|addr\| .{
	4596	.base = .{ .reg = .ds },
	4597	.disp = math.cast(i32, @bitCast(i64, addr)) orelse
	4598	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4599	@tagName(lhs_mcv),
	4600	@tagName(rhs_mcv),
	4601	}),
	4602	},
	4603	.indirect => \|reg_off\| .{
	4604	.base = .{ .reg = reg_off.reg },
	4605	.disp = reg_off.off,
	4606	},
	4607	.load_frame => \|frame_addr\| .{
	4608	.base = .{ .frame = frame_addr.index },
	4609	.disp = frame_addr.off,
	4610	},
	4611	else => unreachable,
	4612	});
	4613	switch (rhs_mcv) {
	4614	.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
	4615	tag,
	4616	lhs_mem,
	4617	Immediate.u(rhs_imm),
	4618	),
	4619	.register => \|rhs_reg\| try self.asmMemoryRegister(
	4620	tag,
	4621	lhs_mem,
	4622	registerAlias(rhs_reg, 1),
	4623	),
	4624	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4625	@tagName(lhs_mcv),
	4626	@tagName(rhs_mcv),
	4627	}),
	4628	}
4346	4629	},
4347	4630	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4348	4631	@tagName(lhs_mcv),
...	...	@@ -4360,22 +4643,31 @@ fn genShiftBinOpMir(
4360	4643	else => unreachable,
4361	4644	};
4362	4645	switch (lhs_mcv) {
4363		.stack_offset => \|dst_off\| switch (rhs_mcv) {
	4646	.load_frame => \|dst_frame_addr\| switch (rhs_mcv) {
4364	4647	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
4365	4648	try self.asmRegisterMemory(
4366	4649	.mov,
4367	4650	tmp_reg,
4368		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4651	Memory.sib(.qword, .{
	4652	.base = .{ .frame = dst_frame_addr.index },
	4653	.disp = dst_frame_addr.off + info.offsets[0],
	4654	}),
4369	4655	);
4370	4656	try self.asmMemoryRegisterImmediate(
4371	4657	info.double_tag,
4372		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4658	Memory.sib(.qword, .{
	4659	.base = .{ .frame = dst_frame_addr.index },
	4660	.disp = dst_frame_addr.off + info.offsets[1],
	4661	}),
4373	4662	tmp_reg,
4374	4663	Immediate.u(rhs_imm),
4375	4664	);
4376	4665	try self.asmMemoryImmediate(
4377	4666	tag,
4378		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4667	Memory.sib(.qword, .{
	4668	.base = .{ .frame = dst_frame_addr.index },
	4669	.disp = dst_frame_addr.off + info.offsets[0],
	4670	}),
4379	4671	Immediate.u(rhs_imm),
4380	4672	);
4381	4673	} else {
...	...	@@ -4383,14 +4675,20 @@ fn genShiftBinOpMir(
4383	4675	try self.asmRegisterMemory(
4384	4676	.mov,
4385	4677	tmp_reg,
4386		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4678	Memory.sib(.qword, .{
	4679	.base = .{ .frame = dst_frame_addr.index },
	4680	.disp = dst_frame_addr.off + info.offsets[0],
	4681	}),
4387	4682	);
4388	4683	if (rhs_imm > 64) {
4389	4684	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
4390	4685	}
4391	4686	try self.asmMemoryRegister(
4392	4687	.mov,
4393		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4688	Memory.sib(.qword, .{
	4689	.base = .{ .frame = dst_frame_addr.index },
	4690	.disp = dst_frame_addr.off + info.offsets[1],
	4691	}),
4394	4692	tmp_reg,
4395	4693	);
4396	4694	switch (tag) {
...	...	@@ -4398,13 +4696,19 @@ fn genShiftBinOpMir(
4398	4696	try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
4399	4697	try self.asmMemoryRegister(
4400	4698	.mov,
4401		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4699	Memory.sib(.qword, .{
	4700	.base = .{ .frame = dst_frame_addr.index },
	4701	.disp = dst_frame_addr.off + info.offsets[0],
	4702	}),
4402	4703	tmp_reg,
4403	4704	);
4404	4705	},
4405	4706	.sar => try self.asmMemoryImmediate(
4406	4707	tag,
4407		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4708	Memory.sib(.qword, .{
	4709	.base = .{ .frame = dst_frame_addr.index },
	4710	.disp = dst_frame_addr.off + info.offsets[0],
	4711	}),
4408	4712	Immediate.u(63),
4409	4713	),
4410	4714	else => unreachable,
...	...	@@ -4419,16 +4723,22 @@ fn genShiftBinOpMir(
4419	4723	const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);
4420	4724	defer self.register_manager.unlockReg(second_lock);
4421	4725
4422		try self.genSetReg(Type.u8, .cl, rhs_mcv);
	4726	try self.genSetReg(.cl, Type.u8, rhs_mcv);
4423	4727	try self.asmRegisterMemory(
4424	4728	.mov,
4425	4729	first_reg,
4426		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4730	Memory.sib(.qword, .{
	4731	.base = .{ .frame = dst_frame_addr.index },
	4732	.disp = dst_frame_addr.off + info.offsets[0],
	4733	}),
4427	4734	);
4428	4735	try self.asmRegisterMemory(
4429	4736	.mov,
4430	4737	second_reg,
4431		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4738	Memory.sib(.qword, .{
	4739	.base = .{ .frame = dst_frame_addr.index },
	4740	.disp = dst_frame_addr.off + info.offsets[1],
	4741	}),
4432	4742	);
4433	4743	switch (tag) {
4434	4744	.shl, .sal, .shr => try self.asmRegisterRegister(
...	...	@@ -4449,12 +4759,18 @@ fn genShiftBinOpMir(
4449	4759	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
4450	4760	try self.asmMemoryRegister(
4451	4761	.mov,
4452		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4762	Memory.sib(.qword, .{
	4763	.base = .{ .frame = dst_frame_addr.index },
	4764	.disp = dst_frame_addr.off + info.offsets[1],
	4765	}),
4453	4766	second_reg,
4454	4767	);
4455	4768	try self.asmMemoryRegister(
4456	4769	.mov,
4457		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4770	Memory.sib(.qword, .{
	4771	.base = .{ .frame = dst_frame_addr.index },
	4772	.disp = dst_frame_addr.off + info.offsets[0],
	4773	}),
4458	4774	first_reg,
4459	4775	);
4460	4776	},
...	...	@@ -4515,7 +4831,7 @@ fn genShiftBinOp(
4515	4831	if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
4516	4832	}
4517	4833	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4518		try self.setRegOrMem(lhs_ty, dst_mcv, lhs_mcv);
	4834	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
4519	4835	break :dst dst_mcv;
4520	4836	};
4521	4837
...	...	@@ -4610,14 +4926,22 @@ fn genMulDivBinOp(
4610	4926	}, dst_abi_size) };
4611	4927
4612	4928	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
4613		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
4614		.base = .rbp,
4615		.disp = 0 - dst_mcv.stack_offset,
4616		}), .rax);
4617		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
4618		.base = .rbp,
4619		.disp = 8 - dst_mcv.stack_offset,
4620		}), .rdx);
	4929	try self.asmMemoryRegister(
	4930	.mov,
	4931	Memory.sib(.qword, .{
	4932	.base = .{ .frame = dst_mcv.load_frame.index },
	4933	.disp = dst_mcv.load_frame.off,
	4934	}),
	4935	.rax,
	4936	);
	4937	try self.asmMemoryRegister(
	4938	.mov,
	4939	Memory.sib(.qword, .{
	4940	.base = .{ .frame = dst_mcv.load_frame.index },
	4941	.disp = dst_mcv.load_frame.off + 8,
	4942	}),
	4943	.rdx,
	4944	);
4621	4945	return dst_mcv;
4622	4946	},
4623	4947
...	...	@@ -4627,8 +4951,24 @@ fn genMulDivBinOp(
4627	4951
4628	4952	switch (signedness) {
4629	4953	.signed => {
	4954	const lhs_lock = switch (lhs) {
	4955	.register => \|reg\| self.register_manager.lockReg(reg),
	4956	else => null,
	4957	};
	4958	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	4959	const rhs_lock = switch (rhs) {
	4960	.register => \|reg\| self.register_manager.lockReg(reg),
	4961	else => null,
	4962	};
	4963	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	4964
	4965	// hack around hazard between rhs and div_floor by copying rhs to another register
	4966	const rhs_copy = try self.copyToTmpRegister(ty, rhs);
	4967	const rhs_copy_lock = self.register_manager.lockRegAssumeUnused(rhs_copy);
	4968	defer self.register_manager.unlockReg(rhs_copy_lock);
	4969
4630	4970	const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);
4631		try self.genIntMulComplexOpMir(ty, div_floor, rhs);
	4971	try self.genIntMulComplexOpMir(ty, div_floor, .{ .register = rhs_copy });
4632	4972	const div_floor_lock = self.register_manager.lockReg(div_floor.register);
4633	4973	defer if (div_floor_lock) \|lock\| self.register_manager.unlockReg(lock);
4634	4974
...	...	@@ -4775,18 +5115,14 @@ fn genBinOp(
4775	5115	}
4776	5116	}
4777	5117	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4778		try self.setRegOrMem(lhs_ty, dst_mcv, switch (lhs) {
4779		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
4780		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4781		else => lhs,
4782		});
	5118	try self.genCopy(lhs_ty, dst_mcv, lhs);
4783	5119	break :dst dst_mcv;
4784	5120	};
4785		const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
	5121	const dst_lock: ?RegisterLock = switch (dst_mcv) {
4786	5122	.register => \|reg\| self.register_manager.lockReg(reg),
4787	5123	else => null,
4788	5124	};
4789		defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
	5125	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
4790	5126
4791	5127	const src_mcv = if (flipped) lhs else rhs;
4792	5128	switch (tag) {
...	...	@@ -4878,10 +5214,21 @@ fn genBinOp(
4878	5214	.max,
4879	5215	=> switch (lhs_ty.zigTypeTag()) {
4880	5216	.Int => {
4881		const mat_src_mcv = switch (src_mcv) {
4882		.immediate => MCValue{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) },
4883		else => src_mcv,
4884		};
	5217	const mat_src_mcv: MCValue = if (switch (src_mcv) {
	5218	.immediate,
	5219	.eflags,
	5220	.register_offset,
	5221	.load_direct,
	5222	.lea_direct,
	5223	.load_got,
	5224	.lea_got,
	5225	.load_tlv,
	5226	.lea_tlv,
	5227	.lea_frame,
	5228	=> true,
	5229	.memory => \|addr\| math.cast(i32, @bitCast(i64, addr)) == null,
	5230	else => false,
	5231	}) .{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) } else src_mcv;
4885	5232	const mat_mcv_lock = switch (mat_src_mcv) {
4886	5233	.register => \|reg\| self.register_manager.lockReg(reg),
4887	5234	else => null,
...	...	@@ -4913,57 +5260,56 @@ fn genBinOp(
4913	5260	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);
4914	5261	switch (mat_src_mcv) {
4915	5262	.none,
4916		.undef,
4917		.dead,
4918	5263	.unreach,
	5264	.dead,
	5265	.undef,
4919	5266	.immediate,
4920	5267	.eflags,
	5268	.register_offset,
4921	5269	.register_overflow,
4922		.ptr_stack_offset,
4923		.load_got,
	5270	.load_direct,
4924	5271	.lea_direct,
	5272	.load_got,
	5273	.lea_got,
	5274	.load_tlv,
4925	5275	.lea_tlv,
	5276	.lea_frame,
	5277	.reserved_frame,
4926	5278	=> unreachable,
4927	5279	.register => \|src_reg\| try self.asmCmovccRegisterRegister(
4928	5280	registerAlias(tmp_reg, cmov_abi_size),
4929	5281	registerAlias(src_reg, cmov_abi_size),
4930	5282	cc,
4931	5283	),
4932		.stack_offset => \|off\| try self.asmCmovccRegisterMemory(
	5284	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
4933	5285	registerAlias(tmp_reg, cmov_abi_size),
4934		Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
4935		.base = .rbp,
4936		.disp = -off,
	5286	Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
	5287	.memory => \|addr\| .{
	5288	.base = .{ .reg = .ds },
	5289	.disp = @intCast(i32, @bitCast(i64, addr)),
	5290	},
	5291	.indirect => \|reg_off\| .{
	5292	.base = .{ .reg = reg_off.reg },
	5293	.disp = reg_off.off,
	5294	},
	5295	.load_frame => \|frame_addr\| .{
	5296	.base = .{ .frame = frame_addr.index },
	5297	.disp = frame_addr.off,
	5298	},
	5299	else => unreachable,
4937	5300	}),
4938	5301	cc,
4939	5302	),
4940		.memory, .load_direct, .load_tlv => {
4941		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4942		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4943		defer self.register_manager.unlockReg(addr_reg_lock);
4944
4945		try self.genSetReg(Type.usize, addr_reg, switch (mat_src_mcv) {
4946		.memory => \|addr\| .{ .immediate = addr },
4947		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
4948		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4949		else => unreachable,
4950		});
4951		try self.asmCmovccRegisterMemory(
4952		registerAlias(tmp_reg, cmov_abi_size),
4953		Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }),
4954		cc,
4955		);
4956		},
4957	5303	}
4958		try self.setRegOrMem(lhs_ty, dst_mcv, .{ .register = tmp_reg });
	5304	try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
4959	5305	},
4960		.Float => try self.genBinOpMir(switch (lhs_ty.tag()) {
4961		.f32 => switch (tag) {
	5306	.Float => try self.genBinOpMir(switch (lhs_ty.floatBits(self.target.*)) {
	5307	32 => switch (tag) {
4962	5308	.min => .minss,
4963	5309	.max => .maxss,
4964	5310	else => unreachable,
4965	5311	},
4966		.f64 => switch (tag) {
	5312	64 => switch (tag) {
4967	5313	.min => .minsd,
4968	5314	.max => .maxsd,
4969	5315	else => unreachable,
...	...	@@ -4981,25 +5327,38 @@ fn genBinOp(
4981	5327	fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
4982	5328	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4983	5329	switch (dst_mcv) {
4984		.none => unreachable,
4985		.undef => unreachable,
4986		.dead, .unreach, .immediate => unreachable,
4987		.eflags => unreachable,
4988		.register_overflow => unreachable,
	5330	.none,
	5331	.unreach,
	5332	.dead,
	5333	.undef,
	5334	.immediate,
	5335	.register_offset,
	5336	.eflags,
	5337	.register_overflow,
	5338	.lea_direct,
	5339	.lea_got,
	5340	.lea_tlv,
	5341	.lea_frame,
	5342	.reserved_frame,
	5343	=> unreachable, // unmodifiable destination
4989	5344	.register => \|dst_reg\| {
4990	5345	const dst_alias = registerAlias(dst_reg, abi_size);
4991	5346	switch (src_mcv) {
4992		.none => unreachable,
4993		.undef => unreachable,
4994		.dead, .unreach => unreachable,
4995		.register_overflow => unreachable,
	5347	.none,
	5348	.unreach,
	5349	.dead,
	5350	.undef,
	5351	.register_overflow,
	5352	.reserved_frame,
	5353	=> unreachable,
4996	5354	.register => \|src_reg\| switch (ty.zigTypeTag()) {
4997	5355	.Float => {
4998		if (intrinsicsAllowed(self.target.*, ty)) {
4999		return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5000		}
5001
5002		return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
	5356	if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))
	5357	return self.fail("TODO genBinOpMir for {s} {} without sse", .{
	5358	@tagName(mir_tag),
	5359	ty.fmt(self.bin_file.options.module.?),
	5360	});
	5361	return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5003	5362	},
5004	5363	else => try self.asmRegisterRegister(
5005	5364	mir_tag,
...	...	@@ -5041,14 +5400,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5041	5400	)),
5042	5401	else => unreachable,
5043	5402	},
5044		.ptr_stack_offset,
	5403	.register_offset,
	5404	.eflags,
5045	5405	.memory,
5046		.load_got,
5047		.lea_direct,
5048	5406	.load_direct,
5049		.lea_tlv,
	5407	.lea_direct,
	5408	.load_got,
	5409	.lea_got,
5050	5410	.load_tlv,
5051		.eflags,
	5411	.lea_tlv,
	5412	.lea_frame,
5052	5413	=> {
5053	5414	assert(abi_size <= 8);
5054	5415	const dst_reg_lock = self.register_manager.lockReg(dst_reg);
...	...	@@ -5057,75 +5418,84 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5057	5418	const reg = try self.copyToTmpRegister(ty, src_mcv);
5058	5419	return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
5059	5420	},
5060		.stack_offset => \|off\| try self.asmRegisterMemory(
	5421	.indirect, .load_frame => try self.asmRegisterMemory(
5061	5422	mir_tag,
5062	5423	registerAlias(dst_reg, abi_size),
5063		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
	5424	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
	5425	.indirect => \|reg_off\| .{
	5426	.base = .{ .reg = reg_off.reg },
	5427	.disp = reg_off.off,
	5428	},
	5429	.load_frame => \|frame_addr\| .{
	5430	.base = .{ .frame = frame_addr.index },
	5431	.disp = frame_addr.off,
	5432	},
	5433	else => unreachable,
	5434	}),
5064	5435	),
5065	5436	}
5066	5437	},
5067		.memory, .load_got, .load_direct, .load_tlv, .stack_offset => {
5068		const dst: ?struct {
5069		addr_reg: Register,
5070		addr_lock: RegisterLock,
5071		} = switch (dst_mcv) {
	5438	.memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => {
	5439	const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock };
	5440	const limb_abi_size = @min(abi_size, 8);
	5441
	5442	const dst_info: OpInfo = switch (dst_mcv) {
5072	5443	else => unreachable,
5073	5444	.memory, .load_got, .load_direct, .load_tlv => dst: {
5074	5445	const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5075	5446	const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
5076	5447	errdefer self.register_manager.unlockReg(dst_addr_lock);
5077	5448
5078		try self.genSetReg(Type.usize, dst_addr_reg, switch (dst_mcv) {
5079		.memory => \|addr\| .{ .immediate = addr },
5080		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
5081		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
5082		else => dst_mcv,
5083		});
5084
	5449	try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address());
5085	5450	break :dst .{
5086	5451	.addr_reg = dst_addr_reg,
5087	5452	.addr_lock = dst_addr_lock,
5088	5453	};
5089	5454	},
5090		.stack_offset => null,
	5455	.load_frame => null,
5091	5456	};
5092		defer if (dst) \|lock\| self.register_manager.unlockReg(lock.addr_lock);
5093
5094		const src: ?struct {
5095		limb_reg: Register,
5096		limb_lock: RegisterLock,
5097		addr_reg: Register,
5098		addr_lock: RegisterLock,
5099		} = switch (src_mcv) {
5100		else => null,
	5457	defer if (dst_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
	5458
	5459	const src_info: OpInfo = switch (src_mcv) {
	5460	.none,
	5461	.unreach,
	5462	.dead,
	5463	.undef,
	5464	.register_overflow,
	5465	.reserved_frame,
	5466	=> unreachable,
	5467	.immediate,
	5468	.register,
	5469	.register_offset,
	5470	.eflags,
	5471	.indirect,
	5472	.lea_direct,
	5473	.lea_got,
	5474	.lea_tlv,
	5475	.load_frame,
	5476	.lea_frame,
	5477	=> null,
5101	5478	.memory, .load_got, .load_direct, .load_tlv => src: {
5102		const src_limb_reg = try self.register_manager.allocReg(null, gp);
5103		const src_limb_lock = self.register_manager.lockRegAssumeUnused(src_limb_reg);
5104		errdefer self.register_manager.unlockReg(src_limb_lock);
	5479	switch (src_mcv) {
	5480	.memory => \|addr\| if (math.cast(i32, @bitCast(i64, addr)) != null and
	5481	math.cast(i32, @bitCast(i64, addr) + abi_size - limb_abi_size) != null)
	5482	break :src null,
	5483	.load_got, .load_direct, .load_tlv => {},
	5484	else => unreachable,
	5485	}
5105	5486
5106	5487	const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5107	5488	const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
5108	5489	errdefer self.register_manager.unlockReg(src_addr_lock);
5109	5490
5110		try self.genSetReg(Type.usize, src_addr_reg, switch (src_mcv) {
5111		.memory => \|addr\| .{ .immediate = addr },
5112		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
5113		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
5114		else => src_mcv,
5115		});
5116
	5491	try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address());
5117	5492	break :src .{
5118	5493	.addr_reg = src_addr_reg,
5119	5494	.addr_lock = src_addr_lock,
5120		.limb_reg = src_limb_reg,
5121		.limb_lock = src_limb_lock,
5122	5495	};
5123	5496	},
5124	5497	};
5125		defer if (src) \|locks\| {
5126		self.register_manager.unlockReg(locks.limb_lock);
5127		self.register_manager.unlockReg(locks.addr_lock);
5128		};
	5498	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
5129	5499
5130	5500	const ty_signedness =
5131	5501	if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
...	...	@@ -5133,7 +5503,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5133	5503	.signed => Type.usize,
5134	5504	.unsigned => Type.isize,
5135	5505	};
5136		const limb_abi_size = @min(abi_size, 8);
5137	5506	var off: i32 = 0;
5138	5507	while (off < abi_size) : (off += 8) {
5139	5508	const mir_limb_tag = switch (off) {
...	...	@@ -5150,35 +5519,45 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5150	5519	const dst_limb_mem = Memory.sib(
5151	5520	Memory.PtrSize.fromSize(limb_abi_size),
5152	5521	switch (dst_mcv) {
5153		else => unreachable,
5154		.stack_offset => \|dst_off\| .{
5155		.base = .rbp,
5156		.disp = off - dst_off,
5157		},
5158	5522	.memory,
5159	5523	.load_got,
5160	5524	.load_direct,
5161	5525	.load_tlv,
5162		=> .{ .base = dst.?.addr_reg, .disp = off },
	5526	=> .{ .base = .{ .reg = dst_info.?.addr_reg }, .disp = off },
	5527	.indirect => \|reg_off\| .{
	5528	.base = .{ .reg = reg_off.reg },
	5529	.disp = reg_off.off + off,
	5530	},
	5531	.load_frame => \|frame_addr\| .{
	5532	.base = .{ .frame = frame_addr.index },
	5533	.disp = frame_addr.off + off,
	5534	},
	5535	else => unreachable,
5163	5536	},
5164	5537	);
5165	5538	switch (src_mcv) {
5166		.none => unreachable,
5167		.undef => unreachable,
5168		.dead, .unreach => unreachable,
5169		.register_overflow => unreachable,
5170		.register => \|src_reg\| {
5171		assert(off == 0);
5172		try self.asmMemoryRegister(
	5539	.none,
	5540	.unreach,
	5541	.dead,
	5542	.undef,
	5543	.register_overflow,
	5544	.reserved_frame,
	5545	=> unreachable,
	5546	.register => \|src_reg\| switch (off) {
	5547	0 => try self.asmMemoryRegister(
5173	5548	mir_limb_tag,
5174	5549	dst_limb_mem,
5175	5550	registerAlias(src_reg, limb_abi_size),
5176		);
	5551	),
	5552	else => unreachable,
5177	5553	},
5178	5554	.immediate => \|src_imm\| {
5179		const imm = if (off == 0) src_imm else switch (ty_signedness) {
5180		.signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),
5181		.unsigned => 0,
	5555	const imm = switch (off) {
	5556	0 => src_imm,
	5557	else => switch (ty_signedness) {
	5558	.signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),
	5559	.unsigned => 0,
	5560	},
5182	5561	};
5183	5562	switch (self.regBitSize(limb_ty)) {
5184	5563	8 => try self.asmMemoryImmediate(
...	...	@@ -5223,47 +5602,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5223	5602	else => unreachable,
5224	5603	}
5225	5604	},
	5605	.register_offset,
	5606	.eflags,
5226	5607	.memory,
5227		.load_got,
	5608	.indirect,
5228	5609	.load_direct,
5229		.load_tlv,
5230		=> {
5231		try self.asmRegisterMemory(
5232		.mov,
5233		registerAlias(src.?.limb_reg, limb_abi_size),
5234		Memory.sib(
5235		Memory.PtrSize.fromSize(limb_abi_size),
5236		.{ .base = src.?.addr_reg, .disp = off },
5237		),
5238		);
5239		try self.asmMemoryRegister(
5240		mir_limb_tag,
5241		dst_limb_mem,
5242		registerAlias(src.?.limb_reg, limb_abi_size),
5243		);
5244		},
5245		.stack_offset,
5246		.ptr_stack_offset,
5247		.eflags,
5248	5610	.lea_direct,
	5611	.load_got,
	5612	.lea_got,
	5613	.load_tlv,
5249	5614	.lea_tlv,
	5615	.load_frame,
	5616	.lea_frame,
5250	5617	=> {
5251		const src_limb_reg = try self.copyToTmpRegister(limb_ty, switch (src_mcv) {
5252		.stack_offset => \|src_off\| .{ .stack_offset = src_off - off },
5253		.ptr_stack_offset,
	5618	const src_limb_reg = try self.copyToTmpRegister(limb_ty, if (src_info) \|info\| .{
	5619	.indirect = .{ .reg = info.addr_reg, .off = off },
	5620	} else switch (src_mcv) {
5254	5621	.eflags,
5255		.load_got,
	5622	.register_offset,
5256	5623	.lea_direct,
	5624	.lea_got,
5257	5625	.lea_tlv,
5258		=> off: {
5259		assert(off == 0);
5260		break :off src_mcv;
	5626	.lea_frame,
	5627	=> switch (off) {
	5628	0 => src_mcv,
	5629	else => .{ .immediate = 0 },
5261	5630	},
	5631	.memory => \|addr\| .{ .memory = @bitCast(u64, @bitCast(i64, addr) + off) },
	5632	.indirect => \|reg_off\| .{ .indirect = .{
	5633	.reg = reg_off.reg,
	5634	.off = reg_off.off + off,
	5635	} },
	5636	.load_frame => \|frame_addr\| .{ .load_frame = .{
	5637	.index = frame_addr.index,
	5638	.off = frame_addr.off + off,
	5639	} },
5262	5640	else => unreachable,
5263	5641	});
5264		const src_limb_lock = self.register_manager.lockReg(src_limb_reg);
5265		defer if (src_limb_lock) \|lock\| self.register_manager.unlockReg(lock);
5266
5267	5642	try self.asmMemoryRegister(
5268	5643	mir_limb_tag,
5269	5644	dst_limb_mem,
...	...	@@ -5273,9 +5648,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5273	5648	}
5274	5649	}
5275	5650	},
5276		.ptr_stack_offset => unreachable,
5277		.lea_tlv => unreachable,
5278		.lea_direct => unreachable,
5279	5651	}
5280	5652	}
5281	5653
...	...	@@ -5284,27 +5656,33 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5284	5656	fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
5285	5657	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
5286	5658	switch (dst_mcv) {
5287		.none => unreachable,
5288		.undef => unreachable,
5289		.dead, .unreach, .immediate => unreachable,
5290		.eflags => unreachable,
5291		.ptr_stack_offset => unreachable,
5292		.lea_direct => unreachable,
5293		.lea_tlv => unreachable,
5294		.register_overflow => unreachable,
	5659	.none,
	5660	.unreach,
	5661	.dead,
	5662	.undef,
	5663	.immediate,
	5664	.register_offset,
	5665	.eflags,
	5666	.register_overflow,
	5667	.lea_direct,
	5668	.lea_got,
	5669	.lea_tlv,
	5670	.lea_frame,
	5671	.reserved_frame,
	5672	=> unreachable, // unmodifiable destination
5295	5673	.register => \|dst_reg\| {
5296	5674	const dst_alias = registerAlias(dst_reg, abi_size);
5297	5675	const dst_lock = self.register_manager.lockReg(dst_reg);
5298	5676	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5299	5677
5300	5678	switch (src_mcv) {
5301		.none => unreachable,
5302		.undef => try self.genSetReg(dst_ty, dst_reg, .undef),
5303		.dead, .unreach => unreachable,
5304		.ptr_stack_offset => unreachable,
5305		.lea_direct => unreachable,
5306		.lea_tlv => unreachable,
5307		.register_overflow => unreachable,
	5679	.none,
	5680	.unreach,
	5681	.dead,
	5682	.undef,
	5683	.register_overflow,
	5684	.reserved_frame,
	5685	=> unreachable,
5308	5686	.register => \|src_reg\| try self.asmRegisterRegister(
5309	5687	.imul,
5310	5688	dst_alias,
...	...	@@ -5323,74 +5701,54 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
5323	5701	return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
5324	5702	}
5325	5703	},
5326		.stack_offset => \|off\| {
5327		try self.asmRegisterMemory(
5328		.imul,
5329		dst_alias,
5330		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
5331		);
5332		},
5333		.memory,
5334		.load_got,
	5704	.register_offset,
	5705	.eflags,
5335	5706	.load_direct,
5336		.load_tlv,
5337		=> {
5338		return self.fail("TODO implement x86 multiply source memory", .{});
5339		},
5340		.eflags => {
5341		return self.fail("TODO implement x86 multiply source eflags", .{});
5342		},
5343		}
5344		},
5345		.stack_offset => \|off\| {
5346		switch (src_mcv) {
5347		.none => unreachable,
5348		.undef => return self.genSetStack(dst_ty, off, .undef, .{}),
5349		.dead, .unreach => unreachable,
5350		.ptr_stack_offset => unreachable,
5351		.lea_direct => unreachable,
5352		.lea_tlv => unreachable,
5353		.register_overflow => unreachable,
5354		.register => \|src_reg\| {
5355		// copy dst to a register
5356		const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5357		try self.asmRegisterRegister(
5358		.imul,
5359		registerAlias(dst_reg, abi_size),
5360		registerAlias(src_reg, abi_size),
5361		);
5362		// copy dst_reg back out
5363		return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
5364		},
5365		.immediate => {
5366		// copy dst to a register
5367		const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5368		const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
5369		defer self.register_manager.unlockReg(dst_reg_lock);
5370
5371		try self.genIntMulComplexOpMir(dst_ty, .{ .register = dst_reg }, src_mcv);
5372
5373		return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
5374		},
5375		.memory,
	5707	.lea_direct,
5376	5708	.load_got,
5377		.load_direct,
	5709	.lea_got,
5378	5710	.load_tlv,
5379		.stack_offset,
5380		=> {
5381		return self.fail("TODO implement x86 multiply source memory", .{});
5382		},
5383		.eflags => {
5384		return self.fail("TODO implement x86 multiply source eflags", .{});
5385		},
	5711	.lea_tlv,
	5712	.lea_frame,
	5713	=> try self.asmRegisterRegister(
	5714	.imul,
	5715	dst_alias,
	5716	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
	5717	),
	5718	.memory, .indirect, .load_frame => try self.asmRegisterMemory(
	5719	.imul,
	5720	dst_alias,
	5721	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
	5722	.memory => \|addr\| .{
	5723	.base = .{ .reg = .ds },
	5724	.disp = math.cast(i32, @bitCast(i64, addr)) orelse
	5725	return self.asmRegisterRegister(
	5726	.imul,
	5727	dst_alias,
	5728	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
	5729	),
	5730	},
	5731	.indirect => \|reg_off\| .{
	5732	.base = .{ .reg = reg_off.reg },
	5733	.disp = reg_off.off,
	5734	},
	5735	.load_frame => \|frame_addr\| .{
	5736	.base = .{ .frame = frame_addr.index },
	5737	.disp = frame_addr.off,
	5738	},
	5739	else => unreachable,
	5740	}),
	5741	),
5386	5742	}
5387	5743	},
5388		.memory,
5389		.load_got,
5390		.load_direct,
5391		.load_tlv,
5392		=> {
5393		return self.fail("TODO implement x86 multiply destination memory", .{});
	5744	.memory, .indirect, .load_direct, .load_got, .load_tlv, .load_frame => {
	5745	const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
	5746	const tmp_mcv = MCValue{ .register = tmp_reg };
	5747	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	5748	defer self.register_manager.unlockReg(tmp_lock);
	5749
	5750	try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv);
	5751	try self.genCopy(dst_ty, dst_mcv, tmp_mcv);
5394	5752	},
5395	5753	}
5396	5754	}
...	...	@@ -5401,24 +5759,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
5401	5759	while (self.args[arg_index] == .none) arg_index += 1;
5402	5760	self.arg_index = arg_index + 1;
5403	5761
5404		const ty = self.air.typeOfIndex(inst);
5405		const mcv = self.args[arg_index];
5406		const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
5407		const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5408
5409	5762	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
5410		const dst_mcv: MCValue = switch (mcv) {
5411		.register => \|reg\| blk: {
5412		self.register_manager.getRegAssumeFree(reg.to64(), inst);
5413		break :blk MCValue{ .register = reg };
5414		},
5415		.stack_offset => \|off\| blk: {
5416		const offset = @intCast(i32, self.max_end_stack) - off + 16;
5417		break :blk MCValue{ .stack_offset = -offset };
5418		},
5419		else => return self.fail("TODO implement arg for {}", .{mcv}),
5420		};
	5763	const dst_mcv = self.args[arg_index];
	5764	switch (dst_mcv) {
	5765	.register => \|reg\| self.register_manager.getRegAssumeFree(reg, inst),
	5766	.load_frame => {},
	5767	else => return self.fail("TODO implement arg for {}", .{dst_mcv}),
	5768	}
	5769
	5770	const ty = self.air.typeOfIndex(inst);
	5771	const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
	5772	const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5421	5773	try self.genArgDbgInfo(ty, name, dst_mcv);
	5774
5422	5775	break :result dst_mcv;
5423	5776	};
5424	5777	return self.finishAir(inst, result, .{ .none, .none, .none });
...	...	@@ -5429,13 +5782,15 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
5429	5782	.dwarf => \|dw\| {
5430	5783	const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5431	5784	.register => \|reg\| .{ .register = reg.dwarfLocOp() },
5432		.stack_offset => \|off\| .{
5433		.stack = .{
5434		// TODO handle -fomit-frame-pointer
5435		.fp_register = Register.rbp.dwarfLocOpDeref(),
5436		.offset = -off,
5437		},
5438		},
	5785	// TODO use a frame index
	5786	.load_frame => return,
	5787	//.stack_offset => \|off\| .{
	5788	// .stack = .{
	5789	// // TODO handle -fomit-frame-pointer
	5790	// .fp_register = Register.rbp.dwarfLocOpDeref(),
	5791	// .offset = -off,
	5792	// },
	5793	//},
5439	5794	else => unreachable, // not a valid function parameter
5440	5795	};
5441	5796	try dw.genArgDbgInfo(name, ty, self.mod_fn.owner_decl, loc);
...	...	@@ -5462,12 +5817,12 @@ fn genVarDbgInfo(
5462	5817	.dwarf => \|dw\| {
5463	5818	const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5464	5819	.register => \|reg\| .{ .register = reg.dwarfLocOp() },
5465		.ptr_stack_offset,
5466		.stack_offset,
5467		=> \|off\| .{ .stack = .{
5468		.fp_register = Register.rbp.dwarfLocOpDeref(),
5469		.offset = -off,
5470		} },
	5820	// TODO use a frame index
	5821	.load_frame, .lea_frame => return,
	5822	//=> \|off\| .{ .stack = .{
	5823	// .fp_register = Register.rbp.dwarfLocOpDeref(),
	5824	// .offset = -off,
	5825	//} },
5471	5826	.memory => \|address\| .{ .memory = address },
5472	5827	.load_got => \|sym_index\| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },
5473	5828	.load_direct => \|sym_index\| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },
...	...	@@ -5498,15 +5853,13 @@ fn airBreakpoint(self: *Self) !void {
5498	5853
5499	5854	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
5500	5855	const dst_mcv = try self.allocRegOrMem(inst, true);
5501		try self.setRegOrMem(Type.usize, dst_mcv, .{
5502		.stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)),
5503		});
	5856	try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
5504	5857	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5505	5858	}
5506	5859
5507	5860	fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
5508	5861	const dst_mcv = try self.allocRegOrMem(inst, true);
5509		try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp });
	5862	try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
5510	5863	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5511	5864	}
5512	5865
...	...	@@ -5534,94 +5887,69 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5534	5887	else => unreachable,
5535	5888	};
5536	5889
5537		var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..]);
	5890	var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..], .call_frame);
5538	5891	defer info.deinit(self);
5539	5892
	5893	// We need a properly aligned and sized call frame to be able to call this function.
	5894	{
	5895	const needed_call_frame =
	5896	FrameAlloc.init(.{ .size = info.stack_byte_count, .alignment = info.stack_align });
	5897	const frame_allocs_slice = self.frame_allocs.slice();
	5898	const stack_frame_size =
	5899	&frame_allocs_slice.items(.abi_size)[@enumToInt(FrameIndex.call_frame)];
	5900	stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
	5901	const stack_frame_align =
	5902	&frame_allocs_slice.items(.abi_align)[@enumToInt(FrameIndex.call_frame)];
	5903	stack_frame_align.* = @max(stack_frame_align.*, needed_call_frame.abi_align);
	5904	}
	5905
5540	5906	try self.spillEflagsIfOccupied();
5541	5907	try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
5542	5908
5543	5909	// set stack arguments first because this can clobber registers
5544	5910	// also clobber spill arguments as we go
5545		if (info.return_value == .stack_offset) {
5546		try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]});
	5911	switch (info.return_value.long) {
	5912	.none, .unreach => {},
	5913	.indirect => \|reg_off\| try self.spillRegisters(&.{reg_off.reg}),
	5914	else => unreachable,
5547	5915	}
5548	5916	for (args, info.args) \|arg, mc_arg\| {
5549	5917	const arg_ty = self.air.typeOf(arg);
5550	5918	const arg_mcv = try self.resolveInst(arg);
5551		// Here we do not use setRegOrMem even though the logic is similar, because
5552		// the function call will move the stack pointer, so the offsets are different.
5553	5919	switch (mc_arg) {
5554	5920	.none => {},
5555	5921	.register => \|reg\| try self.spillRegisters(&.{reg}),
5556		.stack_offset => \|off\| {
5557		// TODO rewrite using `genSetStack`
5558		try self.genSetStackArg(arg_ty, off, arg_mcv);
5559		},
5560		.ptr_stack_offset => {
5561		return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
5562		},
5563		.undef => unreachable,
5564		.immediate => unreachable,
5565		.unreach => unreachable,
5566		.dead => unreachable,
5567		.memory => unreachable,
5568		.eflags => unreachable,
5569		.register_overflow => unreachable,
5570		.load_got => unreachable,
5571		.lea_direct => unreachable,
5572		.load_direct => unreachable,
5573		.lea_tlv => unreachable,
5574		.load_tlv => unreachable,
	5922	.load_frame => try self.genCopy(arg_ty, mc_arg, arg_mcv),
	5923	else => unreachable,
5575	5924	}
5576	5925	}
5577	5926
5578	5927	// now we are free to set register arguments
5579		const ret_reg_lock: ?RegisterLock = blk: {
5580		if (info.return_value == .stack_offset) {
	5928	const ret_lock = switch (info.return_value.long) {
	5929	.none, .unreach => null,
	5930	.indirect => \|reg_off\| lock: {
5581	5931	const ret_ty = fn_ty.fnReturnType();
5582		const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
5583		const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
5584		const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));
5585		log.debug("airCall: return value on stack at offset {}", .{stack_offset});
5586
5587		const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
5588		try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset });
5589		const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg);
5590
5591		info.return_value.stack_offset = stack_offset;
5592
5593		break :blk ret_reg_lock;
5594		}
5595		break :blk null;
	5932	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, self.target.*));
	5933	try self.genSetReg(reg_off.reg, Type.usize, .{
	5934	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
	5935	});
	5936	info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
	5937	break :lock self.register_manager.lockRegAssumeUnused(reg_off.reg);
	5938	},
	5939	else => unreachable,
5596	5940	};
5597		defer if (ret_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	5941	defer if (ret_lock) \|lock\| self.register_manager.unlockReg(lock);
5598	5942
5599	5943	for (args, info.args) \|arg, mc_arg\| {
5600	5944	const arg_ty = self.air.typeOf(arg);
5601	5945	const arg_mcv = try self.resolveInst(arg);
5602	5946	switch (mc_arg) {
5603		.none, .stack_offset, .ptr_stack_offset => {},
5604		.register => \|reg\| try self.genSetReg(arg_ty, reg, arg_mcv),
5605		.undef => unreachable,
5606		.immediate => unreachable,
5607		.unreach => unreachable,
5608		.dead => unreachable,
5609		.memory => unreachable,
5610		.eflags => unreachable,
5611		.register_overflow => unreachable,
5612		.load_got => unreachable,
5613		.lea_direct => unreachable,
5614		.load_direct => unreachable,
5615		.lea_tlv => unreachable,
5616		.load_tlv => unreachable,
	5947	.none, .load_frame => {},
	5948	.register => try self.genCopy(arg_ty, mc_arg, arg_mcv),
	5949	else => unreachable,
5617	5950	}
5618	5951	}
5619	5952
5620		if (info.stack_byte_count > 0) {
5621		// Adjust the stack
5622		try self.asmRegisterImmediate(.sub, .rsp, Immediate.u(info.stack_byte_count));
5623		}
5624
5625	5953	// Due to incremental compilation, how function calls are generated depends
5626	5954	// on linking.
5627	5955	const mod = self.bin_file.options.module.?;
...	...	@@ -5635,16 +5963,16 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5635	5963	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
5636	5964	const got_addr = atom.getOffsetTableAddress(elf_file);
5637	5965	try self.asmMemory(.call, Memory.sib(.qword, .{
5638		.base = .ds,
	5966	.base = .{ .reg = .ds },
5639	5967	.disp = @intCast(i32, got_addr),
5640	5968	}));
5641	5969	} else if (self.bin_file.cast(link.File.Coff)) \|_\| {
5642	5970	const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5643		try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });
	5971	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5644	5972	try self.asmRegister(.call, .rax);
5645	5973	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
5646	5974	const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5647		try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });
	5975	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5648	5976	try self.asmRegister(.call, .rax);
5649	5977	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
5650	5978	const decl_block_index = try p9.seeDecl(func.owner_decl);
...	...	@@ -5655,7 +5983,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5655	5983	const got_index = decl_block.got_index.?;
5656	5984	const fn_got_addr = got_addr + got_index * ptr_bytes;
5657	5985	try self.asmMemory(.call, Memory.sib(.qword, .{
5658		.base = .ds,
	5986	.base = .{ .reg = .ds },
5659	5987	.disp = @intCast(i32, fn_got_addr),
5660	5988	}));
5661	5989	} else unreachable;
...	...	@@ -5696,41 +6024,15 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5696	6024	} else {
5697	6025	assert(ty.zigTypeTag() == .Pointer);
5698	6026	const mcv = try self.resolveInst(callee);
5699		try self.genSetReg(Type.usize, .rax, mcv);
	6027	try self.genSetReg(.rax, Type.usize, mcv);
5700	6028	try self.asmRegister(.call, .rax);
5701	6029	}
5702	6030
5703		if (info.stack_byte_count > 0) {
5704		// Readjust the stack
5705		try self.asmRegisterImmediate(.add, .rsp, Immediate.u(info.stack_byte_count));
5706		}
5707
5708		const result: MCValue = result: {
5709		if (self.liveness.isUnused(inst)) break :result .unreach;
5710
5711		switch (info.return_value) {
5712		.register => {
5713		// Save function return value in a new register
5714		break :result try self.copyToRegisterWithInstTracking(
5715		inst,
5716		self.air.typeOfIndex(inst),
5717		info.return_value,
5718		);
5719		},
5720		else => {},
5721		}
5722		break :result info.return_value;
5723		};
5724
5725		if (args.len <= Liveness.bpi - 2) {
5726		var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
5727		buf[0] = callee;
5728		std.mem.copy(Air.Inst.Ref, buf[1..], args);
5729		return self.finishAir(inst, result, buf);
5730		}
5731	6031	var bt = self.liveness.iterateBigTomb(inst);
5732	6032	self.feed(&bt, callee);
5733	6033	for (args) \|arg\| self.feed(&bt, arg);
	6034
	6035	const result = if (self.liveness.isUnused(inst)) .unreach else info.return_value.short;
5734	6036	return self.finishAirResult(inst, result);
5735	6037	}
5736	6038
...	...	@@ -5738,18 +6040,19 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
5738	6040	const un_op = self.air.instructions.items(.data)[inst].un_op;
5739	6041	const operand = try self.resolveInst(un_op);
5740	6042	const ret_ty = self.fn_type.fnReturnType();
5741		switch (self.ret_mcv) {
5742		.immediate => {
5743		assert(ret_ty.isError());
5744		},
5745		.stack_offset => {
5746		try self.store(self.ret_mcv, operand, Type.usize, ret_ty);
5747		},
5748		else => {
5749		try self.setRegOrMem(ret_ty, self.ret_mcv, operand);
	6043	switch (self.ret_mcv.short) {
	6044	.none => {},
	6045	.register => try self.genCopy(ret_ty, self.ret_mcv.short, operand),
	6046	.indirect => \|reg_off\| {
	6047	try self.register_manager.getReg(reg_off.reg, null);
	6048	const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
	6049	defer self.register_manager.unlockReg(lock);
	6050
	6051	try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
	6052	try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);
5750	6053	},
	6054	else => unreachable,
5751	6055	}
5752		// TODO when implementing defer, this will need to jump to the appropriate defer expression.
5753	6056	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5754	6057	// which is available if the jump is 127 bytes or less forward.
5755	6058	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -5761,25 +6064,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
5761	6064	const un_op = self.air.instructions.items(.data)[inst].un_op;
5762	6065	const ptr = try self.resolveInst(un_op);
5763	6066	const ptr_ty = self.air.typeOf(un_op);
5764		const elem_ty = ptr_ty.elemType();
5765		const abi_size = elem_ty.abiSize(self.target.*);
5766		switch (self.ret_mcv) {
5767		.immediate => {
5768		assert(elem_ty.isError());
5769		},
5770		.stack_offset => {
5771		const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
5772		const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5773		defer self.register_manager.unlockReg(reg_lock);
5774
5775		try self.genInlineMemcpy(.{ .register = reg }, ptr, .{ .immediate = abi_size }, .{});
5776		},
5777		else => {
5778		try self.load(self.ret_mcv, ptr, ptr_ty);
5779		try self.setRegOrMem(elem_ty, self.ret_mcv, self.ret_mcv);
5780		},
	6067	switch (self.ret_mcv.short) {
	6068	.none => {},
	6069	.register => try self.load(self.ret_mcv.short, ptr_ty, ptr),
	6070	.indirect => \|reg_off\| try self.genSetReg(reg_off.reg, ptr_ty, ptr),
	6071	else => unreachable,
5781	6072	}
5782		// TODO when implementing defer, this will need to jump to the appropriate defer expression.
5783	6073	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5784	6074	// which is available if the jump is 127 bytes or less forward.
5785	6075	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -5822,7 +6112,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
5822	6112	break :dst rhs_mcv;
5823	6113	} else if (dst_mem_ok) dst: {
5824	6114	const dst_mcv = try self.allocTempRegOrMem(ty, true);
5825		try self.setRegOrMem(ty, dst_mcv, lhs_mcv);
	6115	try self.genCopy(ty, dst_mcv, lhs_mcv);
5826	6116	break :dst dst_mcv;
5827	6117	} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
5828	6118	const dst_lock = switch (dst_mcv) {
...	...	@@ -5865,26 +6155,27 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5865	6155	const atom = elf_file.getAtom(atom_index);
5866	6156	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
5867	6157	const got_addr = atom.getOffsetTableAddress(elf_file);
5868		try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
5869		.base = .ds,
5870		.disp = @intCast(i32, got_addr),
5871		}));
	6158	try self.asmRegisterMemory(
	6159	.mov,
	6160	addr_reg.to64(),
	6161	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
	6162	);
5872	6163	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
5873	6164	const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
5874	6165	.{ .kind = .const_data, .ty = Type.anyerror },
5875	6166	4, // dword alignment
5876	6167	);
5877	6168	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
5878		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	6169	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5879	6170	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
5880	6171	const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
5881	6172	.{ .kind = .const_data, .ty = Type.anyerror },
5882	6173	4, // dword alignment
5883	6174	);
5884	6175	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
5885		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	6176	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5886	6177	} else {
5887		return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
	6178	return self.fail("TODO implement airCmpLtErrorsLen for x86_64 {s}", .{@tagName(self.bin_file.tag)});
5888	6179	}
5889	6180
5890	6181	try self.spillEflagsIfOccupied();
...	...	@@ -5900,7 +6191,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5900	6191	try self.asmRegisterMemory(
5901	6192	.cmp,
5902	6193	registerAlias(dst_reg, op_abi_size),
5903		Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = addr_reg }),
	6194	Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
5904	6195	);
5905	6196	const result = MCValue{ .eflags = .b };
5906	6197	return self.finishAir(inst, result, .{ un_op, .none, .none });
...	...	@@ -5971,14 +6262,13 @@ fn genTry(
5971	6262
5972	6263	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
5973	6264	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
5974		const payload = try self.addExtra(Mir.DbgLineColumn{
5975		.line = dbg_stmt.line,
5976		.column = dbg_stmt.column,
5977		});
5978	6265	_ = try self.addInst(.{
5979	6266	.tag = .dbg_line,
5980	6267	.ops = undefined,
5981		.data = .{ .payload = payload },
	6268	.data = .{ .line_column = .{
	6269	.line = dbg_stmt.line,
	6270	.column = dbg_stmt.column,
	6271	} },
5982	6272	});
5983	6273	return self.finishAirBookkeeping();
5984	6274	}
...	...	@@ -6002,8 +6292,6 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
6002	6292	const ty = self.air.typeOf(operand);
6003	6293	const mcv = try self.resolveInst(operand);
6004	6294
6005		log.debug("airDbgVar: %{d}: {}, {}", .{ inst, ty.fmtDebug(), mcv });
6006
6007	6295	const name = self.air.nullTerminatedString(pl_op.payload);
6008	6296
6009	6297	const tag = self.air.instructions.items(.tag)[inst];
...	...	@@ -6025,7 +6313,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
6025	6313	return self.asmJccReloc(undefined, .e);
6026	6314	},
6027	6315	.immediate,
6028		.stack_offset,
	6316	.load_frame,
6029	6317	=> {
6030	6318	try self.spillEflagsIfOccupied();
6031	6319	if (abi_size <= 8) {
...	...	@@ -6118,11 +6406,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6118	6406	.dead,
6119	6407	.undef,
6120	6408	.immediate,
6121		.register_overflow,
6122		.ptr_stack_offset,
6123	6409	.eflags,
	6410	.register_offset,
	6411	.register_overflow,
6124	6412	.lea_direct,
	6413	.lea_got,
6125	6414	.lea_tlv,
	6415	.lea_frame,
	6416	.reserved_frame,
6126	6417	=> unreachable,
6127	6418
6128	6419	.register => \|opt_reg\| {
...	...	@@ -6152,27 +6443,36 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6152	6443	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6153	6444	defer self.register_manager.unlockReg(addr_reg_lock);
6154	6445
6155		try self.genSetReg(Type.usize, addr_reg, switch (opt_mcv) {
6156		.memory => \|addr\| .{ .immediate = addr },
6157		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
6158		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
6159		else => opt_mcv,
6160		});
6161
	6446	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
6162	6447	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6163		try self.asmMemoryImmediate(.cmp, Memory.sib(
6164		Memory.PtrSize.fromSize(some_abi_size),
6165		.{ .base = addr_reg, .disp = some_info.off },
6166		), Immediate.u(0));
	6448	try self.asmMemoryImmediate(
	6449	.cmp,
	6450	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	6451	.base = .{ .reg = addr_reg },
	6452	.disp = some_info.off,
	6453	}),
	6454	Immediate.u(0),
	6455	);
6167	6456	return .{ .eflags = .e };
6168	6457	},
6169	6458
6170		.stack_offset => \|off\| {
	6459	.indirect, .load_frame => {
6171	6460	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6172		try self.asmMemoryImmediate(.cmp, Memory.sib(
6173		Memory.PtrSize.fromSize(some_abi_size),
6174		.{ .base = .rbp, .disp = some_info.off - off },
6175		), Immediate.u(0));
	6461	try self.asmMemoryImmediate(
	6462	.cmp,
	6463	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
	6464	.indirect => \|reg_off\| .{
	6465	.base = .{ .reg = reg_off.reg },
	6466	.disp = reg_off.off + some_info.off,
	6467	},
	6468	.load_frame => \|frame_addr\| .{
	6469	.base = .{ .frame = frame_addr.index },
	6470	.disp = frame_addr.off + some_info.off,
	6471	},
	6472	else => unreachable,
	6473	}),
	6474	Immediate.u(0),
	6475	);
6176	6476	return .{ .eflags = .e };
6177	6477	},
6178	6478	}
...	...	@@ -6200,10 +6500,14 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
6200	6500	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
6201	6501
6202	6502	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6203		try self.asmMemoryImmediate(.cmp, Memory.sib(
6204		Memory.PtrSize.fromSize(some_abi_size),
6205		.{ .base = ptr_reg, .disp = some_info.off },
6206		), Immediate.u(0));
	6503	try self.asmMemoryImmediate(
	6504	.cmp,
	6505	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	6506	.base = .{ .reg = ptr_reg },
	6507	.disp = some_info.off,
	6508	}),
	6509	Immediate.u(0),
	6510	);
6207	6511	return .{ .eflags = .e };
6208	6512	}
6209	6513
...	...	@@ -6221,10 +6525,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
6221	6525
6222	6526	const err_off = errUnionErrorOffset(ty.errorUnionPayload(), self.target.*);
6223	6527	switch (operand) {
6224		.stack_offset => \|off\| {
6225		const offset = off - @intCast(i32, err_off);
6226		try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
6227		},
6228	6528	.register => \|reg\| {
6229	6529	const eu_lock = self.register_manager.lockReg(reg);
6230	6530	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -6238,6 +6538,15 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
6238	6538	}
6239	6539	try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
6240	6540	},
	6541	.load_frame => \|frame_addr\| try self.genBinOpMir(
	6542	.cmp,
	6543	Type.anyerror,
	6544	.{ .load_frame = .{
	6545	.index = frame_addr.index,
	6546	.off = frame_addr.off + @intCast(i32, err_off),
	6547	} },
	6548	.{ .immediate = 0 },
	6549	),
6241	6550	else => return self.fail("TODO implement isErr for {}", .{operand}),
6242	6551	}
6243	6552
...	...	@@ -6324,9 +6633,9 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6324	6633	}
6325	6634	};
6326	6635	const ptr_ty = self.air.typeOf(un_op);
6327		try self.load(operand, operand_ptr, ptr_ty);
	6636	try self.load(operand, ptr_ty, operand_ptr);
6328	6637
6329		const result = try self.isErr(inst, ptr_ty.elemType(), operand);
	6638	const result = try self.isErr(inst, ptr_ty.childType(), operand);
6330	6639
6331	6640	return self.finishAir(inst, result, .{ un_op, .none, .none });
6332	6641	}
...	...	@@ -6358,9 +6667,9 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6358	6667	}
6359	6668	};
6360	6669	const ptr_ty = self.air.typeOf(un_op);
6361		try self.load(operand, operand_ptr, ptr_ty);
	6670	try self.load(operand, ptr_ty, operand_ptr);
6362	6671
6363		const result = try self.isNonErr(inst, ptr_ty.elemType(), operand);
	6672	const result = try self.isNonErr(inst, ptr_ty.childType(), operand);
6364	6673
6365	6674	return self.finishAir(inst, result, .{ un_op, .none, .none });
6366	6675	}
...	...	@@ -6389,6 +6698,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
6389	6698
6390	6699	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6391	6700	// A block is a setup to be able to jump to the end.
	6701	const inst_tracking_i = self.inst_tracking.count();
6392	6702	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
6393	6703
6394	6704	self.scope_generation += 1;
...	...	@@ -6412,8 +6722,9 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6412	6722	for (block_data.value.relocs.items) \|reloc\| try self.performReloc(reloc);
6413	6723	}
6414	6724
6415		const tracking = self.inst_tracking.getPtr(inst).?;
6416		if (self.liveness.isUnused(inst)) tracking.die(self);
	6725	if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i);
	6726	const tracking = &self.inst_tracking.values()[inst_tracking_i];
	6727	if (self.liveness.isUnused(inst)) tracking.die(self, inst);
6417	6728	self.getValue(tracking.short, inst);
6418	6729	self.finishAirBookkeeping();
6419	6730	}
...	...	@@ -6520,35 +6831,38 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6520	6831	!block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);
6521	6832	const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
6522	6833	const block_data = self.blocks.getPtr(br.block_inst).?;
	6834	const first_br = block_data.relocs.items.len == 0;
	6835	const block_result = result: {
	6836	if (block_unused) break :result .none;
	6837
	6838	if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
	6839	if (first_br) break :result src_mcv;
	6840
	6841	if (block_tracking.getReg()) \|block_reg\|
	6842	try self.register_manager.getReg(block_reg, br.block_inst);
	6843	// .long = .none to avoid merging operand and block result stack frames.
	6844	var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
	6845	try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
	6846	if (src_mcv.getReg()) \|src_reg\| self.register_manager.freeReg(src_reg);
	6847	break :result block_tracking.short;
	6848	}
6523	6849
6524		if (block_data.relocs.items.len == 0) {
6525		block_tracking.* = InstTracking.init(result: {
6526		if (block_unused) break :result .none;
6527		if (self.reuseOperand(inst, br.operand, 0, src_mcv)) {
6528		// Fix instruction tracking
6529		switch (src_mcv) {
6530		.register => \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|index\| {
6531		self.register_manager.registers[index] = br.block_inst;
6532		},
6533		else => {},
6534		}
6535		break :result src_mcv;
6536		}
6537
6538		const new_mcv = try self.allocRegOrMem(br.block_inst, true);
6539		try self.setRegOrMem(block_ty, new_mcv, src_mcv);
6540		break :result new_mcv;
6541		});
6542		} else if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);
	6850	const dst_mcv = if (first_br) try self.allocRegOrMem(br.block_inst, true) else dst: {
	6851	self.getValue(block_tracking.short, br.block_inst);
	6852	break :dst block_tracking.short;
	6853	};
	6854	try self.genCopy(block_ty, dst_mcv, src_mcv);
	6855	break :result dst_mcv;
	6856	};
6543	6857
6544	6858	// Process operand death so that it is properly accounted for in the State below.
6545	6859	if (self.liveness.operandDies(inst, 0)) {
6546	6860	if (Air.refToIndex(br.operand)) \|op_inst\| self.processDeath(op_inst);
6547	6861	}
6548	6862
6549		if (block_data.relocs.items.len == 0) {
	6863	if (first_br) {
	6864	block_tracking.* = InstTracking.init(block_result);
6550	6865	try self.saveRetroactiveState(&block_data.state);
6551		block_tracking.die(self);
6552	6866	} else try self.restoreState(block_data.state, &.{}, .{
6553	6867	.emit_instructions = true,
6554	6868	.update_tracking = false,
...	...	@@ -6556,6 +6870,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6556	6870	.close_scope = false,
6557	6871	});
6558	6872
	6873	// Stop tracking block result without forgetting tracking info
	6874	self.freeValue(block_tracking.short);
	6875
6559	6876	// Emit a jump with a relocation. It will be patched up after the block ends.
6560	6877	// Leave the jump offset undefined
6561	6878	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -6634,7 +6951,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6634	6951
6635	6952	const arg_mcv = try self.resolveInst(input);
6636	6953	try self.register_manager.getReg(reg, null);
6637		try self.genSetReg(self.air.typeOf(input), reg, arg_mcv);
	6954	try self.genSetReg(reg, self.air.typeOf(input), arg_mcv);
6638	6955	}
6639	6956
6640	6957	{
...	...	@@ -6666,11 +6983,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6666	6983	.qword
6667	6984	else
6668	6985	null;
6669		const mnem = std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
6670		(if (mnem_size) \|_\|
6671		std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])
6672		else
6673		null) orelse return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
	6986	const mnem = mnem: {
	6987	if (mnem_size) \|_\| {
	6988	if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) \|mnem\| {
	6989	break :mnem mnem;
	6990	}
	6991	}
	6992	break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
	6993	return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
	6994	};
6674	6995
6675	6996	var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
6676	6997	var ops = [1]encoder.Instruction.Operand{.none} ** 4;
...	...	@@ -6686,7 +7007,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6686	7007	return self.fail("Invalid displacement: '{s}'", .{op_str});
6687	7008	op.* = .{ .mem = Memory.sib(
6688	7009	mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),
6689		.{ .base = reg, .disp = disp },
	7010	.{ .base = .{ .reg = reg }, .disp = disp },
6690	7011	) };
6691	7012	} else {
6692	7013	if (mnem_size) \|size\| if (reg.bitSize() != size.bitSize())
...	...	@@ -6805,335 +7126,326 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6805	7126	return self.finishAirResult(inst, result);
6806	7127	}
6807	7128
6808		/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
6809		fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
6810		switch (loc) {
6811		.none => return,
6812		.immediate => unreachable,
6813		.register => \|reg\| return self.genSetReg(ty, reg, val),
6814		.stack_offset => \|off\| return self.genSetStack(ty, off, val, .{}),
6815		.memory => {
6816		return self.fail("TODO implement setRegOrMem for memory", .{});
	7129	fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {
	7130	return switch (ty.zigTypeTag()) {
	7131	else => .mov,
	7132	.Float => switch (ty.floatBits(self.target.*)) {
	7133	16 => .mov,
	7134	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) .movss else .mov,
	7135	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) .movsd else .mov,
	7136	else => return self.fail("TODO movMirTag for {}", .{
	7137	ty.fmt(self.bin_file.options.module.?),
	7138	}),
6817	7139	},
6818		else => {
6819		return self.fail("TODO implement setRegOrMem for {}", .{loc});
	7140	};
	7141	}
	7142
	7143	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
	7144	const src_lock = switch (src_mcv) {
	7145	.register => \|reg\| self.register_manager.lockReg(reg),
	7146	.register_overflow => \|ro\| self.register_manager.lockReg(ro.reg),
	7147	else => null,
	7148	};
	7149	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
	7150
	7151	switch (dst_mcv) {
	7152	.none,
	7153	.unreach,
	7154	.dead,
	7155	.undef,
	7156	.immediate,
	7157	.eflags,
	7158	.register_overflow,
	7159	.lea_direct,
	7160	.lea_got,
	7161	.lea_tlv,
	7162	.lea_frame,
	7163	.reserved_frame,
	7164	=> unreachable, // unmodifiable destination
	7165	.register => \|reg\| try self.genSetReg(reg, ty, src_mcv),
	7166	.register_offset => \|dst_reg_off\| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) {
	7167	.none,
	7168	.unreach,
	7169	.dead,
	7170	.undef,
	7171	.register_overflow,
	7172	.reserved_frame,
	7173	=> unreachable,
	7174	.immediate,
	7175	.register,
	7176	.register_offset,
	7177	.lea_frame,
	7178	=> src_mcv.offset(-dst_reg_off.off),
	7179	else => .{ .register_offset = .{
	7180	.reg = try self.copyToTmpRegister(ty, src_mcv),
	7181	.off = -dst_reg_off.off,
	7182	} },
	7183	}),
	7184	.indirect => \|reg_off\| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
	7185	.memory, .load_direct, .load_got, .load_tlv => {
	7186	switch (dst_mcv) {
	7187	.memory => \|addr\| if (math.cast(i32, @bitCast(i64, addr))) \|small_addr\|
	7188	return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv),
	7189	.load_direct, .load_got, .load_tlv => {},
	7190	else => unreachable,
	7191	}
	7192
	7193	const addr_reg = try self.copyToTmpRegister(Type.usize, dst_mcv.address());
	7194	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	7195	defer self.register_manager.unlockReg(addr_lock);
	7196
	7197	try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv);
6820	7198	},
	7199	.load_frame => \|frame_addr\| try self.genSetMem(
	7200	.{ .frame = frame_addr.index },
	7201	frame_addr.off,
	7202	ty,
	7203	src_mcv,
	7204	),
6821	7205	}
6822	7206	}
6823	7207
6824		fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {
	7208	fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
6825	7209	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6826		switch (mcv) {
6827		.dead => unreachable,
6828		.unreach, .none => return,
6829		.undef => {
6830		if (!self.wantSafety()) return; // The already existing value will do just fine.
6831		if (abi_size <= 8) {
6832		const reg = try self.copyToTmpRegister(ty, mcv);
6833		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
	7210	if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
	7211	switch (src_mcv) {
	7212	.none,
	7213	.unreach,
	7214	.dead,
	7215	.register_overflow,
	7216	.reserved_frame,
	7217	=> unreachable,
	7218	.undef => if (self.wantSafety())
	7219	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
	7220	.eflags => \|cc\| try self.asmSetccRegister(dst_reg.to8(), cc),
	7221	.immediate => \|imm\| {
	7222	if (imm == 0) {
	7223	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
	7224	// register is the fastest way to zero a register.
	7225	try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());
	7226	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
	7227	// 32-bit moves zero-extend to 64-bit.
	7228	try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));
	7229	} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
	7230	try self.asmRegisterImmediate(
	7231	.mov,
	7232	registerAlias(dst_reg, abi_size),
	7233	Immediate.s(@intCast(i32, @bitCast(i64, imm))),
	7234	);
	7235	} else {
	7236	try self.asmRegisterImmediate(
	7237	.mov,
	7238	registerAlias(dst_reg, abi_size),
	7239	Immediate.u(imm),
	7240	);
6834	7241	}
6835		try self.genInlineMemset(
6836		.{ .ptr_stack_offset = stack_offset },
6837		.{ .immediate = 0xaa },
6838		.{ .immediate = abi_size },
6839		.{ .dest_stack_base = .rsp },
6840		);
6841		},
6842		.register_overflow => return self.fail("TODO genSetStackArg for register with overflow bit", .{}),
6843		.eflags => {
6844		const reg = try self.copyToTmpRegister(ty, mcv);
6845		return self.genSetStackArg(ty, stack_offset, .{ .register = reg });
6846	7242	},
6847		.immediate => \|imm\| {
6848		switch (abi_size) {
6849		1, 2, 4 => {
6850		// TODO
6851		// We have a positive stack offset value but we want a twos complement negative
6852		// offset from rbp, which is at the top of the stack frame.
6853		const immediate = if (ty.isSignedInt())
6854		Immediate.s(@intCast(i32, @bitCast(i64, imm)))
6855		else
6856		Immediate.u(@intCast(u32, imm));
6857		try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6858		.base = .rsp,
6859		.disp = -stack_offset,
6860		}), immediate);
	7243	.register => \|src_reg\| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
	7244	if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
	7245	try self.movMirTag(ty)
	7246	else switch (abi_size) {
	7247	4 => .movd,
	7248	8 => .movq,
	7249	else => return self.fail(
	7250	"unsupported register copy from {s} to {s}",
	7251	.{ @tagName(src_reg), @tagName(dst_reg) },
	7252	),
	7253	},
	7254	registerAlias(dst_reg, abi_size),
	7255	registerAlias(src_reg, abi_size),
	7256	),
	7257	.register_offset, .indirect, .load_frame, .lea_frame => try self.asmRegisterMemory(
	7258	switch (src_mcv) {
	7259	.register_offset => \|reg_off\| switch (reg_off.off) {
	7260	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
	7261	else => .lea,
6861	7262	},
6862		8 => {
6863		const reg = try self.copyToTmpRegister(ty, mcv);
6864		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
	7263	.indirect, .load_frame => try self.movMirTag(ty),
	7264	.lea_frame => .lea,
	7265	else => unreachable,
	7266	},
	7267	registerAlias(dst_reg, abi_size),
	7268	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
	7269	.register_offset, .indirect => \|reg_off\| .{
	7270	.base = .{ .reg = reg_off.reg },
	7271	.disp = reg_off.off,
6865	7272	},
6866		else => return self.fail("TODO implement inputs on stack for {} with abi size > 8", .{mcv}),
6867		}
6868		},
6869		.memory,
6870		.load_direct,
6871		.load_tlv,
6872		=> {
6873		if (abi_size <= 8) {
6874		const reg = try self.copyToTmpRegister(ty, mcv);
6875		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
	7273	.load_frame, .lea_frame => \|frame_addr\| .{
	7274	.base = .{ .frame = frame_addr.index },
	7275	.disp = frame_addr.off,
	7276	},
	7277	else => unreachable,
	7278	}),
	7279	),
	7280	.memory, .load_direct, .load_got, .load_tlv => {
	7281	switch (src_mcv) {
	7282	.memory => \|addr\| if (math.cast(i32, @bitCast(i64, addr))) \|small_addr\|
	7283	return self.asmRegisterMemory(
	7284	try self.movMirTag(ty),
	7285	registerAlias(dst_reg, abi_size),
	7286	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	7287	.base = .{ .reg = .ds },
	7288	.disp = small_addr,
	7289	}),
	7290	),
	7291	.load_direct => \|sym_index\| if (try self.movMirTag(ty) == .mov) {
	7292	const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
	7293	_ = try self.addInst(.{
	7294	.tag = .mov_linker,
	7295	.ops = .direct_reloc,
	7296	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
	7297	.reg = @enumToInt(dst_reg.to64()),
	7298	.atom_index = atom_index,
	7299	.sym_index = sym_index,
	7300	}) },
	7301	});
	7302	return;
	7303	},
	7304	.load_got, .load_tlv => {},
	7305	else => unreachable,
6876	7306	}
6877	7307
6878		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	7308	const addr_reg = try self.copyToTmpRegister(Type.usize, src_mcv.address());
6879	7309	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6880	7310	defer self.register_manager.unlockReg(addr_lock);
6881	7311
6882		try self.genSetReg(Type.usize, addr_reg, switch (mcv) {
6883		.memory => \|addr\| .{ .immediate = addr },
6884		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
6885		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
6886		else => unreachable,
6887		});
6888		try self.genInlineMemcpy(
6889		.{ .ptr_stack_offset = stack_offset },
6890		.{ .register = addr_reg },
6891		.{ .immediate = abi_size },
6892		.{ .dest_stack_base = .rsp },
	7312	try self.asmRegisterMemory(
	7313	try self.movMirTag(ty),
	7314	registerAlias(dst_reg, abi_size),
	7315	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
6893	7316	);
6894	7317	},
6895		.register => \|reg\| {
6896		switch (ty.zigTypeTag()) {
6897		.Float => {
6898		if (intrinsicsAllowed(self.target.*, ty)) {
6899		const tag: Mir.Inst.Tag = switch (ty.tag()) {
6900		.f32 => .movss,
6901		.f64 => .movsd,
6902		else => return self.fail(
6903		"TODO genSetStackArg for register for type {}",
6904		.{ty.fmtDebug()},
6905		),
6906		};
6907		// TODO verify this
6908		const ptr_size: Memory.PtrSize = switch (ty.tag()) {
6909		.f32 => .dword,
6910		.f64 => .qword,
6911		else => unreachable,
6912		};
6913		return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
6914		.base = .rsp,
6915		.disp = -stack_offset,
6916		}), reg.to128());
6917		}
6918
6919		return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
	7318	.lea_direct, .lea_got => \|sym_index\| {
	7319	const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
	7320	_ = try self.addInst(.{
	7321	.tag = switch (src_mcv) {
	7322	.lea_direct => .lea_linker,
	7323	.lea_got => .mov_linker,
	7324	else => unreachable,
6920	7325	},
6921		else => {
6922		try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6923		.base = .rsp,
6924		.disp = -stack_offset,
6925		}), registerAlias(reg, abi_size));
	7326	.ops = switch (src_mcv) {
	7327	.lea_direct => .direct_reloc,
	7328	.lea_got => .got_reloc,
	7329	else => unreachable,
6926	7330	},
6927		}
6928		},
6929		.ptr_stack_offset,
6930		.load_got,
6931		.lea_direct,
6932		.lea_tlv,
6933		=> {
6934		const reg = try self.copyToTmpRegister(ty, mcv);
6935		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
	7331	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
	7332	.reg = @enumToInt(dst_reg.to64()),
	7333	.atom_index = atom_index,
	7334	.sym_index = sym_index,
	7335	}) },
	7336	});
6936	7337	},
6937		.stack_offset => \|mcv_off\| {
6938		if (abi_size <= 8) {
6939		const reg = try self.copyToTmpRegister(ty, mcv);
6940		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6941		}
6942
6943		try self.genInlineMemcpy(
6944		.{ .ptr_stack_offset = stack_offset },
6945		.{ .ptr_stack_offset = mcv_off },
6946		.{ .immediate = abi_size },
6947		.{ .dest_stack_base = .rsp },
6948		);
	7338	.lea_tlv => \|sym_index\| {
	7339	const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
	7340	if (self.bin_file.cast(link.File.MachO)) \|_\| {
	7341	_ = try self.addInst(.{
	7342	.tag = .lea_linker,
	7343	.ops = .tlv_reloc,
	7344	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
	7345	.reg = @enumToInt(Register.rdi),
	7346	.atom_index = atom_index,
	7347	.sym_index = sym_index,
	7348	}) },
	7349	});
	7350	// TODO: spill registers before calling
	7351	try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
	7352	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
	7353	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
	7354	@tagName(self.bin_file.tag),
	7355	});
6949	7356	},
6950	7357	}
6951	7358	}
6952	7359
6953		fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {
6954		const base_reg = opts.dest_stack_base orelse .rbp;
	7360	fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCValue) InnerError!void {
6955	7361	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6956		switch (mcv) {
6957		.dead => unreachable,
6958		.unreach, .none => return, // Nothing to do.
6959		.undef => {
6960		if (!self.wantSafety()) return; // The already existing value will do just fine.
6961		// TODO Upgrade this to a memset call when we have that available.
6962		switch (abi_size) {
6963		1, 2, 4 => {
6964		const value: u64 = switch (abi_size) {
6965		1 => 0xaa,
6966		2 => 0xaaaa,
6967		4 => 0xaaaaaaaa,
6968		else => unreachable,
6969		};
6970		return self.asmMemoryImmediate(.mov, Memory.sib(
6971		Memory.PtrSize.fromSize(abi_size),
6972		.{ .base = base_reg, .disp = -stack_offset },
6973		), Immediate.u(value));
6974		},
6975		8 => return self.genSetStack(
6976		ty,
6977		stack_offset,
6978		.{ .immediate = 0xaaaaaaaaaaaaaaaa },
6979		opts,
6980		),
6981		else => \|x\| return self.genInlineMemset(
6982		.{ .ptr_stack_offset = stack_offset },
6983		.{ .immediate = 0xaa },
6984		.{ .immediate = x },
6985		opts,
6986		),
6987		}
	7362	const dst_ptr_mcv: MCValue = switch (base) {
	7363	.none => .{ .immediate = @bitCast(u64, @as(i64, disp)) },
	7364	.reg => \|base_reg\| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
	7365	.frame => \|base_frame_index\| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
	7366	};
	7367	switch (src_mcv) {
	7368	.none, .unreach, .dead, .reserved_frame => unreachable,
	7369	.undef => if (self.wantSafety())
	7370	try self.genInlineMemset(dst_ptr_mcv, .{ .immediate = 0xaa }, .{ .immediate = abi_size }),
	7371	.immediate => \|imm\| switch (abi_size) {
	7372	1, 2, 4 => {
	7373	const immediate = if (ty.isSignedInt())
	7374	Immediate.s(@truncate(i32, @bitCast(i64, imm)))
	7375	else
	7376	Immediate.u(@intCast(u32, imm));
	7377	try self.asmMemoryImmediate(
	7378	.mov,
	7379	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	7380	immediate,
	7381	);
	7382	},
	7383	3, 5...7 => unreachable,
	7384	else => if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
	7385	try self.asmMemoryImmediate(
	7386	.mov,
	7387	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	7388	Immediate.s(small),
	7389	);
	7390	} else {
	7391	var offset: i32 = 0;
	7392	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
	7393	.mov,
	7394	Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
	7395	if (ty.isSignedInt())
	7396	Immediate.s(@truncate(
	7397	i32,
	7398	@bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
	7399	))
	7400	else
	7401	Immediate.u(@truncate(
	7402	u32,
	7403	if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0,
	7404	)),
	7405	);
	7406	},
6988	7407	},
	7408	.eflags => \|cc\| try self.asmSetccMemory(Memory.sib(.byte, .{ .base = base, .disp = disp }), cc),
	7409	.register => \|reg\| try self.asmMemoryRegister(
	7410	try self.movMirTag(ty),
	7411	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	7412	registerAlias(reg, abi_size),
	7413	),
6989	7414	.register_overflow => \|ro\| {
6990		const reg_lock = self.register_manager.lockReg(ro.reg);
6991		defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
6992
6993		const wrapped_ty = ty.structFieldType(0);
6994		try self.genSetStack(wrapped_ty, stack_offset, .{ .register = ro.reg }, .{});
6995
6996		const overflow_bit_ty = ty.structFieldType(1);
6997		const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
6998		const tmp_reg = try self.register_manager.allocReg(null, gp);
6999		try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
7000
7001		return self.genSetStack(
7002		overflow_bit_ty,
7003		stack_offset - @intCast(i32, overflow_bit_offset),
7004		.{ .register = tmp_reg.to8() },
7005		.{},
	7415	try self.genSetMem(
	7416	base,
	7417	disp + @intCast(i32, ty.structFieldOffset(0, self.target.*)),
	7418	ty.structFieldType(0),
	7419	.{ .register = ro.reg },
	7420	);
	7421	try self.genSetMem(
	7422	base,
	7423	disp + @intCast(i32, ty.structFieldOffset(1, self.target.*)),
	7424	ty.structFieldType(1),
	7425	.{ .eflags = ro.eflags },
7006	7426	);
7007	7427	},
7008		.eflags => \|cc\| try self.asmSetccMemory(
7009		Memory.sib(.byte, .{ .base = base_reg, .disp = -stack_offset }),
7010		cc,
7011		),
7012		.immediate => \|imm\| {
7013		// TODO
7014		switch (abi_size) {
7015		0 => {
7016		assert(ty.isError());
7017		try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
7018		.base = base_reg,
7019		.disp = -stack_offset,
7020		}), Immediate.u(@truncate(u8, imm)));
7021		},
7022		1, 2, 4 => {
7023		const immediate = if (ty.isSignedInt())
7024		Immediate.s(@truncate(i32, @bitCast(i64, imm)))
7025		else
7026		Immediate.u(@intCast(u32, imm));
7027		try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7028		.base = base_reg,
7029		.disp = -stack_offset,
7030		}), immediate);
7031		},
7032		3, 5...7 => unreachable,
7033		else => {
7034		// 64 bit write to memory would take two mov's anyways so we
7035		// insted just use two 32 bit writes to avoid register allocation
7036		if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
7037		try self.asmMemoryImmediate(.mov, Memory.sib(
7038		Memory.PtrSize.fromSize(abi_size),
7039		.{ .base = base_reg, .disp = -stack_offset },
7040		), Immediate.s(small));
7041		} else {
7042		var offset: i32 = 0;
7043		while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
7044		.mov,
7045		Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }),
7046		if (ty.isSignedInt())
7047		Immediate.s(@truncate(
7048		i32,
7049		@bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
7050		))
7051		else
7052		Immediate.u(@truncate(
7053		u32,
7054		if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0,
7055		)),
7056		);
7057		}
7058		},
7059		}
7060		},
7061		.register => \|reg\| {
7062		switch (ty.zigTypeTag()) {
7063		.Float => {
7064		if (intrinsicsAllowed(self.target.*, ty)) {
7065		const tag: Mir.Inst.Tag = switch (ty.tag()) {
7066		.f32 => .movss,
7067		.f64 => .movsd,
7068		else => return self.fail(
7069		"TODO genSetStack for register for type {}",
7070		.{ty.fmtDebug()},
7071		),
7072		};
7073		const ptr_size: Memory.PtrSize = switch (ty.tag()) {
7074		.f32 => .dword,
7075		.f64 => .qword,
7076		else => unreachable,
7077		};
7078		return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
7079		.base = base_reg.to64(),
7080		.disp = -stack_offset,
7081		}), reg.to128());
7082		}
7083
7084		return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
7085		},
7086		else => {
7087		try self.genInlineMemcpyRegisterRegister(ty, base_reg, reg, stack_offset);
7088		},
7089		}
7090		},
	7428	.register_offset,
7091	7429	.memory,
	7430	.indirect,
7092	7431	.load_direct,
7093		.load_tlv,
7094	7432	.lea_direct,
7095		.lea_tlv,
7096		=> if (abi_size <= 8) {
7097		const reg = try self.copyToTmpRegister(ty, mcv);
7098		return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7099		} else {
7100		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7101		const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7102		defer self.register_manager.unlockReg(addr_lock);
7103
7104		try self.genSetReg(Type.usize, addr_reg, switch (mcv) {
7105		.memory => \|addr\| .{ .immediate = addr },
7106		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7107		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7108		else => mcv,
7109		});
7110		try self.genInlineMemcpy(
7111		.{ .ptr_stack_offset = stack_offset },
7112		.{ .register = addr_reg },
7113		.{ .immediate = abi_size },
7114		.{},
7115		);
7116		},
7117		.stack_offset => \|off\| if (abi_size <= 8) {
7118		const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7119		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7120		defer self.register_manager.unlockReg(tmp_lock);
7121
7122		try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7123		} else try self.genInlineMemcpy(
7124		.{ .ptr_stack_offset = stack_offset },
7125		.{ .ptr_stack_offset = off },
7126		.{ .immediate = abi_size },
7127		.{},
7128		),
7129		.ptr_stack_offset,
7130	7433	.load_got,
7131		=> {
7132		const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7133		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7134		defer self.register_manager.unlockReg(tmp_lock);
7135
7136		try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
	7434	.lea_got,
	7435	.load_tlv,
	7436	.lea_tlv,
	7437	.load_frame,
	7438	.lea_frame,
	7439	=> switch (abi_size) {
	7440	0 => {},
	7441	1, 2, 4, 8 => {
	7442	const src_reg = try self.copyToTmpRegister(ty, src_mcv);
	7443	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	7444	defer self.register_manager.unlockReg(src_lock);
	7445
	7446	try self.genSetMem(base, disp, ty, .{ .register = src_reg });
	7447	},
	7448	else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }),
7137	7449	},
7138	7450	}
7139	7451	}
...	...	@@ -7165,10 +7477,14 @@ fn genInlineMemcpyRegisterRegister(
7165	7477	var remainder = abi_size;
7166	7478	while (remainder > 0) {
7167	7479	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
7168		try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
7169		.base = dst_reg,
7170		.disp = -next_offset,
7171		}), registerAlias(tmp_reg, nearest_power_of_two));
	7480	try self.asmMemoryRegister(
	7481	.mov,
	7482	Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
	7483	.base = dst_reg,
	7484	.disp = -next_offset,
	7485	}),
	7486	registerAlias(tmp_reg, nearest_power_of_two),
	7487	);
7172	7488
7173	7489	if (nearest_power_of_two > 1) {
7174	7490	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
...	...	@@ -7191,121 +7507,11 @@ fn genInlineMemcpyRegisterRegister(
7191	7507	}
7192	7508	}
7193	7509
7194		const InlineMemcpyOpts = struct {
7195		source_stack_base: ?Register = null,
7196		dest_stack_base: ?Register = null,
7197		};
7198
7199		fn genInlineMemcpy(
7200		self: *Self,
7201		dst_ptr: MCValue,
7202		src_ptr: MCValue,
7203		len: MCValue,
7204		opts: InlineMemcpyOpts,
7205		) InnerError!void {
7206		const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) \|reg\|
7207		self.register_manager.lockReg(reg)
7208		else
7209		null;
7210		defer if (ssbase_lock) \|reg\| self.register_manager.unlockReg(reg);
7211
7212		const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) \|reg\|
7213		self.register_manager.lockReg(reg)
7214		else
7215		null;
7216		defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
7217
	7510	fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
7218	7511	try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
7219
7220		switch (dst_ptr) {
7221		.lea_tlv,
7222		.load_tlv,
7223		=> {
7224		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7225		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7226		else => dst_ptr,
7227		});
7228		},
7229		.memory,
7230		.load_got,
7231		.lea_direct,
7232		.load_direct,
7233		=> {
7234		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7235		.memory => \|addr\| .{ .immediate = addr },
7236		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7237		else => dst_ptr,
7238		});
7239		// Load the pointer, which is stored in memory
7240		try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7241		},
7242		.stack_offset, .ptr_stack_offset => \|off\| {
7243		try self.asmRegisterMemory(switch (dst_ptr) {
7244		.stack_offset => .mov,
7245		.ptr_stack_offset => .lea,
7246		else => unreachable,
7247		}, .rdi, Memory.sib(.qword, .{
7248		.base = opts.dest_stack_base orelse .rbp,
7249		.disp = -off,
7250		}));
7251		},
7252		.register => \|reg\| {
7253		try self.asmRegisterRegister(
7254		.mov,
7255		registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7256		reg,
7257		);
7258		},
7259		else => {
7260		return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
7261		},
7262		}
7263
7264		switch (src_ptr) {
7265		.lea_tlv,
7266		.load_tlv,
7267		=> {
7268		try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7269		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7270		else => dst_ptr,
7271		});
7272		},
7273		.memory,
7274		.load_got,
7275		.lea_direct,
7276		.load_direct,
7277		=> {
7278		try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7279		.memory => \|addr\| .{ .immediate = addr },
7280		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7281		else => src_ptr,
7282		});
7283		// Load the pointer, which is stored in memory
7284		try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi }));
7285		},
7286		.stack_offset, .ptr_stack_offset => \|off\| {
7287		try self.asmRegisterMemory(switch (src_ptr) {
7288		.stack_offset => .mov,
7289		.ptr_stack_offset => .lea,
7290		else => unreachable,
7291		}, .rsi, Memory.sib(.qword, .{
7292		.base = opts.source_stack_base orelse .rbp,
7293		.disp = -off,
7294		}));
7295		},
7296		.register => \|reg\| {
7297		try self.asmRegisterRegister(
7298		.mov,
7299		registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7300		reg,
7301		);
7302		},
7303		else => {
7304		return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
7305		},
7306		}
7307
7308		try self.genSetReg(Type.usize, .rcx, len);
	7512	try self.genSetReg(.rdi, Type.usize, dst_ptr);
	7513	try self.genSetReg(.rsi, Type.usize, src_ptr);
	7514	try self.genSetReg(.rcx, Type.usize, len);
7309	7515	_ = try self.addInst(.{
7310	7516	.tag = .movs,
7311	7517	.ops = .string,
...	...	@@ -7313,67 +7519,11 @@ fn genInlineMemcpy(
7313	7519	});
7314	7520	}
7315	7521
7316		fn genInlineMemset(
7317		self: *Self,
7318		dst_ptr: MCValue,
7319		value: MCValue,
7320		len: MCValue,
7321		opts: InlineMemcpyOpts,
7322		) InnerError!void {
7323		const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) \|reg\|
7324		self.register_manager.lockReg(reg)
7325		else
7326		null;
7327		defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
7328
	7522	fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
7329	7523	try self.spillRegisters(&.{ .rdi, .al, .rcx });
7330
7331		switch (dst_ptr) {
7332		.lea_tlv,
7333		.load_tlv,
7334		=> {
7335		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7336		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7337		else => dst_ptr,
7338		});
7339		},
7340		.load_got => try self.genSetReg(Type.usize, .rdi, dst_ptr),
7341		.memory,
7342		.lea_direct,
7343		.load_direct,
7344		=> {
7345		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7346		.memory => \|addr\| .{ .immediate = addr },
7347		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7348		else => dst_ptr,
7349		});
7350		// Load the pointer, which is stored in memory
7351		try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7352		},
7353		.stack_offset, .ptr_stack_offset => \|off\| {
7354		try self.asmRegisterMemory(switch (dst_ptr) {
7355		.stack_offset => .mov,
7356		.ptr_stack_offset => .lea,
7357		else => unreachable,
7358		}, .rdi, Memory.sib(.qword, .{
7359		.base = opts.dest_stack_base orelse .rbp,
7360		.disp = -off,
7361		}));
7362		},
7363		.register => \|reg\| {
7364		try self.asmRegisterRegister(
7365		.mov,
7366		registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7367		reg,
7368		);
7369		},
7370		else => {
7371		return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr});
7372		},
7373		}
7374
7375		try self.genSetReg(Type.u8, .al, value);
7376		try self.genSetReg(Type.usize, .rcx, len);
	7524	try self.genSetReg(.rdi, Type.usize, dst_ptr);
	7525	try self.genSetReg(.al, Type.u8, value);
	7526	try self.genSetReg(.rcx, Type.usize, len);
7377	7527	_ = try self.addInst(.{
7378	7528	.tag = .stos,
7379	7529	.ops = .string,
...	...	@@ -7381,307 +7531,6 @@ fn genInlineMemset(
7381	7531	});
7382	7532	}
7383	7533
7384		fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
7385		const abi_size = @intCast(u32, ty.abiSize(self.target.*));
7386		if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
7387		switch (mcv) {
7388		.dead => unreachable,
7389		.register_overflow => unreachable,
7390		.ptr_stack_offset => \|off\| {
7391		try self.asmRegisterMemory(
7392		.lea,
7393		registerAlias(reg, abi_size),
7394		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7395		);
7396		},
7397		.unreach, .none => return, // Nothing to do.
7398		.undef => {
7399		if (!self.wantSafety()) return; // The already existing value will do just fine.
7400		// Write the debug undefined value.
7401		switch (self.regBitSize(ty)) {
7402		8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
7403		16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
7404		32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
7405		64 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
7406		else => unreachable,
7407		}
7408		},
7409		.eflags => \|cc\| {
7410		return self.asmSetccRegister(reg.to8(), cc);
7411		},
7412		.immediate => \|imm\| {
7413		if (imm == 0) {
7414		// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7415		// register is the fastest way to zero a register.
7416		try self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
7417		} else if (abi_size > 4 and math.cast(u32, imm) != null) {
7418		// 32-bit moves zero-extend to 64-bit.
7419		try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm));
7420		} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7421		try self.asmRegisterImmediate(
7422		.mov,
7423		registerAlias(reg, abi_size),
7424		Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7425		);
7426		} else {
7427		try self.asmRegisterImmediate(
7428		.mov,
7429		registerAlias(reg, abi_size),
7430		Immediate.u(imm),
7431		);
7432		}
7433		},
7434		.register => \|src_reg\| {
7435		// If the registers are the same, nothing to do.
7436		if (src_reg.id() == reg.id())
7437		return;
7438
7439		switch (ty.zigTypeTag()) {
7440		.Int => switch (ty.intInfo(self.target.*).signedness) {
7441		.signed => {
7442		if (abi_size <= 4) {
7443		return self.asmRegisterRegister(
7444		.movsx,
7445		reg.to64(),
7446		registerAlias(src_reg, abi_size),
7447		);
7448		}
7449		},
7450		.unsigned => {
7451		if (abi_size <= 2) {
7452		return self.asmRegisterRegister(
7453		.movzx,
7454		reg.to64(),
7455		registerAlias(src_reg, abi_size),
7456		);
7457		}
7458		},
7459		},
7460		.Float => {
7461		if (intrinsicsAllowed(self.target.*, ty)) {
7462		const tag: Mir.Inst.Tag = switch (ty.tag()) {
7463		.f32 => .movss,
7464		.f64 => .movsd,
7465		else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
7466		};
7467		return self.asmRegisterRegister(tag, reg.to128(), src_reg.to128());
7468		}
7469		return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
7470		},
7471		else => {},
7472		}
7473
7474		try self.asmRegisterRegister(.mov, registerAlias(reg, abi_size), registerAlias(src_reg, abi_size));
7475		},
7476		.memory => \|addr\| switch (ty.zigTypeTag()) {
7477		.Float => {
7478		const base_reg = (try self.register_manager.allocReg(null, gp)).to64();
7479		try self.genSetReg(Type.usize, base_reg, .{ .immediate = addr });
7480
7481		if (intrinsicsAllowed(self.target.*, ty)) {
7482		return self.asmRegisterMemory(
7483		switch (ty.tag()) {
7484		.f32 => .movss,
7485		.f64 => .movsd,
7486		else => return self.fail("TODO genSetReg from memory for {}", .{
7487		ty.fmt(self.bin_file.options.module.?),
7488		}),
7489		},
7490		reg.to128(),
7491		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7492		);
7493		}
7494
7495		return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7496		},
7497		else => {
7498		if (addr <= math.maxInt(i32)) {
7499		try self.asmRegisterMemory(
7500		.mov,
7501		registerAlias(reg, abi_size),
7502		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7503		.base = .ds,
7504		.disp = @intCast(i32, addr),
7505		}),
7506		);
7507		} else {
7508		if (reg.to64() == .rax) {
7509		// If this is RAX, we can use a direct load.
7510		// Otherwise, we need to load the address, then indirectly load the value.
7511		_ = try self.addInst(.{
7512		.tag = .mov_moffs,
7513		.ops = .rax_moffs,
7514		.data = .{ .payload = try self.addExtra(Mir.MemoryMoffs.encode(.ds, addr)) },
7515		});
7516		} else {
7517		// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
7518		try self.genSetReg(Type.usize, reg, MCValue{ .immediate = addr });
7519		try self.asmRegisterMemory(
7520		.mov,
7521		registerAlias(reg, abi_size),
7522		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
7523		);
7524		}
7525		}
7526		},
7527		},
7528		.load_got, .lea_direct => \|sym_index\| {
7529		const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7530		_ = try self.addInst(.{
7531		.tag = switch (mcv) {
7532		.load_got => .mov_linker,
7533		.lea_direct => .lea_linker,
7534		else => unreachable,
7535		},
7536		.ops = switch (mcv) {
7537		.load_got => .got_reloc,
7538		.lea_direct => .direct_reloc,
7539		else => unreachable,
7540		},
7541		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7542		.reg = @enumToInt(reg),
7543		.atom_index = atom_index,
7544		.sym_index = sym_index,
7545		}) },
7546		});
7547		},
7548		.load_direct => \|sym_index\| {
7549		switch (ty.zigTypeTag()) {
7550		.Float => {
7551		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7552		try self.genSetReg(Type.usize, addr_reg, .{ .lea_direct = sym_index });
7553
7554		if (intrinsicsAllowed(self.target.*, ty)) {
7555		return self.asmRegisterMemory(
7556		switch (ty.tag()) {
7557		.f32 => .movss,
7558		.f64 => .movsd,
7559		else => return self.fail("TODO genSetReg from memory for {}", .{
7560		ty.fmt(self.bin_file.options.module.?),
7561		}),
7562		},
7563		reg.to128(),
7564		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
7565		);
7566		}
7567
7568		return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7569		},
7570		else => {
7571		const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7572		_ = try self.addInst(.{
7573		.tag = .mov_linker,
7574		.ops = .direct_reloc,
7575		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7576		.reg = @enumToInt(registerAlias(reg, abi_size)),
7577		.atom_index = atom_index,
7578		.sym_index = sym_index,
7579		}) },
7580		});
7581		},
7582		}
7583		},
7584		.lea_tlv => \|sym_index\| {
7585		const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7586		if (self.bin_file.cast(link.File.MachO)) \|_\| {
7587		_ = try self.addInst(.{
7588		.tag = .lea_linker,
7589		.ops = .tlv_reloc,
7590		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7591		.reg = @enumToInt(Register.rdi),
7592		.atom_index = atom_index,
7593		.sym_index = sym_index,
7594		}) },
7595		});
7596		// TODO: spill registers before calling
7597		try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .rdi }));
7598		try self.genSetReg(Type.usize, reg, .{ .register = .rax });
7599		} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{@tagName(self.bin_file.tag)});
7600		},
7601		.load_tlv => \|sym_index\| {
7602		const base_reg = switch (ty.zigTypeTag()) {
7603		.Float => (try self.register_manager.allocReg(null, gp)).to64(),
7604		else => reg.to64(),
7605		};
7606		try self.genSetReg(Type.usize, base_reg, .{ .lea_tlv = sym_index });
7607		switch (ty.zigTypeTag()) {
7608		.Float => if (intrinsicsAllowed(self.target.*, ty)) {
7609		return self.asmRegisterMemory(
7610		switch (ty.tag()) {
7611		.f32 => .movss,
7612		.f64 => .movsd,
7613		else => return self.fail("TODO genSetReg from memory for {}", .{
7614		ty.fmt(self.bin_file.options.module.?),
7615		}),
7616		},
7617		reg.to128(),
7618		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7619		);
7620		} else return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}),
7621		else => try self.asmRegisterMemory(
7622		.mov,
7623		registerAlias(reg, abi_size),
7624		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7625		),
7626		}
7627		},
7628		.stack_offset => \|off\| {
7629		switch (ty.zigTypeTag()) {
7630		.Int => switch (ty.intInfo(self.target.*).signedness) {
7631		.signed => {
7632		if (abi_size <= 4) {
7633		return self.asmRegisterMemory(
7634		.movsx,
7635		reg.to64(),
7636		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7637		.base = .rbp,
7638		.disp = -off,
7639		}),
7640		);
7641		}
7642		},
7643		.unsigned => {
7644		if (abi_size <= 2) {
7645		return self.asmRegisterMemory(
7646		.movzx,
7647		reg.to64(),
7648		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7649		.base = .rbp,
7650		.disp = -off,
7651		}),
7652		);
7653		}
7654		},
7655		},
7656		.Float => {
7657		if (intrinsicsAllowed(self.target.*, ty)) {
7658		const tag: Mir.Inst.Tag = switch (ty.tag()) {
7659		.f32 => .movss,
7660		.f64 => .movsd,
7661		else => return self.fail(
7662		"TODO genSetReg from stack offset for {}",
7663		.{ty.fmtDebug()},
7664		),
7665		};
7666		return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(
7667		Memory.PtrSize.fromSize(abi_size),
7668		.{ .base = .rbp, .disp = -off },
7669		));
7670		}
7671		return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
7672		},
7673		else => {},
7674		}
7675
7676		try self.asmRegisterMemory(
7677		.mov,
7678		registerAlias(reg, abi_size),
7679		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7680		);
7681		},
7682		}
7683		}
7684
7685	7534	fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7686	7535	const un_op = self.air.instructions.items(.data)[inst].un_op;
7687	7536	const result = result: {
...	...	@@ -7691,7 +7540,7 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7691	7540
7692	7541	const dst_mcv = try self.allocRegOrMem(inst, true);
7693	7542	const dst_ty = self.air.typeOfIndex(inst);
7694		try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);
	7543	try self.genCopy(dst_ty, dst_mcv, src_mcv);
7695	7544	break :result dst_mcv;
7696	7545	};
7697	7546	return self.finishAir(inst, result, .{ un_op, .none, .none });
...	...	@@ -7699,9 +7548,14 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7699	7548
7700	7549	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7701	7550	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	7551	const dst_ty = self.air.typeOfIndex(inst);
	7552	const src_ty = self.air.typeOf(ty_op.operand);
	7553
7702	7554	const result = result: {
	7555	const dst_rc = try self.regClassForType(dst_ty);
	7556	const src_rc = try self.regClassForType(src_ty);
7703	7557	const operand = try self.resolveInst(ty_op.operand);
7704		if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
	7558	if (dst_rc.eql(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
7705	7559
7706	7560	const operand_lock = switch (operand) {
7707	7561	.register => \|reg\| self.register_manager.lockReg(reg),
...	...	@@ -7711,25 +7565,31 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7711	7565	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
7712	7566
7713	7567	const dest = try self.allocRegOrMem(inst, true);
7714		try self.setRegOrMem(self.air.typeOfIndex(inst), dest, operand);
	7568	try self.genCopy(self.air.typeOfIndex(inst), dest, operand);
7715	7569	break :result dest;
7716	7570	};
7717		log.debug("airBitCast(%{d}): {}", .{ inst, result });
7718	7571	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7719	7572	}
7720	7573
7721	7574	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
7722	7575	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	7576
	7577	const slice_ty = self.air.typeOfIndex(inst);
7723	7578	const ptr_ty = self.air.typeOf(ty_op.operand);
7724	7579	const ptr = try self.resolveInst(ty_op.operand);
7725	7580	const array_ty = ptr_ty.childType();
7726	7581	const array_len = array_ty.arrayLen();
7727	7582
7728		const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
7729		try self.genSetStack(ptr_ty, stack_offset, ptr, .{});
7730		try self.genSetStack(Type.u64, stack_offset - 8, .{ .immediate = array_len }, .{});
	7583	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
	7584	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
	7585	try self.genSetMem(
	7586	.{ .frame = frame_index },
	7587	@intCast(i32, ptr_ty.abiSize(self.target.*)),
	7588	Type.usize,
	7589	.{ .immediate = array_len },
	7590	);
7731	7591
7732		const result = MCValue{ .stack_offset = stack_offset };
	7592	const result = MCValue{ .load_frame = .{ .index = frame_index } };
7733	7593	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7734	7594	}
7735	7595
...	...	@@ -7758,29 +7618,31 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
7758	7618	}
7759	7619
7760	7620	// move float src to ST(0)
7761		const stack_offset = switch (operand) {
7762		.stack_offset, .ptr_stack_offset => \|offset\| offset,
7763		else => blk: {
7764		const offset = @intCast(i32, try self.allocMem(
7765		inst,
7766		src_abi_size,
7767		src_ty.abiAlignment(self.target.*),
7768		));
7769		try self.genSetStack(src_ty, offset, operand, .{});
7770		break :blk offset;
	7621	const frame_addr: FrameAddr = switch (operand) {
	7622	.load_frame => \|frame_addr\| frame_addr,
	7623	else => frame_addr: {
	7624	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(src_ty, self.target.*));
	7625	try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, operand);
	7626	break :frame_addr .{ .index = frame_index };
7771	7627	},
7772	7628	};
7773		try self.asmMemory(.fld, Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
7774		.base = .rbp,
7775		.disp = -stack_offset,
7776		}));
	7629	try self.asmMemory(
	7630	.fld,
	7631	Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
	7632	.base = .{ .frame = frame_addr.index },
	7633	.disp = frame_addr.off,
	7634	}),
	7635	);
7777	7636
7778	7637	// convert
7779	7638	const stack_dst = try self.allocRegOrMem(inst, false);
7780		try self.asmMemory(.fisttp, Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
7781		.base = .rbp,
7782		.disp = -stack_dst.stack_offset,
7783		}));
	7639	try self.asmMemory(
	7640	.fisttp,
	7641	Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
	7642	.base = .{ .frame = stack_dst.load_frame.index },
	7643	.disp = stack_dst.load_frame.off,
	7644	}),
	7645	);
7784	7646
7785	7647	return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
7786	7648	}
...	...	@@ -7800,20 +7662,32 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7800	7662
7801	7663	const exp_mcv = try self.resolveInst(extra.expected_value);
7802	7664	if (val_abi_size > 8) switch (exp_mcv) {
7803		.stack_offset => \|exp_off\| {
7804		try self.genSetReg(Type.usize, .rax, .{ .stack_offset = exp_off - 0 });
7805		try self.genSetReg(Type.usize, .rdx, .{ .stack_offset = exp_off - 8 });
	7665	.load_frame => \|frame_addr\| {
	7666	try self.genSetReg(.rax, Type.usize, .{ .load_frame = .{
	7667	.index = frame_addr.index,
	7668	.off = frame_addr.off + 0,
	7669	} });
	7670	try self.genSetReg(.rdx, Type.usize, .{ .load_frame = .{
	7671	.index = frame_addr.index,
	7672	.off = frame_addr.off + 8,
	7673	} });
7806	7674	},
7807	7675	else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
7808		} else try self.genSetReg(val_ty, .rax, exp_mcv);
	7676	} else try self.genSetReg(.rax, val_ty, exp_mcv);
7809	7677	const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
7810	7678	defer self.register_manager.unlockReg(rax_lock);
7811	7679
7812	7680	const new_mcv = try self.resolveInst(extra.new_value);
7813	7681	const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {
7814		.stack_offset => \|new_off\| new: {
7815		try self.genSetReg(Type.usize, .rbx, .{ .stack_offset = new_off - 0 });
7816		try self.genSetReg(Type.usize, .rcx, .{ .stack_offset = new_off - 8 });
	7682	.load_frame => \|frame_addr\| new: {
	7683	try self.genSetReg(.rbx, Type.usize, .{ .load_frame = .{
	7684	.index = frame_addr.index,
	7685	.off = frame_addr.off + 0,
	7686	} });
	7687	try self.genSetReg(.rcx, Type.usize, .{ .load_frame = .{
	7688	.index = frame_addr.index,
	7689	.off = frame_addr.off + 8,
	7690	} });
7817	7691	break :new undefined;
7818	7692	},
7819	7693	else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
...	...	@@ -7823,11 +7697,19 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7823	7697
7824	7698	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7825	7699	const ptr_mem = switch (ptr_mcv) {
7826		.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg }),
7827		.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
7828		else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
	7700	.register => \|reg\| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
	7701	.lea_frame => \|frame_addr\| Memory.sib(ptr_size, .{
	7702	.base = .{ .frame = frame_addr.index },
	7703	.disp = frame_addr.off,
	7704	}),
	7705	else => Memory.sib(ptr_size, .{ .base = .{
	7706	.reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
	7707	} }),
	7708	};
	7709	const mem_lock = switch (ptr_mem.base()) {
	7710	.none, .frame => null,
	7711	.reg => \|reg\| self.register_manager.lockReg(reg),
7829	7712	};
7830		const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
7831	7713	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
7832	7714
7833	7715	try self.spillEflagsIfOccupied();
...	...	@@ -7851,9 +7733,24 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7851	7733	}
7852	7734
7853	7735	const dst_mcv = try self.allocRegOrMem(inst, false);
7854		try self.genSetStack(Type.bool, dst_mcv.stack_offset - 16, .{ .eflags = .ne }, .{});
7855		try self.genSetStack(Type.usize, dst_mcv.stack_offset - 8, .{ .register = .rdx }, .{});
7856		try self.genSetStack(Type.usize, dst_mcv.stack_offset - 0, .{ .register = .rax }, .{});
	7736	try self.genSetMem(
	7737	.{ .frame = dst_mcv.load_frame.index },
	7738	dst_mcv.load_frame.off + 16,
	7739	Type.bool,
	7740	.{ .eflags = .ne },
	7741	);
	7742	try self.genSetMem(
	7743	.{ .frame = dst_mcv.load_frame.index },
	7744	dst_mcv.load_frame.off + 8,
	7745	Type.usize,
	7746	.{ .register = .rdx },
	7747	);
	7748	try self.genSetMem(
	7749	.{ .frame = dst_mcv.load_frame.index },
	7750	dst_mcv.load_frame.off + 0,
	7751	Type.usize,
	7752	.{ .register = .rax },
	7753	);
7857	7754	break :result dst_mcv;
7858	7755	};
7859	7756	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
...	...	@@ -7884,11 +7781,19 @@ fn atomicOp(
7884	7781	const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
7885	7782	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7886	7783	const ptr_mem = switch (ptr_mcv) {
7887		.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg }),
7888		.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
7889		else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
	7784	.register => \|reg\| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
	7785	.lea_frame => \|frame_addr\| Memory.sib(ptr_size, .{
	7786	.base = .{ .frame = frame_addr.index },
	7787	.disp = frame_addr.off,
	7788	}),
	7789	else => Memory.sib(ptr_size, .{ .base = .{
	7790	.reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
	7791	} }),
	7792	};
	7793	const mem_lock = switch (ptr_mem.base()) {
	7794	.none, .frame => null,
	7795	.reg => \|reg\| self.register_manager.lockReg(reg),
7890	7796	};
7891		const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
7892	7797	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
7893	7798
7894	7799	const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())
...	...	@@ -7928,7 +7833,7 @@ fn atomicOp(
7928	7833	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
7929	7834	defer self.register_manager.unlockReg(dst_lock);
7930	7835
7931		try self.genSetReg(val_ty, dst_reg, val_mcv);
	7836	try self.genSetReg(dst_reg, val_ty, val_mcv);
7932	7837	if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
7933	7838	try self.genUnOpMir(.neg, val_ty, dst_mcv);
7934	7839	}
...	...	@@ -7952,10 +7857,10 @@ fn atomicOp(
7952	7857	try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
7953	7858	const loop = @intCast(u32, self.mir_instructions.len);
7954	7859	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
7955		try self.genSetReg(val_ty, tmp_reg, .{ .register = .rax });
	7860	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
7956	7861	}
7957	7862	if (rmw_op) \|op\| switch (op) {
7958		.Xchg => try self.genSetReg(val_ty, tmp_reg, val_mcv),
	7863	.Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
7959	7864	.Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
7960	7865	.Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
7961	7866	.And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
...	...	@@ -7990,12 +7895,12 @@ fn atomicOp(
7990	7895	registerAlias(val_reg, cmov_abi_size),
7991	7896	cc,
7992	7897	),
7993		.stack_offset => \|val_off\| try self.asmCmovccRegisterMemory(
	7898	.load_frame => \|frame_addr\| try self.asmCmovccRegisterMemory(
7994	7899	registerAlias(tmp_reg, cmov_abi_size),
7995		Memory.sib(
7996		Memory.PtrSize.fromSize(cmov_abi_size),
7997		.{ .base = .rbp, .disp = -val_off },
7998		),
	7900	Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
	7901	.base = .{ .frame = frame_addr.index },
	7902	.disp = frame_addr.off,
	7903	}),
7999	7904	cc,
8000	7905	),
8001	7906	else => {
...	...	@@ -8028,9 +7933,15 @@ fn atomicOp(
8028	7933	}));
8029	7934	const loop = @intCast(u32, self.mir_instructions.len);
8030	7935	switch (val_mcv) {
8031		.stack_offset => \|val_off\| {
8032		const val_lo_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - val_off });
8033		const val_hi_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - val_off });
	7936	.load_frame => \|frame_addr\| {
	7937	const val_lo_mem = Memory.sib(.qword, .{
	7938	.base = .{ .frame = frame_addr.index },
	7939	.disp = frame_addr.off + 0,
	7940	});
	7941	const val_hi_mem = Memory.sib(.qword, .{
	7942	.base = .{ .frame = frame_addr.index },
	7943	.disp = frame_addr.off + 8,
	7944	});
8034	7945
8035	7946	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8036	7947	try self.asmRegisterRegister(.mov, .rbx, .rax);
...	...	@@ -8087,12 +7998,18 @@ fn atomicOp(
8087	7998	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
8088	7999	try self.asmMemoryRegister(
8089	8000	.mov,
8090		Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - dst_mcv.stack_offset }),
	8001	Memory.sib(.qword, .{
	8002	.base = .{ .frame = dst_mcv.load_frame.index },
	8003	.disp = dst_mcv.load_frame.off + 0,
	8004	}),
8091	8005	.rax,
8092	8006	);
8093	8007	try self.asmMemoryRegister(
8094	8008	.mov,
8095		Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - dst_mcv.stack_offset }),
	8009	Memory.sib(.qword, .{
	8010	.base = .{ .frame = dst_mcv.load_frame.index },
	8011	.disp = dst_mcv.load_frame.off + 8,
	8012	}),
8096	8013	.rdx,
8097	8014	);
8098	8015	return dst_mcv;
...	...	@@ -8139,7 +8056,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
8139	8056	else
8140	8057	try self.allocRegOrMem(inst, true);
8141	8058
8142		try self.load(dst_mcv, ptr_mcv, ptr_ty);
	8059	try self.load(dst_mcv, ptr_ty, ptr_mcv);
8143	8060	return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
8144	8061	}
8145	8062
...	...	@@ -8184,16 +8101,16 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8184	8101	const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));
8185	8102
8186	8103	if (elem_abi_size == 1) {
8187		const ptr = switch (dst_ptr_ty.ptrSize()) {
	8104	const ptr: MCValue = switch (dst_ptr_ty.ptrSize()) {
8188	8105	// TODO: this only handles slices stored in the stack
8189		.Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 }),
	8106	.Slice => dst_ptr,
8190	8107	.One => dst_ptr,
8191	8108	.C, .Many => unreachable,
8192	8109	};
8193		const len = switch (dst_ptr_ty.ptrSize()) {
	8110	const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
8194	8111	// TODO: this only handles slices stored in the stack
8195		.Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),
8196		.One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),
	8112	.Slice => dst_ptr.address().offset(8).deref(),
	8113	.One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8197	8114	.C, .Many => unreachable,
8198	8115	};
8199	8116	const len_lock: ?RegisterLock = switch (len) {
...	...	@@ -8202,7 +8119,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8202	8119	};
8203	8120	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
8204	8121
8205		try self.genInlineMemset(ptr, src_val, len, .{});
	8122	try self.genInlineMemset(ptr, src_val, len);
8206	8123	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8207	8124	}
8208	8125
...	...	@@ -8215,8 +8132,8 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8215	8132	const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(&buf);
8216	8133
8217	8134	// TODO: this only handles slices stored in the stack
8218		const ptr = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 });
8219		const len = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 });
	8135	const ptr = dst_ptr;
	8136	const len = dst_ptr.address().offset(8).deref();
8220	8137
8221	8138	// Used to store the number of elements for comparison.
8222	8139	// After comparison, updated to store number of bytes needed to copy.
...	...	@@ -8225,55 +8142,51 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8225	8142	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
8226	8143	defer self.register_manager.unlockReg(len_lock);
8227	8144
8228		try self.asmRegisterMemory(.mov, len_reg, Memory.sib(.qword, .{
8229		.base = .rbp,
8230		.disp = -len.stack_offset,
8231		}));
	8145	try self.genSetReg(len_reg, Type.usize, len);
8232	8146
8233	8147	const skip_reloc = try self.asmJccReloc(undefined, .z);
8234		try self.store(ptr, src_val, slice_ptr_ty, elem_ty);
	8148	try self.store(slice_ptr_ty, ptr, src_val);
8235	8149
8236	8150	const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8237	8151	const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8238	8152	const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8239	8153	defer self.register_manager.unlockReg(second_elem_ptr_lock);
8240	8154
8241		try self.asmRegisterMemory(
8242		.lea,
8243		second_elem_ptr_reg,
8244		Memory.sib(.qword, .{
8245		.base = try self.copyToTmpRegister(Type.usize, ptr),
8246		.disp = elem_abi_size,
8247		}),
8248		);
	8155	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
	8156	.reg = try self.copyToTmpRegister(Type.usize, ptr),
	8157	.off = elem_abi_size,
	8158	} });
8249	8159
8250	8160	try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });
8251	8161	try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));
8252		try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv, .{});
	8162	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
8253	8163
8254	8164	try self.performReloc(skip_reloc);
8255	8165	},
8256	8166	.One => {
	8167	var elem_ptr_pl = Type.Payload.ElemType{
	8168	.base = .{ .tag = .single_mut_pointer },
	8169	.data = elem_ty,
	8170	};
	8171	const elem_ptr_ty = Type.initPayload(&elem_ptr_pl.base);
	8172
8257	8173	const len = dst_ptr_ty.childType().arrayLen();
	8174
8258	8175	assert(len != 0); // prevented by Sema
8259		try self.store(dst_ptr, src_val, dst_ptr_ty, elem_ty);
	8176	try self.store(elem_ptr_ty, dst_ptr, src_val);
8260	8177
8261	8178	const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8262	8179	const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8263	8180	const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8264	8181	defer self.register_manager.unlockReg(second_elem_ptr_lock);
8265	8182
8266		try self.asmRegisterMemory(
8267		.lea,
8268		second_elem_ptr_reg,
8269		Memory.sib(.qword, .{
8270		.base = try self.copyToTmpRegister(Type.usize, dst_ptr),
8271		.disp = elem_abi_size,
8272		}),
8273		);
	8183	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
	8184	.reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
	8185	.off = elem_abi_size,
	8186	} });
8274	8187
8275	8188	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
8276		try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy, .{});
	8189	try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
8277	8190	},
8278	8191	.C, .Many => unreachable,
8279	8192	}
...	...	@@ -8299,9 +8212,9 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8299	8212	};
8300	8213	defer if (src_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
8301	8214
8302		const len = switch (dst_ptr_ty.ptrSize()) {
8303		.Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),
8304		.One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),
	8215	const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
	8216	.Slice => dst_ptr.address().offset(8).deref(),
	8217	.One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8305	8218	.C, .Many => unreachable,
8306	8219	};
8307	8220	const len_lock: ?RegisterLock = switch (len) {
...	...	@@ -8311,7 +8224,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8311	8224	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
8312	8225
8313	8226	// TODO: dst_ptr and src_ptr could be slices rather than raw pointers
8314		try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});
	8227	try self.genInlineMemcpy(dst_ptr, src_ptr, len);
8315	8228
8316	8229	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8317	8230	}
...	...	@@ -8345,24 +8258,25 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8345	8258	const atom = elf_file.getAtom(atom_index);
8346	8259	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
8347	8260	const got_addr = atom.getOffsetTableAddress(elf_file);
8348		try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
8349		.base = .ds,
8350		.disp = @intCast(i32, got_addr),
8351		}));
	8261	try self.asmRegisterMemory(
	8262	.mov,
	8263	addr_reg.to64(),
	8264	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
	8265	);
8352	8266	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
8353	8267	const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
8354	8268	.{ .kind = .const_data, .ty = Type.anyerror },
8355	8269	4, // dword alignment
8356	8270	);
8357	8271	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
8358		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	8272	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8359	8273	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
8360	8274	const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
8361	8275	.{ .kind = .const_data, .ty = Type.anyerror },
8362	8276	4, // dword alignment
8363	8277	);
8364	8278	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
8365		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	8279	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8366	8280	} else {
8367	8281	return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
8368	8282	}
...	...	@@ -8377,36 +8291,60 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8377	8291
8378	8292	try self.truncateRegister(err_ty, err_reg.to32());
8379	8293
8380		try self.asmRegisterMemory(.mov, start_reg.to32(), Memory.sib(.dword, .{
8381		.base = addr_reg.to64(),
8382		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
8383		.disp = 4,
8384		}));
8385		try self.asmRegisterMemory(.mov, end_reg.to32(), Memory.sib(.dword, .{
8386		.base = addr_reg.to64(),
8387		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
8388		.disp = 8,
8389		}));
	8294	try self.asmRegisterMemory(
	8295	.mov,
	8296	start_reg.to32(),
	8297	Memory.sib(.dword, .{
	8298	.base = .{ .reg = addr_reg.to64() },
	8299	.scale_index = .{ .scale = 4, .index = err_reg.to64() },
	8300	.disp = 4,
	8301	}),
	8302	);
	8303	try self.asmRegisterMemory(
	8304	.mov,
	8305	end_reg.to32(),
	8306	Memory.sib(.dword, .{
	8307	.base = .{ .reg = addr_reg.to64() },
	8308	.scale_index = .{ .scale = 4, .index = err_reg.to64() },
	8309	.disp = 8,
	8310	}),
	8311	);
8390	8312	try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
8391		try self.asmRegisterMemory(.lea, start_reg.to64(), Memory.sib(.byte, .{
8392		.base = addr_reg.to64(),
8393		.scale_index = .{ .scale = 1, .index = start_reg.to64() },
8394		.disp = 0,
8395		}));
8396		try self.asmRegisterMemory(.lea, end_reg.to32(), Memory.sib(.byte, .{
8397		.base = end_reg.to64(),
8398		.disp = -1,
8399		}));
	8313	try self.asmRegisterMemory(
	8314	.lea,
	8315	start_reg.to64(),
	8316	Memory.sib(.byte, .{
	8317	.base = .{ .reg = addr_reg.to64() },
	8318	.scale_index = .{ .scale = 1, .index = start_reg.to64() },
	8319	.disp = 0,
	8320	}),
	8321	);
	8322	try self.asmRegisterMemory(
	8323	.lea,
	8324	end_reg.to32(),
	8325	Memory.sib(.byte, .{
	8326	.base = .{ .reg = end_reg.to64() },
	8327	.disp = -1,
	8328	}),
	8329	);
8400	8330
8401	8331	const dst_mcv = try self.allocRegOrMem(inst, false);
8402		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
8403		.base = .rbp,
8404		.disp = 0 - dst_mcv.stack_offset,
8405		}), start_reg.to64());
8406		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
8407		.base = .rbp,
8408		.disp = 8 - dst_mcv.stack_offset,
8409		}), end_reg.to64());
	8332	try self.asmMemoryRegister(
	8333	.mov,
	8334	Memory.sib(.qword, .{
	8335	.base = .{ .frame = dst_mcv.load_frame.index },
	8336	.disp = dst_mcv.load_frame.off,
	8337	}),
	8338	start_reg.to64(),
	8339	);
	8340	try self.asmMemoryRegister(
	8341	.mov,
	8342	Memory.sib(.qword, .{
	8343	.base = .{ .frame = dst_mcv.load_frame.index },
	8344	.disp = dst_mcv.load_frame.off + 8,
	8345	}),
	8346	end_reg.to64(),
	8347	);
8410	8348
8411	8349	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
8412	8350	}
...	...	@@ -8445,19 +8383,17 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8445	8383	const len = @intCast(usize, result_ty.arrayLen());
8446	8384	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
8447	8385	const elements = @ptrCast([]const Air.Inst.Ref, self.air.extra[ty_pl.payload..][0..len]);
8448		const abi_size = @intCast(u32, result_ty.abiSize(self.target.*));
8449		const abi_align = result_ty.abiAlignment(self.target.*);
8450	8386	const result: MCValue = result: {
8451	8387	switch (result_ty.zigTypeTag()) {
8452	8388	.Struct => {
8453		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
	8389	const frame_index =
	8390	try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8454	8391	if (result_ty.containerLayout() == .Packed) {
8455	8392	const struct_obj = result_ty.castTag(.@"struct").?.data;
8456	8393	try self.genInlineMemset(
8457		.{ .ptr_stack_offset = stack_offset },
	8394	.{ .lea_frame = .{ .index = frame_index } },
8458	8395	.{ .immediate = 0 },
8459		.{ .immediate = abi_size },
8460		.{},
	8396	.{ .immediate = result_ty.abiSize(self.target.*) },
8461	8397	);
8462	8398	for (elements, 0..) \|elem, elem_i\| {
8463	8399	if (result_ty.structFieldValueComptime(elem_i) != null) continue;
...	...	@@ -8465,7 +8401,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8465	8401	const elem_ty = result_ty.structFieldType(elem_i);
8466	8402	const elem_bit_size = @intCast(u32, elem_ty.bitSize(self.target.*));
8467	8403	if (elem_bit_size > 64) {
8468		return self.fail("TODO airAggregateInit implement packed structs with large fields", .{});
	8404	return self.fail(
	8405	"TODO airAggregateInit implement packed structs with large fields",
	8406	.{},
	8407	);
8469	8408	}
8470	8409	const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
8471	8410	const elem_abi_bits = elem_abi_size * 8;
...	...	@@ -8503,7 +8442,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8503	8442	try self.genBinOpMir(
8504	8443	.@"or",
8505	8444	elem_ty,
8506		.{ .stack_offset = stack_offset - elem_byte_off },
	8445	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
8507	8446	.{ .register = elem_reg },
8508	8447	);
8509	8448	if (elem_bit_off > elem_extra_bits) {
...	...	@@ -8520,8 +8459,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8520	8459	try self.genBinOpMir(
8521	8460	.@"or",
8522	8461	elem_ty,
8523		.{ .stack_offset = stack_offset - elem_byte_off -
8524		@intCast(i32, elem_abi_size) },
	8462	.{ .load_frame = .{
	8463	.index = frame_index,
	8464	.off = elem_byte_off + @intCast(i32, elem_abi_size),
	8465	} },
8525	8466	.{ .register = reg },
8526	8467	);
8527	8468	}
...	...	@@ -8536,12 +8477,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8536	8477	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
8537	8478	else => elem_mcv,
8538	8479	};
8539		try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});
	8480	try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8540	8481	}
8541		break :result .{ .stack_offset = stack_offset };
	8482	break :result .{ .load_frame = .{ .index = frame_index } };
8542	8483	},
8543	8484	.Array => {
8544		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
	8485	const frame_index =
	8486	try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8545	8487	const elem_ty = result_ty.childType();
8546	8488	const elem_size = @intCast(u32, elem_ty.abiSize(self.target.*));
8547	8489
...	...	@@ -8552,9 +8494,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8552	8494	else => elem_mcv,
8553	8495	};
8554	8496	const elem_off = @intCast(i32, elem_size * elem_i);
8555		try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});
	8497	try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8556	8498	}
8557		break :result MCValue{ .stack_offset = stack_offset };
	8499	break :result .{ .load_frame = .{ .index = frame_index } };
8558	8500	},
8559	8501	.Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),
8560	8502	else => unreachable,
...	...	@@ -8596,7 +8538,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
8596	8538	const ty = self.air.typeOf(ref);
8597	8539
8598	8540	// If the type has no codegen bits, no need to store it.
8599		if (!ty.hasRuntimeBitsIgnoreComptime() and !ty.isError()) return .none;
	8541	if (!ty.hasRuntimeBitsIgnoreComptime()) return .none;
8600	8542
8601	8543	if (Air.refToIndex(ref)) \|inst\| {
8602	8544	const mcv = switch (self.air.instructions.items(.tag)[inst]) {
...	...	@@ -8666,8 +8608,8 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
8666	8608	.immediate => \|imm\| .{ .immediate = imm },
8667	8609	.memory => \|addr\| .{ .memory = addr },
8668	8610	.load_direct => \|sym_index\| .{ .load_direct = sym_index },
8669		.load_got => \|sym_index\| .{ .load_got = sym_index },
8670		.load_tlv => \|sym_index\| .{ .load_tlv = sym_index },
	8611	.load_got => \|sym_index\| .{ .lea_got = sym_index },
	8612	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
8671	8613	},
8672	8614	.fail => \|msg\| {
8673	8615	self.err_msg = msg;
...	...	@@ -8679,9 +8621,9 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
8679	8621
8680	8622	const CallMCValues = struct {
8681	8623	args: []MCValue,
8682		return_value: MCValue,
8683		stack_byte_count: u32,
8684		stack_align: u32,
	8624	return_value: InstTracking,
	8625	stack_byte_count: u31,
	8626	stack_align: u31,
8685	8627
8686	8628	fn deinit(self: CallMCValues, func: Self) void {
8687	8629	func.gpa.free(self.args);
...	...	@@ -8694,6 +8636,7 @@ fn resolveCallingConventionValues(
8694	8636	self: *Self,
8695	8637	fn_ty: Type,
8696	8638	var_args: []const Air.Inst.Ref,
	8639	stack_frame_base: FrameIndex,
8697	8640	) !CallMCValues {
8698	8641	const cc = fn_ty.fnCallingConvention();
8699	8642	const param_len = fn_ty.fnParamLen();
...	...	@@ -8706,7 +8649,7 @@ fn resolveCallingConventionValues(
8706	8649	.args = try self.gpa.alloc(MCValue, param_types.len),
8707	8650	// These undefined values must be populated before returning from this function.
8708	8651	.return_value = undefined,
8709		.stack_byte_count = undefined,
	8652	.stack_byte_count = 0,
8710	8653	.stack_align = undefined,
8711	8654	};
8712	8655	errdefer self.gpa.free(result.args);
...	...	@@ -8716,40 +8659,46 @@ fn resolveCallingConventionValues(
8716	8659	switch (cc) {
8717	8660	.Naked => {
8718	8661	assert(result.args.len == 0);
8719		result.return_value = .unreach;
8720		result.stack_byte_count = 0;
8721		result.stack_align = 1;
8722		return result;
	8662	result.return_value = InstTracking.init(.unreach);
	8663	result.stack_align = 8;
8723	8664	},
8724	8665	.C => {
	8666	var param_reg_i: usize = 0;
	8667	result.stack_align = 16;
	8668
	8669	switch (self.target.os.tag) {
	8670	.windows => {
	8671	// Align the stack to 16bytes before allocating shadow stack space (if any).
	8672	result.stack_byte_count += @intCast(u31, 4 * Type.usize.abiSize(self.target.*));
	8673	},
	8674	else => {},
	8675	}
	8676
8725	8677	// Return values
8726	8678	if (ret_ty.zigTypeTag() == .NoReturn) {
8727		result.return_value = .unreach;
8728		} else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
	8679	result.return_value = InstTracking.init(.unreach);
	8680	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
8729	8681	// TODO: is this even possible for C calling convention?
8730		result.return_value = .none;
	8682	result.return_value = InstTracking.init(.none);
8731	8683	} else {
8732		const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
8733		if (ret_ty_size == 0) {
8734		assert(ret_ty.isError());
8735		result.return_value = .{ .immediate = 0 };
8736		} else if (ret_ty_size <= 8) {
8737		const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8738		result.return_value = .{ .register = aliased_reg };
	8684	const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
	8685	const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
	8686	if (ret_ty_size <= 8) {
	8687	const aliased_reg = registerAlias(ret_reg, ret_ty_size);
	8688	result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8739	8689	} else {
8740		// TODO: return argument cell should go first
8741		result.return_value = .{ .stack_offset = 0 };
	8690	const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
	8691	param_reg_i += 1;
	8692	result.return_value = .{
	8693	.short = .{ .indirect = .{ .reg = ret_reg } },
	8694	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
	8695	};
8742	8696	}
8743	8697	}
8744	8698
8745	8699	// Input params
8746		var next_stack_offset: u32 = switch (result.return_value) {
8747		.stack_offset => \|off\| @intCast(u32, off),
8748		else => 0,
8749		};
8750
8751		for (param_types, result.args, 0..) \|ty, *arg, i\| {
8752		assert(ty.hasRuntimeBits());
	8700	for (param_types, result.args) \|ty, *arg\| {
	8701	assert(ty.hasRuntimeBitsIgnoreComptime());
8753	8702
8754	8703	const classes: []const abi.Class = switch (self.target.os.tag) {
8755	8704	.windows => &[1]abi.Class{abi.classifyWindows(ty, self.target.*)},
...	...	@@ -8760,8 +8709,10 @@ fn resolveCallingConventionValues(
8760	8709	}
8761	8710	switch (classes[0]) {
8762	8711	.integer => blk: {
8763		if (i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk; // fallthrough
8764		arg.* = .{ .register = abi.getCAbiIntParamRegs(self.target.*)[i] };
	8712	if (param_reg_i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk;
	8713	const param_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
	8714	param_reg_i += 1;
	8715	arg.* = .{ .register = param_reg };
8765	8716	continue;
8766	8717	},
8767	8718	.memory => {}, // fallthrough
...	...	@@ -8770,82 +8721,61 @@ fn resolveCallingConventionValues(
8770	8721	}),
8771	8722	}
8772	8723
8773		const param_size = @intCast(u32, ty.abiSize(self.target.*));
8774		const param_align = @intCast(u32, ty.abiAlignment(self.target.*));
8775		const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);
8776		arg.* = .{ .stack_offset = @intCast(i32, offset) };
8777		next_stack_offset = offset;
8778		}
8779
8780		// Align the stack to 16bytes before allocating shadow stack space (if any).
8781		const aligned_next_stack_offset = mem.alignForwardGeneric(u32, next_stack_offset, 16);
8782		const padding = aligned_next_stack_offset - next_stack_offset;
8783		if (padding > 0) {
8784		for (result.args) \|*arg\| {
8785		if (arg.isRegister()) continue;
8786		arg.stack_offset += @intCast(i32, padding);
8787		}
	8724	const param_size = @intCast(u31, ty.abiSize(self.target.*));
	8725	const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
	8726	result.stack_byte_count =
	8727	mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
	8728	arg.* = .{ .load_frame = .{
	8729	.index = stack_frame_base,
	8730	.off = result.stack_byte_count,
	8731	} };
	8732	result.stack_byte_count += param_size;
8788	8733	}
8789
8790		const shadow_stack_space: u32 = switch (self.target.os.tag) {
8791		.windows => @intCast(u32, 4 * @sizeOf(u64)),
8792		else => 0,
8793		};
8794
8795		// alignment padding \| args ... \| shadow stack space (if any) \| ret addr \| $rbp \|
8796		result.stack_byte_count = aligned_next_stack_offset + shadow_stack_space;
8797		result.stack_align = 16;
8798	8734	},
8799	8735	.Unspecified => {
	8736	result.stack_align = 16;
	8737
8800	8738	// Return values
8801	8739	if (ret_ty.zigTypeTag() == .NoReturn) {
8802		result.return_value = .unreach;
8803		} else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
8804		result.return_value = .none;
	8740	result.return_value = InstTracking.init(.unreach);
	8741	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
	8742	result.return_value = InstTracking.init(.none);
8805	8743	} else {
8806		const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
8807		if (ret_ty_size == 0) {
8808		assert(ret_ty.isError());
8809		result.return_value = .{ .immediate = 0 };
8810		} else if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
8811		const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8812		result.return_value = .{ .register = aliased_reg };
	8744	const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
	8745	const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
	8746	if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
	8747	const aliased_reg = registerAlias(ret_reg, ret_ty_size);
	8748	result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8813	8749	} else {
8814		// We simply make the return MCValue a stack offset. However, the actual value
8815		// for the offset will be populated later. We will also push the stack offset
8816		// value into an appropriate register when we resolve the offset.
8817		result.return_value = .{ .stack_offset = 0 };
	8750	const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
	8751	result.return_value = .{
	8752	.short = .{ .indirect = .{ .reg = ret_reg } },
	8753	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
	8754	};
8818	8755	}
8819	8756	}
8820	8757
8821	8758	// Input params
8822		var next_stack_offset: u32 = switch (result.return_value) {
8823		.stack_offset => \|off\| @intCast(u32, off),
8824		else => 0,
8825		};
8826
8827	8759	for (param_types, result.args) \|ty, *arg\| {
8828		if (!ty.hasRuntimeBits()) {
	8760	if (!ty.hasRuntimeBitsIgnoreComptime()) {
8829	8761	arg.* = .none;
8830	8762	continue;
8831	8763	}
8832		const param_size = @intCast(u32, ty.abiSize(self.target.*));
8833		const param_align = @intCast(u32, ty.abiAlignment(self.target.*));
8834		const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);
8835		arg.* = .{ .stack_offset = @intCast(i32, offset) };
8836		next_stack_offset = offset;
	8764	const param_size = @intCast(u31, ty.abiSize(self.target.*));
	8765	const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
	8766	result.stack_byte_count =
	8767	mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
	8768	arg.* = .{ .load_frame = .{
	8769	.index = stack_frame_base,
	8770	.off = result.stack_byte_count,
	8771	} };
	8772	result.stack_byte_count += param_size;
8837	8773	}
8838
8839		result.stack_align = 16;
8840		// TODO fix this so that the 16byte alignment padding is at the current value of $rsp, and push
8841		// the args onto the stack so that there is no padding between the first argument and
8842		// the standard preamble.
8843		// alignment padding \| args ... \| ret addr \| $rbp \|
8844		result.stack_byte_count = mem.alignForwardGeneric(u32, next_stack_offset, result.stack_align);
8845	8774	},
8846	8775	else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
8847	8776	}
8848	8777
	8778	result.stack_byte_count = mem.alignForwardGeneric(u31, result.stack_byte_count, result.stack_align);
8849	8779	return result;
8850	8780	}
8851	8781
...	...	@@ -8953,15 +8883,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 {
8953	8883	return self.regBitSize(ty) - ty.bitSize(self.target.*);
8954	8884	}
8955	8885
8956		fn intrinsicsAllowed(target: Target, ty: Type) bool {
8957		return switch (ty.tag()) {
8958		.f32,
8959		.f64,
8960		=> Target.x86.featureSetHasAny(target.cpu.features, .{ .sse2, .avx, .avx2 }),
8961		else => unreachable, // TODO finish this off
8962		};
8963		}
8964
8965	8886	fn hasAvxSupport(target: Target) bool {
8966	8887	return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
8967	8888	}

src/arch/x86_64/Emit.zig+8-7

...	...	@@ -13,7 +13,9 @@ prev_di_pc: usize,
13	13	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
14	14	relocs: std.ArrayListUnmanaged(Reloc) = .{},
15	15
16		pub const Error = Lower.Error \|\| error{EmitFail};
	16	pub const Error = Lower.Error \|\| error{
	17	EmitFail,
	18	};
17	19
18	20	pub fn emitMir(emit: *Emit) Error!void {
19	21	for (0..emit.lower.mir.instructions.len) \|i\| {
...	...	@@ -22,7 +24,7 @@ pub fn emitMir(emit: *Emit) Error!void {
22	24
23	25	const start_offset = @intCast(u32, emit.code.items.len);
24	26	try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);
25		for (try emit.lower.lowerMir(inst)) \|lower_inst\| try lower_inst.encode(emit.code.writer());
	27	for (try emit.lower.lowerMir(inst)) \|lower_inst\| try lower_inst.encode(emit.code.writer(), .{});
26	28	const end_offset = @intCast(u32, emit.code.items.len);
27	29
28	30	switch (inst.tag) {
...	...	@@ -120,11 +122,10 @@ pub fn emitMir(emit: *Emit) Error!void {
120	122	.length = 6,
121	123	}),
122	124
123		.dbg_line => {
124		const dbg_line_column =
125		emit.lower.mir.extraData(Mir.DbgLineColumn, inst.data.payload).data;
126		try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
127		},
	125	.dbg_line => try emit.dbgAdvancePCAndLine(
	126	inst.data.line_column.line,
	127	inst.data.line_column.column,
	128	),
128	129
129	130	.dbg_prologue_end => {
130	131	switch (emit.debug_output) {

src/arch/x86_64/Encoding.zig+30-25

...	...	@@ -47,17 +47,8 @@ pub fn findByMnemonic(
47	47	},
48	48	else => {},
49	49	} else false;
50		const rex_extended = for (ops) \|op\| switch (op) {
51		.reg => \|r\| if (r.isExtended()) break true,
52		.mem => \|m\| {
53		if (m.base()) \|base\| {
54		if (base.isExtended()) break true;
55		}
56		if (m.scaleIndex()) \|si\| {
57		if (si.index.isExtended()) break true;
58		}
59		},
60		else => {},
	50	const rex_extended = for (ops) \|op\| {
	51	if (op.isBaseExtended() or op.isIndexExtended()) break true;
61	52	} else false;
62	53
63	54	if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;
...	...	@@ -67,11 +58,11 @@ pub fn findByMnemonic(
67	58	next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) \|data\| {
68	59	switch (data.mode) {
69	60	.rex => if (!rex_required) continue,
70		.long => {},
	61	.long, .sse2_long => {},
71	62	else => if (rex_required) continue,
72	63	}
73	64	for (input_ops, data.ops) \|input_op, data_op\|
74		if (!input_op.isSubset(data_op, data.mode)) continue :next;
	65	if (!input_op.isSubset(data_op)) continue :next;
75	66
76	67	const enc = Encoding{ .mnemonic = mnemonic, .data = data };
77	68	if (shortest_enc) \|previous_shortest_enc\| {
...	...	@@ -99,7 +90,7 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
99	90	if (prefixes.rex.w) {
100	91	switch (data.mode) {
101	92	.short, .fpu, .sse, .sse2, .sse4_1, .none => continue,
102		.long, .rex => {},
	93	.long, .sse2_long, .rex => {},
103	94	}
104	95	} else if (prefixes.rex.present and !prefixes.rex.isSet()) {
105	96	switch (data.mode) {
...	...	@@ -147,7 +138,7 @@ pub fn modRmExt(encoding: Encoding) u3 {
147	138	pub fn operandBitSize(encoding: Encoding) u64 {
148	139	switch (encoding.data.mode) {
149	140	.short => return 16,
150		.long => return 64,
	141	.long, .sse2_long => return 64,
151	142	else => {},
152	143	}
153	144	const bit_size: u64 = switch (encoding.data.op_en) {
...	...	@@ -172,7 +163,7 @@ pub fn format(
172	163	_ = options;
173	164	_ = fmt;
174	165	switch (encoding.data.mode) {
175		.long => try writer.writeAll("REX.W + "),
	166	.long, .sse2_long => try writer.writeAll("REX.W + "),
176	167	else => {},
177	168	}
178	169
...	...	@@ -273,6 +264,8 @@ pub const Mnemonic = enum {
273	264	@"test", tzcnt,
274	265	ud2,
275	266	xadd, xchg, xor,
	267	// MMX
	268	movd,
276	269	// SSE
277	270	addss,
278	271	cmpss,
...	...	@@ -287,7 +280,7 @@ pub const Mnemonic = enum {
287	280	//cmpsd,
288	281	divsd,
289	282	maxsd, minsd,
290		movq, //movsd,
	283	movq, //movd, movsd,
291	284	mulsd,
292	285	subsd,
293	286	ucomisd,
...	...	@@ -470,6 +463,17 @@ pub const Op = enum {
470	463	};
471	464	}
472	465
	466	pub fn class(op: Op) bits.Register.Class {
	467	return switch (op) {
	468	else => unreachable,
	469	.al, .ax, .eax, .rax, .cl => .general_purpose,
	470	.r8, .r16, .r32, .r64 => .general_purpose,
	471	.rm8, .rm16, .rm32, .rm64 => .general_purpose,
	472	.sreg => .segment,
	473	.xmm, .xmm_m32, .xmm_m64 => .floating_point,
	474	};
	475	}
	476
473	477	pub fn isFloatingPointRegister(op: Op) bool {
474	478	return switch (op) {
475	479	.xmm, .xmm_m32, .xmm_m64 => true,
...	...	@@ -478,7 +482,7 @@ pub const Op = enum {
478	482	}
479	483
480	484	/// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.
481		pub fn isSubset(op: Op, target: Op, mode: Mode) bool {
	485	pub fn isSubset(op: Op, target: Op) bool {
482	486	switch (op) {
483	487	.m, .o16, .o32, .o64 => unreachable,
484	488	.moffs, .sreg => return op == target,
...	...	@@ -488,13 +492,13 @@ pub const Op = enum {
488	492	},
489	493	else => {
490	494	if (op.isRegister() and target.isRegister()) {
491		switch (mode) {
492		.sse, .sse2, .sse4_1 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
493		else => switch (target) {
494		.cl, .al, .ax, .eax, .rax => return op == target,
495		else => return op.bitSize() == target.bitSize(),
	495	return switch (target) {
	496	.cl, .al, .ax, .eax, .rax => op == target,
	497	else => op.class() == target.class() and switch (target.class()) {
	498	.floating_point => true,
	499	else => op.bitSize() == target.bitSize(),
496	500	},
497		}
	501	};
498	502	}
499	503	if (op.isMemory() and target.isMemory()) {
500	504	switch (target) {
...	...	@@ -532,6 +536,7 @@ pub const Mode = enum {
532	536	long,
533	537	sse,
534	538	sse2,
	539	sse2_long,
535	540	sse4_1,
536	541	};
537	542
...	...	@@ -544,7 +549,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
544	549	std.mem.copy(Operand, &inst.ops, ops);
545	550
546	551	var cwriter = std.io.countingWriter(std.io.null_writer);
547		inst.encode(cwriter.writer()) catch unreachable; // Not allowed to fail here unless OOM.
	552	inst.encode(cwriter.writer(), .{ .allow_frame_loc = true }) catch unreachable; // Not allowed to fail here unless OOM.
548	553	return @intCast(usize, cwriter.bytes_written);
549	554	}
550	555

src/arch/x86_64/Lower.zig+14-19

...	...	@@ -60,6 +60,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
60	60	.mfence,
61	61	.mov,
62	62	.movbe,
	63	.movd,
	64	.movq,
63	65	.movzx,
64	66	.mul,
65	67	.neg,
...	...	@@ -136,7 +138,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
136	138	.setcc => try lower.mirSetcc(inst),
137	139	.jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
138	140
139		.push_regs, .pop_regs => try lower.mirPushPopRegisterList(inst),
	141	.push_regs => try lower.mirPushPopRegisterList(inst, .push),
	142	.pop_regs => try lower.mirPushPopRegisterList(inst, .pop),
140	143
141	144	.dbg_line,
142	145	.dbg_prologue_end,
...	...	@@ -190,7 +193,7 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
190	193	}
191	194
192	195	fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
193		return switch (ops) {
	196	return lower.mir.resolveFrameLoc(switch (ops) {
194	197	.rm_sib,
195	198	.rm_sib_cc,
196	199	.m_sib,
...	...	@@ -227,7 +230,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
227	230	=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
228	231
229	232	else => unreachable,
230		};
	233	});
231	234	}
232	235
233	236	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
...	...	@@ -417,23 +420,15 @@ fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
417	420	}
418	421	}
419	422
420		fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst) Error!void {
421		const save_reg_list = lower.mir.extraData(Mir.SaveRegisterList, inst.data.payload).data;
422		const base = @intToEnum(Register, save_reg_list.base_reg);
423		var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
424		const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
	423	fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {
	424	const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
425	425	const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
426		for (callee_preserved_regs) \|callee_preserved_reg\| {
427		if (!reg_list.isSet(callee_preserved_regs, callee_preserved_reg)) continue;
428		const reg_op = Operand{ .reg = callee_preserved_reg };
429		const mem_op = Operand{ .mem = Memory.sib(.qword, .{ .base = base, .disp = disp }) };
430		try lower.emit(.none, .mov, switch (inst.tag) {
431		.push_regs => &.{ mem_op, reg_op },
432		.pop_regs => &.{ reg_op, mem_op },
433		else => unreachable,
434		});
435		disp += 8;
436		}
	426	var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
	427	.push => .reverse,
	428	.pop => .forward,
	429	else => unreachable,
	430	} });
	431	while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
437	432	}
438	433
439	434	fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {

src/arch/x86_64/Mir.zig+69-39

...	...	@@ -23,6 +23,7 @@ const Register = bits.Register;
23	23	instructions: std.MultiArrayList(Inst).Slice,
24	24	/// The meaning of this data is determined by `Inst.Tag` value.
25	25	extra: []const u32,
	26	frame_locs: std.MultiArrayList(FrameLoc).Slice,
26	27
27	28	pub const Inst = struct {
28	29	tag: Tag,
...	...	@@ -98,6 +99,10 @@ pub const Inst = struct {
98	99	mov,
99	100	/// Move data after swapping bytes
100	101	movbe,
	102	/// Move doubleword
	103	movd,
	104	/// Move quadword
	105	movq,
101	106	/// Move with sign extension
102	107	movsx,
103	108	/// Move with zero extension
...	...	@@ -241,13 +246,13 @@ pub const Inst = struct {
241	246	/// Start of epilogue
242	247	dbg_epilogue_begin,
243	248	/// Update debug line
244		/// Uses `payload` payload with data of type `DbgLineColumn`.
	249	/// Uses `line_column` payload containing the line and column.
245	250	dbg_line,
246	251	/// Push registers
247		/// Uses `payload` payload with data of type `SaveRegisterList`.
	252	/// Uses `payload` payload containing `RegisterList.asInt` directly.
248	253	push_regs,
249	254	/// Pop registers
250		/// Uses `payload` payload with data of type `SaveRegisterList`.
	255	/// Uses `payload` payload containing `RegisterList.asInt` directly.
251	256	pop_regs,
252	257
253	258	/// Tombstone
...	...	@@ -500,6 +505,11 @@ pub const Inst = struct {
500	505	/// Index into the linker's symbol table.
501	506	sym_index: u32,
502	507	},
	508	/// Debug line and column position
	509	line_column: struct {
	510	line: u32,
	511	column: u32,
	512	},
503	513	/// Index into `extra`. Meaning of what can be found there is context-dependent.
504	514	payload: u32,
505	515	};
...	...	@@ -522,12 +532,11 @@ pub const LeaRegisterReloc = struct {
522	532	sym_index: u32,
523	533	};
524	534
525		/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers
526		/// in a compact manner.
	535	/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
527	536	pub const RegisterList = struct {
528	537	bitset: BitSet = BitSet.initEmpty(),
529	538
530		const BitSet = IntegerBitSet(@ctz(@as(u32, 0)));
	539	const BitSet = IntegerBitSet(32);
531	540	const Self = @This();
532	541
533	542	fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
...	...	@@ -547,6 +556,10 @@ pub const RegisterList = struct {
547	556	return self.bitset.isSet(index);
548	557	}
549	558
	559	pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) {
	560	return self.bitset.iterator(options);
	561	}
	562
550	563	pub fn asInt(self: Self) u32 {
551	564	return self.bitset.mask;
552	565	}
...	...	@@ -562,14 +575,6 @@ pub const RegisterList = struct {
562	575	}
563	576	};
564	577
565		pub const SaveRegisterList = struct {
566		/// Base register
567		base_reg: u32,
568		/// Use `RegisterList` to populate.
569		register_list: u32,
570		stack_end: u32,
571		};
572
573	578	pub const Imm64 = struct {
574	579	msb: u32,
575	580	lsb: u32,
...	...	@@ -593,41 +598,51 @@ pub const Imm64 = struct {
593	598	pub const MemorySib = struct {
594	599	/// Size of the pointer.
595	600	ptr_size: u32,
596		/// Base register. -1 means null, or no base register.
597		base: i32,
598		/// Scale for index register. -1 means null, or no scale.
599		/// This has to be in sync with `index` field.
600		scale: i32,
601		/// Index register. -1 means null, or no index register.
602		/// This has to be in sync with `scale` field.
603		index: i32,
	601	/// Base register tag of type Memory.Base.Tag
	602	base_tag: u32,
	603	/// Base register of type Register or FrameIndex
	604	base: u32,
	605	/// Scale starting at bit 0 and index register starting at bit 4.
	606	scale_index: u32,
604	607	/// Displacement value.
605	608	disp: i32,
606	609
607	610	pub fn encode(mem: Memory) MemorySib {
608	611	const sib = mem.sib;
	612	assert(sib.scale_index.scale == 0 or std.math.isPowerOfTwo(sib.scale_index.scale));
609	613	return .{
610	614	.ptr_size = @enumToInt(sib.ptr_size),
611		.base = if (sib.base) \|r\| @enumToInt(r) else -1,
612		.scale = if (sib.scale_index) \|si\| si.scale else -1,
613		.index = if (sib.scale_index) \|si\| @enumToInt(si.index) else -1,
	615	.base_tag = @enumToInt(@as(Memory.Base.Tag, sib.base)),
	616	.base = switch (sib.base) {
	617	.none => undefined,
	618	.reg => \|r\| @enumToInt(r),
	619	.frame => \|fi\| @enumToInt(fi),
	620	},
	621	.scale_index = @as(u32, sib.scale_index.scale) << 0 \|
	622	@as(u32, if (sib.scale_index.scale > 0)
	623	@enumToInt(sib.scale_index.index)
	624	else
	625	undefined) << 4,
614	626	.disp = sib.disp,
615	627	};
616	628	}
617	629
618	630	pub fn decode(msib: MemorySib) Memory {
619		const base: ?Register = if (msib.base == -1) null else @intToEnum(Register, msib.base);
620		const scale_index: ?Memory.ScaleIndex = if (msib.index == -1) null else .{
621		.scale = @intCast(u4, msib.scale),
622		.index = @intToEnum(Register, msib.index),
623		};
624		const mem: Memory = .{ .sib = .{
	631	const scale = @truncate(u4, msib.scale_index);
	632	assert(scale == 0 or std.math.isPowerOfTwo(scale));
	633	return .{ .sib = .{
625	634	.ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),
626		.base = base,
627		.scale_index = scale_index,
	635	.base = switch (@intToEnum(Memory.Base.Tag, msib.base_tag)) {
	636	.none => .none,
	637	.reg => .{ .reg = @intToEnum(Register, msib.base) },
	638	.frame => .{ .frame = @intToEnum(bits.FrameIndex, msib.base) },
	639	},
	640	.scale_index = .{
	641	.scale = scale,
	642	.index = if (scale > 0) @intToEnum(Register, msib.scale_index >> 4) else undefined,
	643	},
628	644	.disp = msib.disp,
629	645	} };
630		return mem;
631	646	}
632	647	};
633	648
...	...	@@ -676,14 +691,10 @@ pub const MemoryMoffs = struct {
676	691	}
677	692	};
678	693
679		pub const DbgLineColumn = struct {
680		line: u32,
681		column: u32,
682		};
683
684	694	pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
685	695	mir.instructions.deinit(gpa);
686	696	gpa.free(mir.extra);
	697	mir.frame_locs.deinit(gpa);
687	698	mir.* = undefined;
688	699	}
689	700
...	...	@@ -704,3 +715,22 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:
704	715	.end = i,
705	716	};
706	717	}
	718
	719	pub const FrameLoc = struct {
	720	base: Register,
	721	disp: i32,
	722	};
	723
	724	pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
	725	return switch (mem) {
	726	.sib => \|sib\| switch (sib.base) {
	727	.none, .reg => mem,
	728	.frame => \|index\| if (mir.frame_locs.len > 0) Memory.sib(sib.ptr_size, .{
	729	.base = .{ .reg = mir.frame_locs.items(.base)[@enumToInt(index)] },
	730	.disp = mir.frame_locs.items(.disp)[@enumToInt(index)] + sib.disp,
	731	.scale_index = mem.scaleIndex(),
	732	}) else mem,
	733	},
	734	.rip, .moffs => mem,
	735	};
	736	}

src/arch/x86_64/bits.zig+68-10

...	...	@@ -405,14 +405,69 @@ test "Register classes" {
405	405	try expect(Register.fs.class() == .segment);
406	406	}
407	407
	408	pub const FrameIndex = enum(u32) {
	409	// This index refers to the start of the arguments passed to this function
	410	args_frame,
	411	// This index refers to the return address pushed by a `call` and popped by a `ret`.
	412	ret_addr,
	413	// This index refers to the base pointer pushed in the prologue and popped in the epilogue.
	414	base_ptr,
	415	// This index refers to the entire stack frame.
	416	stack_frame,
	417	// This index refers to the start of the call frame for arguments passed to called functions
	418	call_frame,
	419	// Other indices are used for local variable stack slots
	420	_,
	421
	422	pub const named_count = @typeInfo(FrameIndex).Enum.fields.len;
	423
	424	pub fn isNamed(fi: FrameIndex) bool {
	425	return @enumToInt(fi) < named_count;
	426	}
	427
	428	pub fn format(
	429	fi: FrameIndex,
	430	comptime fmt: []const u8,
	431	options: std.fmt.FormatOptions,
	432	writer: anytype,
	433	) @TypeOf(writer).Error!void {
	434	try writer.writeAll("FrameIndex");
	435	if (fi.isNamed()) {
	436	try writer.writeByte('.');
	437	try writer.writeAll(@tagName(fi));
	438	} else {
	439	try writer.writeByte('(');
	440	try std.fmt.formatType(@enumToInt(fi), fmt, options, writer, 0);
	441	try writer.writeByte(')');
	442	}
	443	}
	444	};
	445
408	446	pub const Memory = union(enum) {
409	447	sib: Sib,
410	448	rip: Rip,
411	449	moffs: Moffs,
412	450
413		pub const ScaleIndex = packed struct {
	451	pub const Base = union(enum) {
	452	none,
	453	reg: Register,
	454	frame: FrameIndex,
	455
	456	pub const Tag = @typeInfo(Base).Union.tag_type.?;
	457
	458	pub fn isExtended(self: Base) bool {
	459	return switch (self) {
	460	.none, .frame => false, // neither rsp nor rbp are extended
	461	.reg => \|reg\| reg.isExtended(),
	462	};
	463	}
	464	};
	465
	466	pub const ScaleIndex = struct {
414	467	scale: u4,
415	468	index: Register,
	469
	470	const none = ScaleIndex{ .scale = 0, .index = undefined };
416	471	};
417	472
418	473	pub const PtrSize = enum {
...	...	@@ -460,8 +515,8 @@ pub const Memory = union(enum) {
460	515
461	516	pub const Sib = struct {
462	517	ptr_size: PtrSize,
463		base: ?Register,
464		scale_index: ?ScaleIndex,
	518	base: Base,
	519	scale_index: ScaleIndex,
465	520	disp: i32,
466	521	};
467	522
...	...	@@ -482,7 +537,7 @@ pub const Memory = union(enum) {
482	537
483	538	pub fn sib(ptr_size: PtrSize, args: struct {
484	539	disp: i32 = 0,
485		base: ?Register = null,
	540	base: Base = .none,
486	541	scale_index: ?ScaleIndex = null,
487	542	}) Memory {
488	543	if (args.scale_index) \|si\| assert(std.math.isPowerOfTwo(si.scale));
...	...	@@ -490,7 +545,7 @@ pub const Memory = union(enum) {
490	545	.base = args.base,
491	546	.disp = args.disp,
492	547	.ptr_size = ptr_size,
493		.scale_index = args.scale_index,
	548	.scale_index = if (args.scale_index) \|si\| si else ScaleIndex.none,
494	549	} };
495	550	}
496	551
...	...	@@ -502,22 +557,25 @@ pub const Memory = union(enum) {
502	557	return switch (mem) {
503	558	.moffs => true,
504	559	.rip => false,
505		.sib => \|s\| if (s.base) \|r\| r.class() == .segment else false,
	560	.sib => \|s\| switch (s.base) {
	561	.none, .frame => false,
	562	.reg => \|reg\| reg.class() == .segment,
	563	},
506	564	};
507	565	}
508	566
509		pub fn base(mem: Memory) ?Register {
	567	pub fn base(mem: Memory) Base {
510	568	return switch (mem) {
511		.moffs => \|m\| m.seg,
	569	.moffs => \|m\| .{ .reg = m.seg },
512	570	.sib => \|s\| s.base,
513		.rip => null,
	571	.rip => .none,
514	572	};
515	573	}
516	574
517	575	pub fn scaleIndex(mem: Memory) ?ScaleIndex {
518	576	return switch (mem) {
519	577	.moffs, .rip => null,
520		.sib => \|s\| s.scale_index,
	578	.sib => \|s\| if (s.scale_index.scale > 0) s.scale_index else null,
521	579	};
522	580	}
523	581

src/arch/x86_64/encoder.zig+117-94

...	...	@@ -54,6 +54,21 @@ pub const Instruction = struct {
54	54	};
55	55	}
56	56
	57	pub fn isBaseExtended(op: Operand) bool {
	58	return switch (op) {
	59	.none, .imm => false,
	60	.reg => \|reg\| reg.isExtended(),
	61	.mem => \|mem\| mem.base().isExtended(),
	62	};
	63	}
	64
	65	pub fn isIndexExtended(op: Operand) bool {
	66	return switch (op) {
	67	.none, .reg, .imm => false,
	68	.mem => \|mem\| if (mem.scaleIndex()) \|si\| si.index.isExtended() else false,
	69	};
	70	}
	71
57	72	fn format(
58	73	op: Operand,
59	74	comptime unused_format_string: []const u8,
...	...	@@ -98,17 +113,24 @@ pub const Instruction = struct {
98	113	try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});
99	114
100	115	if (mem.isSegmentRegister()) {
101		return writer.print("{s}:0x{x}", .{ @tagName(sib.base.?), sib.disp });
	116	return writer.print("{s}:0x{x}", .{ @tagName(sib.base.reg), sib.disp });
102	117	}
103	118
104	119	try writer.writeByte('[');
105	120
106	121	var any = false;
107		if (sib.base) \|base\| {
108		try writer.print("{s}", .{@tagName(base)});
109		any = true;
	122	switch (sib.base) {
	123	.none => {},
	124	.reg => \|reg\| {
	125	try writer.print("{s}", .{@tagName(reg)});
	126	any = true;
	127	},
	128	.frame => \|frame\| {
	129	try writer.print("{}", .{frame});
	130	any = true;
	131	},
110	132	}
111		if (sib.scale_index) \|si\| {
	133	if (mem.scaleIndex()) \|si\| {
112	134	if (any) try writer.writeAll(" + ");
113	135	try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });
114	136	any = true;
...	...	@@ -182,8 +204,8 @@ pub const Instruction = struct {
182	204	}
183	205	}
184	206
185		pub fn encode(inst: Instruction, writer: anytype) !void {
186		const encoder = Encoder(@TypeOf(writer)){ .writer = writer };
	207	pub fn encode(inst: Instruction, writer: anytype, comptime opts: Options) !void {
	208	const encoder = Encoder(@TypeOf(writer), opts){ .writer = writer };
187	209	const enc = inst.encoding;
188	210	const data = enc.data;
189	211
...	...	@@ -269,22 +291,24 @@ pub const Instruction = struct {
269	291
270	292	const segment_override: ?Register = switch (op_en) {
271	293	.i, .zi, .o, .oi, .d, .np => null,
272		.fd => inst.ops[1].mem.base().?,
273		.td => inst.ops[0].mem.base().?,
274		.rm, .rmi => if (inst.ops[1].isSegmentRegister()) blk: {
275		break :blk switch (inst.ops[1]) {
276		.reg => \|r\| r,
277		.mem => \|m\| m.base().?,
	294	.fd => inst.ops[1].mem.base().reg,
	295	.td => inst.ops[0].mem.base().reg,
	296	.rm, .rmi => if (inst.ops[1].isSegmentRegister())
	297	switch (inst.ops[1]) {
	298	.reg => \|reg\| reg,
	299	.mem => \|mem\| mem.base().reg,
278	300	else => unreachable,
279		};
280		} else null,
281		.m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister()) blk: {
282		break :blk switch (inst.ops[0]) {
283		.reg => \|r\| r,
284		.mem => \|m\| m.base().?,
	301	}
	302	else
	303	null,
	304	.m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister())
	305	switch (inst.ops[0]) {
	306	.reg => \|reg\| reg,
	307	.mem => \|mem\| mem.base().reg,
285	308	else => unreachable,
286		};
287		} else null,
	309	}
	310	else
	311	null,
288	312	};
289	313	if (segment_override) \|seg\| {
290	314	legacy.setSegmentOverride(seg);
...	...	@@ -298,7 +322,10 @@ pub const Instruction = struct {
298	322
299	323	var rex = Rex{};
300	324	rex.present = inst.encoding.data.mode == .rex;
301		rex.w = inst.encoding.data.mode == .long;
	325	switch (inst.encoding.data.mode) {
	326	.long, .sse2_long => rex.w = true,
	327	else => {},
	328	}
302	329
303	330	switch (op_en) {
304	331	.np, .i, .zi, .fd, .td, .d => {},
...	...	@@ -307,27 +334,17 @@ pub const Instruction = struct {
307	334	const r_op = switch (op_en) {
308	335	.rm, .rmi => inst.ops[0],
309	336	.mr, .mri, .mrc => inst.ops[1],
310		else => null,
	337	else => .none,
311	338	};
312		if (r_op) \|op\| {
313		rex.r = op.reg.isExtended();
314		}
	339	rex.r = r_op.isBaseExtended();
315	340
316	341	const b_x_op = switch (op_en) {
317	342	.rm, .rmi => inst.ops[1],
318	343	.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
319	344	else => unreachable,
320	345	};
321		switch (b_x_op) {
322		.reg => \|r\| {
323		rex.b = r.isExtended();
324		},
325		.mem => \|mem\| {
326		rex.b = if (mem.base()) \|base\| base.isExtended() else false;
327		rex.x = if (mem.scaleIndex()) \|si\| si.index.isExtended() else false;
328		},
329		else => unreachable,
330		}
	346	rex.b = b_x_op.isBaseExtended();
	347	rex.x = b_x_op.isIndexExtended();
331	348	},
332	349	}
333	350
...	...	@@ -348,72 +365,75 @@ pub const Instruction = struct {
348	365
349	366	switch (mem) {
350	367	.moffs => unreachable,
351		.sib => \|sib\| {
352		if (sib.base) \|base\| {
353		if (base.class() == .segment) {
354		// TODO audit this wrt SIB
355		try encoder.modRm_SIBDisp0(operand_enc);
356		if (sib.scale_index) \|si\| {
357		const scale = math.log2_int(u4, si.scale);
358		try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
359		} else {
360		try encoder.sib_disp32();
361		}
362		try encoder.disp32(sib.disp);
	368	.sib => \|sib\| switch (sib.base) {
	369	.none => {
	370	try encoder.modRm_SIBDisp0(operand_enc);
	371	if (mem.scaleIndex()) \|si\| {
	372	const scale = math.log2_int(u4, si.scale);
	373	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
363	374	} else {
364		assert(base.class() == .general_purpose);
365		const dst = base.lowEnc();
366		const src = operand_enc;
367		if (dst == 4 or sib.scale_index != null) {
368		if (sib.disp == 0 and dst != 5) {
369		try encoder.modRm_SIBDisp0(src);
370		if (sib.scale_index) \|si\| {
371		const scale = math.log2_int(u4, si.scale);
372		try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
373		} else {
374		try encoder.sib_base(dst);
375		}
376		} else if (math.cast(i8, sib.disp)) \|_\| {
377		try encoder.modRm_SIBDisp8(src);
378		if (sib.scale_index) \|si\| {
379		const scale = math.log2_int(u4, si.scale);
380		try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
381		} else {
382		try encoder.sib_baseDisp8(dst);
383		}
384		try encoder.disp8(@truncate(i8, sib.disp));
385		} else {
386		try encoder.modRm_SIBDisp32(src);
387		if (sib.scale_index) \|si\| {
388		const scale = math.log2_int(u4, si.scale);
389		try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
390		} else {
391		try encoder.sib_baseDisp32(dst);
392		}
393		try encoder.disp32(sib.disp);
394		}
395		} else {
396		if (sib.disp == 0 and dst != 5) {
397		try encoder.modRm_indirectDisp0(src, dst);
398		} else if (math.cast(i8, sib.disp)) \|_\| {
399		try encoder.modRm_indirectDisp8(src, dst);
400		try encoder.disp8(@truncate(i8, sib.disp));
401		} else {
402		try encoder.modRm_indirectDisp32(src, dst);
403		try encoder.disp32(sib.disp);
404		}
405		}
	375	try encoder.sib_disp32();
406	376	}
407		} else {
	377	try encoder.disp32(sib.disp);
	378	},
	379	.reg => \|base\| if (base.class() == .segment) {
	380	// TODO audit this wrt SIB
408	381	try encoder.modRm_SIBDisp0(operand_enc);
409		if (sib.scale_index) \|si\| {
	382	if (mem.scaleIndex()) \|si\| {
410	383	const scale = math.log2_int(u4, si.scale);
411	384	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
412	385	} else {
413	386	try encoder.sib_disp32();
414	387	}
415	388	try encoder.disp32(sib.disp);
416		}
	389	} else {
	390	assert(base.class() == .general_purpose);
	391	const dst = base.lowEnc();
	392	const src = operand_enc;
	393	if (dst == 4 or mem.scaleIndex() != null) {
	394	if (sib.disp == 0 and dst != 5) {
	395	try encoder.modRm_SIBDisp0(src);
	396	if (mem.scaleIndex()) \|si\| {
	397	const scale = math.log2_int(u4, si.scale);
	398	try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
	399	} else {
	400	try encoder.sib_base(dst);
	401	}
	402	} else if (math.cast(i8, sib.disp)) \|_\| {
	403	try encoder.modRm_SIBDisp8(src);
	404	if (mem.scaleIndex()) \|si\| {
	405	const scale = math.log2_int(u4, si.scale);
	406	try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
	407	} else {
	408	try encoder.sib_baseDisp8(dst);
	409	}
	410	try encoder.disp8(@truncate(i8, sib.disp));
	411	} else {
	412	try encoder.modRm_SIBDisp32(src);
	413	if (mem.scaleIndex()) \|si\| {
	414	const scale = math.log2_int(u4, si.scale);
	415	try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
	416	} else {
	417	try encoder.sib_baseDisp32(dst);
	418	}
	419	try encoder.disp32(sib.disp);
	420	}
	421	} else {
	422	if (sib.disp == 0 and dst != 5) {
	423	try encoder.modRm_indirectDisp0(src, dst);
	424	} else if (math.cast(i8, sib.disp)) \|_\| {
	425	try encoder.modRm_indirectDisp8(src, dst);
	426	try encoder.disp8(@truncate(i8, sib.disp));
	427	} else {
	428	try encoder.modRm_indirectDisp32(src, dst);
	429	try encoder.disp32(sib.disp);
	430	}
	431	}
	432	},
	433	.frame => if (@TypeOf(encoder).options.allow_frame_loc) {
	434	try encoder.modRm_indirectDisp32(operand_enc, undefined);
	435	try encoder.disp32(undefined);
	436	} else return error.CannotEncode,
417	437	},
418	438	.rip => \|rip\| {
419	439	try encoder.modRm_RIPDisp32(operand_enc);
...	...	@@ -482,11 +502,14 @@ pub const LegacyPrefixes = packed struct {
482	502	}
483	503	};
484	504
485		fn Encoder(comptime T: type) type {
	505	pub const Options = struct { allow_frame_loc: bool = false };
	506
	507	fn Encoder(comptime T: type, comptime opts: Options) type {
486	508	return struct {
487	509	writer: T,
488	510
489	511	const Self = @This();
	512	pub const options = opts;
490	513
491	514	// --------
492	515	// Prefixes

src/arch/x86_64/encodings.zig+6

...	...	@@ -860,6 +860,12 @@ pub const table = [_]Entry{
860	860
861	861	.{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },
862	862
	863	.{ .movd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2 },
	864	.{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2 },
	865
	866	.{ .movq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2_long },
	867	.{ .movq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2_long },
	868
863	869	.{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },
864	870	.{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },
865	871

src/link/Coff.zig+19-7

...	...	@@ -492,7 +492,7 @@ fn growSection(self: *Coff, sect_id: u32, needed_size: u32) !void {
492	492
493	493	const sect_vm_capacity = self.allocatedVirtualSize(header.virtual_address);
494	494	if (needed_size > sect_vm_capacity) {
495		self.markRelocsDirtyByAddress(header.virtual_address + needed_size);
	495	self.markRelocsDirtyByAddress(header.virtual_address + header.virtual_size);
496	496	try self.growSectionVirtualMemory(sect_id, needed_size);
497	497	}
498	498
...	...	@@ -759,7 +759,9 @@ fn writeAtom(self: *Coff, atom_index: Atom.Index, code: []u8) !void {
759	759	if (self.relocs.getPtr(atom_index)) \|rels\| {
760	760	try relocs.ensureTotalCapacityPrecise(rels.items.len);
761	761	for (rels.items) \|*reloc\| {
762		if (reloc.isResolvable(self)) relocs.appendAssumeCapacity(reloc);
	762	if (reloc.isResolvable(self) and reloc.dirty) {
	763	relocs.appendAssumeCapacity(reloc);
	764	}
763	765	}
764	766	}
765	767
...	...	@@ -904,18 +906,28 @@ fn markRelocsDirtyByTarget(self: *Coff, target: SymbolWithLoc) void {
904	906	}
905	907
906	908	fn markRelocsDirtyByAddress(self: *Coff, addr: u32) void {
	909	const got_moved = blk: {
	910	const sect_id = self.got_section_index orelse break :blk false;
	911	break :blk self.sections.items(.header)[sect_id].virtual_address > addr;
	912	};
	913
	914	// TODO: dirty relocations targeting import table if that got moved in memory
	915
907	916	for (self.relocs.values()) \|*relocs\| {
908	917	for (relocs.items) \|*reloc\| {
909		const target_vaddr = reloc.getTargetAddress(self) orelse continue;
910		if (target_vaddr < addr) continue;
911		reloc.dirty = true;
	918	if (reloc.isGotIndirection()) {
	919	reloc.dirty = reloc.dirty or got_moved;
	920	} else {
	921	const target_vaddr = reloc.getTargetAddress(self) orelse continue;
	922	if (target_vaddr > addr) reloc.dirty = true;
	923	}
912	924	}
913	925	}
914	926
915	927	// TODO: dirty only really affected GOT cells
916	928	for (self.got_table.entries.items) \|entry\| {
917	929	const target_addr = self.getSymbol(entry).value;
918		if (target_addr >= addr) {
	930	if (target_addr > addr) {
919	931	self.got_table_contents_dirty = true;
920	932	break;
921	933	}
...	...	@@ -1624,7 +1636,7 @@ pub fn flushModule(self: Coff, comp: Compilation, prog_node: *std.Progress.Nod
1624	1636
1625	1637	for (self.relocs.keys(), self.relocs.values()) \|atom_index, relocs\| {
1626	1638	const needs_update = for (relocs.items) \|reloc\| {
1627		if (reloc.isResolvable(self)) break true;
	1639	if (reloc.dirty) break true;
1628	1640	} else false;
1629	1641
1630	1642	if (!needs_update) continue;

src/link/Coff/Relocation.zig+18-12

...	...	@@ -45,6 +45,19 @@ pcrel: bool,
45	45	length: u2,
46	46	dirty: bool = true,
47	47
	48	/// Returns true if and only if the reloc can be resolved.
	49	pub fn isResolvable(self: Relocation, coff_file: *Coff) bool {
	50	_ = self.getTargetAddress(coff_file) orelse return false;
	51	return true;
	52	}
	53
	54	pub fn isGotIndirection(self: Relocation) bool {
	55	return switch (self.type) {
	56	.got, .got_page, .got_pageoff => true,
	57	else => false,
	58	};
	59	}
	60
48	61	/// Returns address of the target if any.
49	62	pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {
50	63	switch (self.type) {
...	...	@@ -53,11 +66,6 @@ pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {
53	66	const header = coff_file.sections.items(.header)[coff_file.got_section_index.?];
54	67	return header.virtual_address + got_index * coff_file.ptr_width.size();
55	68	},
56		.direct, .page, .pageoff => {
57		const target_atom_index = coff_file.getAtomIndexForSymbol(self.target) orelse return null;
58		const target_atom = coff_file.getAtom(target_atom_index);
59		return target_atom.getSymbol(coff_file).value;
60		},
61	69	.import, .import_page, .import_pageoff => {
62	70	const sym = coff_file.getSymbol(self.target);
63	71	const index = coff_file.import_tables.getIndex(sym.value) orelse return null;
...	...	@@ -68,16 +76,14 @@ pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {
68	76	.name_off = sym.value,
69	77	});
70	78	},
	79	else => {
	80	const target_atom_index = coff_file.getAtomIndexForSymbol(self.target) orelse return null;
	81	const target_atom = coff_file.getAtom(target_atom_index);
	82	return target_atom.getSymbol(coff_file).value;
	83	},
71	84	}
72	85	}
73	86
74		/// Returns true if and only if the reloc is dirty AND the target address is available.
75		pub fn isResolvable(self: Relocation, coff_file: *Coff) bool {
76		const addr = self.getTargetAddress(coff_file) orelse return false;
77		if (addr == 0) return false;
78		return self.dirty;
79		}
80
81	87	pub fn resolve(self: Relocation, atom_index: Atom.Index, code: []u8, image_base: u64, coff_file: *Coff) void {
82	88	const atom = coff_file.getAtom(atom_index);
83	89	const source_sym = atom.getSymbol(coff_file);

src/link/MachO.zig+26-8

...	...	@@ -1146,7 +1146,9 @@ pub fn writeAtom(self: *MachO, atom_index: Atom.Index, code: []u8) !void {
1146	1146	if (self.relocs.getPtr(atom_index)) \|rels\| {
1147	1147	try relocs.ensureTotalCapacityPrecise(rels.items.len);
1148	1148	for (rels.items) \|*reloc\| {
1149		if (reloc.isResolvable(self)) relocs.appendAssumeCapacity(reloc);
	1149	if (reloc.isResolvable(self) and reloc.dirty) {
	1150	relocs.appendAssumeCapacity(reloc);
	1151	}
1150	1152	}
1151	1153	}
1152	1154
...	...	@@ -1332,18 +1334,33 @@ fn markRelocsDirtyByTarget(self: *MachO, target: SymbolWithLoc) void {
1332	1334
1333	1335	fn markRelocsDirtyByAddress(self: *MachO, addr: u64) void {
1334	1336	log.debug("marking relocs dirty by address: {x}", .{addr});
	1337
	1338	const got_moved = blk: {
	1339	const sect_id = self.got_section_index orelse break :blk false;
	1340	break :blk self.sections.items(.header)[sect_id].addr > addr;
	1341	};
	1342	const stubs_moved = blk: {
	1343	const sect_id = self.stubs_section_index orelse break :blk false;
	1344	break :blk self.sections.items(.header)[sect_id].addr > addr;
	1345	};
	1346
1335	1347	for (self.relocs.values()) \|*relocs\| {
1336	1348	for (relocs.items) \|*reloc\| {
1337		const target_addr = reloc.getTargetBaseAddress(self) orelse continue;
1338		if (target_addr < addr) continue;
1339		reloc.dirty = true;
	1349	if (reloc.isGotIndirection()) {
	1350	reloc.dirty = reloc.dirty or got_moved;
	1351	} else if (reloc.isStubTrampoline(self)) {
	1352	reloc.dirty = reloc.dirty or stubs_moved;
	1353	} else {
	1354	const target_addr = reloc.getTargetBaseAddress(self) orelse continue;
	1355	if (target_addr > addr) reloc.dirty = true;
	1356	}
1340	1357	}
1341	1358	}
1342	1359
1343	1360	// TODO: dirty only really affected GOT cells
1344	1361	for (self.got_table.entries.items) \|entry\| {
1345	1362	const target_addr = self.getSymbol(entry).n_value;
1346		if (target_addr >= addr) {
	1363	if (target_addr > addr) {
1347	1364	self.got_table_contents_dirty = true;
1348	1365	break;
1349	1366	}
...	...	@@ -1353,7 +1370,7 @@ fn markRelocsDirtyByAddress(self: *MachO, addr: u64) void {
1353	1370	const stubs_addr = self.getSegment(self.stubs_section_index.?).vmaddr;
1354	1371	const stub_helper_addr = self.getSegment(self.stub_helper_section_index.?).vmaddr;
1355	1372	const laptr_addr = self.getSegment(self.la_symbol_ptr_section_index.?).vmaddr;
1356		if (stubs_addr >= addr or stub_helper_addr >= addr or laptr_addr >= addr)
	1373	if (stubs_addr > addr or stub_helper_addr > addr or laptr_addr > addr)
1357	1374	self.stub_table_contents_dirty = true;
1358	1375	}
1359	1376	}
...	...	@@ -2794,7 +2811,7 @@ fn growSection(self: *MachO, sect_id: u8, needed_size: u64) !void {
2794	2811
2795	2812	const sect_vm_capacity = self.allocatedVirtualSize(segment.vmaddr);
2796	2813	if (needed_size > sect_vm_capacity) {
2797		self.markRelocsDirtyByAddress(segment.vmaddr + needed_size);
	2814	self.markRelocsDirtyByAddress(segment.vmaddr + segment.vmsize);
2798	2815	try self.growSectionVirtualMemory(sect_id, needed_size);
2799	2816	}
2800	2817
...	...	@@ -4067,11 +4084,12 @@ pub fn findFirst(comptime T: type, haystack: []align(1) const T, start: usize, p
4067	4084	pub fn logSections(self: *MachO) void {
4068	4085	log.debug("sections:", .{});
4069	4086	for (self.sections.items(.header), 0..) \|header, i\| {
4070		log.debug(" sect({d}): {s},{s} @{x}, sizeof({x})", .{
	4087	log.debug(" sect({d}): {s},{s} @{x} ({x}), sizeof({x})", .{
4071	4088	i + 1,
4072	4089	header.segName(),
4073	4090	header.sectName(),
4074	4091	header.offset,
	4092	header.addr,
4075	4093	header.size,
4076	4094	});
4077	4095	}

src/link/MachO/Relocation.zig+27-14

...	...	@@ -37,14 +37,33 @@ pub const Type = enum {
37	37	tlv_initializer,
38	38	};
39	39
40		/// Returns true if and only if the reloc is dirty AND the target address is available.
	40	/// Returns true if and only if the reloc can be resolved.
41	41	pub fn isResolvable(self: Relocation, macho_file: *MachO) bool {
42		const addr = self.getTargetBaseAddress(macho_file) orelse return false;
43		if (addr == 0) return false;
44		return self.dirty;
	42	_ = self.getTargetBaseAddress(macho_file) orelse return false;
	43	return true;
	44	}
	45
	46	pub fn isGotIndirection(self: Relocation) bool {
	47	return switch (self.type) {
	48	.got, .got_page, .got_pageoff => true,
	49	else => false,
	50	};
	51	}
	52
	53	pub fn isStubTrampoline(self: Relocation, macho_file: *MachO) bool {
	54	return switch (self.type) {
	55	.branch => macho_file.getSymbol(self.target).undf(),
	56	else => false,
	57	};
45	58	}
46	59
47	60	pub fn getTargetBaseAddress(self: Relocation, macho_file: *MachO) ?u64 {
	61	if (self.isStubTrampoline(macho_file)) {
	62	const index = macho_file.stub_table.lookup.get(self.target) orelse return null;
	63	const header = macho_file.sections.items(.header)[macho_file.stubs_section_index.?];
	64	return header.addr +
	65	index * @import("stubs.zig").calcStubEntrySize(macho_file.base.options.target.cpu.arch);
	66	}
48	67	switch (self.type) {
49	68	.got, .got_page, .got_pageoff => {
50	69	const got_index = macho_file.got_table.lookup.get(self.target) orelse return null;
...	...	@@ -56,17 +75,11 @@ pub fn getTargetBaseAddress(self: Relocation, macho_file: *MachO) ?u64 {
56	75	const atom = macho_file.getAtom(atom_index);
57	76	return atom.getSymbol(macho_file).n_value;
58	77	},
59		.branch => {
60		if (macho_file.stub_table.lookup.get(self.target)) \|index\| {
61		const header = macho_file.sections.items(.header)[macho_file.stubs_section_index.?];
62		return header.addr +
63		index * @import("stubs.zig").calcStubEntrySize(macho_file.base.options.target.cpu.arch);
64		}
65		const atom_index = macho_file.getAtomIndexForSymbol(self.target) orelse return null;
66		const atom = macho_file.getAtom(atom_index);
67		return atom.getSymbol(macho_file).n_value;
	78	else => {
	79	const target_atom_index = macho_file.getAtomIndexForSymbol(self.target) orelse return null;
	80	const target_atom = macho_file.getAtom(target_atom_index);
	81	return target_atom.getSymbol(macho_file).n_value;
68	82	},
69		else => return macho_file.getSymbol(self.target).n_value,
70	83	}
71	84	}
72	85

test/behavior/align.zig+4-1

...	...	@@ -491,7 +491,10 @@ test "read 128-bit field from default aligned struct in global memory" {
491	491	}
492	492
493	493	test "struct field explicit alignment" {
494		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	494	if (builtin.zig_backend == .stage2_x86_64) {
	495	// Careful enabling this test, fails randomly.
	496	return error.SkipZigTest;
	497	}
495	498	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
496	499	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
497	500	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/basic.zig-1

...	...	@@ -785,7 +785,6 @@ fn manyptrConcat(comptime s: [:0]const u8) [:0]const u8 {
785	785	test "comptime manyptr concatenation" {
786	786	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
787	787	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
788		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
789	788	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
790	789
791	790	const s = "epic";

test/behavior/bugs/12571.zig-1

...	...	@@ -12,7 +12,6 @@ const Entry = packed struct {
12	12	};
13	13
14	14	test {
15		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
16	15	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
17	16	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
18	17

test/behavior/bugs/12776.zig+4-1

...	...	@@ -32,7 +32,10 @@ test {
32	32	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
33	33	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
34	34	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
35		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	35	if (builtin.zig_backend == .stage2_x86_64) {
	36	// Careful enabling this test, fails randomly.
	37	return error.SkipZigTest;
	38	}
36	39
37	40	var ram = try RAM.new();
38	41	var cpu = try CPU.new(&ram);

test/behavior/bugs/13069.zig-1

...	...	@@ -4,7 +4,6 @@ const expect = std.testing.expect;
4	4
5	5	test {
6	6	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
7		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8	7	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9	8	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
10	9

test/behavior/eval.zig-1

...	...	@@ -532,7 +532,6 @@ test "@tagName of @typeInfo" {
532	532	}
533	533
534	534	test "static eval list init" {
535		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
536	535	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
537	536	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
538	537	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/fn.zig-2

...	...	@@ -335,7 +335,6 @@ fn numberLiteralArg(a: anytype) !void {
335	335	}
336	336
337	337	test "function call with anon list literal" {
338		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
339	338	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
340	339	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
341	340	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -356,7 +355,6 @@ test "function call with anon list literal" {
356	355	}
357	356
358	357	test "function call with anon list literal - 2D" {
359		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
360	358	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
361	359	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
362	360	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/math.zig-2

...	...	@@ -734,7 +734,6 @@ test "small int addition" {
734	734	test "basic @mulWithOverflow" {
735	735	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
736	736	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
737		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
738	737
739	738	{
740	739	var a: u8 = 86;
...	...	@@ -1126,7 +1125,6 @@ test "allow signed integer division/remainder when values are comptime-known and
1126	1125
1127	1126	test "quad hex float literal parsing accurate" {
1128	1127	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1129		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1130	1128	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1131	1129	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1132	1130	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/packed-struct.zig-2

...	...	@@ -232,7 +232,6 @@ test "nested packed structs" {
232	232	}
233	233
234	234	test "regular in irregular packed struct" {
235		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
236	235	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
237	236	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
238	237	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -621,7 +620,6 @@ test "store undefined to packed result location" {
621	620	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
622	621	if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
623	622	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
624		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
625	623
626	624	var x: u4 = 0;
627	625	var s = packed struct { x: u4, y: u4 }{ .x = x, .y = if (x > 0) x else undefined };

test/behavior/pointers.zig-1

...	...	@@ -343,7 +343,6 @@ test "pointer sentinel with optional element" {
343	343	test "pointer sentinel with +inf" {
344	344	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
345	345	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
346		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
347	346	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
348	347	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
349	348

test/behavior/popcount.zig-1

...	...	@@ -4,7 +4,6 @@ const expect = std.testing.expect;
4	4	const expectEqual = std.testing.expectEqual;
5	5
6	6	test "@popCount integers" {
7		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8	7	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9	8	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10	9	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/union.zig-1

...	...	@@ -26,7 +26,6 @@ fn setFloat(foo: *FooWithFloats, x: f64) void {
26	26	}
27	27
28	28	test "init union with runtime value - floats" {
29		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
30	29	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
31	30	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
32	31	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/vector.zig+4-4

...	...	@@ -318,7 +318,6 @@ test "vector @splat" {
318	318	}
319	319
320	320	test "load vector elements via comptime index" {
321		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
322	321	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
323	322	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
324	323	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -340,7 +339,6 @@ test "load vector elements via comptime index" {
340	339	}
341	340
342	341	test "store vector elements via comptime index" {
343		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
344	342	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
345	343	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
346	344	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -368,7 +366,6 @@ test "store vector elements via comptime index" {
368	366	}
369	367
370	368	test "load vector elements via runtime index" {
371		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
372	369	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
373	370	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
374	371	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -390,7 +387,6 @@ test "load vector elements via runtime index" {
390	387	}
391	388
392	389	test "store vector elements via runtime index" {
393		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
394	390	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
395	391	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
396	392	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -1100,6 +1096,7 @@ test "loading the second vector from a slice of vectors" {
1100	1096	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1101	1097	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1102	1098	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
	1099	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1103	1100
1104	1101	@setRuntimeSafety(false);
1105	1102	var small_bases = [2]@Vector(2, u8){
...	...	@@ -1200,6 +1197,7 @@ test "zero multiplicand" {
1200	1197	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1201	1198	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1202	1199	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
	1200	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1203	1201
1204	1202	const zeros = @Vector(2, u32){ 0.0, 0.0 };
1205	1203	var ones = @Vector(2, u32){ 1.0, 1.0 };
...	...	@@ -1259,6 +1257,7 @@ test "load packed vector element" {
1259	1257	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1260	1258	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1261	1259	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
	1260	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1262	1261
1263	1262	var x: @Vector(2, u15) = .{ 1, 4 };
1264	1263	try expect((&x[0]).* == 1);
...	...	@@ -1297,6 +1296,7 @@ test "store to vector in slice" {
1297	1296	test "addition of vectors represented as strings" {
1298	1297	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1299	1298	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
	1299	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1300	1300
1301	1301	const V = @Vector(3, u8);
1302	1302	const foo: V = "foo".*;

test/src/Cases.zig+7

...	...	@@ -390,6 +390,13 @@ fn addFromDirInner(
390	390	// Cross-product to get all possible test combinations
391	391	for (backends) \|backend\| {
392	392	for (targets) \|target\| {
	393	if (backend == .stage2 and
	394	target.getCpuArch() != .wasm32 and target.getCpuArch() != .x86_64)
	395	{
	396	// Other backends don't support new liveness format
	397	continue;
	398	}
	399
393	400	const next = ctx.cases.items.len;
394	401	try ctx.cases.append(.{
395	402	.name = std.fs.path.stem(filename),