Commit 26d4f9b69e087f3a9ef818f878f12affc94317f9 | zig

author	Jacob Young <jacobly@ziglang.org> 2023-04-23 22:25:53-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-04-26 19:05:17-04:00
log	26d4f9b69e087f3a9ef818f878f12affc94317f9
tree	6f9029aca24e97d961fcfbb10c09191b568eae83
parent	f618398b24acdc3317e6fd81f486d49176ffcef9

x86_64: add frame indices

This allows abstract references to the stack frame which enables:
 * deferred frame layout until after the function has been processed
 * frame compaction (sorting by alignment)
 * being able to overalign the stack
 * references that change based on an overaligned stack
 * moving callee saved spills to the usual part (smaller code size)
 * shared call frame (avoids stack adjustments before and after calls)

7 files changed, 2503 insertions(+), 2542 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    bits.zig
    encoder.zig

src/arch/x86_64/CodeGen.zig+2234-2362

...	...	@@ -6,6 +6,7 @@ const codegen = @import("../../codegen.zig");
6	6	const leb128 = std.leb;
7	7	const link = @import("../../link.zig");
8	8	const log = std.log.scoped(.codegen);
	9	const tracking_log = std.log.scoped(.tracking);
9	10	const math = std.math;
10	11	const mem = std.mem;
11	12	const trace = @import("../../tracy.zig").trace;
...	...	@@ -40,6 +41,7 @@ const Memory = bits.Memory;
40	41	const Register = bits.Register;
41	42	const RegisterManager = abi.RegisterManager;
42	43	const RegisterLock = RegisterManager.RegisterLock;
	44	const FrameIndex = bits.FrameIndex;
43	45
44	46	const gp = abi.RegisterClass.gp;
45	47	const sse = abi.RegisterClass.sse;
...	...	@@ -47,6 +49,7 @@ const sse = abi.RegisterClass.sse;
47	49	const InnerError = CodeGenError \|\| error{OutOfRegisters};
48	50
49	51	const debug_wip_mir = false;
	52	const debug_tracking = false;
50	53
51	54	gpa: Allocator,
52	55	air: Air,
...	...	@@ -57,11 +60,10 @@ target: *const std.Target,
57	60	mod_fn: *const Module.Fn,
58	61	err_msg: ?*ErrorMsg,
59	62	args: []MCValue,
60		ret_mcv: MCValue,
	63	ret_mcv: InstTracking,
61	64	fn_type: Type,
62	65	arg_index: u32,
63	66	src_loc: Module.SrcLoc,
64		stack_align: u32,
65	67
66	68	eflags_inst: ?Air.Inst.Index = null,
67	69
...	...	@@ -86,17 +88,13 @@ inst_tracking: InstTrackingMap = .{},
86	88	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
87	89
88	90	register_manager: RegisterManager = .{},
89		/// Maps offset to what is stored there.
90		stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
91	91
92	92	/// Generation of the current scope, increments by 1 for every entered scope.
93	93	scope_generation: u32 = 0,
94	94
95		/// Offset from the stack base, representing the end of the stack frame.
96		max_end_stack: u32 = 0,
97		/// Represents the current end stack offset. If there is no existing slot
98		/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
99		next_stack_offset: u32 = 0,
	95	frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
	96	free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .{},
	97	frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
100	98
101	99	/// Debug field, used to find bugs in the compiler.
102	100	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
...	...	@@ -108,6 +106,9 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}
108	106
109	107	const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged(Mir.Inst.Index, Air.Inst.Index){} else {};
110	108
	109	const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
	110	const RegisterOffset = struct { reg: Register, off: i32 = 0 };
	111
111	112	pub const MCValue = union(enum) {
112	113	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
113	114	/// TODO Look into deleting this tag and using `dead` instead, since every use
...	...	@@ -123,46 +124,59 @@ pub const MCValue = union(enum) {
123	124	/// A pointer-sized integer that fits in a register.
124	125	/// If the type is a pointer, this is the pointer address in virtual address space.
125	126	immediate: u64,
126		/// The value is in a GP register.
	127	/// The value resides in the EFLAGS register.
	128	eflags: Condition,
	129	/// The value is in a register.
127	130	register: Register,
	131	/// The value is a constant offset from the value in a register.
	132	register_offset: RegisterOffset,
128	133	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
129	134	register_overflow: struct { reg: Register, eflags: Condition },
130	135	/// The value is in memory at a hard-coded address.
131	136	/// If the type is a pointer, it means the pointer address is at this memory location.
132	137	memory: u64,
	138	/// The value is in memory at a constant offset from the address in a register.
	139	indirect: RegisterOffset,
133	140	/// The value is in memory.
134	141	/// Payload is a symbol index.
135	142	load_direct: u32,
136		/// The value is a pointer to value in memory.
	143	/// The value is a pointer to a value in memory.
137	144	/// Payload is a symbol index.
138	145	lea_direct: u32,
139	146	/// The value is in memory referenced indirectly via GOT.
140	147	/// Payload is a symbol index.
141	148	load_got: u32,
	149	/// The value is a pointer to a value referenced indirectly via GOT.
	150	/// Payload is a symbol index.
	151	lea_got: u32,
142	152	/// The value is a threadlocal variable.
143	153	/// Payload is a symbol index.
144	154	load_tlv: u32,
145		/// The value is a pointer to threadlocal variable.
	155	/// The value is a pointer to a threadlocal variable.
146	156	/// Payload is a symbol index.
147	157	lea_tlv: u32,
148		/// The value is one of the stack variables.
149		/// If the type is a pointer, it means the pointer address is in the stack at this offset.
150		stack_offset: i32,
151		/// The value is a pointer to one of the stack variables (payload is stack offset).
152		ptr_stack_offset: i32,
153		/// The value resides in the EFLAGS register.
154		eflags: Condition,
	158	/// The value stored at an offset from a frame index
	159	/// Payload is a frame address.
	160	load_frame: FrameAddr,
	161	/// The address of an offset from a frame index
	162	/// Payload is a frame address.
	163	lea_frame: FrameAddr,
	164	/// This indicates that we have already allocated a frame index for this instruction,
	165	/// but it has not been spilled there yet in the current control flow.
	166	/// Payload is a frame index.
	167	reserved_frame: FrameIndex,
155	168
156	169	fn isMemory(mcv: MCValue) bool {
157	170	return switch (mcv) {
158	171	.memory,
159		.stack_offset,
160		.ptr_stack_offset,
161	172	.load_direct,
162	173	.lea_direct,
163	174	.load_got,
	175	.lea_got,
164	176	.load_tlv,
165	177	.lea_tlv,
	178	.load_frame,
	179	.lea_frame,
166	180	=> true,
167	181	else => false,
168	182	};
...	...	@@ -181,6 +195,134 @@ pub const MCValue = union(enum) {
181	195	else => false,
182	196	};
183	197	}
	198
	199	fn getReg(mcv: MCValue) ?Register {
	200	return switch (mcv) {
	201	.register => \|reg\| reg,
	202	.register_offset, .indirect => \|ro\| ro.reg,
	203	.register_overflow => \|ro\| ro.reg,
	204	else => null,
	205	};
	206	}
	207
	208	fn getCondition(mcv: MCValue) ?Condition {
	209	return switch (mcv) {
	210	.eflags => \|cc\| cc,
	211	.register_overflow => \|reg_ov\| reg_ov.eflags,
	212	else => null,
	213	};
	214	}
	215
	216	fn address(mcv: MCValue) MCValue {
	217	return switch (mcv) {
	218	.none,
	219	.unreach,
	220	.dead,
	221	.undef,
	222	.immediate,
	223	.register,
	224	.register_offset,
	225	.eflags,
	226	.register_overflow,
	227	.lea_direct,
	228	.lea_got,
	229	.lea_tlv,
	230	.lea_frame,
	231	.reserved_frame,
	232	=> unreachable, // not in memory
	233	.memory => \|addr\| .{ .immediate = addr },
	234	.indirect => \|reg_off\| switch (reg_off.off) {
	235	0 => .{ .register = reg_off.reg },
	236	else => .{ .register_offset = reg_off },
	237	},
	238	.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
	239	.load_got => \|sym_index\| .{ .lea_got = sym_index },
	240	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
	241	.load_frame => \|frame_addr\| .{ .lea_frame = frame_addr },
	242	};
	243	}
	244
	245	fn deref(mcv: MCValue) MCValue {
	246	return switch (mcv) {
	247	.none,
	248	.unreach,
	249	.dead,
	250	.undef,
	251	.eflags,
	252	.register_overflow,
	253	.memory,
	254	.indirect,
	255	.load_direct,
	256	.load_got,
	257	.load_tlv,
	258	.load_frame,
	259	.reserved_frame,
	260	=> unreachable, // not a dereferenceable
	261	.immediate => \|addr\| .{ .memory = addr },
	262	.register => \|reg\| .{ .indirect = .{ .reg = reg } },
	263	.register_offset => \|reg_off\| .{ .indirect = reg_off },
	264	.lea_direct => \|sym_index\| .{ .load_direct = sym_index },
	265	.lea_got => \|sym_index\| .{ .load_got = sym_index },
	266	.lea_tlv => \|sym_index\| .{ .load_tlv = sym_index },
	267	.lea_frame => \|frame_addr\| .{ .load_frame = frame_addr },
	268	};
	269	}
	270
	271	fn offset(mcv: MCValue, off: i32) MCValue {
	272	return switch (mcv) {
	273	.none,
	274	.unreach,
	275	.dead,
	276	.undef,
	277	.eflags,
	278	.register_overflow,
	279	.memory,
	280	.indirect,
	281	.load_direct,
	282	.lea_direct,
	283	.load_got,
	284	.lea_got,
	285	.load_tlv,
	286	.lea_tlv,
	287	.load_frame,
	288	.reserved_frame,
	289	=> unreachable, // not offsettable
	290	.immediate => \|imm\| .{ .immediate = @bitCast(u64, @bitCast(i64, imm) +% off) },
	291	.register => \|reg\| .{ .register_offset = .{ .reg = reg, .off = off } },
	292	.register_offset => \|reg_off\| .{
	293	.register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off },
	294	},
	295	.lea_frame => \|frame_addr\| .{
	296	.lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
	297	},
	298	};
	299	}
	300
	301	pub fn format(
	302	mcv: MCValue,
	303	comptime _: []const u8,
	304	_: std.fmt.FormatOptions,
	305	writer: anytype,
	306	) @TypeOf(writer).Error!void {
	307	switch (mcv) {
	308	.none, .unreach, .dead, .undef => try writer.print("({s})", .{@tagName(mcv)}),
	309	.immediate => \|pl\| try writer.print("0x{x}", .{pl}),
	310	.memory => \|pl\| try writer.print("[ds:0x{x}]", .{pl}),
	311	inline .eflags, .register => \|pl\| try writer.print("{s}", .{@tagName(pl)}),
	312	.register_offset => \|pl\| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }),
	313	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{ @tagName(pl.eflags), @tagName(pl.reg) }),
	314	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
	315	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),
	316	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),
	317	.load_got => \|pl\| try writer.print("[got:{d}]", .{pl}),
	318	.lea_got => \|pl\| try writer.print("got:{d}", .{pl}),
	319	.load_tlv => \|pl\| try writer.print("[tlv:{d}]", .{pl}),
	320	.lea_tlv => \|pl\| try writer.print("tlv:{d}", .{pl}),
	321	.load_frame => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }),
	322	.lea_frame => \|pl\| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
	323	.reserved_frame => \|pl\| try writer.print("(dead:{})", .{pl}),
	324	}
	325	}
184	326	};
185	327
186	328	const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);
...	...	@@ -189,89 +331,211 @@ const InstTracking = struct {
189	331	short: MCValue,
190	332
191	333	fn init(result: MCValue) InstTracking {
192		return .{ .long = result, .short = result };
	334	return .{ .long = switch (result) {
	335	.none,
	336	.unreach,
	337	.undef,
	338	.immediate,
	339	.memory,
	340	.load_direct,
	341	.lea_direct,
	342	.load_got,
	343	.lea_got,
	344	.load_tlv,
	345	.lea_tlv,
	346	.load_frame,
	347	.lea_frame,
	348	=> result,
	349	.dead,
	350	.reserved_frame,
	351	=> unreachable,
	352	.eflags,
	353	.register,
	354	.register_offset,
	355	.register_overflow,
	356	.indirect,
	357	=> .none,
	358	}, .short = result };
193	359	}
194	360
195	361	fn getReg(self: InstTracking) ?Register {
196		return switch (self.short) {
197		.register => \|reg\| reg,
198		.register_overflow => \|ro\| ro.reg,
199		else => null,
200		};
	362	return self.short.getReg();
201	363	}
202	364
203	365	fn getCondition(self: InstTracking) ?Condition {
204		return switch (self.short) {
205		.eflags => \|eflags\| eflags,
206		.register_overflow => \|ro\| ro.eflags,
207		else => null,
208		};
	366	return self.short.getCondition();
209	367	}
210	368
211	369	fn spill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
	370	if (std.meta.eql(self.long, self.short)) return; // Already spilled
	371	// Allocate or reuse frame index
	372	switch (self.long) {
	373	.none => self.long = try function.allocRegOrMem(inst, false),
	374	.load_frame => {},
	375	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
	376	else => unreachable,
	377	}
	378	tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
	379	try function.genCopy(function.air.typeOfIndex(inst), self.long, self.short);
	380	}
	381
	382	fn reuseFrame(self: *InstTracking) void {
212	383	switch (self.long) {
	384	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
	385	else => {},
	386	}
	387	self.short = switch (self.long) {
213	388	.none,
214		.dead,
215	389	.unreach,
216		=> unreachable,
	390	.undef,
	391	.immediate,
	392	.memory,
	393	.load_direct,
	394	.lea_direct,
	395	.load_got,
	396	.lea_got,
	397	.load_tlv,
	398	.lea_tlv,
	399	.load_frame,
	400	.lea_frame,
	401	=> self.long,
	402	.dead,
	403	.eflags,
217	404	.register,
	405	.register_offset,
218	406	.register_overflow,
219		.eflags,
220		=> self.long = try function.allocRegOrMem(inst, self.short == .eflags),
221		.stack_offset => {},
	407	.indirect,
	408	.reserved_frame,
	409	=> unreachable,
	410	};
	411	}
	412
	413	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
	414	function.freeValue(self.short);
	415	self.reuseFrame();
	416	tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
	417	}
	418
	419	fn verifyMaterialize(self: *InstTracking, target: InstTracking) void {
	420	switch (self.long) {
	421	.none,
	422	.unreach,
222	423	.undef,
223	424	.immediate,
224	425	.memory,
225	426	.load_direct,
226	427	.lea_direct,
227	428	.load_got,
	429	.lea_got,
228	430	.load_tlv,
229	431	.lea_tlv,
230		.ptr_stack_offset,
231		=> return, // these can be rematerialized without using a stack slot
	432	.lea_frame,
	433	=> assert(std.meta.eql(self.long, target.long)),
	434	.load_frame,
	435	.reserved_frame,
	436	=> switch (target.long) {
	437	.none,
	438	.load_frame,
	439	.reserved_frame,
	440	=> {},
	441	else => unreachable,
	442	},
	443	.dead,
	444	.eflags,
	445	.register,
	446	.register_offset,
	447	.register_overflow,
	448	.indirect,
	449	=> unreachable,
232	450	}
233		log.debug("spilling %{d} from {} to {}", .{ inst, self.short, self.long });
234		const ty = function.air.typeOfIndex(inst);
235		try function.setRegOrMem(ty, self.long, self.short);
236	451	}
237	452
238		fn trackSpill(self: InstTracking, function: Self) void {
239		if (self.getReg()) \|reg\| function.register_manager.freeReg(reg);
240		switch (self.short) {
241		.none, .dead, .unreach => unreachable,
242		else => {},
243		}
244		self.short = self.long;
	453	fn materialize(
	454	self: *InstTracking,
	455	function: *Self,
	456	inst: Air.Inst.Index,
	457	target: InstTracking,
	458	) !void {
	459	self.verifyMaterialize(target);
	460	try self.materializeUnsafe(function, inst, target);
245	461	}
246	462
247		fn materialize(self: InstTracking, function: Self, inst: Air.Inst.Index, reg: Register) !void {
	463	fn materializeUnsafe(
	464	self: *InstTracking,
	465	function: *Self,
	466	inst: Air.Inst.Index,
	467	target: InstTracking,
	468	) !void {
248	469	const ty = function.air.typeOfIndex(inst);
249		try function.genSetReg(ty, reg, self.long);
	470	if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame)
	471	try function.genCopy(ty, target.long, self.short);
	472	try function.genCopy(ty, target.short, self.short);
250	473	}
251	474
252		fn trackMaterialize(self: InstTracking, function: Self, inst: Air.Inst.Index, reg: Register) void {
253		assert(inst == function.register_manager.registers[
254		RegisterManager.indexOfRegIntoTracked(reg).?
255		]);
256		self.short = .{ .register = reg };
	475	fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
	476	self.verifyMaterialize(target);
	477	// Don't clobber reserved frame indices
	478	self.long = if (target.long == .none) switch (self.long) {
	479	.load_frame => \|addr\| .{ .reserved_frame = addr.index },
	480	.reserved_frame => self.long,
	481	else => target.long,
	482	} else target.long;
	483	self.short = target.short;
	484	tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });
257	485	}
258	486
259		fn resurrect(self: *InstTracking, scope_generation: u32) void {
	487	fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
260	488	switch (self.short) {
261	489	.dead => \|die_generation\| if (die_generation >= scope_generation) {
262		self.short = self.long;
	490	self.reuseFrame();
	491	tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });
263	492	},
264	493	else => {},
265	494	}
266	495	}
267	496
268		fn die(self: InstTracking, function: Self) void {
	497	fn die(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
269	498	function.freeValue(self.short);
270		self.reuse(function);
	499	self.short = .{ .dead = function.scope_generation };
	500	tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
271	501	}
272	502
273		fn reuse(self: InstTracking, function: Self) void {
	503	fn reuse(
	504	self: *InstTracking,
	505	function: *Self,
	506	new_inst: Air.Inst.Index,
	507	old_inst: Air.Inst.Index,
	508	) void {
274	509	self.short = .{ .dead = function.scope_generation };
	510	tracking_log.debug("%{d} => {} (reuse %{d})", .{ new_inst, self.*, old_inst });
	511	}
	512
	513	pub fn format(
	514	self: InstTracking,
	515	comptime _: []const u8,
	516	_: std.fmt.FormatOptions,
	517	writer: anytype,
	518	) @TypeOf(writer).Error!void {
	519	if (!std.meta.eql(self.long, self.short)) try writer.print("\|{}\| ", .{self.long});
	520	try writer.print("{}", .{self.short});
	521	}
	522	};
	523
	524	const FrameAlloc = struct {
	525	abi_size: u31,
	526	abi_align: u5,
	527	ref_count: u16,
	528
	529	fn init(alloc_abi: struct { size: u64, alignment: u32 }) FrameAlloc {
	530	assert(math.isPowerOfTwo(alloc_abi.alignment));
	531	return .{
	532	.abi_size = @intCast(u31, alloc_abi.size),
	533	.abi_align = math.log2_int(u32, alloc_abi.alignment),
	534	.ref_count = 0,
	535	};
	536	}
	537	fn initType(ty: Type, target: Target) FrameAlloc {
	538	return init(.{ .size = ty.abiSize(target), .alignment = ty.abiAlignment(target) });
275	539	}
276	540	};
277	541
...	...	@@ -332,12 +596,13 @@ pub fn generate(
332	596	.fn_type = fn_type,
333	597	.arg_index = 0,
334	598	.src_loc = src_loc,
335		.stack_align = undefined,
336	599	.end_di_line = module_fn.rbrace_line,
337	600	.end_di_column = module_fn.rbrace_column,
338	601	};
339	602	defer {
340		function.stack.deinit(gpa);
	603	function.frame_allocs.deinit(gpa);
	604	function.free_frame_indices.deinit(gpa);
	605	function.frame_locs.deinit(gpa);
341	606	var block_it = function.blocks.valueIterator();
342	607	while (block_it.next()) \|block\| block.deinit(gpa);
343	608	function.blocks.deinit(gpa);
...	...	@@ -349,7 +614,23 @@ pub fn generate(
349	614	if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);
350	615	}
351	616
352		var call_info = function.resolveCallingConventionValues(fn_type, &.{}) catch \|err\| switch (err) {
	617	try function.frame_allocs.resize(gpa, FrameIndex.named_count);
	618	function.frame_allocs.set(
	619	@enumToInt(FrameIndex.stack_frame),
	620	FrameAlloc.init(.{
	621	.size = 0,
	622	.alignment = if (mod.align_stack_fns.get(module_fn)) \|set_align_stack\|
	623	set_align_stack.alignment
	624	else
	625	1,
	626	}),
	627	);
	628	function.frame_allocs.set(
	629	@enumToInt(FrameIndex.call_frame),
	630	FrameAlloc.init(.{ .size = 0, .alignment = 1 }),
	631	);
	632
	633	var call_info = function.resolveCallingConventionValues(fn_type, &.{}, .args_frame) catch \|err\| switch (err) {
353	634	error.CodegenFail => return Result{ .fail = function.err_msg.? },
354	635	error.OutOfRegisters => return Result{
355	636	.fail = try ErrorMsg.create(
...	...	@@ -365,8 +646,18 @@ pub fn generate(
365	646
366	647	function.args = call_info.args;
367	648	function.ret_mcv = call_info.return_value;
368		function.stack_align = call_info.stack_align;
369		function.max_end_stack = call_info.stack_byte_count;
	649	function.frame_allocs.set(@enumToInt(FrameIndex.ret_addr), FrameAlloc.init(.{
	650	.size = Type.usize.abiSize(function.target.*),
	651	.alignment = @min(Type.usize.abiAlignment(function.target.*), call_info.stack_align),
	652	}));
	653	function.frame_allocs.set(@enumToInt(FrameIndex.base_ptr), FrameAlloc.init(.{
	654	.size = Type.usize.abiSize(function.target.*),
	655	.alignment = @min(Type.usize.abiAlignment(function.target.) 2, call_info.stack_align),
	656	}));
	657	function.frame_allocs.set(
	658	@enumToInt(FrameIndex.args_frame),
	659	FrameAlloc.init(.{ .size = call_info.stack_byte_count, .alignment = call_info.stack_align }),
	660	);
370	661
371	662	function.gen() catch \|err\| switch (err) {
372	663	error.CodegenFail => return Result{ .fail = function.err_msg.? },
...	...	@@ -379,6 +670,7 @@ pub fn generate(
379	670	var mir = Mir{
380	671	.instructions = function.mir_instructions.toOwnedSlice(),
381	672	.extra = try function.mir_extra.toOwnedSlice(bin_file.allocator),
	673	.frame_locs = function.frame_locs.toOwnedSlice(),
382	674	};
383	675	defer mir.deinit(bin_file.allocator);
384	676
...	...	@@ -432,6 +724,7 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
432	724	.mir = .{
433	725	.instructions = self.mir_instructions.slice(),
434	726	.extra = self.mir_extra.items,
	727	.frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
435	728	},
436	729	.target = self.target,
437	730	.src_loc = self.src_loc,
...	...	@@ -458,6 +751,14 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
458	751	}
459	752	}
460	753
	754	fn dumpTracking(self: *Self) !void {
	755	if (!debug_tracking) return;
	756	const stderr = std.io.getStdErr().writer();
	757
	758	var it = self.inst_tracking.iterator();
	759	while (it.next()) \|entry\| try stderr.print("%{d} = {}\n", .{ entry.key_ptr., entry.value_ptr. });
	760	}
	761
461	762	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
462	763	const gpa = self.gpa;
463	764	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
...	...	@@ -550,6 +851,14 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
550	851	});
551	852	}
552	853
	854	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
	855	return self.addInst(.{
	856	.tag = .dead,
	857	.ops = undefined,
	858	.data = undefined,
	859	});
	860	}
	861
553	862	fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
554	863	_ = try self.addInst(.{
555	864	.tag = tag,
...	...	@@ -765,44 +1074,32 @@ fn gen(self: *Self) InnerError!void {
765	1074	const cc = self.fn_type.fnCallingConvention();
766	1075	if (cc != .Naked) {
767	1076	try self.asmRegister(.push, .rbp);
	1077	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
768	1078	try self.asmRegisterRegister(.mov, .rbp, .rsp);
769
770		// We want to subtract the aligned stack frame size from rsp here, but we don't
771		// yet know how big it will be, so we leave room for a 4-byte stack size.
772		// TODO During semantic analysis, check if there are no function calls. If there
773		// are none, here we can omit the part where we subtract and then add rsp.
774		const backpatch_stack_sub = try self.addInst(.{
775		.tag = .dead,
776		.ops = undefined,
777		.data = undefined,
778		});
779
780		if (self.ret_mcv == .stack_offset) {
781		// The address where to store the return value for the caller is in a
782		// register which the callee is free to clobber. Therefore, we purposely
783		// spill it to stack immediately.
784		const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + 8, 8);
785		self.next_stack_offset = stack_offset;
786		self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);
787
788		const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
789		try self.genSetStack(Type.usize, @intCast(i32, stack_offset), MCValue{ .register = ret_reg }, .{});
790		self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };
791		log.debug("gen: spilling {s} to stack at offset {}", .{ @tagName(ret_reg), stack_offset });
	1079	const backpatch_frame_align = try self.asmPlaceholder();
	1080	const backpatch_stack_alloc = try self.asmPlaceholder();
	1081
	1082	switch (self.ret_mcv.long) {
	1083	.none, .unreach => {},
	1084	.indirect => {
	1085	// The address where to store the return value for the caller is in a
	1086	// register which the callee is free to clobber. Therefore, we purposely
	1087	// spill it to stack immediately.
	1088	const frame_index =
	1089	try self.allocFrameIndex(FrameAlloc.initType(Type.usize, self.target.*));
	1090	try self.genSetMem(
	1091	.{ .frame = frame_index },
	1092	0,
	1093	Type.usize,
	1094	self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
	1095	);
	1096	self.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } };
	1097	tracking_log.debug("spill {} to {}", .{ self.ret_mcv.long, frame_index });
	1098	},
	1099	else => unreachable,
792	1100	}
793	1101
794		_ = try self.addInst(.{
795		.tag = .dbg_prologue_end,
796		.ops = undefined,
797		.data = undefined,
798		});
799
800		// Push callee-preserved regs that were used actually in use.
801		const backpatch_push_callee_preserved_regs = try self.addInst(.{
802		.tag = .dead,
803		.ops = undefined,
804		.data = undefined,
805		});
	1102	try self.asmOpOnly(.dbg_prologue_end);
806	1103
807	1104	try self.genBody(self.air.getMainBody());
808	1105
...	...	@@ -814,69 +1111,43 @@ fn gen(self: *Self) InnerError!void {
814	1111	// }
815	1112	// Eliding the reloc will cause a miscompilation in this case.
816	1113	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
817		self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);
	1114	self.mir_instructions.items(.data)[jmp_reloc].inst =
	1115	@intCast(u32, self.mir_instructions.len);
818	1116	}
819	1117
820		// Create list of registers to save in the prologue.
821		// TODO handle register classes
822		var reg_list = Mir.RegisterList{};
823		const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
824		for (callee_preserved_regs) \|reg\| {
825		if (self.register_manager.isRegAllocated(reg)) {
826		reg_list.push(callee_preserved_regs, reg);
827		}
828		}
829		const saved_regs_stack_space: u32 = reg_list.count() * 8;
830
831		// Pop saved callee-preserved regs.
832		const backpatch_pop_callee_preserved_regs = try self.addInst(.{
833		.tag = .dead,
834		.ops = undefined,
835		.data = undefined,
836		});
837
838		_ = try self.addInst(.{
839		.tag = .dbg_epilogue_begin,
840		.ops = undefined,
841		.data = undefined,
842		});
843
844		// Maybe add rsp, x if required. This is backpatched later.
845		const backpatch_stack_add = try self.addInst(.{
846		.tag = .dead,
847		.ops = undefined,
848		.data = undefined,
849		});
850
	1118	try self.asmOpOnly(.dbg_epilogue_begin);
	1119	const backpatch_stack_dealloc = try self.asmPlaceholder();
	1120	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
851	1121	try self.asmRegister(.pop, .rbp);
852	1122	try self.asmOpOnly(.ret);
853	1123
854		// Adjust the stack
855		if (self.max_end_stack > math.maxInt(i32)) {
856		return self.failSymbol("too much stack used in call parameters", .{});
	1124	const frame_layout = try self.computeFrameLayout();
	1125	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
	1126	const need_stack_adjust = frame_layout.stack_adjust > 0;
	1127	const need_save_reg = frame_layout.save_reg_list.count() > 0;
	1128	if (need_frame_align) {
	1129	self.mir_instructions.set(backpatch_frame_align, .{
	1130	.tag = .@"and",
	1131	.ops = .ri_s,
	1132	.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },
	1133	});
857	1134	}
858
859		const aligned_stack_end = @intCast(
860		u32,
861		mem.alignForward(self.max_end_stack + saved_regs_stack_space, self.stack_align),
862		);
863		if (aligned_stack_end > 0) {
864		self.mir_instructions.set(backpatch_stack_sub, .{
	1135	if (need_stack_adjust) {
	1136	self.mir_instructions.set(backpatch_stack_alloc, .{
865	1137	.tag = .sub,
866		.ops = .ri_u,
867		.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
	1138	.ops = .ri_s,
	1139	.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },
868	1140	});
869		self.mir_instructions.set(backpatch_stack_add, .{
870		.tag = .add,
871		.ops = .ri_u,
872		.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
873		});
874
875		const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
876		.base_reg = @enumToInt(Register.rbp),
877		.register_list = reg_list.asInt(),
878		.stack_end = aligned_stack_end,
	1141	}
	1142	if (need_frame_align or need_stack_adjust) {
	1143	self.mir_instructions.set(backpatch_stack_dealloc, .{
	1144	.tag = .mov,
	1145	.ops = .rr,
	1146	.data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },
879	1147	});
	1148	}
	1149	if (need_save_reg) {
	1150	const save_reg_list = frame_layout.save_reg_list.asInt();
880	1151	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
881	1152	.tag = .push_regs,
882	1153	.ops = undefined,
...	...	@@ -889,30 +1160,19 @@ fn gen(self: *Self) InnerError!void {
889	1160	});
890	1161	}
891	1162	} else {
892		_ = try self.addInst(.{
893		.tag = .dbg_prologue_end,
894		.ops = undefined,
895		.data = undefined,
896		});
897
	1163	try self.asmOpOnly(.dbg_prologue_end);
898	1164	try self.genBody(self.air.getMainBody());
899
900		_ = try self.addInst(.{
901		.tag = .dbg_epilogue_begin,
902		.ops = undefined,
903		.data = undefined,
904		});
	1165	try self.asmOpOnly(.dbg_epilogue_begin);
905	1166	}
906	1167
907	1168	// Drop them off at the rbrace.
908		const payload = try self.addExtra(Mir.DbgLineColumn{
909		.line = self.end_di_line,
910		.column = self.end_di_column,
911		});
912	1169	_ = try self.addInst(.{
913	1170	.tag = .dbg_line,
914	1171	.ops = undefined,
915		.data = .{ .payload = payload },
	1172	.data = .{ .line_column = .{
	1173	.line = self.end_di_line,
	1174	.column = self.end_di_column,
	1175	} },
916	1176	});
917	1177	}
918	1178
...	...	@@ -932,6 +1192,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
932	1192	self.air,
933	1193	self.liveness,
934	1194	);
	1195	self.dumpTracking() catch {};
935	1196
936	1197	const old_air_bookkeeping = self.air_bookkeeping;
937	1198	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
...	...	@@ -1187,23 +1448,16 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1187	1448	while (it.next()) \|index\| {
1188	1449	const tracked_inst = self.register_manager.registers[index];
1189	1450	const tracking = self.getResolvedInstValue(tracked_inst);
1190		assert(RegisterManager.indexOfRegIntoTracked(switch (tracking.short) {
1191		.register => \|reg\| reg,
1192		.register_overflow => \|ro\| ro.reg,
1193		else => unreachable,
1194		}).? == index);
	1451	assert(RegisterManager.indexOfRegIntoTracked(tracking.getReg().?).? == index);
1195	1452	}
1196	1453	}
1197	1454	}
1198	1455	}
	1456	self.dumpTracking() catch {};
1199	1457	}
1200	1458
1201	1459	fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
1202		const reg = switch (value) {
1203		.register => \|reg\| reg,
1204		.register_overflow => \|ro\| ro.reg,
1205		else => return,
1206		};
	1460	const reg = value.getReg() orelse return;
1207	1461	if (self.register_manager.isRegFree(reg)) {
1208	1462	self.register_manager.getRegAssumeFree(reg, inst);
1209	1463	}
...	...	@@ -1214,8 +1468,11 @@ fn freeValue(self: *Self, value: MCValue) void {
1214	1468	.register => \|reg\| {
1215	1469	self.register_manager.freeReg(reg);
1216	1470	},
1217		.register_overflow => \|ro\| {
1218		self.register_manager.freeReg(ro.reg);
	1471	.register_offset => \|reg_off\| {
	1472	self.register_manager.freeReg(reg_off.reg);
	1473	},
	1474	.register_overflow => \|reg_ov\| {
	1475	self.register_manager.freeReg(reg_ov.reg);
1219	1476	self.eflags_inst = null;
1220	1477	},
1221	1478	.eflags => {
...	...	@@ -1231,10 +1488,10 @@ fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) void {
1231	1488
1232	1489	/// Asserts there is already capacity to insert into top branch inst_table.
1233	1490	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
1234		const air_tags = self.air.instructions.items(.tag);
1235		if (air_tags[inst] == .constant) return;
1236		log.debug("%{d} => {}", .{ inst, MCValue.dead });
1237		self.inst_tracking.getPtr(inst).?.die(self);
	1491	switch (self.air.instructions.items(.tag)[inst]) {
	1492	.constant, .const_ty => unreachable,
	1493	else => self.inst_tracking.getPtr(inst).?.die(self, inst),
	1494	}
1238	1495	}
1239	1496
1240	1497	/// Called when there are no operands, and the instruction is always unreferenced.
...	...	@@ -1249,7 +1506,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
1249	1506	.none, .dead, .unreach => {},
1250	1507	else => unreachable, // Why didn't the result die?
1251	1508	} else {
1252		log.debug("%{d} => {}", .{ inst, result });
	1509	tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
1253	1510	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
1254	1511	// In some cases, an operand may be reused as the result.
1255	1512	// If that operand died and was a register, it was freed by
...	...	@@ -1273,36 +1530,123 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
1273	1530	self.finishAirResult(inst, result);
1274	1531	}
1275	1532
1276		fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
1277		if (abi_align > self.stack_align)
1278		self.stack_align = abi_align;
1279		// TODO find a free slot instead of always appending
1280		const offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);
1281		self.next_stack_offset = offset;
1282		self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);
1283		try self.stack.putNoClobber(self.gpa, offset, .{
1284		.inst = inst,
1285		.size = abi_size,
1286		});
1287		return offset;
	1533	const FrameLayout = struct {
	1534	stack_mask: u32,
	1535	stack_adjust: u32,
	1536	save_reg_list: Mir.RegisterList,
	1537	};
	1538
	1539	fn setFrameLoc(
	1540	self: *Self,
	1541	frame_index: FrameIndex,
	1542	base: Register,
	1543	offset: *i32,
	1544	comptime aligned: bool,
	1545	) void {
	1546	const frame_i = @enumToInt(frame_index);
	1547	if (aligned) {
	1548	const alignment = @as(i32, 1) << self.frame_allocs.items(.abi_align)[frame_i];
	1549	offset.* = mem.alignForwardGeneric(i32, offset.*, alignment);
	1550	}
	1551	self.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* });
	1552	offset.* += self.frame_allocs.items(.abi_size)[frame_i];
	1553	}
	1554
	1555	fn computeFrameLayout(self: *Self) !FrameLayout {
	1556	const frame_allocs_len = self.frame_allocs.len;
	1557	try self.frame_locs.resize(self.gpa, frame_allocs_len);
	1558	const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
	1559	defer self.gpa.free(stack_frame_order);
	1560
	1561	const frame_size = self.frame_allocs.items(.abi_size);
	1562	const frame_align = self.frame_allocs.items(.abi_align);
	1563	const frame_offset = self.frame_locs.items(.disp);
	1564
	1565	for (stack_frame_order, FrameIndex.named_count..) \|*frame_order, frame_index\|
	1566	frame_order.* = @intToEnum(FrameIndex, frame_index);
	1567	{
	1568	const SortContext = struct {
	1569	frame_align: @TypeOf(frame_align),
	1570	pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool {
	1571	return context.frame_align[@enumToInt(lhs)] > context.frame_align[@enumToInt(rhs)];
	1572	}
	1573	};
	1574	const sort_context = SortContext{ .frame_align = frame_align };
	1575	std.sort.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
	1576	}
	1577
	1578	const call_frame_align = frame_align[@enumToInt(FrameIndex.call_frame)];
	1579	const stack_frame_align = frame_align[@enumToInt(FrameIndex.stack_frame)];
	1580	const args_frame_align = frame_align[@enumToInt(FrameIndex.args_frame)];
	1581	const needed_align = @max(call_frame_align, stack_frame_align);
	1582	const need_align_stack = needed_align > args_frame_align;
	1583
	1584	// Create list of registers to save in the prologue.
	1585	// TODO handle register classes
	1586	var save_reg_list = Mir.RegisterList{};
	1587	const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
	1588	for (callee_preserved_regs) \|reg\| {
	1589	if (self.register_manager.isRegAllocated(reg)) {
	1590	save_reg_list.push(callee_preserved_regs, reg);
	1591	}
	1592	}
	1593
	1594	var rbp_offset = @intCast(i32, save_reg_list.count() * 8);
	1595	self.setFrameLoc(.base_ptr, .rbp, &rbp_offset, false);
	1596	self.setFrameLoc(.ret_addr, .rbp, &rbp_offset, false);
	1597	self.setFrameLoc(.args_frame, .rbp, &rbp_offset, false);
	1598	const stack_frame_align_offset =
	1599	if (need_align_stack) 0 else frame_offset[@enumToInt(FrameIndex.args_frame)];
	1600
	1601	var rsp_offset: i32 = 0;
	1602	self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true);
	1603	self.setFrameLoc(.stack_frame, .rsp, &rsp_offset, true);
	1604	for (stack_frame_order) \|frame_index\| self.setFrameLoc(frame_index, .rsp, &rsp_offset, true);
	1605	rsp_offset += stack_frame_align_offset;
	1606	rsp_offset = mem.alignForwardGeneric(i32, rsp_offset, @as(i32, 1) << needed_align);
	1607	rsp_offset -= stack_frame_align_offset;
	1608	frame_size[@enumToInt(FrameIndex.call_frame)] =
	1609	@intCast(u31, rsp_offset - frame_offset[@enumToInt(FrameIndex.stack_frame)]);
	1610
	1611	return .{
	1612	.stack_mask = @as(u32, math.maxInt(u32)) << (if (need_align_stack) needed_align else 0),
	1613	.stack_adjust = @intCast(u32, rsp_offset - frame_offset[@enumToInt(FrameIndex.call_frame)]),
	1614	.save_reg_list = save_reg_list,
	1615	};
1288	1616	}
1289	1617
1290		/// Use a pointer instruction as the basis for allocating stack memory.
1291		fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
1292		const ptr_ty = self.air.typeOfIndex(inst);
1293		const elem_ty = ptr_ty.elemType();
	1618	fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
	1619	const frame_allocs_slice = self.frame_allocs.slice();
	1620	const frame_size = frame_allocs_slice.items(.abi_size);
	1621	const frame_align = frame_allocs_slice.items(.abi_align);
	1622	for (self.free_frame_indices.keys(), 0..) \|frame_index, free_i\| {
	1623	const abi_size = frame_size[@enumToInt(frame_index)];
	1624	if (abi_size != alloc.abi_size) continue;
	1625	const abi_align = &frame_align[@enumToInt(frame_index)];
	1626	abi_align.* = @max(abi_align.*, alloc.abi_align);
1294	1627
1295		if (!elem_ty.hasRuntimeBitsIgnoreComptime()) {
1296		return self.allocMem(inst, @sizeOf(usize), @alignOf(usize));
	1628	const stack_frame_align = &frame_align[@enumToInt(FrameIndex.stack_frame)];
	1629	stack_frame_align.* = @max(stack_frame_align.*, alloc.abi_align);
	1630
	1631	_ = self.free_frame_indices.swapRemoveAt(free_i);
	1632	return frame_index;
1297	1633	}
	1634	const frame_index = @intToEnum(FrameIndex, self.frame_allocs.len);
	1635	try self.frame_allocs.append(self.gpa, alloc);
	1636	return frame_index;
	1637	}
1298	1638
1299		const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) orelse {
1300		const mod = self.bin_file.options.module.?;
1301		return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});
1302		};
1303		// TODO swap this for inst.ty.ptrAlign
1304		const abi_align = ptr_ty.ptrAlignment(self.target.*);
1305		return self.allocMem(inst, abi_size, abi_align);
	1639	/// Use a pointer instruction as the basis for allocating stack memory.
	1640	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
	1641	const ptr_ty = self.air.typeOfIndex(inst);
	1642	const val_ty = ptr_ty.childType();
	1643	return self.allocFrameIndex(FrameAlloc.init(.{
	1644	.size = math.cast(u32, val_ty.abiSize(self.target.*)) orelse {
	1645	const mod = self.bin_file.options.module.?;
	1646	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(mod)});
	1647	},
	1648	.alignment = @max(ptr_ty.ptrAlignment(self.target.*), 1),
	1649	}));
1306	1650	}
1307	1651
1308	1652	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
...	...	@@ -1320,42 +1664,35 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
1320	1664	};
1321	1665
1322	1666	if (reg_ok) {
1323		switch (elem_ty.zigTypeTag()) {
1324		.Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}),
1325		.Float => {
1326		if (intrinsicsAllowed(self.target.*, elem_ty)) {
1327		const ptr_bytes: u64 = 32;
1328		if (abi_size <= ptr_bytes) {
1329		if (self.register_manager.tryAllocReg(inst, sse)) \|reg\| {
1330		return MCValue{ .register = registerAlias(reg, abi_size) };
1331		}
1332		}
1333		}
1334
1335		return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{});
1336		},
1337		else => {
1338		// Make sure the type can fit in a register before we try to allocate one.
1339		const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1340		const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1341		if (abi_size <= ptr_bytes) {
1342		if (self.register_manager.tryAllocReg(inst, gp)) \|reg\| {
1343		return MCValue{ .register = registerAlias(reg, abi_size) };
1344		}
1345		}
1346		},
	1667	// Make sure the type can fit in a register before we try to allocate one.
	1668	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
	1669	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
	1670	if (abi_size <= ptr_bytes) {
	1671	if (self.register_manager.tryAllocReg(inst, try self.regClassForType(elem_ty))) \|reg\| {
	1672	return MCValue{ .register = registerAlias(reg, abi_size) };
	1673	}
1347	1674	}
1348	1675	}
1349	1676
1350		const abi_align = elem_ty.abiAlignment(self.target.*);
1351		if (abi_align > self.stack_align)
1352		self.stack_align = abi_align;
1353		const stack_offset = try self.allocMem(inst, abi_size, abi_align);
1354		return MCValue{ .stack_offset = @intCast(i32, stack_offset) };
	1677	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(elem_ty, self.target.*));
	1678	return .{ .load_frame = .{ .index = frame_index } };
	1679	}
	1680
	1681	fn regClassForType(self: *Self, ty: Type) !RegisterManager.RegisterBitSet {
	1682	return switch (ty.zigTypeTag()) {
	1683	.Vector => self.fail("TODO regClassForType for {}", .{ty.fmt(self.bin_file.options.module.?)}),
	1684	.Float => switch (ty.floatBits(self.target.*)) {
	1685	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) sse else gp,
	1686	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) sse else gp,
	1687	else => gp,
	1688	},
	1689	else => gp,
	1690	};
1355	1691	}
1356	1692
1357	1693	const State = struct {
1358	1694	registers: RegisterManager.TrackedRegisters,
	1695	reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
1359	1696	free_registers: RegisterManager.RegisterBitSet,
1360	1697	inst_tracking_len: u32,
1361	1698	scope_generation: u32,
...	...	@@ -1370,8 +1707,14 @@ fn initRetroactiveState(self: *Self) State {
1370	1707
1371	1708	fn saveRetroactiveState(self: Self, state: State) !void {
1372	1709	try self.spillEflagsIfOccupied();
1373		state.registers = self.register_manager.registers;
1374		state.free_registers = self.register_manager.free_registers;
	1710	const free_registers = self.register_manager.free_registers;
	1711	var it = free_registers.iterator(.{ .kind = .unset });
	1712	while (it.next()) \|index\| {
	1713	const tracked_inst = self.register_manager.registers[index];
	1714	state.registers[index] = tracked_inst;
	1715	state.reg_tracking[index] = self.inst_tracking.get(tracked_inst).?;
	1716	}
	1717	state.free_registers = free_registers;
1375	1718	}
1376	1719
1377	1720	fn saveState(self: *Self) !State {
...	...	@@ -1387,18 +1730,18 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1387	1730	close_scope: bool,
1388	1731	}) !void {
1389	1732	if (opts.close_scope) {
1390		for (self.inst_tracking.values()[state.inst_tracking_len..]) \|*tracking\| tracking.die(self);
	1733	for (
	1734	self.inst_tracking.keys()[state.inst_tracking_len..],
	1735	self.inst_tracking.values()[state.inst_tracking_len..],
	1736	) \|inst, *tracking\| tracking.die(self, inst);
1391	1737	self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
1392	1738	}
1393	1739
1394		if (opts.resurrect) for (self.inst_tracking.values()[0..state.inst_tracking_len]) \|*tracking\|
1395		tracking.resurrect(state.scope_generation);
1396		const air_tags = self.air.instructions.items(.tag);
1397		for (deaths) \|death\| switch (air_tags[death]) {
1398		.constant => {},
1399		.const_ty => unreachable,
1400		else => self.inst_tracking.getPtr(death).?.die(self),
1401		};
	1740	if (opts.resurrect) for (
	1741	self.inst_tracking.keys()[0..state.inst_tracking_len],
	1742	self.inst_tracking.values()[0..state.inst_tracking_len],
	1743	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);
	1744	for (deaths) \|death\| self.processDeath(death);
1402	1745
1403	1746	for (0..state.registers.len) \|index\| {
1404	1747	const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
...	...	@@ -1411,26 +1754,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1411	1754	state.registers[index];
1412	1755	if (std.debug.runtime_safety) if (target_maybe_inst) \|target_inst\|
1413	1756	assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
1414		if (current_maybe_inst == target_maybe_inst) continue;
1415		const reg = RegisterManager.regAtTrackedIndex(
1416		@intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
1417		);
1418	1757	if (opts.emit_instructions) {
1419	1758	if (current_maybe_inst) \|current_inst\| {
1420	1759	try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
1421	1760	}
1422	1761	if (target_maybe_inst) \|target_inst\| {
1423		try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg);
	1762	try self.inst_tracking.getPtr(target_inst).?.materialize(
	1763	self,
	1764	target_inst,
	1765	state.reg_tracking[index],
	1766	);
1424	1767	}
1425	1768	}
1426	1769	if (opts.update_tracking) {
1427	1770	if (current_maybe_inst) \|current_inst\| {
1428		self.inst_tracking.getPtr(current_inst).?.trackSpill(self);
	1771	self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
	1772	}
	1773	{
	1774	const reg = RegisterManager.regAtTrackedIndex(
	1775	@intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
	1776	);
	1777	self.register_manager.freeReg(reg);
	1778	self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1429	1779	}
1430		self.register_manager.freeReg(reg);
1431		self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1432	1780	if (target_maybe_inst) \|target_inst\| {
1433		self.inst_tracking.getPtr(target_inst).?.trackMaterialize(self, target_inst, reg);
	1781	self.inst_tracking.getPtr(target_inst).?.trackMaterialize(
	1782	target_inst,
	1783	state.reg_tracking[index],
	1784	);
1434	1785	}
1435	1786	}
1436	1787	}
...	...	@@ -1438,7 +1789,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1438	1789	try self.inst_tracking.getPtr(inst).?.spill(self, inst);
1439	1790	if (opts.update_tracking) if (self.eflags_inst) \|inst\| {
1440	1791	self.eflags_inst = null;
1441		self.inst_tracking.getPtr(inst).?.trackSpill(self);
	1792	self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
1442	1793	};
1443	1794
1444	1795	if (opts.update_tracking and std.debug.runtime_safety) {
...	...	@@ -1452,9 +1803,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1452	1803
1453	1804	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1454	1805	const tracking = self.inst_tracking.getPtr(inst).?;
1455		assert(tracking.getReg().?.to64() == reg.to64());
	1806	assert(tracking.getReg().?.id() == reg.id());
1456	1807	try tracking.spill(self, inst);
1457		tracking.trackSpill(self);
	1808	tracking.trackSpill(self, inst);
1458	1809	}
1459	1810
1460	1811	pub fn spillEflagsIfOccupied(self: *Self) !void {
...	...	@@ -1463,7 +1814,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
1463	1814	const tracking = self.inst_tracking.getPtr(inst).?;
1464	1815	assert(tracking.getCondition() != null);
1465	1816	try tracking.spill(self, inst);
1466		tracking.trackSpill(self);
	1817	tracking.trackSpill(self, inst);
1467	1818	}
1468	1819	}
1469	1820
...	...	@@ -1477,15 +1828,8 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {
1477	1828	/// allocated. A second call to `copyToTmpRegister` may return the same register.
1478	1829	/// This can have a side effect of spilling instructions to the stack to free up a register.
1479	1830	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1480		const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
1481		.Float => blk: {
1482		if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
1483		return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
1484		},
1485		else => gp,
1486		};
1487		const reg: Register = try self.register_manager.allocReg(null, reg_class);
1488		try self.genSetReg(ty, reg, mcv);
	1831	const reg: Register = try self.register_manager.allocReg(null, try self.regClassForType(ty));
	1832	try self.genSetReg(reg, ty, mcv);
1489	1833	return reg;
1490	1834	}
1491	1835
...	...	@@ -1493,28 +1837,35 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1493	1837	/// `reg_owner` is the instruction that gets associated with the register in the register table.
1494	1838	/// This can have a side effect of spilling instructions to the stack to free up a register.
1495	1839	/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
1496		fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {
1497		const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
1498		.Float => blk: {
1499		if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
1500		return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
1501		},
1502		else => gp,
1503		};
1504		const reg: Register = try self.register_manager.allocReg(reg_owner, reg_class);
1505		try self.genSetReg(ty, reg, mcv);
	1840	fn copyToRegisterWithInstTracking(
	1841	self: *Self,
	1842	reg_owner: Air.Inst.Index,
	1843	ty: Type,
	1844	mcv: MCValue,
	1845	) !MCValue {
	1846	const reg: Register = try self.register_manager.allocReg(reg_owner, try self.regClassForType(ty));
	1847	try self.genSetReg(reg, ty, mcv);
1506	1848	return MCValue{ .register = reg };
1507	1849	}
1508	1850
1509	1851	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1510		const stack_offset = try self.allocMemPtr(inst);
1511		const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };
	1852	const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
1512	1853	return self.finishAir(inst, result, .{ .none, .none, .none });
1513	1854	}
1514	1855
1515	1856	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1516		const stack_offset = try self.allocMemPtr(inst);
1517		const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };
	1857	const result: MCValue = switch (self.ret_mcv.long) {
	1858	else => unreachable,
	1859	.none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
	1860	.load_frame => .{ .register_offset = .{
	1861	.reg = (try self.copyToRegisterWithInstTracking(
	1862	inst,
	1863	self.air.typeOfIndex(inst),
	1864	self.ret_mcv.long,
	1865	)).register,
	1866	.off = self.ret_mcv.short.indirect.off,
	1867	} },
	1868	};
1518	1869	return self.finishAir(inst, result, .{ .none, .none, .none });
1519	1870	}
1520	1871
...	...	@@ -1577,12 +1928,14 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1577	1928	registerAlias(src_reg, min_abi_size),
1578	1929	);
1579	1930	},
1580		.stack_offset => \|src_off\| {
1581		try self.asmRegisterMemory(tag, dst_alias, Memory.sib(
1582		Memory.PtrSize.fromSize(min_abi_size),
1583		.{ .base = .rbp, .disp = -src_off },
1584		));
1585		},
	1931	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
	1932	tag,
	1933	dst_alias,
	1934	Memory.sib(Memory.PtrSize.fromSize(min_abi_size), .{
	1935	.base = .{ .frame = frame_addr.index },
	1936	.disp = frame_addr.off,
	1937	}),
	1938	),
1586	1939	else => return self.fail("TODO airIntCast from {s} to {s}", .{
1587	1940	@tagName(src_mcv),
1588	1941	@tagName(dst_mcv),
...	...	@@ -1591,7 +1944,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1591	1944	if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg);
1592	1945	},
1593	1946	else => {
1594		try self.setRegOrMem(min_ty, dst_mcv, src_mcv);
	1947	try self.genCopy(min_ty, dst_mcv, src_mcv);
1595	1948	const extra = dst_abi_size * 8 - dst_int_info.bits;
1596	1949	if (extra > 0) {
1597	1950	try self.genShiftBinOpMir(switch (signedness) {
...	...	@@ -1653,16 +2006,22 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1653	2006	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1654	2007	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1655	2008
	2009	const slice_ty = self.air.typeOfIndex(inst);
1656	2010	const ptr = try self.resolveInst(bin_op.lhs);
1657	2011	const ptr_ty = self.air.typeOf(bin_op.lhs);
1658	2012	const len = try self.resolveInst(bin_op.rhs);
1659	2013	const len_ty = self.air.typeOf(bin_op.rhs);
1660	2014
1661		const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
1662		try self.genSetStack(ptr_ty, stack_offset, ptr, .{});
1663		try self.genSetStack(len_ty, stack_offset - 8, len, .{});
1664		const result = MCValue{ .stack_offset = stack_offset };
	2015	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
	2016	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
	2017	try self.genSetMem(
	2018	.{ .frame = frame_index },
	2019	@intCast(i32, ptr_ty.abiSize(self.target.*)),
	2020	len_ty,
	2021	len,
	2022	);
1665	2023
	2024	const result = MCValue{ .load_frame = .{ .index = frame_index } };
1666	2025	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1667	2026	}
1668	2027
...	...	@@ -1778,14 +2137,14 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1778	2137
1779	2138	const reg_bits = self.regBitSize(ty);
1780	2139	const cc: Condition = if (ty.isSignedInt()) cc: {
1781		try self.genSetReg(ty, limit_reg, dst_mcv);
	2140	try self.genSetReg(limit_reg, ty, dst_mcv);
1782	2141	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1783	2142	try self.genBinOpMir(.xor, ty, limit_mcv, .{
1784	2143	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1785	2144	});
1786	2145	break :cc .o;
1787	2146	} else cc: {
1788		try self.genSetReg(ty, limit_reg, .{
	2147	try self.genSetReg(limit_reg, ty, .{
1789	2148	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1790	2149	});
1791	2150	break :cc .c;
...	...	@@ -1829,14 +2188,14 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1829	2188
1830	2189	const reg_bits = self.regBitSize(ty);
1831	2190	const cc: Condition = if (ty.isSignedInt()) cc: {
1832		try self.genSetReg(ty, limit_reg, dst_mcv);
	2191	try self.genSetReg(limit_reg, ty, dst_mcv);
1833	2192	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1834	2193	try self.genBinOpMir(.xor, ty, limit_mcv, .{
1835	2194	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1836	2195	});
1837	2196	break :cc .o;
1838	2197	} else cc: {
1839		try self.genSetReg(ty, limit_reg, .{ .immediate = 0 });
	2198	try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
1840	2199	break :cc .c;
1841	2200	};
1842	2201	try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
...	...	@@ -1880,7 +2239,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1880	2239
1881	2240	const reg_bits = self.regBitSize(ty);
1882	2241	const cc: Condition = if (ty.isSignedInt()) cc: {
1883		try self.genSetReg(ty, limit_reg, lhs_mcv);
	2242	try self.genSetReg(limit_reg, ty, lhs_mcv);
1884	2243	try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
1885	2244	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1886	2245	try self.genBinOpMir(.xor, ty, limit_mcv, .{
...	...	@@ -1888,7 +2247,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1888	2247	});
1889	2248	break :cc .o;
1890	2249	} else cc: {
1891		try self.genSetReg(ty, limit_reg, .{
	2250	try self.genSetReg(limit_reg, ty, .{
1892	2251	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1893	2252	});
1894	2253	break :cc .c;
...	...	@@ -1916,17 +2275,18 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1916	2275	.Int => {
1917	2276	try self.spillEflagsIfOccupied();
1918	2277
1919		const partial_mcv = switch (tag) {
1920		.add_with_overflow => try self.genBinOp(null, .add, bin_op.lhs, bin_op.rhs),
1921		.sub_with_overflow => try self.genBinOp(null, .sub, bin_op.lhs, bin_op.rhs),
	2278	const partial_mcv = try self.genBinOp(null, switch (tag) {
	2279	.add_with_overflow => .add,
	2280	.sub_with_overflow => .sub,
1922	2281	else => unreachable,
1923		};
	2282	}, bin_op.lhs, bin_op.rhs);
1924	2283	const int_info = ty.intInfo(self.target.*);
1925	2284	const cc: Condition = switch (int_info.signedness) {
1926	2285	.unsigned => .c,
1927	2286	.signed => .o,
1928	2287	};
1929	2288
	2289	const tuple_ty = self.air.typeOfIndex(inst);
1930	2290	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
1931	2291	switch (partial_mcv) {
1932	2292	.register => \|reg\| {
...	...	@@ -1936,27 +2296,32 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1936	2296	else => {},
1937	2297	}
1938	2298
1939		const abi_size = @intCast(i32, ty.abiSize(self.target.*));
1940		const dst_mcv = try self.allocRegOrMem(inst, false);
1941		try self.genSetStack(
	2299	const frame_index =
	2300	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2301	try self.genSetMem(
	2302	.{ .frame = frame_index },
	2303	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
1942	2304	Type.u1,
1943		dst_mcv.stack_offset - abi_size,
1944	2305	.{ .eflags = cc },
1945		.{},
1946	2306	);
1947		try self.genSetStack(ty, dst_mcv.stack_offset, partial_mcv, .{});
1948		break :result dst_mcv;
	2307	try self.genSetMem(
	2308	.{ .frame = frame_index },
	2309	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2310	ty,
	2311	partial_mcv,
	2312	);
	2313	break :result .{ .load_frame = .{ .index = frame_index } };
1949	2314	}
1950	2315
1951		const tuple_ty = self.air.typeOfIndex(inst);
1952		const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
1953		const tuple_align = tuple_ty.abiAlignment(self.target.*);
1954		const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));
1955		const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
1956
1957		try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);
1958
1959		break :result .{ .stack_offset = stack_offset };
	2316	const frame_index =
	2317	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2318	try self.genSetFrameTruncatedOverflowCompare(
	2319	tuple_ty,
	2320	frame_index,
	2321	partial_mcv.register,
	2322	cc,
	2323	);
	2324	break :result .{ .load_frame = .{ .index = frame_index } };
1960	2325	},
1961	2326	else => unreachable,
1962	2327	}
...	...	@@ -1998,6 +2363,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1998	2363	try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);
1999	2364	const cc = Condition.ne;
2000	2365
	2366	const tuple_ty = self.air.typeOfIndex(inst);
2001	2367	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
2002	2368	switch (partial_mcv) {
2003	2369	.register => \|reg\| {
...	...	@@ -2007,27 +2373,32 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2007	2373	else => {},
2008	2374	}
2009	2375
2010		const abi_size = @intCast(i32, lhs_ty.abiSize(self.target.*));
2011		const dst_mcv = try self.allocRegOrMem(inst, false);
2012		try self.genSetStack(
2013		Type.u1,
2014		dst_mcv.stack_offset - abi_size,
	2376	const frame_index =
	2377	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2378	try self.genSetMem(
	2379	.{ .frame = frame_index },
	2380	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
	2381	tuple_ty.structFieldType(1),
2015	2382	.{ .eflags = cc },
2016		.{},
2017	2383	);
2018		try self.genSetStack(lhs_ty, dst_mcv.stack_offset, partial_mcv, .{});
2019		break :result dst_mcv;
	2384	try self.genSetMem(
	2385	.{ .frame = frame_index },
	2386	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2387	tuple_ty.structFieldType(0),
	2388	partial_mcv,
	2389	);
	2390	break :result .{ .load_frame = .{ .index = frame_index } };
2020	2391	}
2021	2392
2022		const tuple_ty = self.air.typeOfIndex(inst);
2023		const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
2024		const tuple_align = tuple_ty.abiAlignment(self.target.*);
2025		const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));
2026		const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
2027
2028		try self.genSetStackTruncatedOverflowCompare(lhs_ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);
2029
2030		break :result .{ .stack_offset = stack_offset };
	2393	const frame_index =
	2394	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2395	try self.genSetFrameTruncatedOverflowCompare(
	2396	tuple_ty,
	2397	frame_index,
	2398	partial_mcv.register,
	2399	cc,
	2400	);
	2401	break :result .{ .load_frame = .{ .index = frame_index } };
2031	2402	},
2032	2403	else => unreachable,
2033	2404	}
...	...	@@ -2035,17 +2406,17 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2035	2406	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2036	2407	}
2037	2408
2038		fn genSetStackTruncatedOverflowCompare(
	2409	fn genSetFrameTruncatedOverflowCompare(
2039	2410	self: *Self,
2040		ty: Type,
2041		stack_offset: i32,
2042		overflow_bit_offset: i32,
	2411	tuple_ty: Type,
	2412	frame_index: FrameIndex,
2043	2413	reg: Register,
2044	2414	cc: Condition,
2045	2415	) !void {
2046	2416	const reg_lock = self.register_manager.lockReg(reg);
2047	2417	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
2048	2418
	2419	const ty = tuple_ty.structFieldType(0);
2049	2420	const int_info = ty.intInfo(self.target.*);
2050	2421	const extended_ty = switch (int_info.signedness) {
2051	2422	.signed => Type.isize,
...	...	@@ -2062,7 +2433,7 @@ fn genSetStackTruncatedOverflowCompare(
2062	2433	try self.asmSetccRegister(overflow_reg.to8(), cc);
2063	2434
2064	2435	const scratch_reg = temp_regs[1];
2065		try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });
	2436	try self.genSetReg(scratch_reg, extended_ty, .{ .register = reg });
2066	2437	try self.truncateRegister(ty, scratch_reg);
2067	2438	try self.genBinOpMir(
2068	2439	.cmp,
...	...	@@ -2080,10 +2451,18 @@ fn genSetStackTruncatedOverflowCompare(
2080	2451	.{ .register = eq_reg },
2081	2452	);
2082	2453
2083		try self.genSetStack(ty, stack_offset, .{ .register = scratch_reg }, .{});
2084		try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{
2085		.register = overflow_reg.to8(),
2086		}, .{});
	2454	try self.genSetMem(
	2455	.{ .frame = frame_index },
	2456	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
	2457	tuple_ty.structFieldType(1),
	2458	.{ .register = overflow_reg.to8() },
	2459	);
	2460	try self.genSetMem(
	2461	.{ .frame = frame_index },
	2462	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2463	ty,
	2464	.{ .register = scratch_reg },
	2465	);
2087	2466	}
2088	2467
2089	2468	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -2101,6 +2480,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2101	2480	.unsigned => .c,
2102	2481	.signed => .o,
2103	2482	};
	2483
	2484	const tuple_ty = self.air.typeOfIndex(inst);
2104	2485	if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {
2105	2486	var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
2106	2487	.signed => .int_signed,
...	...	@@ -2127,16 +2508,21 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2127	2508
2128	2509	// For now, this is the only supported multiply that doesn't fit in a register.
2129	2510	assert(dst_info.bits == 128 and src_pl.data == 64);
2130		const dst_abi_size = @intCast(i32, dst_ty.abiSize(self.target.*));
2131		const dst_mcv = try self.allocRegOrMem(inst, false);
2132		try self.genSetStack(
2133		Type.u1,
2134		dst_mcv.stack_offset - dst_abi_size,
2135		.{ .immediate = 0 }, // 64x64 -> 128 never overflows
2136		.{},
	2511	const frame_index =
	2512	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2513	try self.genSetMem(
	2514	.{ .frame = frame_index },
	2515	@intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
	2516	tuple_ty.structFieldType(1),
	2517	.{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
2137	2518	);
2138		try self.genSetStack(dst_ty, dst_mcv.stack_offset, partial_mcv, .{});
2139		break :result dst_mcv;
	2519	try self.genSetMem(
	2520	.{ .frame = frame_index },
	2521	@intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
	2522	tuple_ty.structFieldType(0),
	2523	partial_mcv,
	2524	);
	2525	break :result .{ .load_frame = .{ .index = frame_index } };
2140	2526	}
2141	2527
2142	2528	const dst_reg: Register = dst_reg: {
...	...	@@ -2184,15 +2570,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2184	2570	}
2185	2571	};
2186	2572
2187		const tuple_ty = self.air.typeOfIndex(inst);
2188		const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
2189		const tuple_align = tuple_ty.abiAlignment(self.target.*);
2190		const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));
2191		const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
2192
2193		try self.genSetStackTruncatedOverflowCompare(dst_ty, stack_offset, overflow_bit_offset, dst_reg, cc);
2194
2195		break :result .{ .stack_offset = stack_offset };
	2573	const frame_index =
	2574	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
	2575	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, dst_reg, cc);
	2576	break :result .{ .load_frame = .{ .index = frame_index } };
2196	2577	},
2197	2578	else => unreachable,
2198	2579	}
...	...	@@ -2220,7 +2601,7 @@ fn genIntMulDivOpMir(
2220	2601	.register => \|reg\| if (reg.to64() == .rax) break :lhs,
2221	2602	else => {},
2222	2603	}
2223		try self.genSetReg(ty, .rax, lhs);
	2604	try self.genSetReg(.rax, ty, lhs);
2224	2605	}
2225	2606
2226	2607	switch (tag) {
...	...	@@ -2230,16 +2611,26 @@ fn genIntMulDivOpMir(
2230	2611	.idiv => try self.asmOpOnly(.cqo),
2231	2612	}
2232	2613
2233		const factor: MCValue = switch (rhs) {
2234		.register, .stack_offset => rhs,
	2614	const mat_rhs: MCValue = switch (rhs) {
	2615	.register, .indirect, .load_frame => rhs,
2235	2616	else => .{ .register = try self.copyToTmpRegister(ty, rhs) },
2236	2617	};
2237		switch (factor) {
	2618	switch (mat_rhs) {
2238	2619	.register => \|reg\| try self.asmRegister(tag, reg),
2239		.stack_offset => \|off\| try self.asmMemory(tag, Memory.sib(
2240		Memory.PtrSize.fromSize(abi_size),
2241		.{ .base = .rbp, .disp = -off },
2242		)),
	2620	.indirect, .load_frame => try self.asmMemory(
	2621	tag,
	2622	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (mat_rhs) {
	2623	.indirect => \|reg_off\| .{
	2624	.base = .{ .reg = reg_off.reg },
	2625	.disp = reg_off.off,
	2626	},
	2627	.load_frame => \|frame_addr\| .{
	2628	.base = .{ .frame = frame_addr.index },
	2629	.disp = frame_addr.off,
	2630	},
	2631	else => unreachable,
	2632	}),
	2633	),
2243	2634	else => unreachable,
2244	2635	}
2245	2636	}
...	...	@@ -2315,7 +2706,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
2315	2706	}
2316	2707
2317	2708	const pl_mcv = try self.allocRegOrMem(inst, true);
2318		try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) {
	2709	try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) {
2319	2710	else => opt_mcv,
2320	2711	.register_overflow => \|ro\| .{ .register = ro.reg },
2321	2712	});
...	...	@@ -2361,11 +2752,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2361	2752
2362	2753	const pl_ty = dst_ty.childType();
2363	2754	const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
2364		try self.asmMemoryImmediate(
2365		.mov,
2366		Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }),
2367		Immediate.u(1),
2368		);
	2755	try self.genSetMem(.{ .reg = dst_mcv.register }, pl_abi_size, Type.bool, .{ .immediate = 1 });
2369	2756	break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
2370	2757	};
2371	2758	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -2389,10 +2776,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2389	2776
2390	2777	const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2391	2778	switch (operand) {
2392		.stack_offset => \|off\| {
2393		const offset = off - @intCast(i32, err_off);
2394		break :result MCValue{ .stack_offset = offset };
2395		},
2396	2779	.register => \|reg\| {
2397	2780	// TODO reuse operand
2398	2781	const eu_lock = self.register_manager.lockReg(reg);
...	...	@@ -2407,6 +2790,10 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2407	2790	}
2408	2791	break :result result;
2409	2792	},
	2793	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
	2794	.index = frame_addr.index,
	2795	.off = frame_addr.off + @intCast(i32, err_off),
	2796	} },
2410	2797	else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),
2411	2798	}
2412	2799	};
...	...	@@ -2434,10 +2821,10 @@ fn genUnwrapErrorUnionPayloadMir(
2434	2821
2435	2822	const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2436	2823	switch (err_union) {
2437		.stack_offset => \|off\| {
2438		const offset = off - @intCast(i32, payload_off);
2439		break :result MCValue{ .stack_offset = offset };
2440		},
	2824	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
	2825	.index = frame_addr.index,
	2826	.off = frame_addr.off + @intCast(i32, payload_off),
	2827	} },
2441	2828	.register => \|reg\| {
2442	2829	// TODO reuse operand
2443	2830	const eu_lock = self.register_manager.lockReg(reg);
...	...	@@ -2488,7 +2875,10 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2488	2875	try self.asmRegisterMemory(
2489	2876	.mov,
2490	2877	registerAlias(dst_reg, err_abi_size),
2491		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
	2878	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	2879	.base = .{ .reg = src_reg },
	2880	.disp = err_off,
	2881	}),
2492	2882	);
2493	2883
2494	2884	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -2523,7 +2913,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2523	2913	try self.asmRegisterMemory(
2524	2914	.lea,
2525	2915	registerAlias(dst_reg, dst_abi_size),
2526		Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
	2916	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2527	2917	);
2528	2918
2529	2919	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -2548,7 +2938,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2548	2938	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
2549	2939	try self.asmMemoryImmediate(
2550	2940	.mov,
2551		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
	2941	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	2942	.base = .{ .reg = src_reg },
	2943	.disp = err_off,
	2944	}),
2552	2945	Immediate.u(0),
2553	2946	);
2554	2947
...	...	@@ -2567,7 +2960,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2567	2960	try self.asmRegisterMemory(
2568	2961	.lea,
2569	2962	registerAlias(dst_reg, dst_abi_size),
2570		Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
	2963	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2571	2964	);
2572	2965	break :result .{ .register = dst_reg };
2573	2966	};
...	...	@@ -2608,7 +3001,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2608	3001	defer if (pl_lock) \|lock\| self.register_manager.unlockReg(lock);
2609	3002
2610	3003	const opt_mcv = try self.allocRegOrMem(inst, true);
2611		try self.setRegOrMem(pl_ty, opt_mcv, pl_mcv);
	3004	try self.genCopy(pl_ty, opt_mcv, pl_mcv);
2612	3005
2613	3006	if (!same_repr) {
2614	3007	const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
...	...	@@ -2621,9 +3014,12 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2621	3014	Immediate.u(@intCast(u6, pl_abi_size * 8)),
2622	3015	),
2623	3016
2624		.stack_offset => \|off\| try self.asmMemoryImmediate(
	3017	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(
2625	3018	.mov,
2626		Memory.sib(.byte, .{ .base = .rbp, .disp = pl_abi_size - off }),
	3019	Memory.sib(.byte, .{
	3020	.base = .{ .frame = frame_addr.index },
	3021	.disp = frame_addr.off + pl_abi_size,
	3022	}),
2627	3023	Immediate.u(1),
2628	3024	),
2629	3025	}
...	...	@@ -2637,26 +3033,21 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2637	3033	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2638	3034	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2639	3035
2640		const error_union_ty = self.air.getRefType(ty_op.ty);
2641		const payload_ty = error_union_ty.errorUnionPayload();
	3036	const eu_ty = self.air.getRefType(ty_op.ty);
	3037	const pl_ty = eu_ty.errorUnionPayload();
	3038	const err_ty = eu_ty.errorUnionSet();
2642	3039	const operand = try self.resolveInst(ty_op.operand);
2643	3040
2644	3041	const result: MCValue = result: {
2645		if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
2646		break :result operand;
2647		}
2648
2649		const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2650		const abi_align = error_union_ty.abiAlignment(self.target.*);
2651		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2652		const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2653		const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2654		try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), operand, .{});
2655		try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), .{ .immediate = 0 }, .{});
	3042	if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result .{ .immediate = 0 };
2656	3043
2657		break :result MCValue{ .stack_offset = stack_offset };
	3044	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
	3045	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
	3046	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
	3047	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
	3048	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
	3049	break :result .{ .load_frame = .{ .index = frame_index } };
2658	3050	};
2659
2660	3051	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2661	3052	}
2662	3053
...	...	@@ -2664,26 +3055,21 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2664	3055	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2665	3056	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2666	3057
2667		const error_union_ty = self.air.getRefType(ty_op.ty);
2668		const payload_ty = error_union_ty.errorUnionPayload();
2669		const operand = try self.resolveInst(ty_op.operand);
	3058	const eu_ty = self.air.getRefType(ty_op.ty);
	3059	const pl_ty = eu_ty.errorUnionPayload();
	3060	const err_ty = eu_ty.errorUnionSet();
2670	3061
2671	3062	const result: MCValue = result: {
2672		if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
2673		break :result operand;
2674		}
2675
2676		const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2677		const abi_align = error_union_ty.abiAlignment(self.target.*);
2678		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2679		const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2680		const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2681		try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), operand, .{});
2682		try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), .undef, .{});
	3063	if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result try self.resolveInst(ty_op.operand);
2683	3064
2684		break :result MCValue{ .stack_offset = stack_offset };
	3065	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
	3066	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
	3067	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
	3068	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
	3069	const operand = try self.resolveInst(ty_op.operand);
	3070	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
	3071	break :result .{ .load_frame = .{ .index = frame_index } };
2685	3072	};
2686
2687	3073	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2688	3074	}
2689	3075
...	...	@@ -2695,7 +3081,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2695	3081
2696	3082	const dst_mcv = try self.allocRegOrMem(inst, true);
2697	3083	const dst_ty = self.air.typeOfIndex(inst);
2698		try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);
	3084	try self.genCopy(dst_ty, dst_mcv, src_mcv);
2699	3085	break :result dst_mcv;
2700	3086	};
2701	3087	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -2707,9 +3093,10 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2707	3093	const operand = try self.resolveInst(ty_op.operand);
2708	3094	const dst_mcv: MCValue = blk: {
2709	3095	switch (operand) {
2710		.stack_offset => \|off\| {
2711		break :blk MCValue{ .stack_offset = off - 8 };
2712		},
	3096	.load_frame => \|frame_addr\| break :blk .{ .load_frame = .{
	3097	.index = frame_addr.index,
	3098	.off = frame_addr.off + 8,
	3099	} },
2713	3100	else => return self.fail("TODO implement slice_len for {}", .{operand}),
2714	3101	}
2715	3102	};
...	...	@@ -2743,7 +3130,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2743	3130	.lea,
2744	3131	registerAlias(dst_reg, dst_abi_size),
2745	3132	Memory.sib(.qword, .{
2746		.base = src_reg,
	3133	.base = .{ .reg = src_reg },
2747	3134	.disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),
2748	3135	}),
2749	3136	);
...	...	@@ -2771,7 +3158,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
2771	3158	// Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
2772	3159	// and set the register directly to the scaled offset as an immediate.
2773	3160	const reg = try self.register_manager.allocReg(null, gp);
2774		try self.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });
	3161	try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size });
2775	3162	break :blk reg;
2776	3163	},
2777	3164	else => {
...	...	@@ -2812,10 +3199,14 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2812	3199
2813	3200	const addr_reg = try self.register_manager.allocReg(null, gp);
2814	3201	switch (slice_mcv) {
2815		.stack_offset => \|off\| try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
2816		.base = .rbp,
2817		.disp = -off,
2818		})),
	3202	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
	3203	.mov,
	3204	addr_reg.to64(),
	3205	Memory.sib(.qword, .{
	3206	.base = .{ .frame = frame_addr.index },
	3207	.disp = frame_addr.off,
	3208	}),
	3209	),
2819	3210	else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
2820	3211	}
2821	3212	// TODO we could allocate register here, but need to expect addr register and potentially
...	...	@@ -2834,7 +3225,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2834	3225	const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
2835	3226	const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
2836	3227	const dst_mcv = try self.allocRegOrMem(inst, false);
2837		try self.load(dst_mcv, elem_ptr, slice_ptr_field_type);
	3228	try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
2838	3229
2839	3230	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2840	3231	}
...	...	@@ -2872,33 +3263,27 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2872	3263	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2873	3264	defer self.register_manager.unlockReg(offset_reg_lock);
2874	3265
2875		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	3266	const addr_reg = try self.register_manager.allocReg(null, gp);
2876	3267	switch (array) {
2877	3268	.register => {
2878		const off = @intCast(i32, try self.allocMem(
2879		inst,
2880		@intCast(u32, array_ty.abiSize(self.target.*)),
2881		array_ty.abiAlignment(self.target.*),
2882		));
2883		try self.genSetStack(array_ty, off, array, .{});
2884		try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{
2885		.base = .rbp,
2886		.disp = -off,
2887		}));
2888		},
2889		.stack_offset => \|off\| {
2890		try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{
2891		.base = .rbp,
2892		.disp = -off,
2893		}));
	3269	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
	3270	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
	3271	try self.asmRegisterMemory(
	3272	.lea,
	3273	addr_reg,
	3274	Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
	3275	);
2894	3276	},
2895		.load_got => try self.genSetReg(array_ty, addr_reg, array),
2896		.memory, .load_direct, .load_tlv => try self.genSetReg(Type.usize, addr_reg, switch (array) {
2897		.memory => \|addr\| .{ .immediate = addr },
2898		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
2899		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
2900		else => unreachable,
2901		}),
	3277	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
	3278	.lea,
	3279	addr_reg,
	3280	Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
	3281	),
	3282	.memory,
	3283	.load_direct,
	3284	.load_got,
	3285	.load_tlv,
	3286	=> try self.genSetReg(addr_reg, Type.usize, array.address()),
2902	3287	.lea_direct, .lea_tlv => unreachable,
2903	3288	else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2904	3289	}
...	...	@@ -2907,7 +3292,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2907	3292	// offset register.
2908	3293	const dst_mcv = try self.allocRegOrMem(inst, false);
2909	3294	try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2910		try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
	3295	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
2911	3296
2912	3297	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2913	3298	}
...	...	@@ -2948,7 +3333,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2948	3333	else => null,
2949	3334	};
2950	3335	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2951		try self.load(dst_mcv, .{ .register = elem_ptr_reg }, ptr_ty);
	3336	try self.load(dst_mcv, ptr_ty, .{ .register = elem_ptr_reg });
2952	3337
2953	3338	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2954	3339	}
...	...	@@ -2987,8 +3372,8 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2987	3372
2988	3373	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2989	3374	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2990		const ptr_ty = self.air.typeOf(bin_op.lhs);
2991		const union_ty = ptr_ty.childType();
	3375	const ptr_union_ty = self.air.typeOf(bin_op.lhs);
	3376	const union_ty = ptr_union_ty.childType();
2992	3377	const tag_ty = self.air.typeOf(bin_op.rhs);
2993	3378	const layout = union_ty.unionGetLayout(self.target.*);
2994	3379
...	...	@@ -3012,12 +3397,15 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3012	3397
3013	3398	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
3014	3399	// TODO reusing the operand
3015		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3016		try self.genBinOpMir(.add, ptr_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
	3400	const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
	3401	try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
3017	3402	break :blk MCValue{ .register = reg };
3018	3403	} else ptr;
3019	3404
3020		try self.store(adjusted_ptr, tag, ptr_ty, tag_ty);
	3405	var ptr_tag_pl = ptr_union_ty.ptrInfo();
	3406	ptr_tag_pl.data.pointee_type = tag_ty;
	3407	const ptr_tag_ty = Type.initPayload(&ptr_tag_pl.base);
	3408	try self.store(ptr_tag_ty, adjusted_ptr, tag);
3021	3409
3022	3410	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3023	3411	}
...	...	@@ -3044,11 +3432,14 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3044	3432	const tag_abi_size = tag_ty.abiSize(self.target.*);
3045	3433	const dst_mcv: MCValue = blk: {
3046	3434	switch (operand) {
3047		.stack_offset => \|off\| {
	3435	.load_frame => \|frame_addr\| {
3048	3436	if (tag_abi_size <= 8) {
3049		const offset: i32 = if (layout.tag_align < layout.payload_align) @intCast(i32, layout.payload_size) else 0;
	3437	const off: i32 = if (layout.tag_align < layout.payload_align)
	3438	@intCast(i32, layout.payload_size)
	3439	else
	3440	0;
3050	3441	break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{
3051		.stack_offset = off - offset,
	3442	.load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
3052	3443	});
3053	3444	}
3054	3445
...	...	@@ -3131,7 +3522,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3131	3522	.nz,
3132	3523	);
3133	3524
3134		try self.genSetReg(dst_ty, dst_reg, .{ .immediate = src_bits - 1 });
	3525	try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
3135	3526	try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
3136	3527	}
3137	3528	break :result dst_mcv;
...	...	@@ -3171,7 +3562,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3171	3562	};
3172	3563	const tmp_mcv = tmp: {
3173	3564	if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;
3174		try self.genSetReg(src_ty, dst_reg, src_mcv);
	3565	try self.genSetReg(dst_reg, src_ty, src_mcv);
3175	3566	break :tmp dst_mcv;
3176	3567	};
3177	3568	try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
...	...	@@ -3345,7 +3736,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
3345	3736	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
3346	3737	defer self.register_manager.unlockReg(dst_lock);
3347	3738
3348		try self.genSetReg(src_ty, dst_mcv.register, src_mcv);
	3739	try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
3349	3740	switch (src_bits) {
3350	3741	else => unreachable,
3351	3742	16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
...	...	@@ -3450,7 +3841,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3450	3841	.lea,
3451	3842	if (src_abi_size > 4) tmp.to64() else tmp.to32(),
3452	3843	Memory.sib(.qword, .{
3453		.base = dst.to64(),
	3844	.base = .{ .reg = dst.to64() },
3454	3845	.scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
3455	3846	}),
3456	3847	);
...	...	@@ -3473,7 +3864,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3473	3864	.lea,
3474	3865	if (src_abi_size > 4) dst.to64() else dst.to32(),
3475	3866	Memory.sib(.qword, .{
3476		.base = tmp.to64(),
	3867	.base = .{ .reg = tmp.to64() },
3477	3868	.scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
3478	3869	}),
3479	3870	);
...	...	@@ -3506,6 +3897,17 @@ fn reuseOperand(
3506	3897	operand: Air.Inst.Ref,
3507	3898	op_index: Liveness.OperandInt,
3508	3899	mcv: MCValue,
	3900	) bool {
	3901	return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
	3902	}
	3903
	3904	fn reuseOperandAdvanced(
	3905	self: *Self,
	3906	inst: Air.Inst.Index,
	3907	operand: Air.Inst.Ref,
	3908	op_index: Liveness.OperandInt,
	3909	mcv: MCValue,
	3910	tracked_inst: Air.Inst.Index,
3509	3911	) bool {
3510	3912	if (!self.liveness.operandDies(inst, op_index))
3511	3913	return false;
...	...	@@ -3516,25 +3918,23 @@ fn reuseOperand(
3516	3918	// new instruction.
3517	3919	if (!self.register_manager.isRegFree(reg)) {
3518	3920	if (RegisterManager.indexOfRegIntoTracked(reg)) \|index\| {
3519		self.register_manager.registers[index] = inst;
	3921	self.register_manager.registers[index] = tracked_inst;
3520	3922	}
3521	3923	}
3522		log.debug("%{d} => {} (reused)", .{ inst, reg });
3523		},
3524		.stack_offset => \|off\| {
3525		log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
3526	3924	},
	3925	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
3527	3926	else => return false,
3528	3927	}
3529	3928
3530	3929	// Prevent the operand deaths processing code from deallocating it.
3531	3930	self.liveness.clearOperandDeath(inst, op_index);
3532		self.getResolvedInstValue(Air.refToIndex(operand).?).reuse(self);
	3931	const op_inst = Air.refToIndex(operand).?;
	3932	self.getResolvedInstValue(op_inst).reuse(self, tracked_inst, op_inst);
3533	3933
3534	3934	return true;
3535	3935	}
3536	3936
3537		fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {
	3937	fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
3538	3938	const ptr_info = ptr_ty.ptrInfo().data;
3539	3939
3540	3940	const val_ty = ptr_info.pointee_type;
...	...	@@ -3564,10 +3964,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3564	3964	if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;
3565	3965	if (load_abi_size <= 8) {
3566	3966	const load_reg = registerAlias(dst_reg, load_abi_size);
3567		try self.asmRegisterMemory(.mov, load_reg, Memory.sib(
3568		Memory.PtrSize.fromSize(load_abi_size),
3569		.{ .base = ptr_reg, .disp = val_byte_off },
3570		));
	3967	try self.asmRegisterMemory(
	3968	.mov,
	3969	load_reg,
	3970	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
	3971	.base = .{ .reg = ptr_reg },
	3972	.disp = val_byte_off,
	3973	}),
	3974	);
3571	3975	try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
3572	3976	} else {
3573	3977	const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
...	...	@@ -3575,85 +3979,60 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3575	3979	defer self.register_manager.unlockReg(tmp_lock);
3576	3980
3577	3981	const dst_alias = registerAlias(dst_reg, val_abi_size);
3578		try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(
3579		Memory.PtrSize.fromSize(val_abi_size),
3580		.{ .base = ptr_reg, .disp = val_byte_off },
3581		));
3582		try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(
3583		Memory.PtrSize.fromSize(val_abi_size),
3584		.{ .base = ptr_reg, .disp = val_byte_off + 1 },
3585		));
	3982	try self.asmRegisterMemory(
	3983	.mov,
	3984	dst_alias,
	3985	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
	3986	.base = .{ .reg = ptr_reg },
	3987	.disp = val_byte_off,
	3988	}),
	3989	);
	3990	try self.asmRegisterMemory(
	3991	.mov,
	3992	tmp_reg,
	3993	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
	3994	.base = .{ .reg = ptr_reg },
	3995	.disp = val_byte_off + 1,
	3996	}),
	3997	);
3586	3998	try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
3587	3999	}
3588	4000
3589	4001	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
3590		try self.setRegOrMem(val_ty, dst_mcv, .{ .register = dst_reg });
3591		}
3592
3593		fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
3594		const elem_ty = ptr_ty.elemType();
3595		const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
3596		switch (ptr) {
3597		.none => unreachable,
3598		.undef => unreachable,
3599		.unreach => unreachable,
3600		.dead => unreachable,
3601		.eflags => unreachable,
3602		.register_overflow => unreachable,
3603		.immediate => \|imm\| {
3604		try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm });
3605		},
3606		.stack_offset => {
3607		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3608		try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
3609		},
3610		.ptr_stack_offset => \|off\| {
3611		try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
3612		},
3613		.register => \|reg\| {
3614		const reg_lock = self.register_manager.lockReg(reg);
3615		defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	4002	try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg });
	4003	}
3616	4004
3617		switch (dst_mcv) {
3618		.dead => unreachable,
3619		.undef => unreachable,
3620		.eflags => unreachable,
3621		.register => \|dst_reg\| {
3622		try self.asmRegisterMemory(
3623		.mov,
3624		registerAlias(dst_reg, abi_size),
3625		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }),
3626		);
3627		},
3628		.stack_offset => \|off\| {
3629		if (abi_size <= 8) {
3630		const tmp_reg = try self.register_manager.allocReg(null, gp);
3631		try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
3632		return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
3633		}
	4005	fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
	4006	const dst_ty = ptr_ty.childType();
	4007	switch (ptr_mcv) {
	4008	.none,
	4009	.unreach,
	4010	.dead,
	4011	.undef,
	4012	.eflags,
	4013	.register_overflow,
	4014	.reserved_frame,
	4015	=> unreachable, // not a valid pointer
	4016	.immediate,
	4017	.register,
	4018	.register_offset,
	4019	.lea_direct,
	4020	.lea_got,
	4021	.lea_tlv,
	4022	.lea_frame,
	4023	=> try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
	4024	.memory,
	4025	.indirect,
	4026	.load_direct,
	4027	.load_got,
	4028	.load_tlv,
	4029	.load_frame,
	4030	=> {
	4031	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
	4032	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	4033	defer self.register_manager.unlockReg(addr_lock);
3634	4034
3635		try self.genInlineMemcpy(
3636		.{ .ptr_stack_offset = off },
3637		ptr,
3638		.{ .immediate = abi_size },
3639		.{},
3640		);
3641		},
3642		else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
3643		}
3644		},
3645		.load_direct => \|sym_index\| {
3646		const addr_reg = try self.copyToTmpRegister(Type.usize, .{ .lea_direct = sym_index });
3647		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3648		defer self.register_manager.unlockReg(addr_reg_lock);
3649		// Load the pointer, which is stored in memory
3650		try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3651		try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
3652		},
3653		.load_tlv => \|sym_index\| try self.load(dst_mcv, .{ .lea_tlv = sym_index }, ptr_ty),
3654		.memory, .load_got, .lea_direct, .lea_tlv => {
3655		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3656		try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
	4035	try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3657	4036	},
3658	4037	}
3659	4038	}
...	...	@@ -3669,53 +4048,48 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
3669	4048	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
3670	4049	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
3671	4050
3672		const ptr = try self.resolveInst(ty_op.operand);
3673		const dst_mcv: MCValue = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr))
	4051	const ptr_mcv = try self.resolveInst(ty_op.operand);
	4052	const dst_mcv = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
3674	4053	// The MCValue that holds the pointer can be re-used as the value.
3675		ptr
	4054	ptr_mcv
3676	4055	else
3677	4056	try self.allocRegOrMem(inst, true);
3678	4057
3679	4058	const ptr_ty = self.air.typeOf(ty_op.operand);
3680	4059	if (ptr_ty.ptrInfo().data.host_size > 0) {
3681		try self.packedLoad(dst_mcv, ptr, ptr_ty);
	4060	try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv);
3682	4061	} else {
3683		try self.load(dst_mcv, ptr, ptr_ty);
	4062	try self.load(dst_mcv, ptr_ty, ptr_mcv);
3684	4063	}
3685	4064	break :result dst_mcv;
3686	4065	};
3687	4066	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3688	4067	}
3689	4068
3690		fn packedStore(
3691		self: *Self,
3692		ptr_mcv: MCValue,
3693		val_mcv: MCValue,
3694		ptr_ty: Type,
3695		val_ty: Type,
3696		) InnerError!void {
	4069	fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
3697	4070	const ptr_info = ptr_ty.ptrInfo().data;
	4071	const src_ty = ptr_ty.childType();
3698	4072
3699	4073	const limb_abi_size = @min(ptr_info.host_size, 8);
3700	4074	const limb_abi_bits = limb_abi_size * 8;
3701	4075
3702		const val_bit_size = val_ty.bitSize(self.target.*);
3703		const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
3704		const val_bit_off = ptr_info.bit_offset % limb_abi_bits;
	4076	const src_bit_size = src_ty.bitSize(self.target.*);
	4077	const src_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
	4078	const src_bit_off = ptr_info.bit_offset % limb_abi_bits;
3705	4079
3706	4080	const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3707	4081	const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
3708	4082	defer self.register_manager.unlockReg(ptr_lock);
3709	4083
3710	4084	var limb_i: u16 = 0;
3711		while (limb_i * limb_abi_bits < val_bit_off + val_bit_size) : (limb_i += 1) {
3712		const part_bit_off = if (limb_i == 0) val_bit_off else 0;
	4085	while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) {
	4086	const part_bit_off = if (limb_i == 0) src_bit_off else 0;
3713	4087	const part_bit_size =
3714		@min(val_bit_off + val_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
3715		const limb_mem = Memory.sib(
3716		Memory.PtrSize.fromSize(limb_abi_size),
3717		.{ .base = ptr_reg, .disp = val_byte_off + limb_i * limb_abi_bits },
3718		);
	4088	@min(src_bit_off + src_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
	4089	const limb_mem = Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), .{
	4090	.base = .{ .reg = ptr_reg },
	4091	.disp = src_byte_off + limb_i * limb_abi_bits,
	4092	});
3719	4093
3720	4094	const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) <<
3721	4095	@intCast(u6, part_bit_off);
...	...	@@ -3731,211 +4105,62 @@ fn packedStore(
3731	4105	try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
3732	4106	}
3733	4107
3734		if (val_bit_size <= 64) {
	4108	if (src_bit_size <= 64) {
3735	4109	const tmp_reg = try self.register_manager.allocReg(null, gp);
3736	4110	const tmp_mcv = MCValue{ .register = tmp_reg };
3737	4111	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3738	4112	defer self.register_manager.unlockReg(tmp_lock);
3739	4113
3740		try self.genSetReg(val_ty, tmp_reg, val_mcv);
	4114	try self.genSetReg(tmp_reg, src_ty, src_mcv);
3741	4115	switch (limb_i) {
3742		0 => try self.genShiftBinOpMir(.shl, val_ty, tmp_mcv, .{ .immediate = val_bit_off }),
3743		1 => try self.genShiftBinOpMir(.shr, val_ty, tmp_mcv, .{
3744		.immediate = limb_abi_bits - val_bit_off,
	4116	0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),
	4117	1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{
	4118	.immediate = limb_abi_bits - src_bit_off,
3745	4119	}),
3746	4120	else => unreachable,
3747	4121	}
3748		try self.genBinOpMir(.@"and", val_ty, tmp_mcv, .{ .immediate = part_mask });
	4122	try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });
3749	4123	try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
3750	4124	} else return self.fail("TODO: implement packed store of {}", .{
3751		val_ty.fmt(self.bin_file.options.module.?),
	4125	src_ty.fmt(self.bin_file.options.module.?),
3752	4126	});
3753	4127	}
3754	4128	}
3755	4129
3756		fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
3757		const abi_size = @intCast(u32, value_ty.abiSize(self.target.*));
3758		switch (ptr) {
3759		.none => unreachable,
3760		.undef => unreachable,
3761		.unreach => unreachable,
3762		.dead => unreachable,
3763		.eflags => unreachable,
3764		.register_overflow => unreachable,
3765		.immediate, .stack_offset => {
3766		const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3767		try self.store(.{ .register = reg }, value, ptr_ty, value_ty);
3768		},
3769		.ptr_stack_offset => \|off\| {
3770		try self.genSetStack(value_ty, off, value, .{});
3771		},
3772		.register => \|reg\| {
3773		const reg_lock = self.register_manager.lockReg(reg);
3774		defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
3775
3776		switch (value) {
3777		.none => unreachable,
3778		.dead => unreachable,
3779		.unreach => unreachable,
3780		.eflags => \|cc\| try self.asmSetccMemory(
3781		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3782		cc,
3783		),
3784		.undef => if (self.wantSafety()) switch (abi_size) {
3785		1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
3786		2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
3787		4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
3788		8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
3789		else => try self.genInlineMemset(
3790		ptr,
3791		.{ .immediate = 0xaa },
3792		.{ .immediate = abi_size },
3793		.{},
3794		),
3795		},
3796		.immediate => \|imm\| switch (self.regBitSize(value_ty)) {
3797		8 => try self.asmMemoryImmediate(
3798		.mov,
3799		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3800		if (math.cast(i8, @bitCast(i64, imm))) \|small\|
3801		Immediate.s(small)
3802		else
3803		Immediate.u(@intCast(u8, imm)),
3804		),
3805		16 => try self.asmMemoryImmediate(
3806		.mov,
3807		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3808		if (math.cast(i16, @bitCast(i64, imm))) \|small\|
3809		Immediate.s(small)
3810		else
3811		Immediate.u(@intCast(u16, imm)),
3812		),
3813		32 => try self.asmMemoryImmediate(
3814		.mov,
3815		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3816		if (math.cast(i32, @bitCast(i64, imm))) \|small\|
3817		Immediate.s(small)
3818		else
3819		Immediate.u(@intCast(u32, imm)),
3820		),
3821		64 => if (math.cast(i32, @bitCast(i64, imm))) \|small\|
3822		try self.asmMemoryImmediate(
3823		.mov,
3824		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3825		Immediate.s(small),
3826		)
3827		else
3828		try self.asmMemoryRegister(
3829		.mov,
3830		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3831		registerAlias(try self.copyToTmpRegister(value_ty, value), abi_size),
3832		),
3833		else => unreachable,
3834		},
3835		.register => \|src_reg\| try self.genInlineMemcpyRegisterRegister(
3836		value_ty,
3837		reg,
3838		src_reg,
3839		0,
3840		),
3841		.register_overflow => \|ro\| {
3842		const ro_reg_lock = self.register_manager.lockReg(ro.reg);
3843		defer if (ro_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
3844
3845		const wrapped_ty = value_ty.structFieldType(0);
3846		try self.genInlineMemcpyRegisterRegister(wrapped_ty, reg, ro.reg, 0);
3847
3848		const overflow_bit_ty = value_ty.structFieldType(1);
3849		const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);
3850		const tmp_reg = try self.register_manager.allocReg(null, gp);
3851		try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
3852		try self.genInlineMemcpyRegisterRegister(
3853		overflow_bit_ty,
3854		reg,
3855		tmp_reg,
3856		-@intCast(i32, overflow_bit_offset),
3857		);
3858		},
3859		.memory, .load_tlv, .load_direct => if (abi_size <= 8) {
3860		const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3861		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3862		defer self.register_manager.unlockReg(tmp_lock);
3863
3864		try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3865		} else {
3866		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3867		const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3868		defer self.register_manager.unlockReg(addr_lock);
3869
3870		try self.genSetReg(Type.usize, addr_reg, switch (value) {
3871		.memory => \|addr\| .{ .immediate = addr },
3872		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
3873		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
3874		else => unreachable,
3875		});
3876		try self.genInlineMemcpy(
3877		ptr,
3878		.{ .register = addr_reg },
3879		.{ .immediate = abi_size },
3880		.{},
3881		);
3882		},
3883		.stack_offset => \|off\| if (abi_size <= 8) {
3884		const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3885		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3886		defer self.register_manager.unlockReg(tmp_lock);
3887
3888		try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3889		} else try self.genInlineMemcpy(
3890		ptr,
3891		.{ .ptr_stack_offset = off },
3892		.{ .immediate = abi_size },
3893		.{},
3894		),
3895		.ptr_stack_offset, .load_got, .lea_direct, .lea_tlv => {
3896		const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3897		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3898		defer self.register_manager.unlockReg(tmp_lock);
	4130	fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
	4131	const src_ty = ptr_ty.childType();
	4132	switch (ptr_mcv) {
	4133	.none,
	4134	.unreach,
	4135	.dead,
	4136	.undef,
	4137	.eflags,
	4138	.register_overflow,
	4139	.reserved_frame,
	4140	=> unreachable, // not a valid pointer
	4141	.immediate,
	4142	.register,
	4143	.register_offset,
	4144	.lea_direct,
	4145	.lea_got,
	4146	.lea_tlv,
	4147	.lea_frame,
	4148	=> try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
	4149	.memory,
	4150	.indirect,
	4151	.load_direct,
	4152	.load_got,
	4153	.load_tlv,
	4154	.load_frame,
	4155	=> {
	4156	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
	4157	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	4158	defer self.register_manager.unlockReg(addr_lock);
3899	4159
3900		try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3901		},
3902		}
	4160	try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
3903	4161	},
3904		.memory, .load_direct, .load_tlv => {
3905		const value_lock: ?RegisterLock = switch (value) {
3906		.register => \|reg\| self.register_manager.lockReg(reg),
3907		else => null,
3908		};
3909		defer if (value_lock) \|lock\| self.register_manager.unlockReg(lock);
3910
3911		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3912		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3913		defer self.register_manager.unlockReg(addr_reg_lock);
3914
3915		switch (ptr) {
3916		.memory => \|addr\| try self.genSetReg(ptr_ty, addr_reg, .{ .immediate = addr }),
3917		.load_direct => \|sym_index\| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_direct = sym_index }),
3918		.load_tlv => \|sym_index\| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_tlv = sym_index }),
3919		else => unreachable,
3920		}
3921		if (ptr != .load_tlv) {
3922		// Load the pointer, which is stored in memory
3923		try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3924		}
3925
3926		const new_ptr = MCValue{ .register = addr_reg };
3927		try self.store(new_ptr, value, ptr_ty, value_ty);
3928		},
3929		.load_got, .lea_direct, .lea_tlv => {
3930		const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
3931		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3932		defer self.register_manager.unlockReg(addr_reg_lock);
3933
3934		const new_ptr = MCValue{ .register = addr_reg };
3935		try self.store(new_ptr, value, ptr_ty, value_ty);
3936		},
3937		}
3938		}
	4162	}
	4163	}
3939	4164
3940	4165	fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3941	4166	if (safety) {
...	...	@@ -3944,15 +4169,13 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3944	4169	// TODO if the value is undef, don't lower this instruction
3945	4170	}
3946	4171	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3947		const ptr = try self.resolveInst(bin_op.lhs);
	4172	const ptr_mcv = try self.resolveInst(bin_op.lhs);
3948	4173	const ptr_ty = self.air.typeOf(bin_op.lhs);
3949		const value = try self.resolveInst(bin_op.rhs);
3950		const value_ty = self.air.typeOf(bin_op.rhs);
3951		log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value });
	4174	const src_mcv = try self.resolveInst(bin_op.rhs);
3952	4175	if (ptr_ty.ptrInfo().data.host_size > 0) {
3953		try self.packedStore(ptr, value, ptr_ty, value_ty);
	4176	try self.packedStore(ptr_ty, ptr_mcv, src_mcv);
3954	4177	} else {
3955		try self.store(ptr, value, ptr_ty, value_ty);
	4178	try self.store(ptr_ty, ptr_mcv, src_mcv);
3956	4179	}
3957	4180	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3958	4181	}
...	...	@@ -3971,67 +4194,65 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
3971	4194	}
3972	4195
3973	4196	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
	4197	const ptr_field_ty = self.air.typeOfIndex(inst);
3974	4198	const mcv = try self.resolveInst(operand);
3975		const ptr_ty = self.air.typeOf(operand);
3976		const container_ty = ptr_ty.childType();
	4199	const ptr_container_ty = self.air.typeOf(operand);
	4200	const container_ty = ptr_container_ty.childType();
3977	4201	const field_offset = switch (container_ty.containerLayout()) {
3978	4202	.Auto, .Extern => @intCast(u32, container_ty.structFieldOffset(index, self.target.*)),
3979		.Packed => if (container_ty.zigTypeTag() == .Struct and ptr_ty.ptrInfo().data.host_size == 0)
	4203	.Packed => if (container_ty.zigTypeTag() == .Struct and
	4204	ptr_field_ty.ptrInfo().data.host_size == 0)
3980	4205	container_ty.packedStructFieldByteOffset(index, self.target.*)
3981	4206	else
3982	4207	0,
3983	4208	};
3984	4209
3985		const dst_mcv: MCValue = result: {
	4210	const result: MCValue = result: {
3986	4211	switch (mcv) {
3987		.stack_offset, .lea_tlv, .load_tlv => {
3988		const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
	4212	.load_frame, .lea_tlv, .load_tlv => {
	4213	const offset_reg = try self.copyToTmpRegister(Type.usize, .{
3989	4214	.immediate = field_offset,
3990	4215	});
3991	4216	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3992	4217	defer self.register_manager.unlockReg(offset_reg_lock);
3993	4218
3994		const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, switch (mcv) {
	4219	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, Type.usize, switch (mcv) {
3995	4220	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
3996	4221	else => mcv,
3997	4222	});
3998		try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
	4223	try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });
3999	4224	break :result dst_mcv;
4000	4225	},
4001		.ptr_stack_offset => \|off\| {
4002		const ptr_stack_offset = off - @intCast(i32, field_offset);
4003		break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };
4004		},
4005		.register => \|reg\| {
4006		const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
4007		defer self.register_manager.unlockReg(reg_lock);
4008
4009		const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
4010		.immediate = field_offset,
4011		});
4012		const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4013		defer self.register_manager.unlockReg(offset_reg_lock);
4014
4015		const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
4016		const result_reg: Register = blk: {
4017		if (can_reuse_operand) {
4018		break :blk reg;
4019		} else {
4020		const result_reg = try self.register_manager.allocReg(inst, gp);
4021		try self.genSetReg(ptr_ty, result_reg, mcv);
4022		break :blk result_reg;
4023		}
4024		};
4025		const result_reg_lock = self.register_manager.lockReg(result_reg);
4026		defer if (result_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	4226	.indirect => \|reg_off\| break :result .{ .indirect = .{
	4227	.reg = reg_off.reg,
	4228	.off = reg_off.off + @intCast(i32, field_offset),
	4229	} },
	4230	.lea_frame => \|frame_addr\| break :result .{ .lea_frame = .{
	4231	.index = frame_addr.index,
	4232	.off = frame_addr.off + @intCast(i32, field_offset),
	4233	} },
	4234	.register, .register_offset => {
	4235	const src_reg = mcv.getReg().?;
	4236	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	4237	defer self.register_manager.unlockReg(src_lock);
4027	4238
4028		try self.genBinOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });
4029		break :result MCValue{ .register = result_reg };
	4239	const dst_mcv: MCValue = if (self.reuseOperand(inst, operand, 0, mcv))
	4240	mcv
	4241	else
	4242	.{ .register = try self.copyToTmpRegister(ptr_field_ty, mcv) };
	4243	break :result .{ .register_offset = .{
	4244	.reg = dst_mcv.getReg().?,
	4245	.off = switch (dst_mcv) {
	4246	.register => 0,
	4247	.register_offset => \|reg_off\| reg_off.off,
	4248	else => unreachable,
	4249	} + @intCast(i32, field_offset),
	4250	} };
4030	4251	},
4031	4252	else => return self.fail("TODO implement fieldPtr for {}", .{mcv}),
4032	4253	}
4033	4254	};
4034		return dst_mcv;
	4255	return result;
4035	4256	}
4036	4257
4037	4258	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4055,7 +4276,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4055	4276	};
4056	4277
4057	4278	switch (src_mcv) {
4058		.stack_offset => \|src_off\| {
	4279	.load_frame => \|frame_addr\| {
4059	4280	const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));
4060	4281	const limb_abi_size = @min(field_abi_size, 8);
4061	4282	const limb_abi_bits = limb_abi_size * 8;
...	...	@@ -4063,11 +4284,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4063	4284	const field_bit_off = field_off % limb_abi_bits;
4064	4285
4065	4286	if (field_bit_off == 0) {
4066		const off_mcv = MCValue{ .stack_offset = src_off - field_byte_off };
	4287	const off_mcv = MCValue{ .load_frame = .{
	4288	.index = frame_addr.index,
	4289	.off = frame_addr.off + field_byte_off,
	4290	} };
4067	4291	if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;
4068	4292
4069	4293	const dst_mcv = try self.allocRegOrMem(inst, true);
4070		try self.setRegOrMem(field_ty, dst_mcv, off_mcv);
	4294	try self.genCopy(field_ty, dst_mcv, off_mcv);
4071	4295	break :result dst_mcv;
4072	4296	}
4073	4297
...	...	@@ -4081,10 +4305,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4081	4305	if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
4082	4306	if (load_abi_size <= 8) {
4083	4307	const load_reg = registerAlias(dst_reg, load_abi_size);
4084		try self.asmRegisterMemory(.mov, load_reg, Memory.sib(
4085		Memory.PtrSize.fromSize(load_abi_size),
4086		.{ .base = .rbp, .disp = field_byte_off - src_off },
4087		));
	4308	try self.asmRegisterMemory(
	4309	.mov,
	4310	load_reg,
	4311	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
	4312	.base = .{ .frame = frame_addr.index },
	4313	.disp = frame_addr.off + field_byte_off,
	4314	}),
	4315	);
4088	4316	try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));
4089	4317	} else {
4090	4318	const tmp_reg = registerAlias(
...	...	@@ -4095,14 +4323,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4095	4323	defer self.register_manager.unlockReg(tmp_lock);
4096	4324
4097	4325	const dst_alias = registerAlias(dst_reg, field_abi_size);
4098		try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(
4099		Memory.PtrSize.fromSize(field_abi_size),
4100		.{ .base = .rbp, .disp = field_byte_off - src_off },
4101		));
4102		try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(
4103		Memory.PtrSize.fromSize(field_abi_size),
4104		.{ .base = .rbp, .disp = field_byte_off + 1 - src_off },
4105		));
	4326	try self.asmRegisterMemory(
	4327	.mov,
	4328	dst_alias,
	4329	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
	4330	.base = .{ .frame = frame_addr.index },
	4331	.disp = frame_addr.off + field_byte_off,
	4332	}),
	4333	);
	4334	try self.asmRegisterMemory(
	4335	.mov,
	4336	tmp_reg,
	4337	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
	4338	.base = .{ .frame = frame_addr.index },
	4339	.disp = frame_addr.off + field_byte_off + @intCast(i32, limb_abi_size),
	4340	}),
	4341	);
4106	4342	try self.asmRegisterRegisterImmediate(
4107	4343	.shrd,
4108	4344	dst_alias,
...	...	@@ -4249,41 +4485,51 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
4249	4485
4250	4486	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
4251	4487	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	4488	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
	4489	mir_tag,
	4490	dst_ty.fmt(self.bin_file.options.module.?),
	4491	});
4252	4492	switch (dst_mcv) {
4253		.none => unreachable,
4254		.undef => unreachable,
4255		.dead, .unreach, .immediate => unreachable,
4256		.eflags => unreachable,
4257		.register_overflow => unreachable,
	4493	.none,
	4494	.unreach,
	4495	.dead,
	4496	.undef,
	4497	.immediate,
	4498	.register_offset,
	4499	.eflags,
	4500	.register_overflow,
	4501	.lea_direct,
	4502	.lea_got,
	4503	.lea_tlv,
	4504	.lea_frame,
	4505	.reserved_frame,
	4506	=> unreachable, // unmodifiable destination
4258	4507	.register => \|dst_reg\| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
4259		.stack_offset => \|off\| {
4260		if (abi_size > 8) {
4261		return self.fail("TODO implement {} for stack dst with large ABI", .{mir_tag});
4262		}
4263
4264		try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4265		.base = .rbp,
4266		.disp = -off,
4267		}));
4268		},
4269		.ptr_stack_offset => unreachable,
4270		.load_got, .lea_direct, .lea_tlv => unreachable,
4271		.memory, .load_direct, .load_tlv => {
4272		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	4508	.memory, .load_got, .load_direct, .load_tlv => {
	4509	const addr_reg = try self.register_manager.allocReg(null, gp);
4273	4510	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4274	4511	defer self.register_manager.unlockReg(addr_reg_lock);
4275	4512
4276		try self.genSetReg(Type.usize, addr_reg, switch (dst_mcv) {
4277		.memory => \|addr\| .{ .immediate = addr },
4278		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
4279		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4280		else => unreachable,
4281		});
	4513	try self.genSetReg(addr_reg, Type.usize, dst_mcv.address());
4282	4514	try self.asmMemory(
4283	4515	mir_tag,
4284		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
	4516	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
4285	4517	);
4286	4518	},
	4519	.indirect, .load_frame => try self.asmMemory(
	4520	mir_tag,
	4521	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (dst_mcv) {
	4522	.indirect => \|reg_off\| .{
	4523	.base = .{ .reg = reg_off.reg },
	4524	.disp = reg_off.off,
	4525	},
	4526	.load_frame => \|frame_addr\| .{
	4527	.base = .{ .frame = frame_addr.index },
	4528	.disp = frame_addr.off,
	4529	},
	4530	else => unreachable,
	4531	}),
	4532	),
4287	4533	}
4288	4534	}
4289	4535
...	...	@@ -4305,7 +4551,7 @@ fn genShiftBinOpMir(
4305	4551	else => {},
4306	4552	}
4307	4553	self.register_manager.getRegAssumeFree(.rcx, null);
4308		try self.genSetReg(Type.u8, .rcx, shift_mcv);
	4554	try self.genSetReg(.cl, Type.u8, shift_mcv);
4309	4555	break :rhs .{ .register = .rcx };
4310	4556	};
4311	4557
...	...	@@ -4328,21 +4574,42 @@ fn genShiftBinOpMir(
4328	4574	@tagName(rhs_mcv),
4329	4575	}),
4330	4576	},
4331		.stack_offset => \|lhs_off\| switch (rhs_mcv) {
4332		.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
4333		tag,
4334		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
4335		Immediate.u(rhs_imm),
4336		),
4337		.register => \|rhs_reg\| try self.asmMemoryRegister(
4338		tag,
4339		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
4340		registerAlias(rhs_reg, 1),
4341		),
4342		else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4343		@tagName(lhs_mcv),
4344		@tagName(rhs_mcv),
4345		}),
	4577	.memory, .indirect, .load_frame => {
	4578	const lhs_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (lhs_mcv) {
	4579	.memory => \|addr\| .{
	4580	.base = .{ .reg = .ds },
	4581	.disp = math.cast(i32, @bitCast(i64, addr)) orelse
	4582	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4583	@tagName(lhs_mcv),
	4584	@tagName(rhs_mcv),
	4585	}),
	4586	},
	4587	.indirect => \|reg_off\| .{
	4588	.base = .{ .reg = reg_off.reg },
	4589	.disp = reg_off.off,
	4590	},
	4591	.load_frame => \|frame_addr\| .{
	4592	.base = .{ .frame = frame_addr.index },
	4593	.disp = frame_addr.off,
	4594	},
	4595	else => unreachable,
	4596	});
	4597	switch (rhs_mcv) {
	4598	.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
	4599	tag,
	4600	lhs_mem,
	4601	Immediate.u(rhs_imm),
	4602	),
	4603	.register => \|rhs_reg\| try self.asmMemoryRegister(
	4604	tag,
	4605	lhs_mem,
	4606	registerAlias(rhs_reg, 1),
	4607	),
	4608	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4609	@tagName(lhs_mcv),
	4610	@tagName(rhs_mcv),
	4611	}),
	4612	}
4346	4613	},
4347	4614	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4348	4615	@tagName(lhs_mcv),
...	...	@@ -4360,22 +4627,31 @@ fn genShiftBinOpMir(
4360	4627	else => unreachable,
4361	4628	};
4362	4629	switch (lhs_mcv) {
4363		.stack_offset => \|dst_off\| switch (rhs_mcv) {
	4630	.load_frame => \|dst_frame_addr\| switch (rhs_mcv) {
4364	4631	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
4365	4632	try self.asmRegisterMemory(
4366	4633	.mov,
4367	4634	tmp_reg,
4368		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4635	Memory.sib(.qword, .{
	4636	.base = .{ .frame = dst_frame_addr.index },
	4637	.disp = dst_frame_addr.off + info.offsets[0],
	4638	}),
4369	4639	);
4370	4640	try self.asmMemoryRegisterImmediate(
4371	4641	info.double_tag,
4372		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4642	Memory.sib(.qword, .{
	4643	.base = .{ .frame = dst_frame_addr.index },
	4644	.disp = dst_frame_addr.off + info.offsets[1],
	4645	}),
4373	4646	tmp_reg,
4374	4647	Immediate.u(rhs_imm),
4375	4648	);
4376	4649	try self.asmMemoryImmediate(
4377	4650	tag,
4378		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4651	Memory.sib(.qword, .{
	4652	.base = .{ .frame = dst_frame_addr.index },
	4653	.disp = dst_frame_addr.off + info.offsets[0],
	4654	}),
4379	4655	Immediate.u(rhs_imm),
4380	4656	);
4381	4657	} else {
...	...	@@ -4383,14 +4659,20 @@ fn genShiftBinOpMir(
4383	4659	try self.asmRegisterMemory(
4384	4660	.mov,
4385	4661	tmp_reg,
4386		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4662	Memory.sib(.qword, .{
	4663	.base = .{ .frame = dst_frame_addr.index },
	4664	.disp = dst_frame_addr.off + info.offsets[0],
	4665	}),
4387	4666	);
4388	4667	if (rhs_imm > 64) {
4389	4668	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
4390	4669	}
4391	4670	try self.asmMemoryRegister(
4392	4671	.mov,
4393		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4672	Memory.sib(.qword, .{
	4673	.base = .{ .frame = dst_frame_addr.index },
	4674	.disp = dst_frame_addr.off + info.offsets[1],
	4675	}),
4394	4676	tmp_reg,
4395	4677	);
4396	4678	switch (tag) {
...	...	@@ -4398,13 +4680,19 @@ fn genShiftBinOpMir(
4398	4680	try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
4399	4681	try self.asmMemoryRegister(
4400	4682	.mov,
4401		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4683	Memory.sib(.qword, .{
	4684	.base = .{ .frame = dst_frame_addr.index },
	4685	.disp = dst_frame_addr.off + info.offsets[0],
	4686	}),
4402	4687	tmp_reg,
4403	4688	);
4404	4689	},
4405	4690	.sar => try self.asmMemoryImmediate(
4406	4691	tag,
4407		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4692	Memory.sib(.qword, .{
	4693	.base = .{ .frame = dst_frame_addr.index },
	4694	.disp = dst_frame_addr.off + info.offsets[0],
	4695	}),
4408	4696	Immediate.u(63),
4409	4697	),
4410	4698	else => unreachable,
...	...	@@ -4419,16 +4707,22 @@ fn genShiftBinOpMir(
4419	4707	const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);
4420	4708	defer self.register_manager.unlockReg(second_lock);
4421	4709
4422		try self.genSetReg(Type.u8, .cl, rhs_mcv);
	4710	try self.genSetReg(.cl, Type.u8, rhs_mcv);
4423	4711	try self.asmRegisterMemory(
4424	4712	.mov,
4425	4713	first_reg,
4426		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4714	Memory.sib(.qword, .{
	4715	.base = .{ .frame = dst_frame_addr.index },
	4716	.disp = dst_frame_addr.off + info.offsets[0],
	4717	}),
4427	4718	);
4428	4719	try self.asmRegisterMemory(
4429	4720	.mov,
4430	4721	second_reg,
4431		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4722	Memory.sib(.qword, .{
	4723	.base = .{ .frame = dst_frame_addr.index },
	4724	.disp = dst_frame_addr.off + info.offsets[1],
	4725	}),
4432	4726	);
4433	4727	switch (tag) {
4434	4728	.shl, .sal, .shr => try self.asmRegisterRegister(
...	...	@@ -4449,12 +4743,18 @@ fn genShiftBinOpMir(
4449	4743	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
4450	4744	try self.asmMemoryRegister(
4451	4745	.mov,
4452		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4746	Memory.sib(.qword, .{
	4747	.base = .{ .frame = dst_frame_addr.index },
	4748	.disp = dst_frame_addr.off + info.offsets[1],
	4749	}),
4453	4750	second_reg,
4454	4751	);
4455	4752	try self.asmMemoryRegister(
4456	4753	.mov,
4457		Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4754	Memory.sib(.qword, .{
	4755	.base = .{ .frame = dst_frame_addr.index },
	4756	.disp = dst_frame_addr.off + info.offsets[0],
	4757	}),
4458	4758	first_reg,
4459	4759	);
4460	4760	},
...	...	@@ -4515,7 +4815,7 @@ fn genShiftBinOp(
4515	4815	if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
4516	4816	}
4517	4817	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4518		try self.setRegOrMem(lhs_ty, dst_mcv, lhs_mcv);
	4818	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
4519	4819	break :dst dst_mcv;
4520	4820	};
4521	4821
...	...	@@ -4610,14 +4910,22 @@ fn genMulDivBinOp(
4610	4910	}, dst_abi_size) };
4611	4911
4612	4912	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
4613		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
4614		.base = .rbp,
4615		.disp = 0 - dst_mcv.stack_offset,
4616		}), .rax);
4617		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
4618		.base = .rbp,
4619		.disp = 8 - dst_mcv.stack_offset,
4620		}), .rdx);
	4913	try self.asmMemoryRegister(
	4914	.mov,
	4915	Memory.sib(.qword, .{
	4916	.base = .{ .frame = dst_mcv.load_frame.index },
	4917	.disp = dst_mcv.load_frame.off,
	4918	}),
	4919	.rax,
	4920	);
	4921	try self.asmMemoryRegister(
	4922	.mov,
	4923	Memory.sib(.qword, .{
	4924	.base = .{ .frame = dst_mcv.load_frame.index },
	4925	.disp = dst_mcv.load_frame.off + 8,
	4926	}),
	4927	.rdx,
	4928	);
4621	4929	return dst_mcv;
4622	4930	},
4623	4931
...	...	@@ -4775,11 +5083,7 @@ fn genBinOp(
4775	5083	}
4776	5084	}
4777	5085	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4778		try self.setRegOrMem(lhs_ty, dst_mcv, switch (lhs) {
4779		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
4780		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4781		else => lhs,
4782		});
	5086	try self.genCopy(lhs_ty, dst_mcv, lhs);
4783	5087	break :dst dst_mcv;
4784	5088	};
4785	5089	const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
...	...	@@ -4878,10 +5182,21 @@ fn genBinOp(
4878	5182	.max,
4879	5183	=> switch (lhs_ty.zigTypeTag()) {
4880	5184	.Int => {
4881		const mat_src_mcv = switch (src_mcv) {
4882		.immediate => MCValue{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) },
4883		else => src_mcv,
4884		};
	5185	const mat_src_mcv: MCValue = if (switch (src_mcv) {
	5186	.immediate,
	5187	.eflags,
	5188	.register_offset,
	5189	.load_direct,
	5190	.lea_direct,
	5191	.load_got,
	5192	.lea_got,
	5193	.load_tlv,
	5194	.lea_tlv,
	5195	.lea_frame,
	5196	=> true,
	5197	.memory => \|addr\| math.cast(i32, @bitCast(i64, addr)) == null,
	5198	else => false,
	5199	}) .{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) } else src_mcv;
4885	5200	const mat_mcv_lock = switch (mat_src_mcv) {
4886	5201	.register => \|reg\| self.register_manager.lockReg(reg),
4887	5202	else => null,
...	...	@@ -4913,57 +5228,56 @@ fn genBinOp(
4913	5228	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);
4914	5229	switch (mat_src_mcv) {
4915	5230	.none,
4916		.undef,
4917		.dead,
4918	5231	.unreach,
	5232	.dead,
	5233	.undef,
4919	5234	.immediate,
4920	5235	.eflags,
	5236	.register_offset,
4921	5237	.register_overflow,
4922		.ptr_stack_offset,
4923		.load_got,
	5238	.load_direct,
4924	5239	.lea_direct,
	5240	.load_got,
	5241	.lea_got,
	5242	.load_tlv,
4925	5243	.lea_tlv,
	5244	.lea_frame,
	5245	.reserved_frame,
4926	5246	=> unreachable,
4927	5247	.register => \|src_reg\| try self.asmCmovccRegisterRegister(
4928	5248	registerAlias(tmp_reg, cmov_abi_size),
4929	5249	registerAlias(src_reg, cmov_abi_size),
4930	5250	cc,
4931	5251	),
4932		.stack_offset => \|off\| try self.asmCmovccRegisterMemory(
	5252	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
4933	5253	registerAlias(tmp_reg, cmov_abi_size),
4934		Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
4935		.base = .rbp,
4936		.disp = -off,
	5254	Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
	5255	.memory => \|addr\| .{
	5256	.base = .{ .reg = .ds },
	5257	.disp = @intCast(i32, @bitCast(i64, addr)),
	5258	},
	5259	.indirect => \|reg_off\| .{
	5260	.base = .{ .reg = reg_off.reg },
	5261	.disp = reg_off.off,
	5262	},
	5263	.load_frame => \|frame_addr\| .{
	5264	.base = .{ .frame = frame_addr.index },
	5265	.disp = frame_addr.off,
	5266	},
	5267	else => unreachable,
4937	5268	}),
4938	5269	cc,
4939	5270	),
4940		.memory, .load_direct, .load_tlv => {
4941		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4942		const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4943		defer self.register_manager.unlockReg(addr_reg_lock);
4944
4945		try self.genSetReg(Type.usize, addr_reg, switch (mat_src_mcv) {
4946		.memory => \|addr\| .{ .immediate = addr },
4947		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
4948		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4949		else => unreachable,
4950		});
4951		try self.asmCmovccRegisterMemory(
4952		registerAlias(tmp_reg, cmov_abi_size),
4953		Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }),
4954		cc,
4955		);
4956		},
4957	5271	}
4958		try self.setRegOrMem(lhs_ty, dst_mcv, .{ .register = tmp_reg });
	5272	try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
4959	5273	},
4960		.Float => try self.genBinOpMir(switch (lhs_ty.tag()) {
4961		.f32 => switch (tag) {
	5274	.Float => try self.genBinOpMir(switch (lhs_ty.floatBits(self.target.*)) {
	5275	32 => switch (tag) {
4962	5276	.min => .minss,
4963	5277	.max => .maxss,
4964	5278	else => unreachable,
4965	5279	},
4966		.f64 => switch (tag) {
	5280	64 => switch (tag) {
4967	5281	.min => .minsd,
4968	5282	.max => .maxsd,
4969	5283	else => unreachable,
...	...	@@ -4981,25 +5295,38 @@ fn genBinOp(
4981	5295	fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
4982	5296	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4983	5297	switch (dst_mcv) {
4984		.none => unreachable,
4985		.undef => unreachable,
4986		.dead, .unreach, .immediate => unreachable,
4987		.eflags => unreachable,
4988		.register_overflow => unreachable,
	5298	.none,
	5299	.unreach,
	5300	.dead,
	5301	.undef,
	5302	.immediate,
	5303	.register_offset,
	5304	.eflags,
	5305	.register_overflow,
	5306	.lea_direct,
	5307	.lea_got,
	5308	.lea_tlv,
	5309	.lea_frame,
	5310	.reserved_frame,
	5311	=> unreachable, // unmodifiable destination
4989	5312	.register => \|dst_reg\| {
4990	5313	const dst_alias = registerAlias(dst_reg, abi_size);
4991	5314	switch (src_mcv) {
4992		.none => unreachable,
4993		.undef => unreachable,
4994		.dead, .unreach => unreachable,
4995		.register_overflow => unreachable,
	5315	.none,
	5316	.unreach,
	5317	.dead,
	5318	.undef,
	5319	.register_overflow,
	5320	.reserved_frame,
	5321	=> unreachable,
4996	5322	.register => \|src_reg\| switch (ty.zigTypeTag()) {
4997	5323	.Float => {
4998		if (intrinsicsAllowed(self.target.*, ty)) {
4999		return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5000		}
5001
5002		return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
	5324	if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))
	5325	return self.fail("TODO genBinOpMir for {s} {} without sse", .{
	5326	@tagName(mir_tag),
	5327	ty.fmt(self.bin_file.options.module.?),
	5328	});
	5329	return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5003	5330	},
5004	5331	else => try self.asmRegisterRegister(
5005	5332	mir_tag,
...	...	@@ -5041,14 +5368,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5041	5368	)),
5042	5369	else => unreachable,
5043	5370	},
5044		.ptr_stack_offset,
	5371	.register_offset,
	5372	.eflags,
5045	5373	.memory,
5046		.load_got,
5047		.lea_direct,
5048	5374	.load_direct,
5049		.lea_tlv,
	5375	.lea_direct,
	5376	.load_got,
	5377	.lea_got,
5050	5378	.load_tlv,
5051		.eflags,
	5379	.lea_tlv,
	5380	.lea_frame,
5052	5381	=> {
5053	5382	assert(abi_size <= 8);
5054	5383	const dst_reg_lock = self.register_manager.lockReg(dst_reg);
...	...	@@ -5057,75 +5386,84 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5057	5386	const reg = try self.copyToTmpRegister(ty, src_mcv);
5058	5387	return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
5059	5388	},
5060		.stack_offset => \|off\| try self.asmRegisterMemory(
	5389	.indirect, .load_frame => try self.asmRegisterMemory(
5061	5390	mir_tag,
5062	5391	registerAlias(dst_reg, abi_size),
5063		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
	5392	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
	5393	.indirect => \|reg_off\| .{
	5394	.base = .{ .reg = reg_off.reg },
	5395	.disp = reg_off.off,
	5396	},
	5397	.load_frame => \|frame_addr\| .{
	5398	.base = .{ .frame = frame_addr.index },
	5399	.disp = frame_addr.off,
	5400	},
	5401	else => unreachable,
	5402	}),
5064	5403	),
5065	5404	}
5066	5405	},
5067		.memory, .load_got, .load_direct, .load_tlv, .stack_offset => {
5068		const dst: ?struct {
5069		addr_reg: Register,
5070		addr_lock: RegisterLock,
5071		} = switch (dst_mcv) {
	5406	.memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => {
	5407	const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock };
	5408	const limb_abi_size = @min(abi_size, 8);
	5409
	5410	const dst_info: OpInfo = switch (dst_mcv) {
5072	5411	else => unreachable,
5073	5412	.memory, .load_got, .load_direct, .load_tlv => dst: {
5074	5413	const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5075	5414	const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
5076	5415	errdefer self.register_manager.unlockReg(dst_addr_lock);
5077	5416
5078		try self.genSetReg(Type.usize, dst_addr_reg, switch (dst_mcv) {
5079		.memory => \|addr\| .{ .immediate = addr },
5080		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
5081		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
5082		else => dst_mcv,
5083		});
5084
	5417	try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address());
5085	5418	break :dst .{
5086	5419	.addr_reg = dst_addr_reg,
5087	5420	.addr_lock = dst_addr_lock,
5088	5421	};
5089	5422	},
5090		.stack_offset => null,
	5423	.load_frame => null,
5091	5424	};
5092		defer if (dst) \|lock\| self.register_manager.unlockReg(lock.addr_lock);
5093
5094		const src: ?struct {
5095		limb_reg: Register,
5096		limb_lock: RegisterLock,
5097		addr_reg: Register,
5098		addr_lock: RegisterLock,
5099		} = switch (src_mcv) {
5100		else => null,
	5425	defer if (dst_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
	5426
	5427	const src_info: OpInfo = switch (src_mcv) {
	5428	.none,
	5429	.unreach,
	5430	.dead,
	5431	.undef,
	5432	.register_overflow,
	5433	.reserved_frame,
	5434	=> unreachable,
	5435	.immediate,
	5436	.register,
	5437	.register_offset,
	5438	.eflags,
	5439	.indirect,
	5440	.lea_direct,
	5441	.lea_got,
	5442	.lea_tlv,
	5443	.load_frame,
	5444	.lea_frame,
	5445	=> null,
5101	5446	.memory, .load_got, .load_direct, .load_tlv => src: {
5102		const src_limb_reg = try self.register_manager.allocReg(null, gp);
5103		const src_limb_lock = self.register_manager.lockRegAssumeUnused(src_limb_reg);
5104		errdefer self.register_manager.unlockReg(src_limb_lock);
	5447	switch (src_mcv) {
	5448	.memory => \|addr\| if (math.cast(i32, @bitCast(i64, addr)) != null and
	5449	math.cast(i32, @bitCast(i64, addr) + abi_size - limb_abi_size) != null)
	5450	break :src null,
	5451	.load_got, .load_direct, .load_tlv => {},
	5452	else => unreachable,
	5453	}
5105	5454
5106	5455	const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5107	5456	const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
5108	5457	errdefer self.register_manager.unlockReg(src_addr_lock);
5109	5458
5110		try self.genSetReg(Type.usize, src_addr_reg, switch (src_mcv) {
5111		.memory => \|addr\| .{ .immediate = addr },
5112		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
5113		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
5114		else => src_mcv,
5115		});
5116
	5459	try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address());
5117	5460	break :src .{
5118	5461	.addr_reg = src_addr_reg,
5119	5462	.addr_lock = src_addr_lock,
5120		.limb_reg = src_limb_reg,
5121		.limb_lock = src_limb_lock,
5122	5463	};
5123	5464	},
5124	5465	};
5125		defer if (src) \|locks\| {
5126		self.register_manager.unlockReg(locks.limb_lock);
5127		self.register_manager.unlockReg(locks.addr_lock);
5128		};
	5466	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
5129	5467
5130	5468	const ty_signedness =
5131	5469	if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
...	...	@@ -5133,7 +5471,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5133	5471	.signed => Type.usize,
5134	5472	.unsigned => Type.isize,
5135	5473	};
5136		const limb_abi_size = @min(abi_size, 8);
5137	5474	var off: i32 = 0;
5138	5475	while (off < abi_size) : (off += 8) {
5139	5476	const mir_limb_tag = switch (off) {
...	...	@@ -5150,35 +5487,45 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5150	5487	const dst_limb_mem = Memory.sib(
5151	5488	Memory.PtrSize.fromSize(limb_abi_size),
5152	5489	switch (dst_mcv) {
5153		else => unreachable,
5154		.stack_offset => \|dst_off\| .{
5155		.base = .rbp,
5156		.disp = off - dst_off,
5157		},
5158	5490	.memory,
5159	5491	.load_got,
5160	5492	.load_direct,
5161	5493	.load_tlv,
5162		=> .{ .base = dst.?.addr_reg, .disp = off },
	5494	=> .{ .base = .{ .reg = dst_info.?.addr_reg }, .disp = off },
	5495	.indirect => \|reg_off\| .{
	5496	.base = .{ .reg = reg_off.reg },
	5497	.disp = reg_off.off + off,
	5498	},
	5499	.load_frame => \|frame_addr\| .{
	5500	.base = .{ .frame = frame_addr.index },
	5501	.disp = frame_addr.off + off,
	5502	},
	5503	else => unreachable,
5163	5504	},
5164	5505	);
5165	5506	switch (src_mcv) {
5166		.none => unreachable,
5167		.undef => unreachable,
5168		.dead, .unreach => unreachable,
5169		.register_overflow => unreachable,
5170		.register => \|src_reg\| {
5171		assert(off == 0);
5172		try self.asmMemoryRegister(
	5507	.none,
	5508	.unreach,
	5509	.dead,
	5510	.undef,
	5511	.register_overflow,
	5512	.reserved_frame,
	5513	=> unreachable,
	5514	.register => \|src_reg\| switch (off) {
	5515	0 => try self.asmMemoryRegister(
5173	5516	mir_limb_tag,
5174	5517	dst_limb_mem,
5175	5518	registerAlias(src_reg, limb_abi_size),
5176		);
	5519	),
	5520	else => unreachable,
5177	5521	},
5178	5522	.immediate => \|src_imm\| {
5179		const imm = if (off == 0) src_imm else switch (ty_signedness) {
5180		.signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),
5181		.unsigned => 0,
	5523	const imm = switch (off) {
	5524	0 => src_imm,
	5525	else => switch (ty_signedness) {
	5526	.signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),
	5527	.unsigned => 0,
	5528	},
5182	5529	};
5183	5530	switch (self.regBitSize(limb_ty)) {
5184	5531	8 => try self.asmMemoryImmediate(
...	...	@@ -5223,47 +5570,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5223	5570	else => unreachable,
5224	5571	}
5225	5572	},
	5573	.register_offset,
	5574	.eflags,
5226	5575	.memory,
5227		.load_got,
	5576	.indirect,
5228	5577	.load_direct,
5229		.load_tlv,
5230		=> {
5231		try self.asmRegisterMemory(
5232		.mov,
5233		registerAlias(src.?.limb_reg, limb_abi_size),
5234		Memory.sib(
5235		Memory.PtrSize.fromSize(limb_abi_size),
5236		.{ .base = src.?.addr_reg, .disp = off },
5237		),
5238		);
5239		try self.asmMemoryRegister(
5240		mir_limb_tag,
5241		dst_limb_mem,
5242		registerAlias(src.?.limb_reg, limb_abi_size),
5243		);
5244		},
5245		.stack_offset,
5246		.ptr_stack_offset,
5247		.eflags,
5248	5578	.lea_direct,
	5579	.load_got,
	5580	.lea_got,
	5581	.load_tlv,
5249	5582	.lea_tlv,
	5583	.load_frame,
	5584	.lea_frame,
5250	5585	=> {
5251		const src_limb_reg = try self.copyToTmpRegister(limb_ty, switch (src_mcv) {
5252		.stack_offset => \|src_off\| .{ .stack_offset = src_off - off },
5253		.ptr_stack_offset,
	5586	const src_limb_reg = try self.copyToTmpRegister(limb_ty, if (src_info) \|info\| .{
	5587	.indirect = .{ .reg = info.addr_reg, .off = off },
	5588	} else switch (src_mcv) {
5254	5589	.eflags,
5255		.load_got,
	5590	.register_offset,
5256	5591	.lea_direct,
	5592	.lea_got,
5257	5593	.lea_tlv,
5258		=> off: {
5259		assert(off == 0);
5260		break :off src_mcv;
	5594	.lea_frame,
	5595	=> switch (off) {
	5596	0 => src_mcv,
	5597	else => .{ .immediate = 0 },
5261	5598	},
	5599	.memory => \|addr\| .{ .memory = @bitCast(u64, @bitCast(i64, addr) + off) },
	5600	.indirect => \|reg_off\| .{ .indirect = .{
	5601	.reg = reg_off.reg,
	5602	.off = reg_off.off + off,
	5603	} },
	5604	.load_frame => \|frame_addr\| .{ .load_frame = .{
	5605	.index = frame_addr.index,
	5606	.off = frame_addr.off + off,
	5607	} },
5262	5608	else => unreachable,
5263	5609	});
5264		const src_limb_lock = self.register_manager.lockReg(src_limb_reg);
5265		defer if (src_limb_lock) \|lock\| self.register_manager.unlockReg(lock);
5266
5267	5610	try self.asmMemoryRegister(
5268	5611	mir_limb_tag,
5269	5612	dst_limb_mem,
...	...	@@ -5273,9 +5616,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5273	5616	}
5274	5617	}
5275	5618	},
5276		.ptr_stack_offset => unreachable,
5277		.lea_tlv => unreachable,
5278		.lea_direct => unreachable,
5279	5619	}
5280	5620	}
5281	5621
...	...	@@ -5284,27 +5624,33 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5284	5624	fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
5285	5625	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
5286	5626	switch (dst_mcv) {
5287		.none => unreachable,
5288		.undef => unreachable,
5289		.dead, .unreach, .immediate => unreachable,
5290		.eflags => unreachable,
5291		.ptr_stack_offset => unreachable,
5292		.lea_direct => unreachable,
5293		.lea_tlv => unreachable,
5294		.register_overflow => unreachable,
	5627	.none,
	5628	.unreach,
	5629	.dead,
	5630	.undef,
	5631	.immediate,
	5632	.register_offset,
	5633	.eflags,
	5634	.register_overflow,
	5635	.lea_direct,
	5636	.lea_got,
	5637	.lea_tlv,
	5638	.lea_frame,
	5639	.reserved_frame,
	5640	=> unreachable, // unmodifiable destination
5295	5641	.register => \|dst_reg\| {
5296	5642	const dst_alias = registerAlias(dst_reg, abi_size);
5297	5643	const dst_lock = self.register_manager.lockReg(dst_reg);
5298	5644	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5299	5645
5300	5646	switch (src_mcv) {
5301		.none => unreachable,
5302		.undef => try self.genSetReg(dst_ty, dst_reg, .undef),
5303		.dead, .unreach => unreachable,
5304		.ptr_stack_offset => unreachable,
5305		.lea_direct => unreachable,
5306		.lea_tlv => unreachable,
5307		.register_overflow => unreachable,
	5647	.none,
	5648	.unreach,
	5649	.dead,
	5650	.undef,
	5651	.register_overflow,
	5652	.reserved_frame,
	5653	=> unreachable,
5308	5654	.register => \|src_reg\| try self.asmRegisterRegister(
5309	5655	.imul,
5310	5656	dst_alias,
...	...	@@ -5323,74 +5669,54 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
5323	5669	return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
5324	5670	}
5325	5671	},
5326		.stack_offset => \|off\| {
5327		try self.asmRegisterMemory(
5328		.imul,
5329		dst_alias,
5330		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
5331		);
5332		},
5333		.memory,
5334		.load_got,
	5672	.register_offset,
	5673	.eflags,
5335	5674	.load_direct,
	5675	.lea_direct,
	5676	.load_got,
	5677	.lea_got,
5336	5678	.load_tlv,
5337		=> {
5338		return self.fail("TODO implement x86 multiply source memory", .{});
5339		},
5340		.eflags => {
5341		return self.fail("TODO implement x86 multiply source eflags", .{});
5342		},
	5679	.lea_tlv,
	5680	.lea_frame,
	5681	=> try self.asmRegisterRegister(
	5682	.imul,
	5683	dst_alias,
	5684	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
	5685	),
	5686	.memory, .indirect, .load_frame => try self.asmRegisterMemory(
	5687	.imul,
	5688	dst_alias,
	5689	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
	5690	.memory => \|addr\| .{
	5691	.base = .{ .reg = .ds },
	5692	.disp = math.cast(i32, @bitCast(i64, addr)) orelse
	5693	return self.asmRegisterRegister(
	5694	.imul,
	5695	dst_alias,
	5696	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
	5697	),
	5698	},
	5699	.indirect => \|reg_off\| .{
	5700	.base = .{ .reg = reg_off.reg },
	5701	.disp = reg_off.off,
	5702	},
	5703	.load_frame => \|frame_addr\| .{
	5704	.base = .{ .frame = frame_addr.index },
	5705	.disp = frame_addr.off,
	5706	},
	5707	else => unreachable,
	5708	}),
	5709	),
5343	5710	}
5344	5711	},
5345		.stack_offset => \|off\| {
5346		switch (src_mcv) {
5347		.none => unreachable,
5348		.undef => return self.genSetStack(dst_ty, off, .undef, .{}),
5349		.dead, .unreach => unreachable,
5350		.ptr_stack_offset => unreachable,
5351		.lea_direct => unreachable,
5352		.lea_tlv => unreachable,
5353		.register_overflow => unreachable,
5354		.register => \|src_reg\| {
5355		// copy dst to a register
5356		const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5357		try self.asmRegisterRegister(
5358		.imul,
5359		registerAlias(dst_reg, abi_size),
5360		registerAlias(src_reg, abi_size),
5361		);
5362		// copy dst_reg back out
5363		return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
5364		},
5365		.immediate => {
5366		// copy dst to a register
5367		const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5368		const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
5369		defer self.register_manager.unlockReg(dst_reg_lock);
5370
5371		try self.genIntMulComplexOpMir(dst_ty, .{ .register = dst_reg }, src_mcv);
	5712	.memory, .indirect, .load_direct, .load_got, .load_tlv, .load_frame => {
	5713	const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
	5714	const tmp_mcv = MCValue{ .register = tmp_reg };
	5715	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	5716	defer self.register_manager.unlockReg(tmp_lock);
5372	5717
5373		return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
5374		},
5375		.memory,
5376		.load_got,
5377		.load_direct,
5378		.load_tlv,
5379		.stack_offset,
5380		=> {
5381		return self.fail("TODO implement x86 multiply source memory", .{});
5382		},
5383		.eflags => {
5384		return self.fail("TODO implement x86 multiply source eflags", .{});
5385		},
5386		}
5387		},
5388		.memory,
5389		.load_got,
5390		.load_direct,
5391		.load_tlv,
5392		=> {
5393		return self.fail("TODO implement x86 multiply destination memory", .{});
	5718	try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv);
	5719	try self.genCopy(dst_ty, dst_mcv, tmp_mcv);
5394	5720	},
5395	5721	}
5396	5722	}
...	...	@@ -5401,24 +5727,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
5401	5727	while (self.args[arg_index] == .none) arg_index += 1;
5402	5728	self.arg_index = arg_index + 1;
5403	5729
5404		const ty = self.air.typeOfIndex(inst);
5405		const mcv = self.args[arg_index];
5406		const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
5407		const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5408
5409	5730	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
5410		const dst_mcv: MCValue = switch (mcv) {
5411		.register => \|reg\| blk: {
5412		self.register_manager.getRegAssumeFree(reg.to64(), inst);
5413		break :blk MCValue{ .register = reg };
5414		},
5415		.stack_offset => \|off\| blk: {
5416		const offset = @intCast(i32, self.max_end_stack) - off + 16;
5417		break :blk MCValue{ .stack_offset = -offset };
5418		},
5419		else => return self.fail("TODO implement arg for {}", .{mcv}),
5420		};
	5731	const dst_mcv = self.args[arg_index];
	5732	switch (dst_mcv) {
	5733	.register => \|reg\| self.register_manager.getRegAssumeFree(reg, inst),
	5734	.load_frame => {},
	5735	else => return self.fail("TODO implement arg for {}", .{dst_mcv}),
	5736	}
	5737
	5738	const ty = self.air.typeOfIndex(inst);
	5739	const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
	5740	const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5421	5741	try self.genArgDbgInfo(ty, name, dst_mcv);
	5742
5422	5743	break :result dst_mcv;
5423	5744	};
5424	5745	return self.finishAir(inst, result, .{ .none, .none, .none });
...	...	@@ -5429,13 +5750,15 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
5429	5750	.dwarf => \|dw\| {
5430	5751	const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5431	5752	.register => \|reg\| .{ .register = reg.dwarfLocOp() },
5432		.stack_offset => \|off\| .{
5433		.stack = .{
5434		// TODO handle -fomit-frame-pointer
5435		.fp_register = Register.rbp.dwarfLocOpDeref(),
5436		.offset = -off,
5437		},
5438		},
	5753	// TODO use a frame index
	5754	.load_frame => return,
	5755	//.stack_offset => \|off\| .{
	5756	// .stack = .{
	5757	// // TODO handle -fomit-frame-pointer
	5758	// .fp_register = Register.rbp.dwarfLocOpDeref(),
	5759	// .offset = -off,
	5760	// },
	5761	//},
5439	5762	else => unreachable, // not a valid function parameter
5440	5763	};
5441	5764	try dw.genArgDbgInfo(name, ty, self.mod_fn.owner_decl, loc);
...	...	@@ -5462,12 +5785,12 @@ fn genVarDbgInfo(
5462	5785	.dwarf => \|dw\| {
5463	5786	const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5464	5787	.register => \|reg\| .{ .register = reg.dwarfLocOp() },
5465		.ptr_stack_offset,
5466		.stack_offset,
5467		=> \|off\| .{ .stack = .{
5468		.fp_register = Register.rbp.dwarfLocOpDeref(),
5469		.offset = -off,
5470		} },
	5788	// TODO use a frame index
	5789	.load_frame, .lea_frame => return,
	5790	//=> \|off\| .{ .stack = .{
	5791	// .fp_register = Register.rbp.dwarfLocOpDeref(),
	5792	// .offset = -off,
	5793	//} },
5471	5794	.memory => \|address\| .{ .memory = address },
5472	5795	.load_got => \|sym_index\| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },
5473	5796	.load_direct => \|sym_index\| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },
...	...	@@ -5498,15 +5821,13 @@ fn airBreakpoint(self: *Self) !void {
5498	5821
5499	5822	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
5500	5823	const dst_mcv = try self.allocRegOrMem(inst, true);
5501		try self.setRegOrMem(Type.usize, dst_mcv, .{
5502		.stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)),
5503		});
	5824	try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
5504	5825	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5505	5826	}
5506	5827
5507	5828	fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
5508	5829	const dst_mcv = try self.allocRegOrMem(inst, true);
5509		try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp });
	5830	try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
5510	5831	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5511	5832	}
5512	5833
...	...	@@ -5534,94 +5855,69 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5534	5855	else => unreachable,
5535	5856	};
5536	5857
5537		var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..]);
	5858	var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..], .call_frame);
5538	5859	defer info.deinit(self);
5539	5860
	5861	// We need a properly aligned and sized call frame to be able to call this function.
	5862	{
	5863	const needed_call_frame =
	5864	FrameAlloc.init(.{ .size = info.stack_byte_count, .alignment = info.stack_align });
	5865	const frame_allocs_slice = self.frame_allocs.slice();
	5866	const stack_frame_size =
	5867	&frame_allocs_slice.items(.abi_size)[@enumToInt(FrameIndex.call_frame)];
	5868	stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
	5869	const stack_frame_align =
	5870	&frame_allocs_slice.items(.abi_align)[@enumToInt(FrameIndex.call_frame)];
	5871	stack_frame_align.* = @max(stack_frame_align.*, needed_call_frame.abi_align);
	5872	}
	5873
5540	5874	try self.spillEflagsIfOccupied();
5541	5875	try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
5542	5876
5543	5877	// set stack arguments first because this can clobber registers
5544	5878	// also clobber spill arguments as we go
5545		if (info.return_value == .stack_offset) {
5546		try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]});
	5879	switch (info.return_value.long) {
	5880	.none, .unreach => {},
	5881	.indirect => \|reg_off\| try self.spillRegisters(&.{reg_off.reg}),
	5882	else => unreachable,
5547	5883	}
5548	5884	for (args, info.args) \|arg, mc_arg\| {
5549	5885	const arg_ty = self.air.typeOf(arg);
5550	5886	const arg_mcv = try self.resolveInst(arg);
5551		// Here we do not use setRegOrMem even though the logic is similar, because
5552		// the function call will move the stack pointer, so the offsets are different.
5553	5887	switch (mc_arg) {
5554	5888	.none => {},
5555	5889	.register => \|reg\| try self.spillRegisters(&.{reg}),
5556		.stack_offset => \|off\| {
5557		// TODO rewrite using `genSetStack`
5558		try self.genSetStackArg(arg_ty, off, arg_mcv);
5559		},
5560		.ptr_stack_offset => {
5561		return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
5562		},
5563		.undef => unreachable,
5564		.immediate => unreachable,
5565		.unreach => unreachable,
5566		.dead => unreachable,
5567		.memory => unreachable,
5568		.eflags => unreachable,
5569		.register_overflow => unreachable,
5570		.load_got => unreachable,
5571		.lea_direct => unreachable,
5572		.load_direct => unreachable,
5573		.lea_tlv => unreachable,
5574		.load_tlv => unreachable,
	5890	.load_frame => try self.genCopy(arg_ty, mc_arg, arg_mcv),
	5891	else => unreachable,
5575	5892	}
5576	5893	}
5577	5894
5578	5895	// now we are free to set register arguments
5579		const ret_reg_lock: ?RegisterLock = blk: {
5580		if (info.return_value == .stack_offset) {
	5896	const ret_lock = switch (info.return_value.long) {
	5897	.none, .unreach => null,
	5898	.indirect => \|reg_off\| lock: {
5581	5899	const ret_ty = fn_ty.fnReturnType();
5582		const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
5583		const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
5584		const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));
5585		log.debug("airCall: return value on stack at offset {}", .{stack_offset});
5586
5587		const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
5588		try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset });
5589		const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg);
5590
5591		info.return_value.stack_offset = stack_offset;
5592
5593		break :blk ret_reg_lock;
5594		}
5595		break :blk null;
	5900	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, self.target.*));
	5901	try self.genSetReg(reg_off.reg, Type.usize, .{
	5902	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
	5903	});
	5904	info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
	5905	break :lock self.register_manager.lockRegAssumeUnused(reg_off.reg);
	5906	},
	5907	else => unreachable,
5596	5908	};
5597		defer if (ret_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	5909	defer if (ret_lock) \|lock\| self.register_manager.unlockReg(lock);
5598	5910
5599	5911	for (args, info.args) \|arg, mc_arg\| {
5600	5912	const arg_ty = self.air.typeOf(arg);
5601	5913	const arg_mcv = try self.resolveInst(arg);
5602	5914	switch (mc_arg) {
5603		.none, .stack_offset, .ptr_stack_offset => {},
5604		.register => \|reg\| try self.genSetReg(arg_ty, reg, arg_mcv),
5605		.undef => unreachable,
5606		.immediate => unreachable,
5607		.unreach => unreachable,
5608		.dead => unreachable,
5609		.memory => unreachable,
5610		.eflags => unreachable,
5611		.register_overflow => unreachable,
5612		.load_got => unreachable,
5613		.lea_direct => unreachable,
5614		.load_direct => unreachable,
5615		.lea_tlv => unreachable,
5616		.load_tlv => unreachable,
	5915	.none, .load_frame => {},
	5916	.register => try self.genCopy(arg_ty, mc_arg, arg_mcv),
	5917	else => unreachable,
5617	5918	}
5618	5919	}
5619	5920
5620		if (info.stack_byte_count > 0) {
5621		// Adjust the stack
5622		try self.asmRegisterImmediate(.sub, .rsp, Immediate.u(info.stack_byte_count));
5623		}
5624
5625	5921	// Due to incremental compilation, how function calls are generated depends
5626	5922	// on linking.
5627	5923	const mod = self.bin_file.options.module.?;
...	...	@@ -5635,16 +5931,16 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5635	5931	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
5636	5932	const got_addr = atom.getOffsetTableAddress(elf_file);
5637	5933	try self.asmMemory(.call, Memory.sib(.qword, .{
5638		.base = .ds,
	5934	.base = .{ .reg = .ds },
5639	5935	.disp = @intCast(i32, got_addr),
5640	5936	}));
5641	5937	} else if (self.bin_file.cast(link.File.Coff)) \|_\| {
5642	5938	const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5643		try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });
	5939	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5644	5940	try self.asmRegister(.call, .rax);
5645	5941	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
5646	5942	const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5647		try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });
	5943	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5648	5944	try self.asmRegister(.call, .rax);
5649	5945	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
5650	5946	const decl_block_index = try p9.seeDecl(func.owner_decl);
...	...	@@ -5655,7 +5951,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5655	5951	const got_index = decl_block.got_index.?;
5656	5952	const fn_got_addr = got_addr + got_index * ptr_bytes;
5657	5953	try self.asmMemory(.call, Memory.sib(.qword, .{
5658		.base = .ds,
	5954	.base = .{ .reg = .ds },
5659	5955	.disp = @intCast(i32, fn_got_addr),
5660	5956	}));
5661	5957	} else unreachable;
...	...	@@ -5696,41 +5992,15 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5696	5992	} else {
5697	5993	assert(ty.zigTypeTag() == .Pointer);
5698	5994	const mcv = try self.resolveInst(callee);
5699		try self.genSetReg(Type.usize, .rax, mcv);
	5995	try self.genSetReg(.rax, Type.usize, mcv);
5700	5996	try self.asmRegister(.call, .rax);
5701	5997	}
5702	5998
5703		if (info.stack_byte_count > 0) {
5704		// Readjust the stack
5705		try self.asmRegisterImmediate(.add, .rsp, Immediate.u(info.stack_byte_count));
5706		}
5707
5708		const result: MCValue = result: {
5709		if (self.liveness.isUnused(inst)) break :result .unreach;
5710
5711		switch (info.return_value) {
5712		.register => {
5713		// Save function return value in a new register
5714		break :result try self.copyToRegisterWithInstTracking(
5715		inst,
5716		self.air.typeOfIndex(inst),
5717		info.return_value,
5718		);
5719		},
5720		else => {},
5721		}
5722		break :result info.return_value;
5723		};
5724
5725		if (args.len <= Liveness.bpi - 2) {
5726		var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
5727		buf[0] = callee;
5728		std.mem.copy(Air.Inst.Ref, buf[1..], args);
5729		return self.finishAir(inst, result, buf);
5730		}
5731	5999	var bt = self.liveness.iterateBigTomb(inst);
5732	6000	self.feed(&bt, callee);
5733	6001	for (args) \|arg\| self.feed(&bt, arg);
	6002
	6003	const result = if (self.liveness.isUnused(inst)) .unreach else info.return_value.short;
5734	6004	return self.finishAirResult(inst, result);
5735	6005	}
5736	6006
...	...	@@ -5738,18 +6008,19 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
5738	6008	const un_op = self.air.instructions.items(.data)[inst].un_op;
5739	6009	const operand = try self.resolveInst(un_op);
5740	6010	const ret_ty = self.fn_type.fnReturnType();
5741		switch (self.ret_mcv) {
5742		.immediate => {
5743		assert(ret_ty.isError());
5744		},
5745		.stack_offset => {
5746		try self.store(self.ret_mcv, operand, Type.usize, ret_ty);
5747		},
5748		else => {
5749		try self.setRegOrMem(ret_ty, self.ret_mcv, operand);
	6011	switch (self.ret_mcv.short) {
	6012	.none => {},
	6013	.register => try self.genCopy(ret_ty, self.ret_mcv.short, operand),
	6014	.indirect => \|reg_off\| {
	6015	try self.register_manager.getReg(reg_off.reg, null);
	6016	const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
	6017	defer self.register_manager.unlockReg(lock);
	6018
	6019	try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
	6020	try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);
5750	6021	},
	6022	else => unreachable,
5751	6023	}
5752		// TODO when implementing defer, this will need to jump to the appropriate defer expression.
5753	6024	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5754	6025	// which is available if the jump is 127 bytes or less forward.
5755	6026	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -5761,25 +6032,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
5761	6032	const un_op = self.air.instructions.items(.data)[inst].un_op;
5762	6033	const ptr = try self.resolveInst(un_op);
5763	6034	const ptr_ty = self.air.typeOf(un_op);
5764		const elem_ty = ptr_ty.elemType();
5765		const abi_size = elem_ty.abiSize(self.target.*);
5766		switch (self.ret_mcv) {
5767		.immediate => {
5768		assert(elem_ty.isError());
5769		},
5770		.stack_offset => {
5771		const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
5772		const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5773		defer self.register_manager.unlockReg(reg_lock);
5774
5775		try self.genInlineMemcpy(.{ .register = reg }, ptr, .{ .immediate = abi_size }, .{});
5776		},
5777		else => {
5778		try self.load(self.ret_mcv, ptr, ptr_ty);
5779		try self.setRegOrMem(elem_ty, self.ret_mcv, self.ret_mcv);
5780		},
	6035	switch (self.ret_mcv.short) {
	6036	.none => {},
	6037	.register => try self.load(self.ret_mcv.short, ptr_ty, ptr),
	6038	.indirect => \|reg_off\| try self.genSetReg(reg_off.reg, ptr_ty, ptr),
	6039	else => unreachable,
5781	6040	}
5782		// TODO when implementing defer, this will need to jump to the appropriate defer expression.
5783	6041	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5784	6042	// which is available if the jump is 127 bytes or less forward.
5785	6043	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -5822,7 +6080,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
5822	6080	break :dst rhs_mcv;
5823	6081	} else if (dst_mem_ok) dst: {
5824	6082	const dst_mcv = try self.allocTempRegOrMem(ty, true);
5825		try self.setRegOrMem(ty, dst_mcv, lhs_mcv);
	6083	try self.genCopy(ty, dst_mcv, lhs_mcv);
5826	6084	break :dst dst_mcv;
5827	6085	} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
5828	6086	const dst_lock = switch (dst_mcv) {
...	...	@@ -5865,26 +6123,27 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5865	6123	const atom = elf_file.getAtom(atom_index);
5866	6124	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
5867	6125	const got_addr = atom.getOffsetTableAddress(elf_file);
5868		try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
5869		.base = .ds,
5870		.disp = @intCast(i32, got_addr),
5871		}));
	6126	try self.asmRegisterMemory(
	6127	.mov,
	6128	addr_reg.to64(),
	6129	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
	6130	);
5872	6131	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
5873	6132	const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
5874	6133	.{ .kind = .const_data, .ty = Type.anyerror },
5875	6134	4, // dword alignment
5876	6135	);
5877	6136	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
5878		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	6137	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5879	6138	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
5880	6139	const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
5881	6140	.{ .kind = .const_data, .ty = Type.anyerror },
5882	6141	4, // dword alignment
5883	6142	);
5884	6143	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
5885		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	6144	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5886	6145	} else {
5887		return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
	6146	return self.fail("TODO implement airCmpLtErrorsLen for x86_64 {s}", .{@tagName(self.bin_file.tag)});
5888	6147	}
5889	6148
5890	6149	try self.spillEflagsIfOccupied();
...	...	@@ -5900,7 +6159,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5900	6159	try self.asmRegisterMemory(
5901	6160	.cmp,
5902	6161	registerAlias(dst_reg, op_abi_size),
5903		Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = addr_reg }),
	6162	Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
5904	6163	);
5905	6164	const result = MCValue{ .eflags = .b };
5906	6165	return self.finishAir(inst, result, .{ un_op, .none, .none });
...	...	@@ -5971,14 +6230,13 @@ fn genTry(
5971	6230
5972	6231	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
5973	6232	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
5974		const payload = try self.addExtra(Mir.DbgLineColumn{
5975		.line = dbg_stmt.line,
5976		.column = dbg_stmt.column,
5977		});
5978	6233	_ = try self.addInst(.{
5979	6234	.tag = .dbg_line,
5980	6235	.ops = undefined,
5981		.data = .{ .payload = payload },
	6236	.data = .{ .line_column = .{
	6237	.line = dbg_stmt.line,
	6238	.column = dbg_stmt.column,
	6239	} },
5982	6240	});
5983	6241	return self.finishAirBookkeeping();
5984	6242	}
...	...	@@ -6002,8 +6260,6 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
6002	6260	const ty = self.air.typeOf(operand);
6003	6261	const mcv = try self.resolveInst(operand);
6004	6262
6005		log.debug("airDbgVar: %{d}: {}, {}", .{ inst, ty.fmtDebug(), mcv });
6006
6007	6263	const name = self.air.nullTerminatedString(pl_op.payload);
6008	6264
6009	6265	const tag = self.air.instructions.items(.tag)[inst];
...	...	@@ -6025,7 +6281,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
6025	6281	return self.asmJccReloc(undefined, .e);
6026	6282	},
6027	6283	.immediate,
6028		.stack_offset,
	6284	.load_frame,
6029	6285	=> {
6030	6286	try self.spillEflagsIfOccupied();
6031	6287	if (abi_size <= 8) {
...	...	@@ -6118,11 +6374,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6118	6374	.dead,
6119	6375	.undef,
6120	6376	.immediate,
6121		.register_overflow,
6122		.ptr_stack_offset,
6123	6377	.eflags,
	6378	.register_offset,
	6379	.register_overflow,
6124	6380	.lea_direct,
	6381	.lea_got,
6125	6382	.lea_tlv,
	6383	.lea_frame,
	6384	.reserved_frame,
6126	6385	=> unreachable,
6127	6386
6128	6387	.register => \|opt_reg\| {
...	...	@@ -6152,27 +6411,36 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6152	6411	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6153	6412	defer self.register_manager.unlockReg(addr_reg_lock);
6154	6413
6155		try self.genSetReg(Type.usize, addr_reg, switch (opt_mcv) {
6156		.memory => \|addr\| .{ .immediate = addr },
6157		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
6158		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
6159		else => opt_mcv,
6160		});
6161
	6414	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
6162	6415	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6163		try self.asmMemoryImmediate(.cmp, Memory.sib(
6164		Memory.PtrSize.fromSize(some_abi_size),
6165		.{ .base = addr_reg, .disp = some_info.off },
6166		), Immediate.u(0));
	6416	try self.asmMemoryImmediate(
	6417	.cmp,
	6418	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	6419	.base = .{ .reg = addr_reg },
	6420	.disp = some_info.off,
	6421	}),
	6422	Immediate.u(0),
	6423	);
6167	6424	return .{ .eflags = .e };
6168	6425	},
6169	6426
6170		.stack_offset => \|off\| {
	6427	.indirect, .load_frame => {
6171	6428	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6172		try self.asmMemoryImmediate(.cmp, Memory.sib(
6173		Memory.PtrSize.fromSize(some_abi_size),
6174		.{ .base = .rbp, .disp = some_info.off - off },
6175		), Immediate.u(0));
	6429	try self.asmMemoryImmediate(
	6430	.cmp,
	6431	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
	6432	.indirect => \|reg_off\| .{
	6433	.base = .{ .reg = reg_off.reg },
	6434	.disp = reg_off.off + some_info.off,
	6435	},
	6436	.load_frame => \|frame_addr\| .{
	6437	.base = .{ .frame = frame_addr.index },
	6438	.disp = frame_addr.off + some_info.off,
	6439	},
	6440	else => unreachable,
	6441	}),
	6442	Immediate.u(0),
	6443	);
6176	6444	return .{ .eflags = .e };
6177	6445	},
6178	6446	}
...	...	@@ -6200,10 +6468,14 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
6200	6468	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
6201	6469
6202	6470	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6203		try self.asmMemoryImmediate(.cmp, Memory.sib(
6204		Memory.PtrSize.fromSize(some_abi_size),
6205		.{ .base = ptr_reg, .disp = some_info.off },
6206		), Immediate.u(0));
	6471	try self.asmMemoryImmediate(
	6472	.cmp,
	6473	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	6474	.base = .{ .reg = ptr_reg },
	6475	.disp = some_info.off,
	6476	}),
	6477	Immediate.u(0),
	6478	);
6207	6479	return .{ .eflags = .e };
6208	6480	}
6209	6481
...	...	@@ -6221,10 +6493,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
6221	6493
6222	6494	const err_off = errUnionErrorOffset(ty.errorUnionPayload(), self.target.*);
6223	6495	switch (operand) {
6224		.stack_offset => \|off\| {
6225		const offset = off - @intCast(i32, err_off);
6226		try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
6227		},
6228	6496	.register => \|reg\| {
6229	6497	const eu_lock = self.register_manager.lockReg(reg);
6230	6498	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -6238,6 +6506,15 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
6238	6506	}
6239	6507	try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
6240	6508	},
	6509	.load_frame => \|frame_addr\| try self.genBinOpMir(
	6510	.cmp,
	6511	Type.anyerror,
	6512	.{ .load_frame = .{
	6513	.index = frame_addr.index,
	6514	.off = frame_addr.off + @intCast(i32, err_off),
	6515	} },
	6516	.{ .immediate = 0 },
	6517	),
6241	6518	else => return self.fail("TODO implement isErr for {}", .{operand}),
6242	6519	}
6243	6520
...	...	@@ -6324,9 +6601,9 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6324	6601	}
6325	6602	};
6326	6603	const ptr_ty = self.air.typeOf(un_op);
6327		try self.load(operand, operand_ptr, ptr_ty);
	6604	try self.load(operand, ptr_ty, operand_ptr);
6328	6605
6329		const result = try self.isErr(inst, ptr_ty.elemType(), operand);
	6606	const result = try self.isErr(inst, ptr_ty.childType(), operand);
6330	6607
6331	6608	return self.finishAir(inst, result, .{ un_op, .none, .none });
6332	6609	}
...	...	@@ -6358,9 +6635,9 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6358	6635	}
6359	6636	};
6360	6637	const ptr_ty = self.air.typeOf(un_op);
6361		try self.load(operand, operand_ptr, ptr_ty);
	6638	try self.load(operand, ptr_ty, operand_ptr);
6362	6639
6363		const result = try self.isNonErr(inst, ptr_ty.elemType(), operand);
	6640	const result = try self.isNonErr(inst, ptr_ty.childType(), operand);
6364	6641
6365	6642	return self.finishAir(inst, result, .{ un_op, .none, .none });
6366	6643	}
...	...	@@ -6389,6 +6666,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
6389	6666
6390	6667	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6391	6668	// A block is a setup to be able to jump to the end.
	6669	const inst_tracking_i = self.inst_tracking.count();
6392	6670	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
6393	6671
6394	6672	self.scope_generation += 1;
...	...	@@ -6412,8 +6690,9 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6412	6690	for (block_data.value.relocs.items) \|reloc\| try self.performReloc(reloc);
6413	6691	}
6414	6692
6415		const tracking = self.inst_tracking.getPtr(inst).?;
6416		if (self.liveness.isUnused(inst)) tracking.die(self);
	6693	if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i);
	6694	const tracking = &self.inst_tracking.values()[inst_tracking_i];
	6695	if (self.liveness.isUnused(inst)) tracking.die(self, inst);
6417	6696	self.getValue(tracking.short, inst);
6418	6697	self.finishAirBookkeeping();
6419	6698	}
...	...	@@ -6520,35 +6799,38 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6520	6799	!block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);
6521	6800	const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
6522	6801	const block_data = self.blocks.getPtr(br.block_inst).?;
	6802	const first_br = block_data.relocs.items.len == 0;
	6803	const block_result = result: {
	6804	if (block_unused) break :result .none;
	6805
	6806	if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
	6807	if (first_br) break :result src_mcv;
	6808
	6809	if (block_tracking.getReg()) \|block_reg\|
	6810	try self.register_manager.getReg(block_reg, br.block_inst);
	6811	// .long = .none to avoid merging operand and block result stack frames.
	6812	var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
	6813	try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
	6814	if (src_mcv.getReg()) \|src_reg\| self.register_manager.freeReg(src_reg);
	6815	break :result block_tracking.short;
	6816	}
6523	6817
6524		if (block_data.relocs.items.len == 0) {
6525		block_tracking.* = InstTracking.init(result: {
6526		if (block_unused) break :result .none;
6527		if (self.reuseOperand(inst, br.operand, 0, src_mcv)) {
6528		// Fix instruction tracking
6529		switch (src_mcv) {
6530		.register => \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|index\| {
6531		self.register_manager.registers[index] = br.block_inst;
6532		},
6533		else => {},
6534		}
6535		break :result src_mcv;
6536		}
6537
6538		const new_mcv = try self.allocRegOrMem(br.block_inst, true);
6539		try self.setRegOrMem(block_ty, new_mcv, src_mcv);
6540		break :result new_mcv;
6541		});
6542		} else if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);
	6818	const dst_mcv = if (first_br) try self.allocRegOrMem(br.block_inst, true) else dst: {
	6819	self.getValue(block_tracking.short, br.block_inst);
	6820	break :dst block_tracking.short;
	6821	};
	6822	try self.genCopy(block_ty, dst_mcv, src_mcv);
	6823	break :result dst_mcv;
	6824	};
6543	6825
6544	6826	// Process operand death so that it is properly accounted for in the State below.
6545	6827	if (self.liveness.operandDies(inst, 0)) {
6546	6828	if (Air.refToIndex(br.operand)) \|op_inst\| self.processDeath(op_inst);
6547	6829	}
6548	6830
6549		if (block_data.relocs.items.len == 0) {
	6831	if (first_br) {
	6832	block_tracking.* = InstTracking.init(block_result);
6550	6833	try self.saveRetroactiveState(&block_data.state);
6551		block_tracking.die(self);
6552	6834	} else try self.restoreState(block_data.state, &.{}, .{
6553	6835	.emit_instructions = true,
6554	6836	.update_tracking = false,
...	...	@@ -6556,6 +6838,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6556	6838	.close_scope = false,
6557	6839	});
6558	6840
	6841	// Stop tracking block result without forgetting tracking info
	6842	self.freeValue(block_tracking.short);
	6843
6559	6844	// Emit a jump with a relocation. It will be patched up after the block ends.
6560	6845	// Leave the jump offset undefined
6561	6846	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -6634,7 +6919,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6634	6919
6635	6920	const arg_mcv = try self.resolveInst(input);
6636	6921	try self.register_manager.getReg(reg, null);
6637		try self.genSetReg(self.air.typeOf(input), reg, arg_mcv);
	6922	try self.genSetReg(reg, self.air.typeOf(input), arg_mcv);
6638	6923	}
6639	6924
6640	6925	{
...	...	@@ -6686,7 +6971,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6686	6971	return self.fail("Invalid displacement: '{s}'", .{op_str});
6687	6972	op.* = .{ .mem = Memory.sib(
6688	6973	mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),
6689		.{ .base = reg, .disp = disp },
	6974	.{ .base = .{ .reg = reg }, .disp = disp },
6690	6975	) };
6691	6976	} else {
6692	6977	if (mnem_size) \|size\| if (reg.bitSize() != size.bitSize())
...	...	@@ -6805,335 +7090,317 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6805	7090	return self.finishAirResult(inst, result);
6806	7091	}
6807	7092
6808		/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
6809		fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
6810		switch (loc) {
6811		.none => return,
6812		.immediate => unreachable,
6813		.register => \|reg\| return self.genSetReg(ty, reg, val),
6814		.stack_offset => \|off\| return self.genSetStack(ty, off, val, .{}),
6815		.memory => {
6816		return self.fail("TODO implement setRegOrMem for memory", .{});
	7093	fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {
	7094	return switch (ty.zigTypeTag()) {
	7095	else => .mov,
	7096	.Float => switch (ty.floatBits(self.target.*)) {
	7097	16 => .mov,
	7098	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) .movss else .mov,
	7099	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) .movsd else .mov,
	7100	else => return self.fail("TODO movMirTag for {}", .{
	7101	ty.fmt(self.bin_file.options.module.?),
	7102	}),
6817	7103	},
6818		else => {
6819		return self.fail("TODO implement setRegOrMem for {}", .{loc});
	7104	};
	7105	}
	7106
	7107	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
	7108	const src_lock = switch (src_mcv) {
	7109	.register => \|reg\| self.register_manager.lockReg(reg),
	7110	.register_overflow => \|ro\| self.register_manager.lockReg(ro.reg),
	7111	else => null,
	7112	};
	7113	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
	7114
	7115	switch (dst_mcv) {
	7116	.none,
	7117	.unreach,
	7118	.dead,
	7119	.undef,
	7120	.immediate,
	7121	.eflags,
	7122	.register_overflow,
	7123	.lea_direct,
	7124	.lea_got,
	7125	.lea_tlv,
	7126	.lea_frame,
	7127	.reserved_frame,
	7128	=> unreachable, // unmodifiable destination
	7129	.register => \|reg\| try self.genSetReg(reg, ty, src_mcv),
	7130	.register_offset => \|dst_reg_off\| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) {
	7131	.none,
	7132	.unreach,
	7133	.dead,
	7134	.undef,
	7135	.register_overflow,
	7136	.reserved_frame,
	7137	=> unreachable,
	7138	.immediate,
	7139	.register,
	7140	.register_offset,
	7141	.lea_frame,
	7142	=> src_mcv.offset(-dst_reg_off.off),
	7143	else => .{ .register_offset = .{
	7144	.reg = try self.copyToTmpRegister(ty, src_mcv),
	7145	.off = -dst_reg_off.off,
	7146	} },
	7147	}),
	7148	.indirect => \|reg_off\| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
	7149	.memory, .load_direct, .load_got, .load_tlv => {
	7150	switch (dst_mcv) {
	7151	.memory => \|addr\| if (math.cast(i32, @bitCast(i64, addr))) \|small_addr\|
	7152	return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv),
	7153	.load_direct, .load_got, .load_tlv => {},
	7154	else => unreachable,
	7155	}
	7156
	7157	const addr_reg = try self.copyToTmpRegister(Type.usize, dst_mcv.address());
	7158	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	7159	defer self.register_manager.unlockReg(addr_lock);
	7160
	7161	try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv);
6820	7162	},
	7163	.load_frame => \|frame_addr\| try self.genSetMem(
	7164	.{ .frame = frame_addr.index },
	7165	frame_addr.off,
	7166	ty,
	7167	src_mcv,
	7168	),
6821	7169	}
6822	7170	}
6823	7171
6824		fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {
	7172	fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
6825	7173	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6826		switch (mcv) {
6827		.dead => unreachable,
6828		.unreach, .none => return,
6829		.undef => {
6830		if (!self.wantSafety()) return; // The already existing value will do just fine.
6831		if (abi_size <= 8) {
6832		const reg = try self.copyToTmpRegister(ty, mcv);
6833		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
	7174	if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
	7175	switch (src_mcv) {
	7176	.none,
	7177	.unreach,
	7178	.dead,
	7179	.register_overflow,
	7180	.reserved_frame,
	7181	=> unreachable,
	7182	.undef => if (self.wantSafety())
	7183	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
	7184	.eflags => \|cc\| try self.asmSetccRegister(dst_reg.to8(), cc),
	7185	.immediate => \|imm\| {
	7186	if (imm == 0) {
	7187	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
	7188	// register is the fastest way to zero a register.
	7189	try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());
	7190	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
	7191	// 32-bit moves zero-extend to 64-bit.
	7192	try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));
	7193	} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
	7194	try self.asmRegisterImmediate(
	7195	.mov,
	7196	registerAlias(dst_reg, abi_size),
	7197	Immediate.s(@intCast(i32, @bitCast(i64, imm))),
	7198	);
	7199	} else {
	7200	try self.asmRegisterImmediate(
	7201	.mov,
	7202	registerAlias(dst_reg, abi_size),
	7203	Immediate.u(imm),
	7204	);
6834	7205	}
6835		try self.genInlineMemset(
6836		.{ .ptr_stack_offset = stack_offset },
6837		.{ .immediate = 0xaa },
6838		.{ .immediate = abi_size },
6839		.{ .dest_stack_base = .rsp },
6840		);
6841		},
6842		.register_overflow => return self.fail("TODO genSetStackArg for register with overflow bit", .{}),
6843		.eflags => {
6844		const reg = try self.copyToTmpRegister(ty, mcv);
6845		return self.genSetStackArg(ty, stack_offset, .{ .register = reg });
6846	7206	},
6847		.immediate => \|imm\| {
6848		switch (abi_size) {
6849		1, 2, 4 => {
6850		// TODO
6851		// We have a positive stack offset value but we want a twos complement negative
6852		// offset from rbp, which is at the top of the stack frame.
6853		const immediate = if (ty.isSignedInt())
6854		Immediate.s(@intCast(i32, @bitCast(i64, imm)))
6855		else
6856		Immediate.u(@intCast(u32, imm));
6857		try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6858		.base = .rsp,
6859		.disp = -stack_offset,
6860		}), immediate);
	7207	.register => \|reg\| if (dst_reg.id() != reg.id()) try self.asmRegisterRegister(
	7208	try self.movMirTag(ty),
	7209	registerAlias(dst_reg, abi_size),
	7210	registerAlias(reg, abi_size),
	7211	),
	7212	.register_offset, .indirect, .load_frame, .lea_frame => try self.asmRegisterMemory(
	7213	switch (src_mcv) {
	7214	.register_offset => \|reg_off\| switch (reg_off.off) {
	7215	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
	7216	else => .lea,
6861	7217	},
6862		8 => {
6863		const reg = try self.copyToTmpRegister(ty, mcv);
6864		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
	7218	.indirect, .load_frame => try self.movMirTag(ty),
	7219	.lea_frame => .lea,
	7220	else => unreachable,
	7221	},
	7222	registerAlias(dst_reg, abi_size),
	7223	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
	7224	.register_offset, .indirect => \|reg_off\| .{
	7225	.base = .{ .reg = reg_off.reg },
	7226	.disp = reg_off.off,
6865	7227	},
6866		else => return self.fail("TODO implement inputs on stack for {} with abi size > 8", .{mcv}),
6867		}
6868		},
6869		.memory,
6870		.load_direct,
6871		.load_tlv,
6872		=> {
6873		if (abi_size <= 8) {
6874		const reg = try self.copyToTmpRegister(ty, mcv);
6875		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
	7228	.load_frame, .lea_frame => \|frame_addr\| .{
	7229	.base = .{ .frame = frame_addr.index },
	7230	.disp = frame_addr.off,
	7231	},
	7232	else => unreachable,
	7233	}),
	7234	),
	7235	.memory, .load_direct, .load_got, .load_tlv => {
	7236	switch (src_mcv) {
	7237	.memory => \|addr\| if (math.cast(i32, @bitCast(i64, addr))) \|small_addr\|
	7238	return self.asmRegisterMemory(
	7239	try self.movMirTag(ty),
	7240	registerAlias(dst_reg, abi_size),
	7241	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	7242	.base = .{ .reg = .ds },
	7243	.disp = small_addr,
	7244	}),
	7245	),
	7246	.load_direct => \|sym_index\| if (try self.movMirTag(ty) == .mov) {
	7247	const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
	7248	_ = try self.addInst(.{
	7249	.tag = .mov_linker,
	7250	.ops = .direct_reloc,
	7251	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
	7252	.reg = @enumToInt(dst_reg.to64()),
	7253	.atom_index = atom_index,
	7254	.sym_index = sym_index,
	7255	}) },
	7256	});
	7257	return;
	7258	},
	7259	.load_got, .load_tlv => {},
	7260	else => unreachable,
6876	7261	}
6877	7262
6878		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	7263	const addr_reg = try self.copyToTmpRegister(Type.usize, src_mcv.address());
6879	7264	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6880	7265	defer self.register_manager.unlockReg(addr_lock);
6881	7266
6882		try self.genSetReg(Type.usize, addr_reg, switch (mcv) {
6883		.memory => \|addr\| .{ .immediate = addr },
6884		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
6885		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
6886		else => unreachable,
6887		});
6888		try self.genInlineMemcpy(
6889		.{ .ptr_stack_offset = stack_offset },
6890		.{ .register = addr_reg },
6891		.{ .immediate = abi_size },
6892		.{ .dest_stack_base = .rsp },
	7267	try self.asmRegisterMemory(
	7268	try self.movMirTag(ty),
	7269	registerAlias(dst_reg, abi_size),
	7270	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
6893	7271	);
6894	7272	},
6895		.register => \|reg\| {
6896		switch (ty.zigTypeTag()) {
6897		.Float => {
6898		if (intrinsicsAllowed(self.target.*, ty)) {
6899		const tag: Mir.Inst.Tag = switch (ty.tag()) {
6900		.f32 => .movss,
6901		.f64 => .movsd,
6902		else => return self.fail(
6903		"TODO genSetStackArg for register for type {}",
6904		.{ty.fmtDebug()},
6905		),
6906		};
6907		// TODO verify this
6908		const ptr_size: Memory.PtrSize = switch (ty.tag()) {
6909		.f32 => .dword,
6910		.f64 => .qword,
6911		else => unreachable,
6912		};
6913		return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
6914		.base = .rsp,
6915		.disp = -stack_offset,
6916		}), reg.to128());
6917		}
6918
6919		return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
	7273	.lea_direct, .lea_got => \|sym_index\| {
	7274	const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
	7275	_ = try self.addInst(.{
	7276	.tag = switch (src_mcv) {
	7277	.lea_direct => .lea_linker,
	7278	.lea_got => .mov_linker,
	7279	else => unreachable,
6920	7280	},
6921		else => {
6922		try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6923		.base = .rsp,
6924		.disp = -stack_offset,
6925		}), registerAlias(reg, abi_size));
	7281	.ops = switch (src_mcv) {
	7282	.lea_direct => .direct_reloc,
	7283	.lea_got => .got_reloc,
	7284	else => unreachable,
6926	7285	},
6927		}
6928		},
6929		.ptr_stack_offset,
6930		.load_got,
6931		.lea_direct,
6932		.lea_tlv,
6933		=> {
6934		const reg = try self.copyToTmpRegister(ty, mcv);
6935		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6936		},
6937		.stack_offset => \|mcv_off\| {
6938		if (abi_size <= 8) {
6939		const reg = try self.copyToTmpRegister(ty, mcv);
6940		return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6941		}
6942
6943		try self.genInlineMemcpy(
6944		.{ .ptr_stack_offset = stack_offset },
6945		.{ .ptr_stack_offset = mcv_off },
6946		.{ .immediate = abi_size },
6947		.{ .dest_stack_base = .rsp },
6948		);
	7286	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
	7287	.reg = @enumToInt(dst_reg.to64()),
	7288	.atom_index = atom_index,
	7289	.sym_index = sym_index,
	7290	}) },
	7291	});
6949	7292	},
6950		}
	7293	.lea_tlv => \|sym_index\| {
	7294	const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
	7295	if (self.bin_file.cast(link.File.MachO)) \|_\| {
	7296	_ = try self.addInst(.{
	7297	.tag = .mov_linker,
	7298	.ops = .tlv_reloc,
	7299	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
	7300	.reg = @enumToInt(Register.rdi),
	7301	.atom_index = atom_index,
	7302	.sym_index = sym_index,
	7303	}) },
	7304	});
	7305	// TODO: spill registers before calling
	7306	try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
	7307	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
	7308	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
	7309	@tagName(self.bin_file.tag),
	7310	});
	7311	},
	7312	}
6951	7313	}
6952	7314
6953		fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {
6954		const base_reg = opts.dest_stack_base orelse .rbp;
	7315	fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCValue) InnerError!void {
6955	7316	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6956		switch (mcv) {
6957		.dead => unreachable,
6958		.unreach, .none => return, // Nothing to do.
6959		.undef => {
6960		if (!self.wantSafety()) return; // The already existing value will do just fine.
6961		// TODO Upgrade this to a memset call when we have that available.
6962		switch (abi_size) {
6963		1, 2, 4 => {
6964		const value: u64 = switch (abi_size) {
6965		1 => 0xaa,
6966		2 => 0xaaaa,
6967		4 => 0xaaaaaaaa,
6968		else => unreachable,
6969		};
6970		return self.asmMemoryImmediate(.mov, Memory.sib(
6971		Memory.PtrSize.fromSize(abi_size),
6972		.{ .base = base_reg, .disp = -stack_offset },
6973		), Immediate.u(value));
6974		},
6975		8 => return self.genSetStack(
6976		ty,
6977		stack_offset,
6978		.{ .immediate = 0xaaaaaaaaaaaaaaaa },
6979		opts,
6980		),
6981		else => \|x\| return self.genInlineMemset(
6982		.{ .ptr_stack_offset = stack_offset },
6983		.{ .immediate = 0xaa },
6984		.{ .immediate = x },
6985		opts,
6986		),
6987		}
	7317	const dst_ptr_mcv: MCValue = switch (base) {
	7318	.none => .{ .immediate = @bitCast(u64, @as(i64, disp)) },
	7319	.reg => \|base_reg\| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
	7320	.frame => \|base_frame_index\| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
	7321	};
	7322	switch (src_mcv) {
	7323	.none, .unreach, .dead, .reserved_frame => unreachable,
	7324	.undef => if (self.wantSafety())
	7325	try self.genInlineMemset(dst_ptr_mcv, .{ .immediate = 0xaa }, .{ .immediate = abi_size }),
	7326	.immediate => \|imm\| switch (abi_size) {
	7327	1, 2, 4 => {
	7328	const immediate = if (ty.isSignedInt())
	7329	Immediate.s(@truncate(i32, @bitCast(i64, imm)))
	7330	else
	7331	Immediate.u(@intCast(u32, imm));
	7332	try self.asmMemoryImmediate(
	7333	.mov,
	7334	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	7335	immediate,
	7336	);
	7337	},
	7338	3, 5...7 => unreachable,
	7339	else => if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
	7340	try self.asmMemoryImmediate(
	7341	.mov,
	7342	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	7343	Immediate.s(small),
	7344	);
	7345	} else {
	7346	var offset: i32 = 0;
	7347	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
	7348	.mov,
	7349	Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
	7350	if (ty.isSignedInt())
	7351	Immediate.s(@truncate(
	7352	i32,
	7353	@bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
	7354	))
	7355	else
	7356	Immediate.u(@truncate(
	7357	u32,
	7358	if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0,
	7359	)),
	7360	);
	7361	},
6988	7362	},
	7363	.eflags => \|cc\| try self.asmSetccMemory(Memory.sib(.byte, .{ .base = base, .disp = disp }), cc),
	7364	.register => \|reg\| try self.asmMemoryRegister(
	7365	try self.movMirTag(ty),
	7366	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	7367	registerAlias(reg, abi_size),
	7368	),
6989	7369	.register_overflow => \|ro\| {
6990		const reg_lock = self.register_manager.lockReg(ro.reg);
6991		defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
6992
6993		const wrapped_ty = ty.structFieldType(0);
6994		try self.genSetStack(wrapped_ty, stack_offset, .{ .register = ro.reg }, .{});
6995
6996		const overflow_bit_ty = ty.structFieldType(1);
6997		const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
6998		const tmp_reg = try self.register_manager.allocReg(null, gp);
6999		try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
7000
7001		return self.genSetStack(
7002		overflow_bit_ty,
7003		stack_offset - @intCast(i32, overflow_bit_offset),
7004		.{ .register = tmp_reg.to8() },
7005		.{},
	7370	try self.genSetMem(
	7371	base,
	7372	disp + @intCast(i32, ty.structFieldOffset(0, self.target.*)),
	7373	ty.structFieldType(0),
	7374	.{ .register = ro.reg },
	7375	);
	7376	try self.genSetMem(
	7377	base,
	7378	disp + @intCast(i32, ty.structFieldOffset(1, self.target.*)),
	7379	ty.structFieldType(1),
	7380	.{ .eflags = ro.eflags },
7006	7381	);
7007	7382	},
7008		.eflags => \|cc\| try self.asmSetccMemory(
7009		Memory.sib(.byte, .{ .base = base_reg, .disp = -stack_offset }),
7010		cc,
7011		),
7012		.immediate => \|imm\| {
7013		// TODO
7014		switch (abi_size) {
7015		0 => {
7016		assert(ty.isError());
7017		try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
7018		.base = base_reg,
7019		.disp = -stack_offset,
7020		}), Immediate.u(@truncate(u8, imm)));
7021		},
7022		1, 2, 4 => {
7023		const immediate = if (ty.isSignedInt())
7024		Immediate.s(@truncate(i32, @bitCast(i64, imm)))
7025		else
7026		Immediate.u(@intCast(u32, imm));
7027		try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7028		.base = base_reg,
7029		.disp = -stack_offset,
7030		}), immediate);
7031		},
7032		3, 5...7 => unreachable,
7033		else => {
7034		// 64 bit write to memory would take two mov's anyways so we
7035		// insted just use two 32 bit writes to avoid register allocation
7036		if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
7037		try self.asmMemoryImmediate(.mov, Memory.sib(
7038		Memory.PtrSize.fromSize(abi_size),
7039		.{ .base = base_reg, .disp = -stack_offset },
7040		), Immediate.s(small));
7041		} else {
7042		var offset: i32 = 0;
7043		while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
7044		.mov,
7045		Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }),
7046		if (ty.isSignedInt())
7047		Immediate.s(@truncate(
7048		i32,
7049		@bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
7050		))
7051		else
7052		Immediate.u(@truncate(
7053		u32,
7054		if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0,
7055		)),
7056		);
7057		}
7058		},
7059		}
7060		},
7061		.register => \|reg\| {
7062		switch (ty.zigTypeTag()) {
7063		.Float => {
7064		if (intrinsicsAllowed(self.target.*, ty)) {
7065		const tag: Mir.Inst.Tag = switch (ty.tag()) {
7066		.f32 => .movss,
7067		.f64 => .movsd,
7068		else => return self.fail(
7069		"TODO genSetStack for register for type {}",
7070		.{ty.fmtDebug()},
7071		),
7072		};
7073		const ptr_size: Memory.PtrSize = switch (ty.tag()) {
7074		.f32 => .dword,
7075		.f64 => .qword,
7076		else => unreachable,
7077		};
7078		return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
7079		.base = base_reg.to64(),
7080		.disp = -stack_offset,
7081		}), reg.to128());
7082		}
7083
7084		return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
7085		},
7086		else => {
7087		try self.genInlineMemcpyRegisterRegister(ty, base_reg, reg, stack_offset);
7088		},
7089		}
7090		},
	7383	.register_offset,
7091	7384	.memory,
	7385	.indirect,
7092	7386	.load_direct,
7093		.load_tlv,
7094	7387	.lea_direct,
7095		.lea_tlv,
7096		=> if (abi_size <= 8) {
7097		const reg = try self.copyToTmpRegister(ty, mcv);
7098		return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7099		} else {
7100		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7101		const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7102		defer self.register_manager.unlockReg(addr_lock);
7103
7104		try self.genSetReg(Type.usize, addr_reg, switch (mcv) {
7105		.memory => \|addr\| .{ .immediate = addr },
7106		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7107		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7108		else => mcv,
7109		});
7110		try self.genInlineMemcpy(
7111		.{ .ptr_stack_offset = stack_offset },
7112		.{ .register = addr_reg },
7113		.{ .immediate = abi_size },
7114		.{},
7115		);
7116		},
7117		.stack_offset => \|off\| if (abi_size <= 8) {
7118		const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7119		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7120		defer self.register_manager.unlockReg(tmp_lock);
7121
7122		try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7123		} else try self.genInlineMemcpy(
7124		.{ .ptr_stack_offset = stack_offset },
7125		.{ .ptr_stack_offset = off },
7126		.{ .immediate = abi_size },
7127		.{},
7128		),
7129		.ptr_stack_offset,
7130	7388	.load_got,
7131		=> {
7132		const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7133		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7134		defer self.register_manager.unlockReg(tmp_lock);
7135
7136		try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
	7389	.lea_got,
	7390	.load_tlv,
	7391	.lea_tlv,
	7392	.load_frame,
	7393	.lea_frame,
	7394	=> switch (abi_size) {
	7395	0 => {},
	7396	1, 2, 4, 8 => {
	7397	const src_reg = try self.copyToTmpRegister(ty, src_mcv);
	7398	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	7399	defer self.register_manager.unlockReg(src_lock);
	7400
	7401	try self.genSetMem(base, disp, ty, .{ .register = src_reg });
	7402	},
	7403	else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }),
7137	7404	},
7138	7405	}
7139	7406	}
...	...	@@ -7165,10 +7432,14 @@ fn genInlineMemcpyRegisterRegister(
7165	7432	var remainder = abi_size;
7166	7433	while (remainder > 0) {
7167	7434	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
7168		try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
7169		.base = dst_reg,
7170		.disp = -next_offset,
7171		}), registerAlias(tmp_reg, nearest_power_of_two));
	7435	try self.asmMemoryRegister(
	7436	.mov,
	7437	Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
	7438	.base = dst_reg,
	7439	.disp = -next_offset,
	7440	}),
	7441	registerAlias(tmp_reg, nearest_power_of_two),
	7442	);
7172	7443
7173	7444	if (nearest_power_of_two > 1) {
7174	7445	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
...	...	@@ -7191,121 +7462,11 @@ fn genInlineMemcpyRegisterRegister(
7191	7462	}
7192	7463	}
7193	7464
7194		const InlineMemcpyOpts = struct {
7195		source_stack_base: ?Register = null,
7196		dest_stack_base: ?Register = null,
7197		};
7198
7199		fn genInlineMemcpy(
7200		self: *Self,
7201		dst_ptr: MCValue,
7202		src_ptr: MCValue,
7203		len: MCValue,
7204		opts: InlineMemcpyOpts,
7205		) InnerError!void {
7206		const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) \|reg\|
7207		self.register_manager.lockReg(reg)
7208		else
7209		null;
7210		defer if (ssbase_lock) \|reg\| self.register_manager.unlockReg(reg);
7211
7212		const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) \|reg\|
7213		self.register_manager.lockReg(reg)
7214		else
7215		null;
7216		defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
7217
	7465	fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
7218	7466	try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
7219
7220		switch (dst_ptr) {
7221		.lea_tlv,
7222		.load_tlv,
7223		=> {
7224		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7225		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7226		else => dst_ptr,
7227		});
7228		},
7229		.memory,
7230		.load_got,
7231		.lea_direct,
7232		.load_direct,
7233		=> {
7234		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7235		.memory => \|addr\| .{ .immediate = addr },
7236		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7237		else => dst_ptr,
7238		});
7239		// Load the pointer, which is stored in memory
7240		try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7241		},
7242		.stack_offset, .ptr_stack_offset => \|off\| {
7243		try self.asmRegisterMemory(switch (dst_ptr) {
7244		.stack_offset => .mov,
7245		.ptr_stack_offset => .lea,
7246		else => unreachable,
7247		}, .rdi, Memory.sib(.qword, .{
7248		.base = opts.dest_stack_base orelse .rbp,
7249		.disp = -off,
7250		}));
7251		},
7252		.register => \|reg\| {
7253		try self.asmRegisterRegister(
7254		.mov,
7255		registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7256		reg,
7257		);
7258		},
7259		else => {
7260		return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
7261		},
7262		}
7263
7264		switch (src_ptr) {
7265		.lea_tlv,
7266		.load_tlv,
7267		=> {
7268		try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7269		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7270		else => dst_ptr,
7271		});
7272		},
7273		.memory,
7274		.load_got,
7275		.lea_direct,
7276		.load_direct,
7277		=> {
7278		try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7279		.memory => \|addr\| .{ .immediate = addr },
7280		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7281		else => src_ptr,
7282		});
7283		// Load the pointer, which is stored in memory
7284		try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi }));
7285		},
7286		.stack_offset, .ptr_stack_offset => \|off\| {
7287		try self.asmRegisterMemory(switch (src_ptr) {
7288		.stack_offset => .mov,
7289		.ptr_stack_offset => .lea,
7290		else => unreachable,
7291		}, .rsi, Memory.sib(.qword, .{
7292		.base = opts.source_stack_base orelse .rbp,
7293		.disp = -off,
7294		}));
7295		},
7296		.register => \|reg\| {
7297		try self.asmRegisterRegister(
7298		.mov,
7299		registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7300		reg,
7301		);
7302		},
7303		else => {
7304		return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
7305		},
7306		}
7307
7308		try self.genSetReg(Type.usize, .rcx, len);
	7467	try self.genSetReg(.rdi, Type.usize, dst_ptr);
	7468	try self.genSetReg(.rsi, Type.usize, src_ptr);
	7469	try self.genSetReg(.rcx, Type.usize, len);
7309	7470	_ = try self.addInst(.{
7310	7471	.tag = .movs,
7311	7472	.ops = .string,
...	...	@@ -7313,67 +7474,11 @@ fn genInlineMemcpy(
7313	7474	});
7314	7475	}
7315	7476
7316		fn genInlineMemset(
7317		self: *Self,
7318		dst_ptr: MCValue,
7319		value: MCValue,
7320		len: MCValue,
7321		opts: InlineMemcpyOpts,
7322		) InnerError!void {
7323		const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) \|reg\|
7324		self.register_manager.lockReg(reg)
7325		else
7326		null;
7327		defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
7328
	7477	fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
7329	7478	try self.spillRegisters(&.{ .rdi, .al, .rcx });
7330
7331		switch (dst_ptr) {
7332		.lea_tlv,
7333		.load_tlv,
7334		=> {
7335		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7336		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
7337		else => dst_ptr,
7338		});
7339		},
7340		.load_got => try self.genSetReg(Type.usize, .rdi, dst_ptr),
7341		.memory,
7342		.lea_direct,
7343		.load_direct,
7344		=> {
7345		try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7346		.memory => \|addr\| .{ .immediate = addr },
7347		.load_direct => \|sym_index\| .{ .lea_direct = sym_index },
7348		else => dst_ptr,
7349		});
7350		// Load the pointer, which is stored in memory
7351		try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7352		},
7353		.stack_offset, .ptr_stack_offset => \|off\| {
7354		try self.asmRegisterMemory(switch (dst_ptr) {
7355		.stack_offset => .mov,
7356		.ptr_stack_offset => .lea,
7357		else => unreachable,
7358		}, .rdi, Memory.sib(.qword, .{
7359		.base = opts.dest_stack_base orelse .rbp,
7360		.disp = -off,
7361		}));
7362		},
7363		.register => \|reg\| {
7364		try self.asmRegisterRegister(
7365		.mov,
7366		registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7367		reg,
7368		);
7369		},
7370		else => {
7371		return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr});
7372		},
7373		}
7374
7375		try self.genSetReg(Type.u8, .al, value);
7376		try self.genSetReg(Type.usize, .rcx, len);
	7479	try self.genSetReg(.rdi, Type.usize, dst_ptr);
	7480	try self.genSetReg(.al, Type.u8, value);
	7481	try self.genSetReg(.rcx, Type.usize, len);
7377	7482	_ = try self.addInst(.{
7378	7483	.tag = .stos,
7379	7484	.ops = .string,
...	...	@@ -7381,307 +7486,6 @@ fn genInlineMemset(
7381	7486	});
7382	7487	}
7383	7488
7384		fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
7385		const abi_size = @intCast(u32, ty.abiSize(self.target.*));
7386		if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
7387		switch (mcv) {
7388		.dead => unreachable,
7389		.register_overflow => unreachable,
7390		.ptr_stack_offset => \|off\| {
7391		try self.asmRegisterMemory(
7392		.lea,
7393		registerAlias(reg, abi_size),
7394		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7395		);
7396		},
7397		.unreach, .none => return, // Nothing to do.
7398		.undef => {
7399		if (!self.wantSafety()) return; // The already existing value will do just fine.
7400		// Write the debug undefined value.
7401		switch (self.regBitSize(ty)) {
7402		8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
7403		16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
7404		32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
7405		64 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
7406		else => unreachable,
7407		}
7408		},
7409		.eflags => \|cc\| {
7410		return self.asmSetccRegister(reg.to8(), cc);
7411		},
7412		.immediate => \|imm\| {
7413		if (imm == 0) {
7414		// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7415		// register is the fastest way to zero a register.
7416		try self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
7417		} else if (abi_size > 4 and math.cast(u32, imm) != null) {
7418		// 32-bit moves zero-extend to 64-bit.
7419		try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm));
7420		} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7421		try self.asmRegisterImmediate(
7422		.mov,
7423		registerAlias(reg, abi_size),
7424		Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7425		);
7426		} else {
7427		try self.asmRegisterImmediate(
7428		.mov,
7429		registerAlias(reg, abi_size),
7430		Immediate.u(imm),
7431		);
7432		}
7433		},
7434		.register => \|src_reg\| {
7435		// If the registers are the same, nothing to do.
7436		if (src_reg.id() == reg.id())
7437		return;
7438
7439		switch (ty.zigTypeTag()) {
7440		.Int => switch (ty.intInfo(self.target.*).signedness) {
7441		.signed => {
7442		if (abi_size <= 4) {
7443		return self.asmRegisterRegister(
7444		.movsx,
7445		reg.to64(),
7446		registerAlias(src_reg, abi_size),
7447		);
7448		}
7449		},
7450		.unsigned => {
7451		if (abi_size <= 2) {
7452		return self.asmRegisterRegister(
7453		.movzx,
7454		reg.to64(),
7455		registerAlias(src_reg, abi_size),
7456		);
7457		}
7458		},
7459		},
7460		.Float => {
7461		if (intrinsicsAllowed(self.target.*, ty)) {
7462		const tag: Mir.Inst.Tag = switch (ty.tag()) {
7463		.f32 => .movss,
7464		.f64 => .movsd,
7465		else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
7466		};
7467		return self.asmRegisterRegister(tag, reg.to128(), src_reg.to128());
7468		}
7469		return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
7470		},
7471		else => {},
7472		}
7473
7474		try self.asmRegisterRegister(.mov, registerAlias(reg, abi_size), registerAlias(src_reg, abi_size));
7475		},
7476		.memory => \|addr\| switch (ty.zigTypeTag()) {
7477		.Float => {
7478		const base_reg = (try self.register_manager.allocReg(null, gp)).to64();
7479		try self.genSetReg(Type.usize, base_reg, .{ .immediate = addr });
7480
7481		if (intrinsicsAllowed(self.target.*, ty)) {
7482		return self.asmRegisterMemory(
7483		switch (ty.tag()) {
7484		.f32 => .movss,
7485		.f64 => .movsd,
7486		else => return self.fail("TODO genSetReg from memory for {}", .{
7487		ty.fmt(self.bin_file.options.module.?),
7488		}),
7489		},
7490		reg.to128(),
7491		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7492		);
7493		}
7494
7495		return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7496		},
7497		else => {
7498		if (addr <= math.maxInt(i32)) {
7499		try self.asmRegisterMemory(
7500		.mov,
7501		registerAlias(reg, abi_size),
7502		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7503		.base = .ds,
7504		.disp = @intCast(i32, addr),
7505		}),
7506		);
7507		} else {
7508		if (reg.to64() == .rax) {
7509		// If this is RAX, we can use a direct load.
7510		// Otherwise, we need to load the address, then indirectly load the value.
7511		_ = try self.addInst(.{
7512		.tag = .mov_moffs,
7513		.ops = .rax_moffs,
7514		.data = .{ .payload = try self.addExtra(Mir.MemoryMoffs.encode(.ds, addr)) },
7515		});
7516		} else {
7517		// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
7518		try self.genSetReg(Type.usize, reg, MCValue{ .immediate = addr });
7519		try self.asmRegisterMemory(
7520		.mov,
7521		registerAlias(reg, abi_size),
7522		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
7523		);
7524		}
7525		}
7526		},
7527		},
7528		.load_got, .lea_direct => \|sym_index\| {
7529		const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7530		_ = try self.addInst(.{
7531		.tag = switch (mcv) {
7532		.load_got => .mov_linker,
7533		.lea_direct => .lea_linker,
7534		else => unreachable,
7535		},
7536		.ops = switch (mcv) {
7537		.load_got => .got_reloc,
7538		.lea_direct => .direct_reloc,
7539		else => unreachable,
7540		},
7541		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7542		.reg = @enumToInt(reg),
7543		.atom_index = atom_index,
7544		.sym_index = sym_index,
7545		}) },
7546		});
7547		},
7548		.load_direct => \|sym_index\| {
7549		switch (ty.zigTypeTag()) {
7550		.Float => {
7551		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7552		try self.genSetReg(Type.usize, addr_reg, .{ .lea_direct = sym_index });
7553
7554		if (intrinsicsAllowed(self.target.*, ty)) {
7555		return self.asmRegisterMemory(
7556		switch (ty.tag()) {
7557		.f32 => .movss,
7558		.f64 => .movsd,
7559		else => return self.fail("TODO genSetReg from memory for {}", .{
7560		ty.fmt(self.bin_file.options.module.?),
7561		}),
7562		},
7563		reg.to128(),
7564		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
7565		);
7566		}
7567
7568		return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7569		},
7570		else => {
7571		const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7572		_ = try self.addInst(.{
7573		.tag = .mov_linker,
7574		.ops = .direct_reloc,
7575		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7576		.reg = @enumToInt(registerAlias(reg, abi_size)),
7577		.atom_index = atom_index,
7578		.sym_index = sym_index,
7579		}) },
7580		});
7581		},
7582		}
7583		},
7584		.lea_tlv => \|sym_index\| {
7585		const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7586		if (self.bin_file.cast(link.File.MachO)) \|_\| {
7587		_ = try self.addInst(.{
7588		.tag = .lea_linker,
7589		.ops = .tlv_reloc,
7590		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7591		.reg = @enumToInt(Register.rdi),
7592		.atom_index = atom_index,
7593		.sym_index = sym_index,
7594		}) },
7595		});
7596		// TODO: spill registers before calling
7597		try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .rdi }));
7598		try self.genSetReg(Type.usize, reg, .{ .register = .rax });
7599		} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{@tagName(self.bin_file.tag)});
7600		},
7601		.load_tlv => \|sym_index\| {
7602		const base_reg = switch (ty.zigTypeTag()) {
7603		.Float => (try self.register_manager.allocReg(null, gp)).to64(),
7604		else => reg.to64(),
7605		};
7606		try self.genSetReg(Type.usize, base_reg, .{ .lea_tlv = sym_index });
7607		switch (ty.zigTypeTag()) {
7608		.Float => if (intrinsicsAllowed(self.target.*, ty)) {
7609		return self.asmRegisterMemory(
7610		switch (ty.tag()) {
7611		.f32 => .movss,
7612		.f64 => .movsd,
7613		else => return self.fail("TODO genSetReg from memory for {}", .{
7614		ty.fmt(self.bin_file.options.module.?),
7615		}),
7616		},
7617		reg.to128(),
7618		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7619		);
7620		} else return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}),
7621		else => try self.asmRegisterMemory(
7622		.mov,
7623		registerAlias(reg, abi_size),
7624		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7625		),
7626		}
7627		},
7628		.stack_offset => \|off\| {
7629		switch (ty.zigTypeTag()) {
7630		.Int => switch (ty.intInfo(self.target.*).signedness) {
7631		.signed => {
7632		if (abi_size <= 4) {
7633		return self.asmRegisterMemory(
7634		.movsx,
7635		reg.to64(),
7636		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7637		.base = .rbp,
7638		.disp = -off,
7639		}),
7640		);
7641		}
7642		},
7643		.unsigned => {
7644		if (abi_size <= 2) {
7645		return self.asmRegisterMemory(
7646		.movzx,
7647		reg.to64(),
7648		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7649		.base = .rbp,
7650		.disp = -off,
7651		}),
7652		);
7653		}
7654		},
7655		},
7656		.Float => {
7657		if (intrinsicsAllowed(self.target.*, ty)) {
7658		const tag: Mir.Inst.Tag = switch (ty.tag()) {
7659		.f32 => .movss,
7660		.f64 => .movsd,
7661		else => return self.fail(
7662		"TODO genSetReg from stack offset for {}",
7663		.{ty.fmtDebug()},
7664		),
7665		};
7666		return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(
7667		Memory.PtrSize.fromSize(abi_size),
7668		.{ .base = .rbp, .disp = -off },
7669		));
7670		}
7671		return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
7672		},
7673		else => {},
7674		}
7675
7676		try self.asmRegisterMemory(
7677		.mov,
7678		registerAlias(reg, abi_size),
7679		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7680		);
7681		},
7682		}
7683		}
7684
7685	7489	fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7686	7490	const un_op = self.air.instructions.items(.data)[inst].un_op;
7687	7491	const result = result: {
...	...	@@ -7691,7 +7495,7 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7691	7495
7692	7496	const dst_mcv = try self.allocRegOrMem(inst, true);
7693	7497	const dst_ty = self.air.typeOfIndex(inst);
7694		try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);
	7498	try self.genCopy(dst_ty, dst_mcv, src_mcv);
7695	7499	break :result dst_mcv;
7696	7500	};
7697	7501	return self.finishAir(inst, result, .{ un_op, .none, .none });
...	...	@@ -7711,7 +7515,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7711	7515	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
7712	7516
7713	7517	const dest = try self.allocRegOrMem(inst, true);
7714		try self.setRegOrMem(self.air.typeOfIndex(inst), dest, operand);
	7518	try self.genCopy(self.air.typeOfIndex(inst), dest, operand);
7715	7519	break :result dest;
7716	7520	};
7717	7521	log.debug("airBitCast(%{d}): {}", .{ inst, result });
...	...	@@ -7720,16 +7524,23 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7720	7524
7721	7525	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
7722	7526	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	7527
	7528	const slice_ty = self.air.typeOfIndex(inst);
7723	7529	const ptr_ty = self.air.typeOf(ty_op.operand);
7724	7530	const ptr = try self.resolveInst(ty_op.operand);
7725	7531	const array_ty = ptr_ty.childType();
7726	7532	const array_len = array_ty.arrayLen();
7727	7533
7728		const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
7729		try self.genSetStack(ptr_ty, stack_offset, ptr, .{});
7730		try self.genSetStack(Type.u64, stack_offset - 8, .{ .immediate = array_len }, .{});
	7534	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
	7535	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
	7536	try self.genSetMem(
	7537	.{ .frame = frame_index },
	7538	@intCast(i32, ptr_ty.abiSize(self.target.*)),
	7539	Type.usize,
	7540	.{ .immediate = array_len },
	7541	);
7731	7542
7732		const result = MCValue{ .stack_offset = stack_offset };
	7543	const result = MCValue{ .load_frame = .{ .index = frame_index } };
7733	7544	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7734	7545	}
7735	7546
...	...	@@ -7758,29 +7569,31 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
7758	7569	}
7759	7570
7760	7571	// move float src to ST(0)
7761		const stack_offset = switch (operand) {
7762		.stack_offset, .ptr_stack_offset => \|offset\| offset,
7763		else => blk: {
7764		const offset = @intCast(i32, try self.allocMem(
7765		inst,
7766		src_abi_size,
7767		src_ty.abiAlignment(self.target.*),
7768		));
7769		try self.genSetStack(src_ty, offset, operand, .{});
7770		break :blk offset;
	7572	const frame_addr: FrameAddr = switch (operand) {
	7573	.load_frame => \|frame_addr\| frame_addr,
	7574	else => frame_addr: {
	7575	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(src_ty, self.target.*));
	7576	try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, operand);
	7577	break :frame_addr .{ .index = frame_index };
7771	7578	},
7772	7579	};
7773		try self.asmMemory(.fld, Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
7774		.base = .rbp,
7775		.disp = -stack_offset,
7776		}));
	7580	try self.asmMemory(
	7581	.fld,
	7582	Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
	7583	.base = .{ .frame = frame_addr.index },
	7584	.disp = frame_addr.off,
	7585	}),
	7586	);
7777	7587
7778	7588	// convert
7779	7589	const stack_dst = try self.allocRegOrMem(inst, false);
7780		try self.asmMemory(.fisttp, Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
7781		.base = .rbp,
7782		.disp = -stack_dst.stack_offset,
7783		}));
	7590	try self.asmMemory(
	7591	.fisttp,
	7592	Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
	7593	.base = .{ .frame = stack_dst.load_frame.index },
	7594	.disp = stack_dst.load_frame.off,
	7595	}),
	7596	);
7784	7597
7785	7598	return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
7786	7599	}
...	...	@@ -7800,20 +7613,32 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7800	7613
7801	7614	const exp_mcv = try self.resolveInst(extra.expected_value);
7802	7615	if (val_abi_size > 8) switch (exp_mcv) {
7803		.stack_offset => \|exp_off\| {
7804		try self.genSetReg(Type.usize, .rax, .{ .stack_offset = exp_off - 0 });
7805		try self.genSetReg(Type.usize, .rdx, .{ .stack_offset = exp_off - 8 });
	7616	.load_frame => \|frame_addr\| {
	7617	try self.genSetReg(.rax, Type.usize, .{ .load_frame = .{
	7618	.index = frame_addr.index,
	7619	.off = frame_addr.off + 0,
	7620	} });
	7621	try self.genSetReg(.rdx, Type.usize, .{ .load_frame = .{
	7622	.index = frame_addr.index,
	7623	.off = frame_addr.off + 8,
	7624	} });
7806	7625	},
7807	7626	else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
7808		} else try self.genSetReg(val_ty, .rax, exp_mcv);
	7627	} else try self.genSetReg(.rax, val_ty, exp_mcv);
7809	7628	const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
7810	7629	defer self.register_manager.unlockReg(rax_lock);
7811	7630
7812	7631	const new_mcv = try self.resolveInst(extra.new_value);
7813	7632	const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {
7814		.stack_offset => \|new_off\| new: {
7815		try self.genSetReg(Type.usize, .rbx, .{ .stack_offset = new_off - 0 });
7816		try self.genSetReg(Type.usize, .rcx, .{ .stack_offset = new_off - 8 });
	7633	.load_frame => \|frame_addr\| new: {
	7634	try self.genSetReg(.rbx, Type.usize, .{ .load_frame = .{
	7635	.index = frame_addr.index,
	7636	.off = frame_addr.off + 0,
	7637	} });
	7638	try self.genSetReg(.rcx, Type.usize, .{ .load_frame = .{
	7639	.index = frame_addr.index,
	7640	.off = frame_addr.off + 8,
	7641	} });
7817	7642	break :new undefined;
7818	7643	},
7819	7644	else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
...	...	@@ -7823,11 +7648,19 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7823	7648
7824	7649	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7825	7650	const ptr_mem = switch (ptr_mcv) {
7826		.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg }),
7827		.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
7828		else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
	7651	.register => \|reg\| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
	7652	.lea_frame => \|frame_addr\| Memory.sib(ptr_size, .{
	7653	.base = .{ .frame = frame_addr.index },
	7654	.disp = frame_addr.off,
	7655	}),
	7656	else => Memory.sib(ptr_size, .{ .base = .{
	7657	.reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
	7658	} }),
	7659	};
	7660	const mem_lock = switch (ptr_mem.base()) {
	7661	.none, .frame => null,
	7662	.reg => \|reg\| self.register_manager.lockReg(reg),
7829	7663	};
7830		const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
7831	7664	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
7832	7665
7833	7666	try self.spillEflagsIfOccupied();
...	...	@@ -7851,9 +7684,24 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7851	7684	}
7852	7685
7853	7686	const dst_mcv = try self.allocRegOrMem(inst, false);
7854		try self.genSetStack(Type.bool, dst_mcv.stack_offset - 16, .{ .eflags = .ne }, .{});
7855		try self.genSetStack(Type.usize, dst_mcv.stack_offset - 8, .{ .register = .rdx }, .{});
7856		try self.genSetStack(Type.usize, dst_mcv.stack_offset - 0, .{ .register = .rax }, .{});
	7687	try self.genSetMem(
	7688	.{ .frame = dst_mcv.load_frame.index },
	7689	dst_mcv.load_frame.off + 16,
	7690	Type.bool,
	7691	.{ .eflags = .ne },
	7692	);
	7693	try self.genSetMem(
	7694	.{ .frame = dst_mcv.load_frame.index },
	7695	dst_mcv.load_frame.off + 8,
	7696	Type.usize,
	7697	.{ .register = .rdx },
	7698	);
	7699	try self.genSetMem(
	7700	.{ .frame = dst_mcv.load_frame.index },
	7701	dst_mcv.load_frame.off + 0,
	7702	Type.usize,
	7703	.{ .register = .rax },
	7704	);
7857	7705	break :result dst_mcv;
7858	7706	};
7859	7707	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
...	...	@@ -7884,11 +7732,19 @@ fn atomicOp(
7884	7732	const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
7885	7733	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7886	7734	const ptr_mem = switch (ptr_mcv) {
7887		.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg }),
7888		.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
7889		else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
	7735	.register => \|reg\| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
	7736	.lea_frame => \|frame_addr\| Memory.sib(ptr_size, .{
	7737	.base = .{ .frame = frame_addr.index },
	7738	.disp = frame_addr.off,
	7739	}),
	7740	else => Memory.sib(ptr_size, .{ .base = .{
	7741	.reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
	7742	} }),
	7743	};
	7744	const mem_lock = switch (ptr_mem.base()) {
	7745	.none, .frame => null,
	7746	.reg => \|reg\| self.register_manager.lockReg(reg),
7890	7747	};
7891		const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
7892	7748	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
7893	7749
7894	7750	const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())
...	...	@@ -7928,7 +7784,7 @@ fn atomicOp(
7928	7784	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
7929	7785	defer self.register_manager.unlockReg(dst_lock);
7930	7786
7931		try self.genSetReg(val_ty, dst_reg, val_mcv);
	7787	try self.genSetReg(dst_reg, val_ty, val_mcv);
7932	7788	if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
7933	7789	try self.genUnOpMir(.neg, val_ty, dst_mcv);
7934	7790	}
...	...	@@ -7952,10 +7808,10 @@ fn atomicOp(
7952	7808	try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
7953	7809	const loop = @intCast(u32, self.mir_instructions.len);
7954	7810	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
7955		try self.genSetReg(val_ty, tmp_reg, .{ .register = .rax });
	7811	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
7956	7812	}
7957	7813	if (rmw_op) \|op\| switch (op) {
7958		.Xchg => try self.genSetReg(val_ty, tmp_reg, val_mcv),
	7814	.Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
7959	7815	.Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
7960	7816	.Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
7961	7817	.And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
...	...	@@ -7990,12 +7846,12 @@ fn atomicOp(
7990	7846	registerAlias(val_reg, cmov_abi_size),
7991	7847	cc,
7992	7848	),
7993		.stack_offset => \|val_off\| try self.asmCmovccRegisterMemory(
	7849	.load_frame => \|frame_addr\| try self.asmCmovccRegisterMemory(
7994	7850	registerAlias(tmp_reg, cmov_abi_size),
7995		Memory.sib(
7996		Memory.PtrSize.fromSize(cmov_abi_size),
7997		.{ .base = .rbp, .disp = -val_off },
7998		),
	7851	Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
	7852	.base = .{ .frame = frame_addr.index },
	7853	.disp = frame_addr.off,
	7854	}),
7999	7855	cc,
8000	7856	),
8001	7857	else => {
...	...	@@ -8028,9 +7884,15 @@ fn atomicOp(
8028	7884	}));
8029	7885	const loop = @intCast(u32, self.mir_instructions.len);
8030	7886	switch (val_mcv) {
8031		.stack_offset => \|val_off\| {
8032		const val_lo_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - val_off });
8033		const val_hi_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - val_off });
	7887	.load_frame => \|frame_addr\| {
	7888	const val_lo_mem = Memory.sib(.qword, .{
	7889	.base = .{ .frame = frame_addr.index },
	7890	.disp = frame_addr.off + 0,
	7891	});
	7892	const val_hi_mem = Memory.sib(.qword, .{
	7893	.base = .{ .frame = frame_addr.index },
	7894	.disp = frame_addr.off + 8,
	7895	});
8034	7896
8035	7897	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8036	7898	try self.asmRegisterRegister(.mov, .rbx, .rax);
...	...	@@ -8087,12 +7949,18 @@ fn atomicOp(
8087	7949	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
8088	7950	try self.asmMemoryRegister(
8089	7951	.mov,
8090		Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - dst_mcv.stack_offset }),
	7952	Memory.sib(.qword, .{
	7953	.base = .{ .frame = dst_mcv.load_frame.index },
	7954	.disp = dst_mcv.load_frame.off + 0,
	7955	}),
8091	7956	.rax,
8092	7957	);
8093	7958	try self.asmMemoryRegister(
8094	7959	.mov,
8095		Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - dst_mcv.stack_offset }),
	7960	Memory.sib(.qword, .{
	7961	.base = .{ .frame = dst_mcv.load_frame.index },
	7962	.disp = dst_mcv.load_frame.off + 8,
	7963	}),
8096	7964	.rdx,
8097	7965	);
8098	7966	return dst_mcv;
...	...	@@ -8139,7 +8007,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
8139	8007	else
8140	8008	try self.allocRegOrMem(inst, true);
8141	8009
8142		try self.load(dst_mcv, ptr_mcv, ptr_ty);
	8010	try self.load(dst_mcv, ptr_ty, ptr_mcv);
8143	8011	return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
8144	8012	}
8145	8013
...	...	@@ -8184,16 +8052,16 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8184	8052	const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));
8185	8053
8186	8054	if (elem_abi_size == 1) {
8187		const ptr = switch (dst_ptr_ty.ptrSize()) {
	8055	const ptr: MCValue = switch (dst_ptr_ty.ptrSize()) {
8188	8056	// TODO: this only handles slices stored in the stack
8189		.Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 }),
	8057	.Slice => dst_ptr,
8190	8058	.One => dst_ptr,
8191	8059	.C, .Many => unreachable,
8192	8060	};
8193		const len = switch (dst_ptr_ty.ptrSize()) {
	8061	const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
8194	8062	// TODO: this only handles slices stored in the stack
8195		.Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),
8196		.One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),
	8063	.Slice => dst_ptr.address().offset(8).deref(),
	8064	.One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8197	8065	.C, .Many => unreachable,
8198	8066	};
8199	8067	const len_lock: ?RegisterLock = switch (len) {
...	...	@@ -8202,7 +8070,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8202	8070	};
8203	8071	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
8204	8072
8205		try self.genInlineMemset(ptr, src_val, len, .{});
	8073	try self.genInlineMemset(ptr, src_val, len);
8206	8074	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8207	8075	}
8208	8076
...	...	@@ -8215,8 +8083,8 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8215	8083	const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(&buf);
8216	8084
8217	8085	// TODO: this only handles slices stored in the stack
8218		const ptr = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 });
8219		const len = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 });
	8086	const ptr = dst_ptr;
	8087	const len = dst_ptr.address().offset(8).deref();
8220	8088
8221	8089	// Used to store the number of elements for comparison.
8222	8090	// After comparison, updated to store number of bytes needed to copy.
...	...	@@ -8225,55 +8093,51 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8225	8093	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
8226	8094	defer self.register_manager.unlockReg(len_lock);
8227	8095
8228		try self.asmRegisterMemory(.mov, len_reg, Memory.sib(.qword, .{
8229		.base = .rbp,
8230		.disp = -len.stack_offset,
8231		}));
	8096	try self.genSetReg(len_reg, Type.usize, len);
8232	8097
8233	8098	const skip_reloc = try self.asmJccReloc(undefined, .z);
8234		try self.store(ptr, src_val, slice_ptr_ty, elem_ty);
	8099	try self.store(slice_ptr_ty, ptr, src_val);
8235	8100
8236	8101	const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8237	8102	const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8238	8103	const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8239	8104	defer self.register_manager.unlockReg(second_elem_ptr_lock);
8240	8105
8241		try self.asmRegisterMemory(
8242		.lea,
8243		second_elem_ptr_reg,
8244		Memory.sib(.qword, .{
8245		.base = try self.copyToTmpRegister(Type.usize, ptr),
8246		.disp = elem_abi_size,
8247		}),
8248		);
	8106	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
	8107	.reg = try self.copyToTmpRegister(Type.usize, ptr),
	8108	.off = elem_abi_size,
	8109	} });
8249	8110
8250	8111	try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });
8251	8112	try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));
8252		try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv, .{});
	8113	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
8253	8114
8254	8115	try self.performReloc(skip_reloc);
8255	8116	},
8256	8117	.One => {
	8118	var elem_ptr_pl = Type.Payload.ElemType{
	8119	.base = .{ .tag = .single_mut_pointer },
	8120	.data = elem_ty,
	8121	};
	8122	const elem_ptr_ty = Type.initPayload(&elem_ptr_pl.base);
	8123
8257	8124	const len = dst_ptr_ty.childType().arrayLen();
	8125
8258	8126	assert(len != 0); // prevented by Sema
8259		try self.store(dst_ptr, src_val, dst_ptr_ty, elem_ty);
	8127	try self.store(elem_ptr_ty, dst_ptr, src_val);
8260	8128
8261	8129	const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8262	8130	const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8263	8131	const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8264	8132	defer self.register_manager.unlockReg(second_elem_ptr_lock);
8265	8133
8266		try self.asmRegisterMemory(
8267		.lea,
8268		second_elem_ptr_reg,
8269		Memory.sib(.qword, .{
8270		.base = try self.copyToTmpRegister(Type.usize, dst_ptr),
8271		.disp = elem_abi_size,
8272		}),
8273		);
	8134	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
	8135	.reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
	8136	.off = elem_abi_size,
	8137	} });
8274	8138
8275	8139	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
8276		try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy, .{});
	8140	try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
8277	8141	},
8278	8142	.C, .Many => unreachable,
8279	8143	}
...	...	@@ -8299,9 +8163,9 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8299	8163	};
8300	8164	defer if (src_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
8301	8165
8302		const len = switch (dst_ptr_ty.ptrSize()) {
8303		.Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),
8304		.One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),
	8166	const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
	8167	.Slice => dst_ptr.address().offset(8).deref(),
	8168	.One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8305	8169	.C, .Many => unreachable,
8306	8170	};
8307	8171	const len_lock: ?RegisterLock = switch (len) {
...	...	@@ -8311,7 +8175,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8311	8175	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
8312	8176
8313	8177	// TODO: dst_ptr and src_ptr could be slices rather than raw pointers
8314		try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});
	8178	try self.genInlineMemcpy(dst_ptr, src_ptr, len);
8315	8179
8316	8180	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8317	8181	}
...	...	@@ -8345,24 +8209,25 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8345	8209	const atom = elf_file.getAtom(atom_index);
8346	8210	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
8347	8211	const got_addr = atom.getOffsetTableAddress(elf_file);
8348		try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
8349		.base = .ds,
8350		.disp = @intCast(i32, got_addr),
8351		}));
	8212	try self.asmRegisterMemory(
	8213	.mov,
	8214	addr_reg.to64(),
	8215	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
	8216	);
8352	8217	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
8353	8218	const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
8354	8219	.{ .kind = .const_data, .ty = Type.anyerror },
8355	8220	4, // dword alignment
8356	8221	);
8357	8222	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
8358		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	8223	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8359	8224	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
8360	8225	const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
8361	8226	.{ .kind = .const_data, .ty = Type.anyerror },
8362	8227	4, // dword alignment
8363	8228	);
8364	8229	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
8365		try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });
	8230	try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8366	8231	} else {
8367	8232	return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
8368	8233	}
...	...	@@ -8377,36 +8242,60 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8377	8242
8378	8243	try self.truncateRegister(err_ty, err_reg.to32());
8379	8244
8380		try self.asmRegisterMemory(.mov, start_reg.to32(), Memory.sib(.dword, .{
8381		.base = addr_reg.to64(),
8382		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
8383		.disp = 4,
8384		}));
8385		try self.asmRegisterMemory(.mov, end_reg.to32(), Memory.sib(.dword, .{
8386		.base = addr_reg.to64(),
8387		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
8388		.disp = 8,
8389		}));
	8245	try self.asmRegisterMemory(
	8246	.mov,
	8247	start_reg.to32(),
	8248	Memory.sib(.dword, .{
	8249	.base = .{ .reg = addr_reg.to64() },
	8250	.scale_index = .{ .scale = 4, .index = err_reg.to64() },
	8251	.disp = 4,
	8252	}),
	8253	);
	8254	try self.asmRegisterMemory(
	8255	.mov,
	8256	end_reg.to32(),
	8257	Memory.sib(.dword, .{
	8258	.base = .{ .reg = addr_reg.to64() },
	8259	.scale_index = .{ .scale = 4, .index = err_reg.to64() },
	8260	.disp = 8,
	8261	}),
	8262	);
8390	8263	try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
8391		try self.asmRegisterMemory(.lea, start_reg.to64(), Memory.sib(.byte, .{
8392		.base = addr_reg.to64(),
8393		.scale_index = .{ .scale = 1, .index = start_reg.to64() },
8394		.disp = 0,
8395		}));
8396		try self.asmRegisterMemory(.lea, end_reg.to32(), Memory.sib(.byte, .{
8397		.base = end_reg.to64(),
8398		.disp = -1,
8399		}));
	8264	try self.asmRegisterMemory(
	8265	.lea,
	8266	start_reg.to64(),
	8267	Memory.sib(.byte, .{
	8268	.base = .{ .reg = addr_reg.to64() },
	8269	.scale_index = .{ .scale = 1, .index = start_reg.to64() },
	8270	.disp = 0,
	8271	}),
	8272	);
	8273	try self.asmRegisterMemory(
	8274	.lea,
	8275	end_reg.to32(),
	8276	Memory.sib(.byte, .{
	8277	.base = .{ .reg = end_reg.to64() },
	8278	.disp = -1,
	8279	}),
	8280	);
8400	8281
8401	8282	const dst_mcv = try self.allocRegOrMem(inst, false);
8402		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
8403		.base = .rbp,
8404		.disp = 0 - dst_mcv.stack_offset,
8405		}), start_reg.to64());
8406		try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{
8407		.base = .rbp,
8408		.disp = 8 - dst_mcv.stack_offset,
8409		}), end_reg.to64());
	8283	try self.asmMemoryRegister(
	8284	.mov,
	8285	Memory.sib(.qword, .{
	8286	.base = .{ .frame = dst_mcv.load_frame.index },
	8287	.disp = dst_mcv.load_frame.off,
	8288	}),
	8289	start_reg.to64(),
	8290	);
	8291	try self.asmMemoryRegister(
	8292	.mov,
	8293	Memory.sib(.qword, .{
	8294	.base = .{ .frame = dst_mcv.load_frame.index },
	8295	.disp = dst_mcv.load_frame.off + 8,
	8296	}),
	8297	end_reg.to64(),
	8298	);
8410	8299
8411	8300	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
8412	8301	}
...	...	@@ -8445,19 +8334,17 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8445	8334	const len = @intCast(usize, result_ty.arrayLen());
8446	8335	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
8447	8336	const elements = @ptrCast([]const Air.Inst.Ref, self.air.extra[ty_pl.payload..][0..len]);
8448		const abi_size = @intCast(u32, result_ty.abiSize(self.target.*));
8449		const abi_align = result_ty.abiAlignment(self.target.*);
8450	8337	const result: MCValue = result: {
8451	8338	switch (result_ty.zigTypeTag()) {
8452	8339	.Struct => {
8453		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
	8340	const frame_index =
	8341	try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8454	8342	if (result_ty.containerLayout() == .Packed) {
8455	8343	const struct_obj = result_ty.castTag(.@"struct").?.data;
8456	8344	try self.genInlineMemset(
8457		.{ .ptr_stack_offset = stack_offset },
	8345	.{ .lea_frame = .{ .index = frame_index } },
8458	8346	.{ .immediate = 0 },
8459		.{ .immediate = abi_size },
8460		.{},
	8347	.{ .immediate = result_ty.abiSize(self.target.*) },
8461	8348	);
8462	8349	for (elements, 0..) \|elem, elem_i\| {
8463	8350	if (result_ty.structFieldValueComptime(elem_i) != null) continue;
...	...	@@ -8465,7 +8352,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8465	8352	const elem_ty = result_ty.structFieldType(elem_i);
8466	8353	const elem_bit_size = @intCast(u32, elem_ty.bitSize(self.target.*));
8467	8354	if (elem_bit_size > 64) {
8468		return self.fail("TODO airAggregateInit implement packed structs with large fields", .{});
	8355	return self.fail(
	8356	"TODO airAggregateInit implement packed structs with large fields",
	8357	.{},
	8358	);
8469	8359	}
8470	8360	const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
8471	8361	const elem_abi_bits = elem_abi_size * 8;
...	...	@@ -8503,7 +8393,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8503	8393	try self.genBinOpMir(
8504	8394	.@"or",
8505	8395	elem_ty,
8506		.{ .stack_offset = stack_offset - elem_byte_off },
	8396	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
8507	8397	.{ .register = elem_reg },
8508	8398	);
8509	8399	if (elem_bit_off > elem_extra_bits) {
...	...	@@ -8520,8 +8410,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8520	8410	try self.genBinOpMir(
8521	8411	.@"or",
8522	8412	elem_ty,
8523		.{ .stack_offset = stack_offset - elem_byte_off -
8524		@intCast(i32, elem_abi_size) },
	8413	.{ .load_frame = .{
	8414	.index = frame_index,
	8415	.off = elem_byte_off + @intCast(i32, elem_abi_size),
	8416	} },
8525	8417	.{ .register = reg },
8526	8418	);
8527	8419	}
...	...	@@ -8536,12 +8428,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8536	8428	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
8537	8429	else => elem_mcv,
8538	8430	};
8539		try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});
	8431	try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8540	8432	}
8541		break :result .{ .stack_offset = stack_offset };
	8433	break :result .{ .load_frame = .{ .index = frame_index } };
8542	8434	},
8543	8435	.Array => {
8544		const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
	8436	const frame_index =
	8437	try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8545	8438	const elem_ty = result_ty.childType();
8546	8439	const elem_size = @intCast(u32, elem_ty.abiSize(self.target.*));
8547	8440
...	...	@@ -8552,9 +8445,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8552	8445	else => elem_mcv,
8553	8446	};
8554	8447	const elem_off = @intCast(i32, elem_size * elem_i);
8555		try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});
	8448	try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8556	8449	}
8557		break :result MCValue{ .stack_offset = stack_offset };
	8450	break :result .{ .load_frame = .{ .index = frame_index } };
8558	8451	},
8559	8452	.Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),
8560	8453	else => unreachable,
...	...	@@ -8596,7 +8489,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
8596	8489	const ty = self.air.typeOf(ref);
8597	8490
8598	8491	// If the type has no codegen bits, no need to store it.
8599		if (!ty.hasRuntimeBitsIgnoreComptime() and !ty.isError()) return .none;
	8492	if (!ty.hasRuntimeBitsIgnoreComptime()) return .none;
8600	8493
8601	8494	if (Air.refToIndex(ref)) \|inst\| {
8602	8495	const mcv = switch (self.air.instructions.items(.tag)[inst]) {
...	...	@@ -8666,7 +8559,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
8666	8559	.immediate => \|imm\| .{ .immediate = imm },
8667	8560	.memory => \|addr\| .{ .memory = addr },
8668	8561	.load_direct => \|sym_index\| .{ .load_direct = sym_index },
8669		.load_got => \|sym_index\| .{ .load_got = sym_index },
	8562	.load_got => \|sym_index\| .{ .lea_got = sym_index },
8670	8563	.load_tlv => \|sym_index\| .{ .load_tlv = sym_index },
8671	8564	},
8672	8565	.fail => \|msg\| {
...	...	@@ -8679,9 +8572,9 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
8679	8572
8680	8573	const CallMCValues = struct {
8681	8574	args: []MCValue,
8682		return_value: MCValue,
8683		stack_byte_count: u32,
8684		stack_align: u32,
	8575	return_value: InstTracking,
	8576	stack_byte_count: u31,
	8577	stack_align: u31,
8685	8578
8686	8579	fn deinit(self: CallMCValues, func: Self) void {
8687	8580	func.gpa.free(self.args);
...	...	@@ -8694,6 +8587,7 @@ fn resolveCallingConventionValues(
8694	8587	self: *Self,
8695	8588	fn_ty: Type,
8696	8589	var_args: []const Air.Inst.Ref,
	8590	stack_frame_base: FrameIndex,
8697	8591	) !CallMCValues {
8698	8592	const cc = fn_ty.fnCallingConvention();
8699	8593	const param_len = fn_ty.fnParamLen();
...	...	@@ -8706,7 +8600,7 @@ fn resolveCallingConventionValues(
8706	8600	.args = try self.gpa.alloc(MCValue, param_types.len),
8707	8601	// These undefined values must be populated before returning from this function.
8708	8602	.return_value = undefined,
8709		.stack_byte_count = undefined,
	8603	.stack_byte_count = 0,
8710	8604	.stack_align = undefined,
8711	8605	};
8712	8606	errdefer self.gpa.free(result.args);
...	...	@@ -8716,40 +8610,46 @@ fn resolveCallingConventionValues(
8716	8610	switch (cc) {
8717	8611	.Naked => {
8718	8612	assert(result.args.len == 0);
8719		result.return_value = .unreach;
8720		result.stack_byte_count = 0;
8721		result.stack_align = 1;
8722		return result;
	8613	result.return_value = InstTracking.init(.unreach);
	8614	result.stack_align = 8;
8723	8615	},
8724	8616	.C => {
	8617	var param_reg_i: usize = 0;
	8618	result.stack_align = 16;
	8619
	8620	switch (self.target.os.tag) {
	8621	.windows => {
	8622	// Align the stack to 16bytes before allocating shadow stack space (if any).
	8623	result.stack_byte_count += @intCast(u31, 4 * Type.usize.abiSize(self.target.*));
	8624	},
	8625	else => {},
	8626	}
	8627
8725	8628	// Return values
8726	8629	if (ret_ty.zigTypeTag() == .NoReturn) {
8727		result.return_value = .unreach;
8728		} else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
	8630	result.return_value = InstTracking.init(.unreach);
	8631	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
8729	8632	// TODO: is this even possible for C calling convention?
8730		result.return_value = .none;
	8633	result.return_value = InstTracking.init(.none);
8731	8634	} else {
8732		const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
8733		if (ret_ty_size == 0) {
8734		assert(ret_ty.isError());
8735		result.return_value = .{ .immediate = 0 };
8736		} else if (ret_ty_size <= 8) {
8737		const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8738		result.return_value = .{ .register = aliased_reg };
	8635	const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
	8636	const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
	8637	if (ret_ty_size <= 8) {
	8638	const aliased_reg = registerAlias(ret_reg, ret_ty_size);
	8639	result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8739	8640	} else {
8740		// TODO: return argument cell should go first
8741		result.return_value = .{ .stack_offset = 0 };
	8641	const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
	8642	param_reg_i += 1;
	8643	result.return_value = .{
	8644	.short = .{ .indirect = .{ .reg = ret_reg } },
	8645	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
	8646	};
8742	8647	}
8743	8648	}
8744	8649
8745	8650	// Input params
8746		var next_stack_offset: u32 = switch (result.return_value) {
8747		.stack_offset => \|off\| @intCast(u32, off),
8748		else => 0,
8749		};
8750
8751		for (param_types, result.args, 0..) \|ty, *arg, i\| {
8752		assert(ty.hasRuntimeBits());
	8651	for (param_types, result.args) \|ty, *arg\| {
	8652	assert(ty.hasRuntimeBitsIgnoreComptime());
8753	8653
8754	8654	const classes: []const abi.Class = switch (self.target.os.tag) {
8755	8655	.windows => &[1]abi.Class{abi.classifyWindows(ty, self.target.*)},
...	...	@@ -8760,8 +8660,10 @@ fn resolveCallingConventionValues(
8760	8660	}
8761	8661	switch (classes[0]) {
8762	8662	.integer => blk: {
8763		if (i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk; // fallthrough
8764		arg.* = .{ .register = abi.getCAbiIntParamRegs(self.target.*)[i] };
	8663	if (param_reg_i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk;
	8664	const param_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
	8665	param_reg_i += 1;
	8666	arg.* = .{ .register = param_reg };
8765	8667	continue;
8766	8668	},
8767	8669	.memory => {}, // fallthrough
...	...	@@ -8770,82 +8672,61 @@ fn resolveCallingConventionValues(
8770	8672	}),
8771	8673	}
8772	8674
8773		const param_size = @intCast(u32, ty.abiSize(self.target.*));
8774		const param_align = @intCast(u32, ty.abiAlignment(self.target.*));
8775		const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);
8776		arg.* = .{ .stack_offset = @intCast(i32, offset) };
8777		next_stack_offset = offset;
8778		}
8779
8780		// Align the stack to 16bytes before allocating shadow stack space (if any).
8781		const aligned_next_stack_offset = mem.alignForwardGeneric(u32, next_stack_offset, 16);
8782		const padding = aligned_next_stack_offset - next_stack_offset;
8783		if (padding > 0) {
8784		for (result.args) \|*arg\| {
8785		if (arg.isRegister()) continue;
8786		arg.stack_offset += @intCast(i32, padding);
8787		}
	8675	const param_size = @intCast(u31, ty.abiSize(self.target.*));
	8676	const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
	8677	result.stack_byte_count =
	8678	mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
	8679	arg.* = .{ .load_frame = .{
	8680	.index = stack_frame_base,
	8681	.off = result.stack_byte_count,
	8682	} };
	8683	result.stack_byte_count += param_size;
8788	8684	}
8789
8790		const shadow_stack_space: u32 = switch (self.target.os.tag) {
8791		.windows => @intCast(u32, 4 * @sizeOf(u64)),
8792		else => 0,
8793		};
8794
8795		// alignment padding \| args ... \| shadow stack space (if any) \| ret addr \| $rbp \|
8796		result.stack_byte_count = aligned_next_stack_offset + shadow_stack_space;
8797		result.stack_align = 16;
8798	8685	},
8799	8686	.Unspecified => {
	8687	result.stack_align = 16;
	8688
8800	8689	// Return values
8801	8690	if (ret_ty.zigTypeTag() == .NoReturn) {
8802		result.return_value = .unreach;
8803		} else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
8804		result.return_value = .none;
	8691	result.return_value = InstTracking.init(.unreach);
	8692	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
	8693	result.return_value = InstTracking.init(.none);
8805	8694	} else {
8806		const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
8807		if (ret_ty_size == 0) {
8808		assert(ret_ty.isError());
8809		result.return_value = .{ .immediate = 0 };
8810		} else if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
8811		const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8812		result.return_value = .{ .register = aliased_reg };
	8695	const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
	8696	const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
	8697	if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
	8698	const aliased_reg = registerAlias(ret_reg, ret_ty_size);
	8699	result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8813	8700	} else {
8814		// We simply make the return MCValue a stack offset. However, the actual value
8815		// for the offset will be populated later. We will also push the stack offset
8816		// value into an appropriate register when we resolve the offset.
8817		result.return_value = .{ .stack_offset = 0 };
	8701	const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
	8702	result.return_value = .{
	8703	.short = .{ .indirect = .{ .reg = ret_reg } },
	8704	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
	8705	};
8818	8706	}
8819	8707	}
8820	8708
8821	8709	// Input params
8822		var next_stack_offset: u32 = switch (result.return_value) {
8823		.stack_offset => \|off\| @intCast(u32, off),
8824		else => 0,
8825		};
8826
8827	8710	for (param_types, result.args) \|ty, *arg\| {
8828		if (!ty.hasRuntimeBits()) {
	8711	if (!ty.hasRuntimeBitsIgnoreComptime()) {
8829	8712	arg.* = .none;
8830	8713	continue;
8831	8714	}
8832		const param_size = @intCast(u32, ty.abiSize(self.target.*));
8833		const param_align = @intCast(u32, ty.abiAlignment(self.target.*));
8834		const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);
8835		arg.* = .{ .stack_offset = @intCast(i32, offset) };
8836		next_stack_offset = offset;
	8715	const param_size = @intCast(u31, ty.abiSize(self.target.*));
	8716	const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
	8717	result.stack_byte_count =
	8718	mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
	8719	arg.* = .{ .load_frame = .{
	8720	.index = stack_frame_base,
	8721	.off = result.stack_byte_count,
	8722	} };
	8723	result.stack_byte_count += param_size;
8837	8724	}
8838
8839		result.stack_align = 16;
8840		// TODO fix this so that the 16byte alignment padding is at the current value of $rsp, and push
8841		// the args onto the stack so that there is no padding between the first argument and
8842		// the standard preamble.
8843		// alignment padding \| args ... \| ret addr \| $rbp \|
8844		result.stack_byte_count = mem.alignForwardGeneric(u32, next_stack_offset, result.stack_align);
8845	8725	},
8846	8726	else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
8847	8727	}
8848	8728
	8729	result.stack_byte_count = mem.alignForwardGeneric(u31, result.stack_byte_count, result.stack_align);
8849	8730	return result;
8850	8731	}
8851	8732
...	...	@@ -8953,15 +8834,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 {
8953	8834	return self.regBitSize(ty) - ty.bitSize(self.target.*);
8954	8835	}
8955	8836
8956		fn intrinsicsAllowed(target: Target, ty: Type) bool {
8957		return switch (ty.tag()) {
8958		.f32,
8959		.f64,
8960		=> Target.x86.featureSetHasAny(target.cpu.features, .{ .sse2, .avx, .avx2 }),
8961		else => unreachable, // TODO finish this off
8962		};
8963		}
8964
8965	8837	fn hasAvxSupport(target: Target) bool {
8966	8838	return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
8967	8839	}

src/arch/x86_64/Emit.zig+8-7

...	...	@@ -13,7 +13,9 @@ prev_di_pc: usize,
13	13	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
14	14	relocs: std.ArrayListUnmanaged(Reloc) = .{},
15	15
16		pub const Error = Lower.Error \|\| error{EmitFail};
	16	pub const Error = Lower.Error \|\| error{
	17	EmitFail,
	18	};
17	19
18	20	pub fn emitMir(emit: *Emit) Error!void {
19	21	for (0..emit.lower.mir.instructions.len) \|i\| {
...	...	@@ -22,7 +24,7 @@ pub fn emitMir(emit: *Emit) Error!void {
22	24
23	25	const start_offset = @intCast(u32, emit.code.items.len);
24	26	try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);
25		for (try emit.lower.lowerMir(inst)) \|lower_inst\| try lower_inst.encode(emit.code.writer());
	27	for (try emit.lower.lowerMir(inst)) \|lower_inst\| try lower_inst.encode(emit.code.writer(), .{});
26	28	const end_offset = @intCast(u32, emit.code.items.len);
27	29
28	30	switch (inst.tag) {
...	...	@@ -120,11 +122,10 @@ pub fn emitMir(emit: *Emit) Error!void {
120	122	.length = 6,
121	123	}),
122	124
123		.dbg_line => {
124		const dbg_line_column =
125		emit.lower.mir.extraData(Mir.DbgLineColumn, inst.data.payload).data;
126		try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
127		},
	125	.dbg_line => try emit.dbgAdvancePCAndLine(
	126	inst.data.line_column.line,
	127	inst.data.line_column.column,
	128	),
128	129
129	130	.dbg_prologue_end => {
130	131	switch (emit.debug_output) {

src/arch/x86_64/Encoding.zig+3-12

...	...	@@ -47,17 +47,8 @@ pub fn findByMnemonic(
47	47	},
48	48	else => {},
49	49	} else false;
50		const rex_extended = for (ops) \|op\| switch (op) {
51		.reg => \|r\| if (r.isExtended()) break true,
52		.mem => \|m\| {
53		if (m.base()) \|base\| {
54		if (base.isExtended()) break true;
55		}
56		if (m.scaleIndex()) \|si\| {
57		if (si.index.isExtended()) break true;
58		}
59		},
60		else => {},
	50	const rex_extended = for (ops) \|op\| {
	51	if (op.isBaseExtended() or op.isIndexExtended()) break true;
61	52	} else false;
62	53
63	54	if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;
...	...	@@ -544,7 +535,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
544	535	std.mem.copy(Operand, &inst.ops, ops);
545	536
546	537	var cwriter = std.io.countingWriter(std.io.null_writer);
547		inst.encode(cwriter.writer()) catch unreachable; // Not allowed to fail here unless OOM.
	538	inst.encode(cwriter.writer(), .{ .allow_frame_loc = true }) catch unreachable; // Not allowed to fail here unless OOM.
548	539	return @intCast(usize, cwriter.bytes_written);
549	540	}
550	541

src/arch/x86_64/Lower.zig+12-19

...	...	@@ -136,7 +136,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
136	136	.setcc => try lower.mirSetcc(inst),
137	137	.jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
138	138
139		.push_regs, .pop_regs => try lower.mirPushPopRegisterList(inst),
	139	.push_regs => try lower.mirPushPopRegisterList(inst, .push),
	140	.pop_regs => try lower.mirPushPopRegisterList(inst, .pop),
140	141
141	142	.dbg_line,
142	143	.dbg_prologue_end,
...	...	@@ -190,7 +191,7 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
190	191	}
191	192
192	193	fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
193		return switch (ops) {
	194	return lower.mir.resolveFrameLoc(switch (ops) {
194	195	.rm_sib,
195	196	.rm_sib_cc,
196	197	.m_sib,
...	...	@@ -227,7 +228,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
227	228	=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
228	229
229	230	else => unreachable,
230		};
	231	});
231	232	}
232	233
233	234	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
...	...	@@ -417,23 +418,15 @@ fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
417	418	}
418	419	}
419	420
420		fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst) Error!void {
421		const save_reg_list = lower.mir.extraData(Mir.SaveRegisterList, inst.data.payload).data;
422		const base = @intToEnum(Register, save_reg_list.base_reg);
423		var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
424		const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
	421	fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {
	422	const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
425	423	const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
426		for (callee_preserved_regs) \|callee_preserved_reg\| {
427		if (!reg_list.isSet(callee_preserved_regs, callee_preserved_reg)) continue;
428		const reg_op = Operand{ .reg = callee_preserved_reg };
429		const mem_op = Operand{ .mem = Memory.sib(.qword, .{ .base = base, .disp = disp }) };
430		try lower.emit(.none, .mov, switch (inst.tag) {
431		.push_regs => &.{ mem_op, reg_op },
432		.pop_regs => &.{ reg_op, mem_op },
433		else => unreachable,
434		});
435		disp += 8;
436		}
	424	var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
	425	.push => .reverse,
	426	.pop => .forward,
	427	else => unreachable,
	428	} });
	429	while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
437	430	}
438	431
439	432	fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {

src/arch/x86_64/Mir.zig+65-39

...	...	@@ -23,6 +23,7 @@ const Register = bits.Register;
23	23	instructions: std.MultiArrayList(Inst).Slice,
24	24	/// The meaning of this data is determined by `Inst.Tag` value.
25	25	extra: []const u32,
	26	frame_locs: std.MultiArrayList(FrameLoc).Slice,
26	27
27	28	pub const Inst = struct {
28	29	tag: Tag,
...	...	@@ -241,13 +242,13 @@ pub const Inst = struct {
241	242	/// Start of epilogue
242	243	dbg_epilogue_begin,
243	244	/// Update debug line
244		/// Uses `payload` payload with data of type `DbgLineColumn`.
	245	/// Uses `line_column` payload containing the line and column.
245	246	dbg_line,
246	247	/// Push registers
247		/// Uses `payload` payload with data of type `SaveRegisterList`.
	248	/// Uses `payload` payload containing `RegisterList.asInt` directly.
248	249	push_regs,
249	250	/// Pop registers
250		/// Uses `payload` payload with data of type `SaveRegisterList`.
	251	/// Uses `payload` payload containing `RegisterList.asInt` directly.
251	252	pop_regs,
252	253
253	254	/// Tombstone
...	...	@@ -500,6 +501,11 @@ pub const Inst = struct {
500	501	/// Index into the linker's symbol table.
501	502	sym_index: u32,
502	503	},
	504	/// Debug line and column position
	505	line_column: struct {
	506	line: u32,
	507	column: u32,
	508	},
503	509	/// Index into `extra`. Meaning of what can be found there is context-dependent.
504	510	payload: u32,
505	511	};
...	...	@@ -522,12 +528,11 @@ pub const LeaRegisterReloc = struct {
522	528	sym_index: u32,
523	529	};
524	530
525		/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers
526		/// in a compact manner.
	531	/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
527	532	pub const RegisterList = struct {
528	533	bitset: BitSet = BitSet.initEmpty(),
529	534
530		const BitSet = IntegerBitSet(@ctz(@as(u32, 0)));
	535	const BitSet = IntegerBitSet(32);
531	536	const Self = @This();
532	537
533	538	fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
...	...	@@ -547,6 +552,10 @@ pub const RegisterList = struct {
547	552	return self.bitset.isSet(index);
548	553	}
549	554
	555	pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) {
	556	return self.bitset.iterator(options);
	557	}
	558
550	559	pub fn asInt(self: Self) u32 {
551	560	return self.bitset.mask;
552	561	}
...	...	@@ -562,14 +571,6 @@ pub const RegisterList = struct {
562	571	}
563	572	};
564	573
565		pub const SaveRegisterList = struct {
566		/// Base register
567		base_reg: u32,
568		/// Use `RegisterList` to populate.
569		register_list: u32,
570		stack_end: u32,
571		};
572
573	574	pub const Imm64 = struct {
574	575	msb: u32,
575	576	lsb: u32,
...	...	@@ -593,41 +594,51 @@ pub const Imm64 = struct {
593	594	pub const MemorySib = struct {
594	595	/// Size of the pointer.
595	596	ptr_size: u32,
596		/// Base register. -1 means null, or no base register.
597		base: i32,
598		/// Scale for index register. -1 means null, or no scale.
599		/// This has to be in sync with `index` field.
600		scale: i32,
601		/// Index register. -1 means null, or no index register.
602		/// This has to be in sync with `scale` field.
603		index: i32,
	597	/// Base register tag of type Memory.Base.Tag
	598	base_tag: u32,
	599	/// Base register of type Register or FrameIndex
	600	base: u32,
	601	/// Scale starting at bit 0 and index register starting at bit 4.
	602	scale_index: u32,
604	603	/// Displacement value.
605	604	disp: i32,
606	605
607	606	pub fn encode(mem: Memory) MemorySib {
608	607	const sib = mem.sib;
	608	assert(sib.scale_index.scale == 0 or std.math.isPowerOfTwo(sib.scale_index.scale));
609	609	return .{
610	610	.ptr_size = @enumToInt(sib.ptr_size),
611		.base = if (sib.base) \|r\| @enumToInt(r) else -1,
612		.scale = if (sib.scale_index) \|si\| si.scale else -1,
613		.index = if (sib.scale_index) \|si\| @enumToInt(si.index) else -1,
	611	.base_tag = @enumToInt(@as(Memory.Base.Tag, sib.base)),
	612	.base = switch (sib.base) {
	613	.none => undefined,
	614	.reg => \|r\| @enumToInt(r),
	615	.frame => \|fi\| @enumToInt(fi),
	616	},
	617	.scale_index = @as(u32, sib.scale_index.scale) << 0 \|
	618	@as(u32, if (sib.scale_index.scale > 0)
	619	@enumToInt(sib.scale_index.index)
	620	else
	621	undefined) << 4,
614	622	.disp = sib.disp,
615	623	};
616	624	}
617	625
618	626	pub fn decode(msib: MemorySib) Memory {
619		const base: ?Register = if (msib.base == -1) null else @intToEnum(Register, msib.base);
620		const scale_index: ?Memory.ScaleIndex = if (msib.index == -1) null else .{
621		.scale = @intCast(u4, msib.scale),
622		.index = @intToEnum(Register, msib.index),
623		};
624		const mem: Memory = .{ .sib = .{
	627	const scale = @truncate(u4, msib.scale_index);
	628	assert(scale == 0 or std.math.isPowerOfTwo(scale));
	629	return .{ .sib = .{
625	630	.ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),
626		.base = base,
627		.scale_index = scale_index,
	631	.base = switch (@intToEnum(Memory.Base.Tag, msib.base_tag)) {
	632	.none => .none,
	633	.reg => .{ .reg = @intToEnum(Register, msib.base) },
	634	.frame => .{ .frame = @intToEnum(bits.FrameIndex, msib.base) },
	635	},
	636	.scale_index = .{
	637	.scale = scale,
	638	.index = if (scale > 0) @intToEnum(Register, msib.scale_index >> 4) else undefined,
	639	},
628	640	.disp = msib.disp,
629	641	} };
630		return mem;
631	642	}
632	643	};
633	644
...	...	@@ -676,14 +687,10 @@ pub const MemoryMoffs = struct {
676	687	}
677	688	};
678	689
679		pub const DbgLineColumn = struct {
680		line: u32,
681		column: u32,
682		};
683
684	690	pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
685	691	mir.instructions.deinit(gpa);
686	692	gpa.free(mir.extra);
	693	mir.frame_locs.deinit(gpa);
687	694	mir.* = undefined;
688	695	}
689	696
...	...	@@ -704,3 +711,22 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:
704	711	.end = i,
705	712	};
706	713	}
	714
	715	pub const FrameLoc = struct {
	716	base: Register,
	717	disp: i32,
	718	};
	719
	720	pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
	721	return switch (mem) {
	722	.sib => \|sib\| switch (sib.base) {
	723	.none, .reg => mem,
	724	.frame => \|index\| if (mir.frame_locs.len > 0) Memory.sib(sib.ptr_size, .{
	725	.base = .{ .reg = mir.frame_locs.items(.base)[@enumToInt(index)] },
	726	.disp = mir.frame_locs.items(.disp)[@enumToInt(index)] + sib.disp,
	727	.scale_index = mem.scaleIndex(),
	728	}) else mem,
	729	},
	730	.rip, .moffs => mem,
	731	};
	732	}

src/arch/x86_64/bits.zig+68-10

...	...	@@ -405,14 +405,69 @@ test "Register classes" {
405	405	try expect(Register.fs.class() == .segment);
406	406	}
407	407
	408	pub const FrameIndex = enum(u32) {
	409	// This index refers to the start of the arguments passed to this function
	410	args_frame,
	411	// This index refers to the return address pushed by a `call` and popped by a `ret`.
	412	ret_addr,
	413	// This index refers to the base pointer pushed in the prologue and popped in the epilogue.
	414	base_ptr,
	415	// This index refers to the entire stack frame.
	416	stack_frame,
	417	// This index refers to the start of the call frame for arguments passed to called functions
	418	call_frame,
	419	// Other indices are used for local variable stack slots
	420	_,
	421
	422	pub const named_count = @typeInfo(FrameIndex).Enum.fields.len;
	423
	424	pub fn isNamed(fi: FrameIndex) bool {
	425	return @enumToInt(fi) < named_count;
	426	}
	427
	428	pub fn format(
	429	fi: FrameIndex,
	430	comptime fmt: []const u8,
	431	options: std.fmt.FormatOptions,
	432	writer: anytype,
	433	) @TypeOf(writer).Error!void {
	434	try writer.writeAll("FrameIndex");
	435	if (fi.isNamed()) {
	436	try writer.writeByte('.');
	437	try writer.writeAll(@tagName(fi));
	438	} else {
	439	try writer.writeByte('(');
	440	try std.fmt.formatType(@enumToInt(fi), fmt, options, writer, 0);
	441	try writer.writeByte(')');
	442	}
	443	}
	444	};
	445
408	446	pub const Memory = union(enum) {
409	447	sib: Sib,
410	448	rip: Rip,
411	449	moffs: Moffs,
412	450
413		pub const ScaleIndex = packed struct {
	451	pub const Base = union(enum) {
	452	none,
	453	reg: Register,
	454	frame: FrameIndex,
	455
	456	pub const Tag = @typeInfo(Base).Union.tag_type.?;
	457
	458	pub fn isExtended(self: Base) bool {
	459	return switch (self) {
	460	.none, .frame => false, // neither rsp nor rbp are extended
	461	.reg => \|reg\| reg.isExtended(),
	462	};
	463	}
	464	};
	465
	466	pub const ScaleIndex = struct {
414	467	scale: u4,
415	468	index: Register,
	469
	470	const none = ScaleIndex{ .scale = 0, .index = undefined };
416	471	};
417	472
418	473	pub const PtrSize = enum {
...	...	@@ -460,8 +515,8 @@ pub const Memory = union(enum) {
460	515
461	516	pub const Sib = struct {
462	517	ptr_size: PtrSize,
463		base: ?Register,
464		scale_index: ?ScaleIndex,
	518	base: Base,
	519	scale_index: ScaleIndex,
465	520	disp: i32,
466	521	};
467	522
...	...	@@ -482,7 +537,7 @@ pub const Memory = union(enum) {
482	537
483	538	pub fn sib(ptr_size: PtrSize, args: struct {
484	539	disp: i32 = 0,
485		base: ?Register = null,
	540	base: Base = .none,
486	541	scale_index: ?ScaleIndex = null,
487	542	}) Memory {
488	543	if (args.scale_index) \|si\| assert(std.math.isPowerOfTwo(si.scale));
...	...	@@ -490,7 +545,7 @@ pub const Memory = union(enum) {
490	545	.base = args.base,
491	546	.disp = args.disp,
492	547	.ptr_size = ptr_size,
493		.scale_index = args.scale_index,
	548	.scale_index = if (args.scale_index) \|si\| si else ScaleIndex.none,
494	549	} };
495	550	}
496	551
...	...	@@ -502,22 +557,25 @@ pub const Memory = union(enum) {
502	557	return switch (mem) {
503	558	.moffs => true,
504	559	.rip => false,
505		.sib => \|s\| if (s.base) \|r\| r.class() == .segment else false,
	560	.sib => \|s\| switch (s.base) {
	561	.none, .frame => false,
	562	.reg => \|reg\| reg.class() == .segment,
	563	},
506	564	};
507	565	}
508	566
509		pub fn base(mem: Memory) ?Register {
	567	pub fn base(mem: Memory) Base {
510	568	return switch (mem) {
511		.moffs => \|m\| m.seg,
	569	.moffs => \|m\| .{ .reg = m.seg },
512	570	.sib => \|s\| s.base,
513		.rip => null,
	571	.rip => .none,
514	572	};
515	573	}
516	574
517	575	pub fn scaleIndex(mem: Memory) ?ScaleIndex {
518	576	return switch (mem) {
519	577	.moffs, .rip => null,
520		.sib => \|s\| s.scale_index,
	578	.sib => \|s\| if (s.scale_index.scale > 0) s.scale_index else null,
521	579	};
522	580	}
523	581

src/arch/x86_64/encoder.zig+113-93

...	...	@@ -54,6 +54,21 @@ pub const Instruction = struct {
54	54	};
55	55	}
56	56
	57	pub fn isBaseExtended(op: Operand) bool {
	58	return switch (op) {
	59	.none, .imm => false,
	60	.reg => \|reg\| reg.isExtended(),
	61	.mem => \|mem\| mem.base().isExtended(),
	62	};
	63	}
	64
	65	pub fn isIndexExtended(op: Operand) bool {
	66	return switch (op) {
	67	.none, .reg, .imm => false,
	68	.mem => \|mem\| if (mem.scaleIndex()) \|si\| si.index.isExtended() else false,
	69	};
	70	}
	71
57	72	fn format(
58	73	op: Operand,
59	74	comptime unused_format_string: []const u8,
...	...	@@ -98,17 +113,24 @@ pub const Instruction = struct {
98	113	try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});
99	114
100	115	if (mem.isSegmentRegister()) {
101		return writer.print("{s}:0x{x}", .{ @tagName(sib.base.?), sib.disp });
	116	return writer.print("{s}:0x{x}", .{ @tagName(sib.base.reg), sib.disp });
102	117	}
103	118
104	119	try writer.writeByte('[');
105	120
106	121	var any = false;
107		if (sib.base) \|base\| {
108		try writer.print("{s}", .{@tagName(base)});
109		any = true;
	122	switch (sib.base) {
	123	.none => {},
	124	.reg => \|reg\| {
	125	try writer.print("{s}", .{@tagName(reg)});
	126	any = true;
	127	},
	128	.frame => \|frame\| {
	129	try writer.print("{}", .{frame});
	130	any = true;
	131	},
110	132	}
111		if (sib.scale_index) \|si\| {
	133	if (mem.scaleIndex()) \|si\| {
112	134	if (any) try writer.writeAll(" + ");
113	135	try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });
114	136	any = true;
...	...	@@ -182,8 +204,8 @@ pub const Instruction = struct {
182	204	}
183	205	}
184	206
185		pub fn encode(inst: Instruction, writer: anytype) !void {
186		const encoder = Encoder(@TypeOf(writer)){ .writer = writer };
	207	pub fn encode(inst: Instruction, writer: anytype, comptime opts: Options) !void {
	208	const encoder = Encoder(@TypeOf(writer), opts){ .writer = writer };
187	209	const enc = inst.encoding;
188	210	const data = enc.data;
189	211
...	...	@@ -269,22 +291,24 @@ pub const Instruction = struct {
269	291
270	292	const segment_override: ?Register = switch (op_en) {
271	293	.i, .zi, .o, .oi, .d, .np => null,
272		.fd => inst.ops[1].mem.base().?,
273		.td => inst.ops[0].mem.base().?,
274		.rm, .rmi => if (inst.ops[1].isSegmentRegister()) blk: {
275		break :blk switch (inst.ops[1]) {
276		.reg => \|r\| r,
277		.mem => \|m\| m.base().?,
	294	.fd => inst.ops[1].mem.base().reg,
	295	.td => inst.ops[0].mem.base().reg,
	296	.rm, .rmi => if (inst.ops[1].isSegmentRegister())
	297	switch (inst.ops[1]) {
	298	.reg => \|reg\| reg,
	299	.mem => \|mem\| mem.base().reg,
278	300	else => unreachable,
279		};
280		} else null,
281		.m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister()) blk: {
282		break :blk switch (inst.ops[0]) {
283		.reg => \|r\| r,
284		.mem => \|m\| m.base().?,
	301	}
	302	else
	303	null,
	304	.m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister())
	305	switch (inst.ops[0]) {
	306	.reg => \|reg\| reg,
	307	.mem => \|mem\| mem.base().reg,
285	308	else => unreachable,
286		};
287		} else null,
	309	}
	310	else
	311	null,
288	312	};
289	313	if (segment_override) \|seg\| {
290	314	legacy.setSegmentOverride(seg);
...	...	@@ -307,27 +331,17 @@ pub const Instruction = struct {
307	331	const r_op = switch (op_en) {
308	332	.rm, .rmi => inst.ops[0],
309	333	.mr, .mri, .mrc => inst.ops[1],
310		else => null,
	334	else => .none,
311	335	};
312		if (r_op) \|op\| {
313		rex.r = op.reg.isExtended();
314		}
	336	rex.r = r_op.isBaseExtended();
315	337
316	338	const b_x_op = switch (op_en) {
317	339	.rm, .rmi => inst.ops[1],
318	340	.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
319	341	else => unreachable,
320	342	};
321		switch (b_x_op) {
322		.reg => \|r\| {
323		rex.b = r.isExtended();
324		},
325		.mem => \|mem\| {
326		rex.b = if (mem.base()) \|base\| base.isExtended() else false;
327		rex.x = if (mem.scaleIndex()) \|si\| si.index.isExtended() else false;
328		},
329		else => unreachable,
330		}
	343	rex.b = b_x_op.isBaseExtended();
	344	rex.x = b_x_op.isIndexExtended();
331	345	},
332	346	}
333	347
...	...	@@ -348,72 +362,75 @@ pub const Instruction = struct {
348	362
349	363	switch (mem) {
350	364	.moffs => unreachable,
351		.sib => \|sib\| {
352		if (sib.base) \|base\| {
353		if (base.class() == .segment) {
354		// TODO audit this wrt SIB
355		try encoder.modRm_SIBDisp0(operand_enc);
356		if (sib.scale_index) \|si\| {
357		const scale = math.log2_int(u4, si.scale);
358		try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
359		} else {
360		try encoder.sib_disp32();
361		}
362		try encoder.disp32(sib.disp);
	365	.sib => \|sib\| switch (sib.base) {
	366	.none => {
	367	try encoder.modRm_SIBDisp0(operand_enc);
	368	if (mem.scaleIndex()) \|si\| {
	369	const scale = math.log2_int(u4, si.scale);
	370	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
363	371	} else {
364		assert(base.class() == .general_purpose);
365		const dst = base.lowEnc();
366		const src = operand_enc;
367		if (dst == 4 or sib.scale_index != null) {
368		if (sib.disp == 0 and dst != 5) {
369		try encoder.modRm_SIBDisp0(src);
370		if (sib.scale_index) \|si\| {
371		const scale = math.log2_int(u4, si.scale);
372		try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
373		} else {
374		try encoder.sib_base(dst);
375		}
376		} else if (math.cast(i8, sib.disp)) \|_\| {
377		try encoder.modRm_SIBDisp8(src);
378		if (sib.scale_index) \|si\| {
379		const scale = math.log2_int(u4, si.scale);
380		try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
381		} else {
382		try encoder.sib_baseDisp8(dst);
383		}
384		try encoder.disp8(@truncate(i8, sib.disp));
385		} else {
386		try encoder.modRm_SIBDisp32(src);
387		if (sib.scale_index) \|si\| {
388		const scale = math.log2_int(u4, si.scale);
389		try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
390		} else {
391		try encoder.sib_baseDisp32(dst);
392		}
393		try encoder.disp32(sib.disp);
394		}
395		} else {
396		if (sib.disp == 0 and dst != 5) {
397		try encoder.modRm_indirectDisp0(src, dst);
398		} else if (math.cast(i8, sib.disp)) \|_\| {
399		try encoder.modRm_indirectDisp8(src, dst);
400		try encoder.disp8(@truncate(i8, sib.disp));
401		} else {
402		try encoder.modRm_indirectDisp32(src, dst);
403		try encoder.disp32(sib.disp);
404		}
405		}
	372	try encoder.sib_disp32();
406	373	}
407		} else {
	374	try encoder.disp32(sib.disp);
	375	},
	376	.reg => \|base\| if (base.class() == .segment) {
	377	// TODO audit this wrt SIB
408	378	try encoder.modRm_SIBDisp0(operand_enc);
409		if (sib.scale_index) \|si\| {
	379	if (mem.scaleIndex()) \|si\| {
410	380	const scale = math.log2_int(u4, si.scale);
411	381	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
412	382	} else {
413	383	try encoder.sib_disp32();
414	384	}
415	385	try encoder.disp32(sib.disp);
416		}
	386	} else {
	387	assert(base.class() == .general_purpose);
	388	const dst = base.lowEnc();
	389	const src = operand_enc;
	390	if (dst == 4 or mem.scaleIndex() != null) {
	391	if (sib.disp == 0 and dst != 5) {
	392	try encoder.modRm_SIBDisp0(src);
	393	if (mem.scaleIndex()) \|si\| {
	394	const scale = math.log2_int(u4, si.scale);
	395	try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
	396	} else {
	397	try encoder.sib_base(dst);
	398	}
	399	} else if (math.cast(i8, sib.disp)) \|_\| {
	400	try encoder.modRm_SIBDisp8(src);
	401	if (mem.scaleIndex()) \|si\| {
	402	const scale = math.log2_int(u4, si.scale);
	403	try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
	404	} else {
	405	try encoder.sib_baseDisp8(dst);
	406	}
	407	try encoder.disp8(@truncate(i8, sib.disp));
	408	} else {
	409	try encoder.modRm_SIBDisp32(src);
	410	if (mem.scaleIndex()) \|si\| {
	411	const scale = math.log2_int(u4, si.scale);
	412	try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
	413	} else {
	414	try encoder.sib_baseDisp32(dst);
	415	}
	416	try encoder.disp32(sib.disp);
	417	}
	418	} else {
	419	if (sib.disp == 0 and dst != 5) {
	420	try encoder.modRm_indirectDisp0(src, dst);
	421	} else if (math.cast(i8, sib.disp)) \|_\| {
	422	try encoder.modRm_indirectDisp8(src, dst);
	423	try encoder.disp8(@truncate(i8, sib.disp));
	424	} else {
	425	try encoder.modRm_indirectDisp32(src, dst);
	426	try encoder.disp32(sib.disp);
	427	}
	428	}
	429	},
	430	.frame => if (@TypeOf(encoder).options.allow_frame_loc) {
	431	try encoder.modRm_indirectDisp32(operand_enc, undefined);
	432	try encoder.disp32(undefined);
	433	} else return error.CannotEncode,
417	434	},
418	435	.rip => \|rip\| {
419	436	try encoder.modRm_RIPDisp32(operand_enc);
...	...	@@ -482,11 +499,14 @@ pub const LegacyPrefixes = packed struct {
482	499	}
483	500	};
484	501
485		fn Encoder(comptime T: type) type {
	502	pub const Options = struct { allow_frame_loc: bool = false };
	503
	504	fn Encoder(comptime T: type, comptime opts: Options) type {
486	505	return struct {
487	506	writer: T,
488	507
489	508	const Self = @This();
	509	pub const options = opts;
490	510
491	511	// --------
492	512	// Prefixes