Commit 1de242cd732b63bbd8b24579b42b0ccfaf40e193 | zig

author	Jacob Young <jacobly@ziglang.org> 2023-10-08 09:15:35-04:00
committer	GitHub <noreply@github.com> 2023-10-08 09:15:35-04:00
log	1de242cd732b63bbd8b24579b42b0ccfaf40e193
tree	894348970d0f8a4eaf415dd147184fd7d2997318
parent	a9b37ac63724a79805f00b4e6966ebb8329151a2
parent	b5dedd7c00604bc687aab373a430058d9a97cdf9
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17416 from jacobly0/x86_64

x86_64: implement more `f80` operations and other features

15 files changed, 1606 insertions(+), 805 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Encoding.zig
    Mir.zig
    abi.zig
    encodings.zig
- register_manager.zig
test
- behavior

src/arch/x86_64/CodeGen.zig+1497-736

...	...	@@ -32,10 +32,10 @@ const Target = std.Target;
32	32	const Type = @import("../../type.zig").Type;
33	33	const TypedValue = @import("../../TypedValue.zig");
34	34	const Value = @import("../../value.zig").Value;
	35	const Instruction = @import("encoder.zig").Instruction;
35	36
36	37	const abi = @import("abi.zig");
37	38	const bits = @import("bits.zig");
38		const encoder = @import("encoder.zig");
39	39	const errUnionErrorOffset = codegen.errUnionErrorOffset;
40	40	const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
41	41
...	...	@@ -47,9 +47,6 @@ const RegisterManager = abi.RegisterManager;
47	47	const RegisterLock = RegisterManager.RegisterLock;
48	48	const FrameIndex = bits.FrameIndex;
49	49
50		const gp = abi.RegisterClass.gp;
51		const sse = abi.RegisterClass.sse;
52
53	50	const InnerError = CodeGenError \|\| error{OutOfRegisters};
54	51
55	52	gpa: Allocator,
...	...	@@ -61,6 +58,15 @@ target: *const std.Target,
61	58	owner: Owner,
62	59	err_msg: ?*ErrorMsg,
63	60	args: []MCValue,
	61	va_info: union {
	62	sysv: struct {
	63	gp_count: u32,
	64	fp_count: u32,
	65	overflow_arg_area: FrameAddr,
	66	reg_save_area: FrameAddr,
	67	},
	68	win64: struct {},
	69	},
64	70	ret_mcv: InstTracking,
65	71	fn_type: Type,
66	72	arg_index: u32,
...	...	@@ -532,8 +538,8 @@ const InstTracking = struct {
532	538	};
533	539	}
534	540
535		fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
536		function.freeValue(self.short);
	541	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
	542	try function.freeValue(self.short);
537	543	self.reuseFrame();
538	544	tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
539	545	}
...	...	@@ -618,8 +624,9 @@ const InstTracking = struct {
618	624	}
619	625	}
620	626
621		fn die(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
622		function.freeValue(self.short);
	627	fn die(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
	628	if (self.short == .dead) return;
	629	try function.freeValue(self.short);
623	630	self.short = .{ .dead = function.scope_generation };
624	631	tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
625	632	}
...	...	@@ -634,6 +641,40 @@ const InstTracking = struct {
634	641	tracking_log.debug("%{d} => {} (reuse %{d})", .{ new_inst, self.*, old_inst });
635	642	}
636	643
	644	fn liveOut(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
	645	for (self.short.getRegs()) \|reg\| {
	646	if (function.register_manager.isRegFree(reg)) {
	647	tracking_log.debug("%{d} => {} (live-out)", .{ inst, self.* });
	648	continue;
	649	}
	650
	651	const index = RegisterManager.indexOfRegIntoTracked(reg).?;
	652	const tracked_inst = function.register_manager.registers[index];
	653	const tracking = function.getResolvedInstValue(tracked_inst);
	654
	655	// Disable death.
	656	var found_reg = false;
	657	var remaining_reg: Register = .none;
	658	for (tracking.short.getRegs()) \|tracked_reg\| if (tracked_reg.id() == reg.id()) {
	659	assert(!found_reg);
	660	found_reg = true;
	661	} else {
	662	assert(remaining_reg == .none);
	663	remaining_reg = tracked_reg;
	664	};
	665	assert(found_reg);
	666	tracking.short = switch (remaining_reg) {
	667	.none => .{ .dead = function.scope_generation },
	668	else => .{ .register = remaining_reg },
	669	};
	670
	671	// Perform side-effects of freeValue manually.
	672	function.register_manager.freeReg(reg);
	673
	674	tracking_log.debug("%{d} => {} (live-out %{d})", .{ inst, self.*, tracked_inst });
	675	}
	676	}
	677
637	678	pub fn format(
638	679	self: InstTracking,
639	680	comptime _: []const u8,
...	...	@@ -713,6 +754,7 @@ pub fn generate(
713	754	.owner = .{ .func_index = func_index },
714	755	.err_msg = null,
715	756	.args = undefined, // populated after `resolveCallingConventionValues`
	757	.va_info = undefined, // populated after `resolveCallingConventionValues`
716	758	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
717	759	.fn_type = fn_type,
718	760	.arg_index = 0,
...	...	@@ -753,6 +795,7 @@ pub fn generate(
753	795	);
754	796
755	797	const fn_info = mod.typeToFunc(fn_type).?;
	798	const cc = abi.resolveCallingConvention(fn_info.cc, function.target.*);
756	799	var call_info = function.resolveCallingConventionValues(fn_info, &.{}, .args_frame) catch \|err\| switch (err) {
757	800	error.CodegenFail => return Result{ .fail = function.err_msg.? },
758	801	error.OutOfRegisters => return Result{
...	...	@@ -787,6 +830,16 @@ pub fn generate(
787	830	.alignment = call_info.stack_align,
788	831	}),
789	832	);
	833	function.va_info = switch (cc) {
	834	.SysV => .{ .sysv = .{
	835	.gp_count = call_info.gp_count,
	836	.fp_count = call_info.fp_count,
	837	.overflow_arg_area = .{ .index = .args_frame, .off = call_info.stack_byte_count },
	838	.reg_save_area = undefined,
	839	} },
	840	.Win64 => .{ .win64 = .{} },
	841	else => undefined,
	842	};
790	843
791	844	function.gen() catch \|err\| switch (err) {
792	845	error.CodegenFail => return Result{ .fail = function.err_msg.? },
...	...	@@ -807,7 +860,7 @@ pub fn generate(
807	860	.lower = .{
808	861	.allocator = bin_file.allocator,
809	862	.mir = mir,
810		.cc = abi.resolveCallingConvention(fn_info.cc, function.target.*),
	863	.cc = cc,
811	864	.src_loc = src_loc,
812	865	},
813	866	.bin_file = bin_file,
...	...	@@ -862,6 +915,7 @@ pub fn generateLazy(
862	915	.owner = .{ .lazy_sym = lazy_sym },
863	916	.err_msg = null,
864	917	.args = undefined,
	918	.va_info = undefined,
865	919	.ret_mcv = undefined,
866	920	.fn_type = undefined,
867	921	.arg_index = undefined,
...	...	@@ -980,7 +1034,6 @@ fn formatWipMir(
980	1034	_: std.fmt.FormatOptions,
981	1035	writer: anytype,
982	1036	) @TypeOf(writer).Error!void {
983		const mod = data.self.bin_file.options.module.?;
984	1037	var lower = Lower{
985	1038	.allocator = data.self.gpa,
986	1039	.mir = .{
...	...	@@ -988,7 +1041,7 @@ fn formatWipMir(
988	1041	.extra = data.self.mir_extra.items,
989	1042	.frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
990	1043	},
991		.cc = mod.typeToFunc(data.self.fn_type).?.cc,
	1044	.cc = .Unspecified,
992	1045	.src_loc = data.self.src_loc,
993	1046	};
994	1047	for ((lower.lowerMir(data.inst) catch \|err\| switch (err) {
...	...	@@ -1144,7 +1197,7 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
1144	1197	} },
1145	1198	.z_and_np, .nz_or_p => .{ .rr = .{
1146	1199	.r1 = reg,
1147		.r2 = (try self.register_manager.allocReg(null, gp)).to8(),
	1200	.r2 = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to8(),
1148	1201	} },
1149	1202	},
1150	1203	});
...	...	@@ -1184,7 +1237,7 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
1184	1237	.payload = payload,
1185	1238	} },
1186	1239	.z_and_np, .nz_or_p => .{ .rx = .{
1187		.r1 = (try self.register_manager.allocReg(null, gp)).to8(),
	1240	.r1 = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to8(),
1188	1241	.payload = payload,
1189	1242	} },
1190	1243	},
...	...	@@ -1222,6 +1275,17 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
1222	1275	});
1223	1276	}
1224	1277
	1278	fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
	1279	_ = try self.addInst(.{
	1280	.tag = tag[1],
	1281	.ops = .inst,
	1282	.data = .{ .inst = .{
	1283	.fixes = tag[0],
	1284	.inst = target,
	1285	} },
	1286	});
	1287	}
	1288
1225	1289	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1226	1290	return self.addInst(.{
1227	1291	.tag = .pseudo,
...	...	@@ -1629,7 +1693,8 @@ fn asmMemoryRegisterImmediate(
1629	1693
1630	1694	fn gen(self: *Self) InnerError!void {
1631	1695	const mod = self.bin_file.options.module.?;
1632		const cc = self.fn_type.fnCallingConvention(mod);
	1696	const fn_info = mod.typeToFunc(self.fn_type).?;
	1697	const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
1633	1698	if (cc != .Naked) {
1634	1699	try self.asmRegister(.{ ._, .push }, .rbp);
1635	1700	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
...	...	@@ -1659,6 +1724,42 @@ fn gen(self: *Self) InnerError!void {
1659	1724	else => unreachable,
1660	1725	}
1661	1726
	1727	if (fn_info.is_var_args) switch (cc) {
	1728	.SysV => {
	1729	const info = &self.va_info.sysv;
	1730	const reg_save_area_fi = try self.allocFrameIndex(FrameAlloc.init(.{
	1731	.size = abi.SysV.c_abi_int_param_regs.len * 8 +
	1732	abi.SysV.c_abi_sse_param_regs.len * 16,
	1733	.alignment = .@"16",
	1734	}));
	1735	info.reg_save_area = .{ .index = reg_save_area_fi };
	1736
	1737	for (abi.SysV.c_abi_int_param_regs[info.gp_count..], info.gp_count..) \|reg, reg_i\|
	1738	try self.genSetMem(
	1739	.{ .frame = reg_save_area_fi },
	1740	@intCast(reg_i * 8),
	1741	Type.usize,
	1742	.{ .register = reg },
	1743	);
	1744
	1745	try self.asmRegisterImmediate(.{ ._, .cmp }, .al, Immediate.u(info.fp_count));
	1746	const skip_sse_reloc = try self.asmJccReloc(undefined, .na);
	1747
	1748	const vec_2_f64 = try mod.vectorType(.{ .len = 2, .child = .f64_type });
	1749	for (abi.SysV.c_abi_sse_param_regs[info.fp_count..], info.fp_count..) \|reg, reg_i\|
	1750	try self.genSetMem(
	1751	.{ .frame = reg_save_area_fi },
	1752	@intCast(abi.SysV.c_abi_int_param_regs.len * 8 + reg_i * 16),
	1753	vec_2_f64,
	1754	.{ .register = reg },
	1755	);
	1756
	1757	try self.performReloc(skip_sse_reloc);
	1758	},
	1759	.Win64 => return self.fail("TODO implement gen var arg function for Win64", .{}),
	1760	else => unreachable,
	1761	};
	1762
1662	1763	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1663	1764
1664	1765	try self.genBody(self.air.getMainBody());
...	...	@@ -2033,10 +2134,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2033	2134	.error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
2034	2135	.vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
2035	2136
2036		.c_va_arg => return self.fail("TODO implement c_va_arg", .{}),
2037		.c_va_copy => return self.fail("TODO implement c_va_copy", .{}),
2038		.c_va_end => return self.fail("TODO implement c_va_end", .{}),
2039		.c_va_start => return self.fail("TODO implement c_va_start", .{}),
	2137	.c_va_arg => try self.airVaArg(inst),
	2138	.c_va_copy => try self.airVaCopy(inst),
	2139	.c_va_end => try self.airVaEnd(inst),
	2140	.c_va_start => try self.airVaStart(inst),
2040	2141
2041	2142	.wasm_memory_size => unreachable,
2042	2143	.wasm_memory_grow => unreachable,
...	...	@@ -2074,7 +2175,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2074	2175	switch (lazy_sym.ty.zigTypeTag(mod)) {
2075	2176	.Enum => {
2076	2177	const enum_ty = lazy_sym.ty;
2077		wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(self.bin_file.options.module.?)});
	2178	wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(mod)});
2078	2179
2079	2180	const resolved_cc = abi.resolveCallingConvention(.Unspecified, self.target.*);
2080	2181	const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
...	...	@@ -2084,10 +2185,10 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2084	2185	const ret_reg = param_regs[0];
2085	2186	const enum_mcv = MCValue{ .register = param_regs[1] };
2086	2187
2087		var exitlude_jump_relocs = try self.gpa.alloc(u32, enum_ty.enumFieldCount(mod));
	2188	var exitlude_jump_relocs = try self.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(mod));
2088	2189	defer self.gpa.free(exitlude_jump_relocs);
2089	2190
2090		const data_reg = try self.register_manager.allocReg(null, gp);
	2191	const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
2091	2192	const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
2092	2193	defer self.register_manager.unlockReg(data_lock);
2093	2194	try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = enum_ty });
...	...	@@ -2122,7 +2223,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2122	2223	},
2123	2224	else => return self.fail(
2124	2225	"TODO implement {s} for {}",
2125		.{ @tagName(lazy_sym.kind), lazy_sym.ty.fmt(self.bin_file.options.module.?) },
	2226	.{ @tagName(lazy_sym.kind), lazy_sym.ty.fmt(mod) },
2126	2227	),
2127	2228	}
2128	2229	}
...	...	@@ -2132,9 +2233,12 @@ fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
2132	2233	self.register_manager.getRegAssumeFree(reg, inst);
2133	2234	}
2134	2235
2135		fn freeValue(self: *Self, value: MCValue) void {
	2236	fn freeValue(self: *Self, value: MCValue) !void {
2136	2237	switch (value) {
2137		.register => \|reg\| self.register_manager.freeReg(reg),
	2238	.register => \|reg\| {
	2239	if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
	2240	self.register_manager.freeReg(reg);
	2241	},
2138	2242	.register_pair => \|regs\| for (regs) \|reg\| self.register_manager.freeReg(reg),
2139	2243	.register_offset => \|reg_off\| self.register_manager.freeReg(reg_off.reg),
2140	2244	.register_overflow => \|reg_ov\| {
...	...	@@ -2146,13 +2250,13 @@ fn freeValue(self: *Self, value: MCValue) void {
2146	2250	}
2147	2251	}
2148	2252
2149		fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) void {
2150		if (bt.feed()) if (Air.refToIndex(operand)) \|inst\| self.processDeath(inst);
	2253	fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) !void {
	2254	if (bt.feed()) if (Air.refToIndex(operand)) \|inst\| try self.processDeath(inst);
2151	2255	}
2152	2256
2153	2257	/// Asserts there is already capacity to insert into top branch inst_table.
2154		fn processDeath(self: *Self, inst: Air.Inst.Index) void {
2155		self.inst_tracking.getPtr(inst).?.die(self, inst);
	2258	fn processDeath(self: *Self, inst: Air.Inst.Index) !void {
	2259	try self.inst_tracking.getPtr(inst).?.die(self, inst);
2156	2260	}
2157	2261
2158	2262	/// Called when there are no operands, and the instruction is always unreferenced.
...	...	@@ -2177,13 +2281,18 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
2177	2281	self.finishAirBookkeeping();
2178	2282	}
2179	2283
2180		fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
	2284	fn finishAir(
	2285	self: *Self,
	2286	inst: Air.Inst.Index,
	2287	result: MCValue,
	2288	operands: [Liveness.bpi - 1]Air.Inst.Ref,
	2289	) !void {
2181	2290	var tomb_bits = self.liveness.getTombBits(inst);
2182	2291	for (operands) \|op\| {
2183	2292	const dies = @as(u1, @truncate(tomb_bits)) != 0;
2184	2293	tomb_bits >>= 1;
2185	2294	if (!dies) continue;
2186		self.processDeath(Air.refToIndexAllowNone(op) orelse continue);
	2295	try self.processDeath(Air.refToIndexAllowNone(op) orelse continue);
2187	2296	}
2188	2297	self.finishAirResult(inst, result);
2189	2298	}
...	...	@@ -2349,7 +2458,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2349	2458	},
2350	2459	else => 8,
2351	2460	})) {
2352		if (self.register_manager.tryAllocReg(inst, regClassForType(ty, mod))) \|reg\| {
	2461	if (self.register_manager.tryAllocReg(inst, self.regClassForType(ty))) \|reg\| {
2353	2462	return MCValue{ .register = registerAlias(reg, abi_size) };
2354	2463	}
2355	2464	}
...	...	@@ -2359,10 +2468,15 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2359	2468	return .{ .load_frame = .{ .index = frame_index } };
2360	2469	}
2361	2470
2362		fn regClassForType(ty: Type, mod: *Module) RegisterManager.RegisterBitSet {
	2471	fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
	2472	const mod = self.bin_file.options.module.?;
2363	2473	return switch (ty.zigTypeTag(mod)) {
2364		.Float, .Vector => sse,
2365		else => gp,
	2474	.Float => switch (ty.floatBits(self.target.*)) {
	2475	80 => abi.RegisterClass.x87,
	2476	else => abi.RegisterClass.sse,
	2477	},
	2478	.Vector => abi.RegisterClass.sse,
	2479	else => abi.RegisterClass.gp,
2366	2480	};
2367	2481	}
2368	2482
...	...	@@ -2409,7 +2523,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2409	2523	for (
2410	2524	self.inst_tracking.keys()[state.inst_tracking_len..],
2411	2525	self.inst_tracking.values()[state.inst_tracking_len..],
2412		) \|inst, *tracking\| tracking.die(self, inst);
	2526	) \|inst, *tracking\| try tracking.die(self, inst);
2413	2527	self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
2414	2528	}
2415	2529
...	...	@@ -2417,7 +2531,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2417	2531	self.inst_tracking.keys()[0..state.inst_tracking_len],
2418	2532	self.inst_tracking.values()[0..state.inst_tracking_len],
2419	2533	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);
2420		for (deaths) \|death\| self.processDeath(death);
	2534	for (deaths) \|death\| try self.processDeath(death);
2421	2535
2422	2536	for (0..state.registers.len) \|index\| {
2423	2537	const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
...	...	@@ -2444,7 +2558,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2444	2558	}
2445	2559	if (opts.update_tracking) {
2446	2560	if (current_maybe_inst) \|current_inst\| {
2447		self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
	2561	try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
2448	2562	}
2449	2563	{
2450	2564	const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
...	...	@@ -2463,7 +2577,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2463	2577	try self.inst_tracking.getPtr(inst).?.spill(self, inst);
2464	2578	if (opts.update_tracking) if (self.eflags_inst) \|inst\| {
2465	2579	self.eflags_inst = null;
2466		self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
	2580	try self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
2467	2581	};
2468	2582
2469	2583	if (opts.update_tracking and std.debug.runtime_safety) {
...	...	@@ -2481,7 +2595,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
2481	2595	if (tracked_reg.id() == reg.id()) break;
2482	2596	} else unreachable; // spilled reg not tracked with spilled instruciton
2483	2597	try tracking.spill(self, inst);
2484		tracking.trackSpill(self, inst);
	2598	try tracking.trackSpill(self, inst);
2485	2599	}
2486	2600
2487	2601	pub fn spillEflagsIfOccupied(self: *Self) !void {
...	...	@@ -2490,7 +2604,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
2490	2604	const tracking = self.inst_tracking.getPtr(inst).?;
2491	2605	assert(tracking.getCondition() != null);
2492	2606	try tracking.spill(self, inst);
2493		tracking.trackSpill(self, inst);
	2607	try tracking.trackSpill(self, inst);
2494	2608	}
2495	2609	}
2496	2610
...	...	@@ -2502,8 +2616,7 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {
2502	2616	/// allocated. A second call to `copyToTmpRegister` may return the same register.
2503	2617	/// This can have a side effect of spilling instructions to the stack to free up a register.
2504	2618	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
2505		const mod = self.bin_file.options.module.?;
2506		const reg = try self.register_manager.allocReg(null, regClassForType(ty, mod));
	2619	const reg = try self.register_manager.allocReg(null, self.regClassForType(ty));
2507	2620	try self.genSetReg(reg, ty, mcv);
2508	2621	return reg;
2509	2622	}
...	...	@@ -2518,8 +2631,7 @@ fn copyToRegisterWithInstTracking(
2518	2631	ty: Type,
2519	2632	mcv: MCValue,
2520	2633	) !MCValue {
2521		const mod = self.bin_file.options.module.?;
2522		const reg: Register = try self.register_manager.allocReg(reg_owner, regClassForType(ty, mod));
	2634	const reg: Register = try self.register_manager.allocReg(reg_owner, self.regClassForType(ty));
2523	2635	try self.genSetReg(reg, ty, mcv);
2524	2636	return MCValue{ .register = reg };
2525	2637	}
...	...	@@ -2568,7 +2680,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2568	2680	80, 128 => true,
2569	2681	else => unreachable,
2570	2682	},
2571		80 => switch (dst_bits) {
	2683	80 => switch (src_bits) {
2572	2684	128 => true,
2573	2685	else => unreachable,
2574	2686	},
...	...	@@ -2872,9 +2984,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2872	2984	else => null,
2873	2985	},
2874	2986	else => null,
2875		}) orelse return self.fail("TODO implement airTrunc for {}", .{
2876		dst_ty.fmt(self.bin_file.options.module.?),
2877		});
	2987	}) orelse return self.fail("TODO implement airTrunc for {}", .{dst_ty.fmt(mod)});
2878	2988
2879	2989	const elem_ty = src_ty.childType(mod);
2880	2990	const mask_val = try mod.intValue(elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
...	...	@@ -3076,7 +3186,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
3076	3186	};
3077	3187	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3078	3188
3079		const limit_reg = try self.register_manager.allocReg(null, gp);
	3189	const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
3080	3190	const limit_mcv = MCValue{ .register = limit_reg };
3081	3191	const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
3082	3192	defer self.register_manager.unlockReg(limit_lock);
...	...	@@ -3159,7 +3269,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
3159	3269	};
3160	3270	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3161	3271
3162		const limit_reg = try self.register_manager.allocReg(null, gp);
	3272	const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
3163	3273	const limit_mcv = MCValue{ .register = limit_reg };
3164	3274	const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
3165	3275	defer self.register_manager.unlockReg(limit_lock);
...	...	@@ -3237,7 +3347,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3237	3347	};
3238	3348	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3239	3349
3240		const limit_reg = try self.register_manager.allocReg(null, gp);
	3350	const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
3241	3351	const limit_mcv = MCValue{ .register = limit_reg };
3242	3352	const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
3243	3353	defer self.register_manager.unlockReg(limit_lock);
...	...	@@ -3425,7 +3535,8 @@ fn genSetFrameTruncatedOverflowCompare(
3425	3535
3426	3536	const rest_ty = try mod.intType(.unsigned, int_info.bits - hi_limb_bits);
3427	3537
3428		const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
	3538	const temp_regs =
	3539	try self.register_manager.allocRegs(3, .{ null, null, null }, abi.RegisterClass.gp);
3429	3540	const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
3430	3541	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
3431	3542
...	...	@@ -3843,7 +3954,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3843	3954	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
3844	3955	defer self.register_manager.unlockReg(src_lock);
3845	3956
3846		const dst_reg = try self.register_manager.allocReg(inst, gp);
	3957	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
3847	3958	const dst_mcv = MCValue{ .register = dst_reg };
3848	3959	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3849	3960	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -3883,7 +3994,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
3883	3994	const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3884	3995	src_reg
3885	3996	else
3886		try self.register_manager.allocReg(inst, gp);
	3997	try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
3887	3998	const dst_mcv = MCValue{ .register = dst_reg };
3888	3999	const dst_lock = self.register_manager.lockReg(dst_reg);
3889	4000	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -3934,7 +4045,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3934	4045	const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3935	4046	src_reg
3936	4047	else
3937		try self.register_manager.allocReg(inst, gp);
	4048	try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
3938	4049	const dst_lock = self.register_manager.lockReg(dst_reg);
3939	4050	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
3940	4051
...	...	@@ -4107,7 +4218,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
4107	4218	const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
4108	4219	src_reg
4109	4220	else
4110		try self.register_manager.allocReg(inst, gp);
	4221	try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
4111	4222	const dst_mcv = MCValue{ .register = dst_reg };
4112	4223	const dst_lock = self.register_manager.lockReg(dst_reg);
4113	4224	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -4144,7 +4255,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
4144	4255	.immediate => \|imm\| {
4145	4256	// Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
4146	4257	// and set the register directly to the scaled offset as an immediate.
4147		const reg = try self.register_manager.allocReg(null, gp);
	4258	const reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4148	4259	try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size });
4149	4260	break :blk reg;
4150	4261	},
...	...	@@ -4184,7 +4295,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
4184	4295	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4185	4296	defer self.register_manager.unlockReg(offset_reg_lock);
4186	4297
4187		const addr_reg = try self.register_manager.allocReg(null, gp);
	4298	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4188	4299	try self.genSetReg(addr_reg, Type.usize, slice_mcv);
4189	4300	// TODO we could allocate register here, but need to expect addr register and potentially
4190	4301	// offset register.
...	...	@@ -4241,7 +4352,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4241	4352	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4242	4353	defer self.register_manager.unlockReg(offset_reg_lock);
4243	4354
4244		const addr_reg = try self.register_manager.allocReg(null, gp);
	4355	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4245	4356	switch (array) {
4246	4357	.register => {
4247	4358	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));
...	...	@@ -4478,7 +4589,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4478	4589	};
4479	4590	defer if (mat_src_lock) \|lock\| self.register_manager.unlockReg(lock);
4480	4591
4481		const dst_reg = try self.register_manager.allocReg(inst, gp);
	4592	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
4482	4593	const dst_mcv = MCValue{ .register = dst_reg };
4483	4594	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4484	4595	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -4502,7 +4613,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4502	4613	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
4503	4614	}
4504	4615	} else if (src_bits <= 128) {
4505		const tmp_reg = try self.register_manager.allocReg(null, gp);
	4616	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4506	4617	const tmp_mcv = MCValue{ .register = tmp_reg };
4507	4618	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4508	4619	defer self.register_manager.unlockReg(tmp_lock);
...	...	@@ -4525,7 +4636,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4525	4636	.{ .immediate = 128 - src_bits },
4526	4637	);
4527	4638	}
4528		} else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
	4639	} else return self.fail("TODO airClz of {}", .{src_ty.fmt(mod)});
4529	4640	break :result dst_mcv;
4530	4641	}
4531	4642
...	...	@@ -4608,7 +4719,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4608	4719	};
4609	4720	defer if (mat_src_lock) \|lock\| self.register_manager.unlockReg(lock);
4610	4721
4611		const dst_reg = try self.register_manager.allocReg(inst, gp);
	4722	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
4612	4723	const dst_mcv = MCValue{ .register = dst_reg };
4613	4724	const dst_lock = self.register_manager.lockReg(dst_reg);
4614	4725	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -4635,7 +4746,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4635	4746	} else mat_src_mcv;
4636	4747	try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv);
4637	4748	} else if (src_bits <= 128) {
4638		const tmp_reg = try self.register_manager.allocReg(null, gp);
	4749	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4639	4750	const tmp_mcv = MCValue{ .register = tmp_reg };
4640	4751	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4641	4752	defer self.register_manager.unlockReg(tmp_lock);
...	...	@@ -4654,12 +4765,11 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4654	4765	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
4655	4766	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv);
4656	4767	try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
4657		} else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
	4768	} else return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)});
4658	4769	break :result dst_mcv;
4659	4770	}
4660	4771
4661		if (src_bits > 64)
4662		return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
	4772	if (src_bits > 64) return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)});
4663	4773
4664	4774	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
4665	4775	const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
...	...	@@ -4710,7 +4820,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4710	4820	if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
4711	4821	src_mcv
4712	4822	else
4713		.{ .register = try self.register_manager.allocReg(inst, gp) };
	4823	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) };
4714	4824
4715	4825	const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
4716	4826	try self.genBinOpMir(.{ ._, .popcnt }, popcnt_ty, dst_mcv, mat_src_mcv);
...	...	@@ -4731,7 +4841,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4731	4841	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4732	4842	defer self.register_manager.unlockReg(dst_lock);
4733	4843
4734		const tmp_reg = try self.register_manager.allocReg(null, gp);
	4844	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4735	4845	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4736	4846	defer self.register_manager.unlockReg(tmp_lock);
4737	4847
...	...	@@ -4739,7 +4849,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4739	4849	const dst = registerAlias(dst_reg, src_abi_size);
4740	4850	const tmp = registerAlias(tmp_reg, src_abi_size);
4741	4851	const imm = if (src_abi_size > 4)
4742		try self.register_manager.allocReg(null, gp)
	4852	try self.register_manager.allocReg(null, abi.RegisterClass.gp)
4743	4853	else
4744	4854	undefined;
4745	4855
...	...	@@ -4840,7 +4950,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4840	4950	const dst_mcv: MCValue = if (mem_ok)
4841	4951	try self.allocRegOrMem(inst, true)
4842	4952	else
4843		.{ .register = try self.register_manager.allocReg(inst, gp) };
	4953	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) };
4844	4954	if (dst_mcv.isRegister()) {
4845	4955	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
4846	4956	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -4855,7 +4965,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4855	4965	return dst_mcv;
4856	4966	}
4857	4967
4858		const dst_reg = try self.register_manager.allocReg(inst, gp);
	4968	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
4859	4969	const dst_mcv = MCValue{ .register = dst_reg };
4860	4970	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4861	4971	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -4898,7 +5008,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4898	5008	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4899	5009	defer self.register_manager.unlockReg(dst_lock);
4900	5010
4901		const tmp_reg = try self.register_manager.allocReg(null, gp);
	5011	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4902	5012	const tmp_lock = self.register_manager.lockReg(tmp_reg);
4903	5013	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);
4904	5014
...	...	@@ -4906,7 +5016,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4906	5016	const dst = registerAlias(dst_reg, src_abi_size);
4907	5017	const tmp = registerAlias(tmp_reg, src_abi_size);
4908	5018	const imm = if (src_abi_size > 4)
4909		try self.register_manager.allocReg(null, gp)
	5019	try self.register_manager.allocReg(null, abi.RegisterClass.gp)
4910	5020	else
4911	5021	undefined;
4912	5022
...	...	@@ -4998,109 +5108,129 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4998	5108	fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {
4999	5109	const mod = self.bin_file.options.module.?;
5000	5110	const tag = self.air.instructions.items(.tag)[inst];
5001		const abi_size: u32 = switch (ty.abiSize(mod)) {
5002		1...16 => 16,
5003		17...32 => 32,
5004		else => return self.fail("TODO implement floatSign for {}", .{
5005		ty.fmt(mod),
5006		}),
5007		};
5008		const scalar_bits = ty.scalarType(mod).floatBits(self.target.*);
5009		if (scalar_bits == 80) return self.fail("TODO implement floatSign for {}", .{
5010		ty.fmt(mod),
5011		});
5012	5111
5013		const src_mcv = try self.resolveInst(operand);
5014		const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
5015		defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
	5112	const result = result: {
	5113	const scalar_bits = ty.scalarType(mod).floatBits(self.target.*);
	5114	if (scalar_bits == 80) {
	5115	if (ty.zigTypeTag(mod) != .Float) return self.fail("TODO implement floatSign for {}", .{
	5116	ty.fmt(mod),
	5117	});
5016	5118
5017		const dst_mcv: MCValue = if (src_mcv.isRegister() and self.reuseOperand(inst, operand, 0, src_mcv))
5018		src_mcv
5019		else if (self.hasFeature(.avx))
5020		.{ .register = try self.register_manager.allocReg(inst, sse) }
5021		else
5022		try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
5023		const dst_reg = dst_mcv.getReg().?;
5024		const dst_lock = self.register_manager.lockReg(dst_reg);
5025		defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	5119	const src_mcv = try self.resolveInst(operand);
	5120	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
	5121	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
5026	5122
5027		const vec_ty = try mod.vectorType(.{
5028		.len = @divExact(abi_size * 8, scalar_bits),
5029		.child = (try mod.intType(.signed, scalar_bits)).ip_index,
5030		});
	5123	const dst_mcv: MCValue = .{ .register = .st0 };
	5124	if (std.meta.eql(src_mcv, dst_mcv) and self.reuseOperand(inst, operand, 0, src_mcv))
	5125	try self.register_manager.getReg(.st0, inst);
5031	5126
5032		const sign_val = switch (tag) {
5033		.neg => try vec_ty.minInt(mod, vec_ty),
5034		.abs => try vec_ty.maxInt(mod, vec_ty),
5035		else => unreachable,
5036		};
	5127	try self.genCopy(ty, dst_mcv, src_mcv);
	5128	switch (tag) {
	5129	.neg => try self.asmOpOnly(.{ .f_, .chs }),
	5130	.abs => try self.asmOpOnly(.{ .f_, .abs }),
	5131	else => unreachable,
	5132	}
	5133	break :result dst_mcv;
	5134	}
5037	5135
5038		const sign_mcv = try self.genTypedValue(.{ .ty = vec_ty, .val = sign_val });
5039		const sign_mem = if (sign_mcv.isMemory())
5040		sign_mcv.mem(Memory.PtrSize.fromSize(abi_size))
5041		else
5042		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
5043		.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
	5136	const abi_size: u32 = switch (ty.abiSize(mod)) {
	5137	1...16 => 16,
	5138	17...32 => 32,
	5139	else => return self.fail("TODO implement floatSign for {}", .{
	5140	ty.fmt(mod),
	5141	}),
	5142	};
	5143
	5144	const src_mcv = try self.resolveInst(operand);
	5145	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
	5146	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
	5147
	5148	const dst_mcv: MCValue = if (src_mcv.isRegister() and
	5149	self.reuseOperand(inst, operand, 0, src_mcv))
	5150	src_mcv
	5151	else if (self.hasFeature(.avx))
	5152	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	5153	else
	5154	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
	5155	const dst_reg = dst_mcv.getReg().?;
	5156	const dst_lock = self.register_manager.lockReg(dst_reg);
	5157	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	5158
	5159	const vec_ty = try mod.vectorType(.{
	5160	.len = @divExact(abi_size * 8, scalar_bits),
	5161	.child = (try mod.intType(.signed, scalar_bits)).ip_index,
5044	5162	});
5045	5163
5046		if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
5047		switch (scalar_bits) {
5048		16, 128 => if (abi_size <= 16 or self.hasFeature(.avx2)) switch (tag) {
5049		.neg => .{ .vp_, .xor },
5050		.abs => .{ .vp_, .@"and" },
5051		else => unreachable,
5052		} else switch (tag) {
5053		.neg => .{ .v_ps, .xor },
5054		.abs => .{ .v_ps, .@"and" },
5055		else => unreachable,
5056		},
5057		32 => switch (tag) {
5058		.neg => .{ .v_ps, .xor },
5059		.abs => .{ .v_ps, .@"and" },
5060		else => unreachable,
5061		},
5062		64 => switch (tag) {
5063		.neg => .{ .v_pd, .xor },
5064		.abs => .{ .v_pd, .@"and" },
5065		else => unreachable,
5066		},
5067		80 => return self.fail("TODO implement floatSign for {}", .{
5068		ty.fmt(self.bin_file.options.module.?),
5069		}),
	5164	const sign_val = switch (tag) {
	5165	.neg => try vec_ty.minInt(mod, vec_ty),
	5166	.abs => try vec_ty.maxInt(mod, vec_ty),
5070	5167	else => unreachable,
5071		},
5072		registerAlias(dst_reg, abi_size),
5073		registerAlias(if (src_mcv.isRegister())
5074		src_mcv.getReg().?
	5168	};
	5169
	5170	const sign_mcv = try self.genTypedValue(.{ .ty = vec_ty, .val = sign_val });
	5171	const sign_mem = if (sign_mcv.isMemory())
	5172	sign_mcv.mem(Memory.PtrSize.fromSize(abi_size))
5075	5173	else
5076		try self.copyToTmpRegister(ty, src_mcv), abi_size),
5077		sign_mem,
5078		) else try self.asmRegisterMemory(
5079		switch (scalar_bits) {
5080		16, 128 => switch (tag) {
5081		.neg => .{ .p_, .xor },
5082		.abs => .{ .p_, .@"and" },
5083		else => unreachable,
5084		},
5085		32 => switch (tag) {
5086		.neg => .{ ._ps, .xor },
5087		.abs => .{ ._ps, .@"and" },
	5174	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5175	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
	5176	});
	5177
	5178	if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
	5179	switch (scalar_bits) {
	5180	16, 128 => if (abi_size <= 16 or self.hasFeature(.avx2)) switch (tag) {
	5181	.neg => .{ .vp_, .xor },
	5182	.abs => .{ .vp_, .@"and" },
	5183	else => unreachable,
	5184	} else switch (tag) {
	5185	.neg => .{ .v_ps, .xor },
	5186	.abs => .{ .v_ps, .@"and" },
	5187	else => unreachable,
	5188	},
	5189	32 => switch (tag) {
	5190	.neg => .{ .v_ps, .xor },
	5191	.abs => .{ .v_ps, .@"and" },
	5192	else => unreachable,
	5193	},
	5194	64 => switch (tag) {
	5195	.neg => .{ .v_pd, .xor },
	5196	.abs => .{ .v_pd, .@"and" },
	5197	else => unreachable,
	5198	},
	5199	80 => return self.fail("TODO implement floatSign for {}", .{ty.fmt(mod)}),
5088	5200	else => unreachable,
5089	5201	},
5090		64 => switch (tag) {
5091		.neg => .{ ._pd, .xor },
5092		.abs => .{ ._pd, .@"and" },
	5202	registerAlias(dst_reg, abi_size),
	5203	registerAlias(if (src_mcv.isRegister())
	5204	src_mcv.getReg().?
	5205	else
	5206	try self.copyToTmpRegister(ty, src_mcv), abi_size),
	5207	sign_mem,
	5208	) else try self.asmRegisterMemory(
	5209	switch (scalar_bits) {
	5210	16, 128 => switch (tag) {
	5211	.neg => .{ .p_, .xor },
	5212	.abs => .{ .p_, .@"and" },
	5213	else => unreachable,
	5214	},
	5215	32 => switch (tag) {
	5216	.neg => .{ ._ps, .xor },
	5217	.abs => .{ ._ps, .@"and" },
	5218	else => unreachable,
	5219	},
	5220	64 => switch (tag) {
	5221	.neg => .{ ._pd, .xor },
	5222	.abs => .{ ._pd, .@"and" },
	5223	else => unreachable,
	5224	},
	5225	80 => return self.fail("TODO implement floatSign for {}", .{ty.fmt(mod)}),
5093	5226	else => unreachable,
5094	5227	},
5095		80 => return self.fail("TODO implement floatSign for {}", .{
5096		ty.fmt(self.bin_file.options.module.?),
5097		}),
5098		else => unreachable,
5099		},
5100		registerAlias(dst_reg, abi_size),
5101		sign_mem,
5102		);
5103		return self.finishAir(inst, dst_mcv, .{ operand, .none, .none });
	5228	registerAlias(dst_reg, abi_size),
	5229	sign_mem,
	5230	);
	5231	break :result dst_mcv;
	5232	};
	5233	return self.finishAir(inst, result, .{ operand, .none, .none });
5104	5234	}
5105	5235
5106	5236	fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5133,21 +5263,29 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {
5133	5263	const un_op = self.air.instructions.items(.data)[inst].un_op;
5134	5264	const ty = self.typeOf(un_op);
5135	5265
5136		const src_mcv = try self.resolveInst(un_op);
5137		const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
5138		src_mcv
5139		else
5140		try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
5141		const dst_reg = dst_mcv.getReg().?;
5142		const dst_lock = self.register_manager.lockReg(dst_reg);
5143		defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5144		try self.genRound(ty, dst_reg, src_mcv, mode);
5145		return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
	5266	const result = result: {
	5267	switch (try self.genRoundLibcall(ty, .{ .air_ref = un_op }, mode)) {
	5268	.none => {},
	5269	else => \|dst_mcv\| break :result dst_mcv,
	5270	}
	5271
	5272	const src_mcv = try self.resolveInst(un_op);
	5273	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
	5274	src_mcv
	5275	else
	5276	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
	5277	const dst_reg = dst_mcv.getReg().?;
	5278	const dst_lock = self.register_manager.lockReg(dst_reg);
	5279	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	5280	try self.genRound(ty, dst_reg, src_mcv, mode);
	5281	break :result dst_mcv;
	5282	};
	5283	return self.finishAir(inst, result, .{ un_op, .none, .none });
5146	5284	}
5147	5285
5148		fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
	5286	fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {
5149	5287	const mod = self.bin_file.options.module.?;
5150		const mir_tag = @as(?Mir.Inst.FixedTag, if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(mod)) {
	5288	return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(mod)) {
5151	5289	.Float => switch (ty.floatBits(self.target.*)) {
5152	5290	32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
5153	5291	64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
...	...	@@ -5174,27 +5312,38 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5174	5312	else => null,
5175	5313	},
5176	5314	else => unreachable,
5177		} else null) orelse {
5178		if (ty.zigTypeTag(mod) != .Float) return self.fail("TODO implement genRound for {}", .{
5179		ty.fmt(self.bin_file.options.module.?),
5180		});
	5315	} else null;
	5316	}
5181	5317
5182		var callee: ["__trunc?".len]u8 = undefined;
5183		const res = try self.genCall(.{ .lib = .{
5184		.return_type = ty.toIntern(),
5185		.param_types = &.{ty.toIntern()},
5186		.callee = std.fmt.bufPrint(&callee, "{s}{s}{s}", .{
5187		floatLibcAbiPrefix(ty),
5188		switch (mode.mode) {
5189		.down => "floor",
5190		.up => "ceil",
5191		.zero => "trunc",
5192		else => unreachable,
5193		},
5194		floatLibcAbiSuffix(ty),
5195		}) catch unreachable,
5196		} }, &.{ty}, &.{src_mcv});
5197		return self.genSetReg(dst_reg, ty, res);
	5318	fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {
	5319	const mod = self.bin_file.options.module.?;
	5320	if (self.getRoundTag(ty)) \|_\| return .none;
	5321
	5322	if (ty.zigTypeTag(mod) != .Float)
	5323	return self.fail("TODO implement genRound for {}", .{ty.fmt(mod)});
	5324
	5325	var callee: ["__trunc?".len]u8 = undefined;
	5326	return try self.genCall(.{ .lib = .{
	5327	.return_type = ty.toIntern(),
	5328	.param_types = &.{ty.toIntern()},
	5329	.callee = std.fmt.bufPrint(&callee, "{s}{s}{s}", .{
	5330	floatLibcAbiPrefix(ty),
	5331	switch (mode.mode) {
	5332	.down => "floor",
	5333	.up => "ceil",
	5334	.zero => "trunc",
	5335	else => unreachable,
	5336	},
	5337	floatLibcAbiSuffix(ty),
	5338	}) catch unreachable,
	5339	} }, &.{ty}, &.{src_mcv});
	5340	}
	5341
	5342	fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
	5343	const mod = self.bin_file.options.module.?;
	5344	const mir_tag = self.getRoundTag(ty) orelse {
	5345	const result = try self.genRoundLibcall(ty, src_mcv, mode);
	5346	return self.genSetReg(dst_reg, ty, result);
5198	5347	};
5199	5348	const abi_size: u32 = @intCast(ty.abiSize(mod));
5200	5349	const dst_alias = registerAlias(dst_reg, abi_size);
...	...	@@ -5236,46 +5385,104 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
5236	5385	const mod = self.bin_file.options.module.?;
5237	5386	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
5238	5387	const ty = self.typeOf(ty_op.operand);
5239		const scalar_ty = ty.scalarType(mod);
5240	5388
5241		switch (scalar_ty.zigTypeTag(mod)) {
5242		.Int => if (ty.zigTypeTag(mod) == .Vector) {
5243		return self.fail("TODO implement airAbs for {}", .{ty.fmt(mod)});
5244		} else {
5245		if (ty.abiSize(mod) > 8) {
5246		return self.fail("TODO implement abs for integer abi sizes larger than 8", .{});
5247		}
5248		const src_mcv = try self.resolveInst(ty_op.operand);
5249		const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
	5389	const result: MCValue = result: {
	5390	const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) {
	5391	else => null,
	5392	.Int => {
	5393	if (ty.abiSize(mod) > 8) {
	5394	return self.fail("TODO implement abs for integer abi sizes larger than 8", .{});
	5395	}
	5396	const src_mcv = try self.resolveInst(ty_op.operand);
	5397	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
5250	5398
5251		try self.genUnOpMir(.{ ._, .neg }, ty, dst_mcv);
	5399	try self.genUnOpMir(.{ ._, .neg }, ty, dst_mcv);
5252	5400
5253		const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
5254		switch (src_mcv) {
5255		.register => \|val_reg\| try self.asmCmovccRegisterRegister(
5256		registerAlias(dst_mcv.register, cmov_abi_size),
5257		registerAlias(val_reg, cmov_abi_size),
5258		.l,
5259		),
5260		.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
5261		registerAlias(dst_mcv.register, cmov_abi_size),
5262		src_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
5263		.l,
5264		),
5265		else => {
5266		const val_reg = try self.copyToTmpRegister(ty, src_mcv);
5267		try self.asmCmovccRegisterRegister(
	5401	const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
	5402	switch (src_mcv) {
	5403	.register => \|val_reg\| try self.asmCmovccRegisterRegister(
5268	5404	registerAlias(dst_mcv.register, cmov_abi_size),
5269	5405	registerAlias(val_reg, cmov_abi_size),
5270	5406	.l,
5271		);
	5407	),
	5408	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
	5409	registerAlias(dst_mcv.register, cmov_abi_size),
	5410	src_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
	5411	.l,
	5412	),
	5413	else => {
	5414	const val_reg = try self.copyToTmpRegister(ty, src_mcv);
	5415	try self.asmCmovccRegisterRegister(
	5416	registerAlias(dst_mcv.register, cmov_abi_size),
	5417	registerAlias(val_reg, cmov_abi_size),
	5418	.l,
	5419	);
	5420	},
	5421	}
	5422	break :result dst_mcv;
	5423	},
	5424	.Float => return self.floatSign(inst, ty_op.operand, ty),
	5425	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	5426	else => null,
	5427	.Int => switch (ty.childType(mod).intInfo(mod).bits) {
	5428	else => null,
	5429	8 => switch (ty.vectorLen(mod)) {
	5430	else => null,
	5431	1...16 => if (self.hasFeature(.avx))
	5432	.{ .vp_b, .abs }
	5433	else if (self.hasFeature(.ssse3))
	5434	.{ .p_b, .abs }
	5435	else
	5436	null,
	5437	17...32 => if (self.hasFeature(.avx2)) .{ .vp_b, .abs } else null,
	5438	},
	5439	16 => switch (ty.vectorLen(mod)) {
	5440	else => null,
	5441	1...8 => if (self.hasFeature(.avx))
	5442	.{ .vp_w, .abs }
	5443	else if (self.hasFeature(.ssse3))
	5444	.{ .p_w, .abs }
	5445	else
	5446	null,
	5447	9...16 => if (self.hasFeature(.avx2)) .{ .vp_w, .abs } else null,
	5448	},
	5449	32 => switch (ty.vectorLen(mod)) {
	5450	else => null,
	5451	1...4 => if (self.hasFeature(.avx))
	5452	.{ .vp_d, .abs }
	5453	else if (self.hasFeature(.ssse3))
	5454	.{ .p_d, .abs }
	5455	else
	5456	null,
	5457	5...8 => if (self.hasFeature(.avx2)) .{ .vp_d, .abs } else null,
	5458	},
5272	5459	},
5273		}
5274		return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
5275		},
5276		.Float => return self.floatSign(inst, ty_op.operand, ty),
5277		else => unreachable,
5278		}
	5460	.Float => return self.floatSign(inst, ty_op.operand, ty),
	5461	},
	5462	}) orelse return self.fail("TODO implement airAbs for {}", .{ty.fmt(mod)});
	5463
	5464	const abi_size: u32 = @intCast(ty.abiSize(mod));
	5465	const src_mcv = try self.resolveInst(ty_op.operand);
	5466	const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	5467	src_mcv.getReg().?
	5468	else
	5469	try self.register_manager.allocReg(inst, self.regClassForType(ty));
	5470	const dst_alias = registerAlias(dst_reg, abi_size);
	5471	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	5472	mir_tag,
	5473	dst_alias,
	5474	src_mcv.mem(self.memPtrSize(ty)),
	5475	) else try self.asmRegisterRegister(
	5476	mir_tag,
	5477	dst_alias,
	5478	registerAlias(if (src_mcv.isRegister())
	5479	src_mcv.getReg().?
	5480	else
	5481	try self.copyToTmpRegister(ty, src_mcv), abi_size),
	5482	);
	5483	break :result .{ .register = dst_reg };
	5484	};
	5485	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5279	5486	}
5280	5487
5281	5488	fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5284,19 +5491,43 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
5284	5491	const ty = self.typeOf(un_op);
5285	5492	const abi_size: u32 = @intCast(ty.abiSize(mod));
5286	5493
5287		const src_mcv = try self.resolveInst(un_op);
5288		const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
5289		src_mcv
5290		else
5291		try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
5292		const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
5293		const dst_lock = self.register_manager.lockReg(dst_reg);
5294		defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5295
5296	5494	const result: MCValue = result: {
	5495	switch (ty.zigTypeTag(mod)) {
	5496	.Float => {
	5497	const float_bits = ty.floatBits(self.target.*);
	5498	if (switch (float_bits) {
	5499	16 => !self.hasFeature(.f16c),
	5500	32, 64 => false,
	5501	80, 128 => true,
	5502	else => unreachable,
	5503	}) {
	5504	var callee: ["__sqrt?".len]u8 = undefined;
	5505	break :result try self.genCall(.{ .lib = .{
	5506	.return_type = ty.toIntern(),
	5507	.param_types = &.{ty.toIntern()},
	5508	.callee = std.fmt.bufPrint(&callee, "{s}sqrt{s}", .{
	5509	floatLibcAbiPrefix(ty),
	5510	floatLibcAbiSuffix(ty),
	5511	}) catch unreachable,
	5512	} }, &.{ty}, &.{.{ .air_ref = un_op }});
	5513	}
	5514	},
	5515	else => {},
	5516	}
	5517
	5518	const src_mcv = try self.resolveInst(un_op);
	5519	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
	5520	src_mcv
	5521	else
	5522	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
	5523	const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
	5524	const dst_lock = self.register_manager.lockReg(dst_reg);
	5525	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	5526
5297	5527	const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) {
5298	5528	.Float => switch (ty.floatBits(self.target.*)) {
5299		16 => if (self.hasFeature(.f16c)) {
	5529	16 => {
	5530	assert(self.hasFeature(.f16c));
5300	5531	const mat_src_reg = if (src_mcv.isRegister())
5301	5532	src_mcv.getReg().?
5302	5533	else
...	...	@@ -5310,10 +5541,9 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
5310	5541	Immediate.u(0b1_00),
5311	5542	);
5312	5543	break :result dst_mcv;
5313		} else null,
	5544	},
5314	5545	32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
5315	5546	64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
5316		80, 128 => null,
5317	5547	else => unreachable,
5318	5548	},
5319	5549	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
...	...	@@ -5387,21 +5617,9 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
5387	5617	else => unreachable,
5388	5618	},
5389	5619	else => unreachable,
5390		}) orelse {
5391		if (ty.zigTypeTag(mod) != .Float) return self.fail("TODO implement airSqrt for {}", .{
5392		ty.fmt(mod),
5393		});
5394
5395		var callee: ["__sqrt?".len]u8 = undefined;
5396		break :result try self.genCall(.{ .lib = .{
5397		.return_type = ty.toIntern(),
5398		.param_types = &.{ty.toIntern()},
5399		.callee = std.fmt.bufPrint(&callee, "{s}sqrt{s}", .{
5400		floatLibcAbiPrefix(ty),
5401		floatLibcAbiSuffix(ty),
5402		}) catch unreachable,
5403		} }, &.{ty}, &.{src_mcv});
5404		};
	5620	}) orelse return self.fail("TODO implement airSqrt for {}", .{
	5621	ty.fmt(mod),
	5622	});
5405	5623	switch (mir_tag[0]) {
5406	5624	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
5407	5625	mir_tag,
...	...	@@ -5518,9 +5736,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
5518	5736	const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;
5519	5737	const val_extra_bits = self.regExtraBits(val_ty);
5520	5738
5521		if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{
5522		val_ty.fmt(self.bin_file.options.module.?),
5523		});
	5739	if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{val_ty.fmt(mod)});
5524	5740
5525	5741	const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
5526	5742	const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
...	...	@@ -5528,7 +5744,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
5528	5744
5529	5745	const dst_reg = switch (dst_mcv) {
5530	5746	.register => \|reg\| reg,
5531		else => try self.register_manager.allocReg(null, gp),
	5747	else => try self.register_manager.allocReg(null, abi.RegisterClass.gp),
5532	5748	};
5533	5749	const dst_lock = self.register_manager.lockReg(dst_reg);
5534	5750	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -5547,7 +5763,8 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
5547	5763	);
5548	5764	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
5549	5765	} else {
5550		const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
	5766	const tmp_reg =
	5767	registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size);
5551	5768	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5552	5769	defer self.register_manager.unlockReg(tmp_lock);
5553	5770
...	...	@@ -5632,8 +5849,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
5632	5849	const ptr_ty = self.typeOf(ty_op.operand);
5633	5850	const elem_size = elem_ty.abiSize(mod);
5634	5851
5635		const elem_rc = regClassForType(elem_ty, mod);
5636		const ptr_rc = regClassForType(ptr_ty, mod);
	5852	const elem_rc = self.regClassForType(elem_ty);
	5853	const ptr_rc = self.regClassForType(ptr_ty);
5637	5854
5638	5855	const ptr_mcv = try self.resolveInst(ty_op.operand);
5639	5856	const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and
...	...	@@ -5688,13 +5905,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
5688	5905	} else if (math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {
5689	5906	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
5690	5907	} else {
5691		const part_mask_reg = try self.register_manager.allocReg(null, gp);
	5908	const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5692	5909	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
5693	5910	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
5694	5911	}
5695	5912
5696	5913	if (src_bit_size <= 64) {
5697		const tmp_reg = try self.register_manager.allocReg(null, gp);
	5914	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5698	5915	const tmp_mcv = MCValue{ .register = tmp_reg };
5699	5916	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5700	5917	defer self.register_manager.unlockReg(tmp_lock);
...	...	@@ -5721,9 +5938,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
5721	5938	limb_mem,
5722	5939	registerAlias(tmp_reg, limb_abi_size),
5723	5940	);
5724		} else return self.fail("TODO: implement packed store of {}", .{
5725		src_ty.fmt(self.bin_file.options.module.?),
5726		});
	5941	} else return self.fail("TODO: implement packed store of {}", .{src_ty.fmt(mod)});
5727	5942	}
5728	5943	}
5729	5944
...	...	@@ -5761,7 +5976,7 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr
5761	5976
5762	5977	try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
5763	5978	},
5764		.air_ref => \|ptr_ref\| try self.store(ptr_ty, ptr_mcv, try self.resolveInst(ptr_ref)),
	5979	.air_ref => \|ptr_ref\| try self.store(ptr_ty, try self.resolveInst(ptr_ref), src_mcv),
5765	5980	}
5766	5981	}
5767	5982
...	...	@@ -5832,11 +6047,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5832	6047	const index = extra.field_index;
5833	6048
5834	6049	const container_ty = self.typeOf(operand);
5835		const container_rc = regClassForType(container_ty, mod);
	6050	const container_rc = self.regClassForType(container_ty);
5836	6051	const field_ty = container_ty.structFieldType(index, mod);
5837	6052	if (!field_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;
5838		const field_rc = regClassForType(field_ty, mod);
5839		const field_is_gp = field_rc.supersetOf(gp);
	6053	const field_rc = self.regClassForType(field_ty);
	6054	const field_is_gp = field_rc.supersetOf(abi.RegisterClass.gp);
5840	6055
5841	6056	const src_mcv = try self.resolveInst(operand);
5842	6057	const field_off: u32 = switch (container_ty.containerLayout(mod)) {
...	...	@@ -5860,8 +6075,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5860	6075	@intCast(field_ty.bitSize(mod)),
5861	6076	);
5862	6077
5863		const dst_reg =
5864		try self.register_manager.allocReg(if (field_is_gp) inst else null, gp);
	6078	const dst_reg = try self.register_manager.allocReg(
	6079	if (field_is_gp) inst else null,
	6080	abi.RegisterClass.gp,
	6081	);
5865	6082	const dst_mcv = MCValue{ .register = dst_reg };
5866	6083	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
5867	6084	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -5890,7 +6107,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5890	6107	return self.fail("TODO implement struct_field_val with large packed field", .{});
5891	6108	}
5892	6109
5893		const dst_reg = try self.register_manager.allocReg(if (field_is_gp) inst else null, gp);
	6110	const dst_reg = try self.register_manager.allocReg(
	6111	if (field_is_gp) inst else null,
	6112	abi.RegisterClass.gp,
	6113	);
5894	6114	const field_extra_bits = self.regExtraBits(field_ty);
5895	6115	const load_abi_size =
5896	6116	if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
...	...	@@ -5907,7 +6127,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5907	6127	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
5908	6128	} else {
5909	6129	const tmp_reg = registerAlias(
5910		try self.register_manager.allocReg(null, gp),
	6130	try self.register_manager.allocReg(null, abi.RegisterClass.gp),
5911	6131	field_abi_size,
5912	6132	);
5913	6133	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...	...	@@ -5967,7 +6187,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5967	6187	);
5968	6188	if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
5969	6189
5970		break :result if (field_rc.supersetOf(gp))
	6190	break :result if (field_rc.supersetOf(abi.RegisterClass.gp))
5971	6191	dst_mcv
5972	6192	else
5973	6193	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
...	...	@@ -5988,7 +6208,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5988	6208	self.eflags_inst = inst;
5989	6209	break :result .{ .eflags = ro.eflags };
5990	6210	} else {
5991		const dst_reg = try self.register_manager.allocReg(inst, gp);
	6211	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
5992	6212	try self.asmSetccRegister(dst_reg.to8(), ro.eflags);
5993	6213	break :result .{ .register = dst_reg.to8() };
5994	6214	},
...	...	@@ -6024,9 +6244,8 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
6024	6244	const mod = self.bin_file.options.module.?;
6025	6245	const src_ty = self.typeOf(src_air);
6026	6246	const src_mcv = try self.resolveInst(src_air);
6027		if (src_ty.zigTypeTag(mod) == .Vector) {
6028		return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)});
6029		}
	6247	if (src_ty.zigTypeTag(mod) == .Vector)
	6248	return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(mod)});
6030	6249
6031	6250	switch (src_mcv) {
6032	6251	.eflags => \|cc\| switch (tag) {
...	...	@@ -6086,10 +6305,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
6086	6305	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
6087	6306	const mod = self.bin_file.options.module.?;
6088	6307	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
6089		if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
6090		mir_tag,
6091		dst_ty.fmt(self.bin_file.options.module.?),
6092		});
	6308	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{ mir_tag, dst_ty.fmt(mod) });
6093	6309	switch (dst_mcv) {
6094	6310	.none,
6095	6311	.unreach,
...	...	@@ -6109,7 +6325,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
6109	6325	.register => \|dst_reg\| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
6110	6326	.register_pair => unreachable, // unimplemented
6111	6327	.memory, .load_got, .load_direct, .load_tlv => {
6112		const addr_reg = try self.register_manager.allocReg(null, gp);
	6328	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6113	6329	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6114	6330	defer self.register_manager.unlockReg(addr_reg_lock);
6115	6331
...	...	@@ -6211,7 +6427,7 @@ fn genShiftBinOpMir(
6211	6427	}),
6212	6428	}
6213	6429	} else if (abi_size <= 16) {
6214		const tmp_reg = try self.register_manager.allocReg(null, gp);
	6430	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6215	6431	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6216	6432	defer self.register_manager.unlockReg(tmp_lock);
6217	6433
...	...	@@ -6289,11 +6505,11 @@ fn genShiftBinOpMir(
6289	6505	}
6290	6506	},
6291	6507	else => {
6292		const first_reg = try self.register_manager.allocReg(null, gp);
	6508	const first_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6293	6509	const first_lock = self.register_manager.lockRegAssumeUnused(first_reg);
6294	6510	defer self.register_manager.unlockReg(first_lock);
6295	6511
6296		const second_reg = try self.register_manager.allocReg(null, gp);
	6512	const second_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6297	6513	const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);
6298	6514	defer self.register_manager.unlockReg(second_lock);
6299	6515
...	...	@@ -6445,7 +6661,7 @@ fn genMulDivBinOp(
6445	6661	.mul, .mul_wrap => dst_abi_size != src_abi_size and dst_abi_size != src_abi_size * 2,
6446	6662	.div_trunc, .div_floor, .div_exact, .rem, .mod => dst_abi_size != src_abi_size,
6447	6663	} or src_abi_size > 8) return self.fail("TODO implement genMulDivBinOp from {} to {}", .{
6448		src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
	6664	src_ty.fmt(mod), dst_ty.fmt(mod),
6449	6665	});
6450	6666	const ty = if (dst_abi_size <= 8) dst_ty else src_ty;
6451	6667	const abi_size = if (dst_abi_size <= 8) dst_abi_size else src_abi_size;
...	...	@@ -6615,31 +6831,53 @@ fn genBinOp(
6615	6831	const rhs_ty = self.typeOf(rhs_air);
6616	6832	const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
6617	6833
6618		if (lhs_ty.isRuntimeFloat() and switch (lhs_ty.floatBits(self.target.*)) {
	6834	if (lhs_ty.isRuntimeFloat() and (air_tag == .rem or switch (lhs_ty.floatBits(self.target.*)) {
6619	6835	16 => !self.hasFeature(.f16c),
6620	6836	32, 64 => false,
6621	6837	80, 128 => true,
6622	6838	else => unreachable,
6623		}) {
	6839	})) {
6624	6840	var callee: ["__add?f3".len]u8 = undefined;
6625		return self.genCall(.{ .lib = .{
	6841	const result = try self.genCall(.{ .lib = .{
6626	6842	.return_type = lhs_ty.toIntern(),
6627	6843	.param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },
6628	6844	.callee = switch (air_tag) {
6629		.add, .sub, .mul, .div_float => std.fmt.bufPrint(&callee, "__{s}{c}f3", .{
	6845	.add,
	6846	.sub,
	6847	.mul,
	6848	.div_float,
	6849	.div_trunc,
	6850	.div_floor,
	6851	=> std.fmt.bufPrint(&callee, "__{s}{c}f3", .{
6630	6852	@tagName(air_tag)[0..3],
6631	6853	floatCompilerRtAbiName(lhs_ty.floatBits(self.target.*)),
6632	6854	}),
6633		.min, .max => std.fmt.bufPrint(&callee, "{s}f{s}{s}", .{
	6855	.rem, .min, .max => std.fmt.bufPrint(&callee, "{s}f{s}{s}", .{
6634	6856	floatLibcAbiPrefix(lhs_ty),
6635		@tagName(air_tag),
	6857	switch (air_tag) {
	6858	.rem => "mod",
	6859	.min => "min",
	6860	.max => "max",
	6861	else => unreachable,
	6862	},
6636	6863	floatLibcAbiSuffix(lhs_ty),
6637	6864	}),
6638	6865	else => return self.fail("TODO implement genBinOp for {s} {}", .{
6639		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	6866	@tagName(air_tag), lhs_ty.fmt(mod),
6640	6867	}),
6641	6868	} catch unreachable,
6642	6869	} }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } });
	6870	return switch (air_tag) {
	6871	.div_trunc, .div_floor => try self.genRoundLibcall(lhs_ty, result, .{
	6872	.mode = switch (air_tag) {
	6873	.div_trunc => .zero,
	6874	.div_floor => .down,
	6875	else => unreachable,
	6876	},
	6877	.precision = .inexact,
	6878	}),
	6879	else => result,
	6880	};
6643	6881	}
6644	6882
6645	6883	if ((lhs_ty.scalarType(mod).isRuntimeFloat() and
...	...	@@ -6655,11 +6893,11 @@ fn genBinOp(
6655	6893	if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {
6656	6894	try self.register_manager.getReg(.xmm0, null);
6657	6895	break :mask .xmm0;
6658		} else try self.register_manager.allocReg(null, sse),
	6896	} else try self.register_manager.allocReg(null, abi.RegisterClass.sse),
6659	6897	abi_size,
6660	6898	) else null,
6661	6899	.rem, .mod => return self.fail("TODO implement genBinOp for {s} {}", .{
6662		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	6900	@tagName(air_tag), lhs_ty.fmt(mod),
6663	6901	}),
6664	6902	};
6665	6903	const mask_lock =
...	...	@@ -6896,7 +7134,7 @@ fn genBinOp(
6896	7134	},
6897	7135
6898	7136	else => return self.fail("TODO implement genBinOp for {s} {}", .{
6899		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7137	@tagName(air_tag), lhs_ty.fmt(mod),
6900	7138	}),
6901	7139	}
6902	7140	return dst_mcv;
...	...	@@ -6908,7 +7146,8 @@ fn genBinOp(
6908	7146	.Float => switch (lhs_ty.floatBits(self.target.*)) {
6909	7147	16 => {
6910	7148	assert(self.hasFeature(.f16c));
6911		const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	7149	const tmp_reg =
	7150	(try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
6912	7151	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6913	7152	defer self.register_manager.unlockReg(tmp_lock);
6914	7153
...	...	@@ -6943,6 +7182,23 @@ fn genBinOp(
6943	7182	dst_reg,
6944	7183	tmp_reg,
6945	7184	);
	7185	switch (air_tag) {
	7186	.div_trunc, .div_floor => try self.asmRegisterRegisterRegisterImmediate(
	7187	.{ .v_ss, .round },
	7188	dst_reg,
	7189	dst_reg,
	7190	dst_reg,
	7191	Immediate.u(@as(u5, @bitCast(RoundMode{
	7192	.mode = switch (air_tag) {
	7193	.div_trunc => .zero,
	7194	.div_floor => .down,
	7195	else => unreachable,
	7196	},
	7197	.precision = .inexact,
	7198	}))),
	7199	),
	7200	else => {},
	7201	}
6946	7202	try self.asmRegisterRegisterImmediate(
6947	7203	.{ .v_, .cvtps2ph },
6948	7204	dst_reg,
...	...	@@ -7215,7 +7471,10 @@ fn genBinOp(
7215	7471	assert(self.hasFeature(.f16c));
7216	7472	switch (lhs_ty.vectorLen(mod)) {
7217	7473	1 => {
7218		const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	7474	const tmp_reg = (try self.register_manager.allocReg(
	7475	null,
	7476	abi.RegisterClass.sse,
	7477	)).to128();
7219	7478	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7220	7479	defer self.register_manager.unlockReg(tmp_lock);
7221	7480
...	...	@@ -7259,7 +7518,10 @@ fn genBinOp(
7259	7518	return dst_mcv;
7260	7519	},
7261	7520	2 => {
7262		const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	7521	const tmp_reg = (try self.register_manager.allocReg(
	7522	null,
	7523	abi.RegisterClass.sse,
	7524	)).to128();
7263	7525	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7264	7526	defer self.register_manager.unlockReg(tmp_lock);
7265	7527
...	...	@@ -7307,7 +7569,10 @@ fn genBinOp(
7307	7569	return dst_mcv;
7308	7570	},
7309	7571	3...4 => {
7310		const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	7572	const tmp_reg = (try self.register_manager.allocReg(
	7573	null,
	7574	abi.RegisterClass.sse,
	7575	)).to128();
7311	7576	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7312	7577	defer self.register_manager.unlockReg(tmp_lock);
7313	7578
...	...	@@ -7347,7 +7612,10 @@ fn genBinOp(
7347	7612	return dst_mcv;
7348	7613	},
7349	7614	5...8 => {
7350		const tmp_reg = (try self.register_manager.allocReg(null, sse)).to256();
	7615	const tmp_reg = (try self.register_manager.allocReg(
	7616	null,
	7617	abi.RegisterClass.sse,
	7618	)).to256();
7351	7619	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7352	7620	defer self.register_manager.unlockReg(tmp_lock);
7353	7621
...	...	@@ -7470,7 +7738,7 @@ fn genBinOp(
7470	7738	},
7471	7739	},
7472	7740	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7473		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7741	@tagName(air_tag), lhs_ty.fmt(mod),
7474	7742	});
7475	7743
7476	7744	const lhs_copy_reg = if (maybe_mask_reg) \|_\| registerAlias(
...	...	@@ -7487,7 +7755,10 @@ fn genBinOp(
7487	7755	mir_tag,
7488	7756	dst_reg,
7489	7757	lhs_reg,
7490		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	7758	src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
	7759	else => Memory.PtrSize.fromSize(abi_size),
	7760	.Vector => Memory.PtrSize.fromBitSize(dst_reg.bitSize()),
	7761	}),
7491	7762	) else try self.asmRegisterRegisterRegister(
7492	7763	mir_tag,
7493	7764	dst_reg,
...	...	@@ -7502,7 +7773,10 @@ fn genBinOp(
7502	7773	if (src_mcv.isMemory()) try self.asmRegisterMemory(
7503	7774	mir_tag,
7504	7775	dst_reg,
7505		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	7776	src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
	7777	else => Memory.PtrSize.fromSize(abi_size),
	7778	.Vector => Memory.PtrSize.fromBitSize(dst_reg.bitSize()),
	7779	}),
7506	7780	) else try self.asmRegisterRegister(
7507	7781	mir_tag,
7508	7782	dst_reg,
...	...	@@ -7515,16 +7789,14 @@ fn genBinOp(
7515	7789
7516	7790	switch (air_tag) {
7517	7791	.add, .add_wrap, .sub, .sub_wrap, .mul, .mul_wrap, .div_float, .div_exact => {},
7518		.div_trunc, .div_floor => try self.genRound(
7519		lhs_ty,
7520		dst_reg,
7521		.{ .register = dst_reg },
7522		.{ .mode = switch (air_tag) {
	7792	.div_trunc, .div_floor => try self.genRound(lhs_ty, dst_reg, .{ .register = dst_reg }, .{
	7793	.mode = switch (air_tag) {
7523	7794	.div_trunc => .zero,
7524	7795	.div_floor => .down,
7525	7796	else => unreachable,
7526		}, .precision = .inexact },
7527		),
	7797	},
	7798	.precision = .inexact,
	7799	}),
7528	7800	.bit_and, .bit_or, .xor => {},
7529	7801	.max, .min => if (maybe_mask_reg) \|mask_reg\| if (self.hasFeature(.avx)) {
7530	7802	const rhs_copy_reg = registerAlias(src_mcv.getReg().?, abi_size);
...	...	@@ -7556,7 +7828,7 @@ fn genBinOp(
7556	7828	},
7557	7829	else => unreachable,
7558	7830	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7559		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7831	@tagName(air_tag), lhs_ty.fmt(mod),
7560	7832	}),
7561	7833	mask_reg,
7562	7834	rhs_copy_reg,
...	...	@@ -7588,7 +7860,7 @@ fn genBinOp(
7588	7860	},
7589	7861	else => unreachable,
7590	7862	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7591		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7863	@tagName(air_tag), lhs_ty.fmt(mod),
7592	7864	}),
7593	7865	dst_reg,
7594	7866	dst_reg,
...	...	@@ -7624,7 +7896,7 @@ fn genBinOp(
7624	7896	},
7625	7897	else => unreachable,
7626	7898	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7627		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7899	@tagName(air_tag), lhs_ty.fmt(mod),
7628	7900	}),
7629	7901	mask_reg,
7630	7902	mask_reg,
...	...	@@ -7655,7 +7927,7 @@ fn genBinOp(
7655	7927	},
7656	7928	else => unreachable,
7657	7929	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7658		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7930	@tagName(air_tag), lhs_ty.fmt(mod),
7659	7931	}),
7660	7932	dst_reg,
7661	7933	lhs_copy_reg.?,
...	...	@@ -7686,7 +7958,7 @@ fn genBinOp(
7686	7958	},
7687	7959	else => unreachable,
7688	7960	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7689		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7961	@tagName(air_tag), lhs_ty.fmt(mod),
7690	7962	}),
7691	7963	dst_reg,
7692	7964	mask_reg,
...	...	@@ -7716,7 +7988,7 @@ fn genBinOp(
7716	7988	},
7717	7989	else => unreachable,
7718	7990	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7719		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	7991	@tagName(air_tag), lhs_ty.fmt(mod),
7720	7992	}),
7721	7993	mask_reg,
7722	7994	lhs_copy_reg.?,
...	...	@@ -7746,7 +8018,7 @@ fn genBinOp(
7746	8018	},
7747	8019	else => unreachable,
7748	8020	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7749		@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	8021	@tagName(air_tag), lhs_ty.fmt(mod),
7750	8022	}),
7751	8023	dst_reg,
7752	8024	mask_reg,
...	...	@@ -7940,7 +8212,8 @@ fn genBinOpMir(
7940	8212	const dst_info: OpInfo = switch (dst_mcv) {
7941	8213	else => unreachable,
7942	8214	.memory, .load_got, .load_direct, .load_tlv => dst: {
7943		const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	8215	const dst_addr_reg =
	8216	(try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64();
7944	8217	const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
7945	8218	errdefer self.register_manager.unlockReg(dst_addr_lock);
7946	8219
...	...	@@ -7985,7 +8258,8 @@ fn genBinOpMir(
7985	8258	else => unreachable,
7986	8259	}
7987	8260
7988		const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	8261	const src_addr_reg =
	8262	(try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64();
7989	8263	const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
7990	8264	errdefer self.register_manager.unlockReg(src_addr_lock);
7991	8265
...	...	@@ -8428,8 +8702,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
8428	8702	const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals);
8429	8703
8430	8704	var bt = self.liveness.iterateBigTomb(inst);
8431		self.feed(&bt, pl_op.operand);
8432		for (arg_refs) \|arg_ref\| self.feed(&bt, arg_ref);
	8705	try self.feed(&bt, pl_op.operand);
	8706	for (arg_refs) \|arg_ref\| try self.feed(&bt, arg_ref);
8433	8707
8434	8708	const result = if (self.liveness.isUnused(inst)) .unreach else ret;
8435	8709	return self.finishAirResult(inst, result);
...	...	@@ -8520,7 +8794,10 @@ fn genCall(self: *Self, info: union(enum) {
8520	8794	try self.spillRegisters(&regs);
8521	8795	try arg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
8522	8796	},
8523		.load_frame => try self.genCopy(arg_ty, dst_arg, src_arg),
	8797	.load_frame => {
	8798	try self.genCopy(arg_ty, dst_arg, src_arg);
	8799	try self.freeValue(src_arg);
	8800	},
8524	8801	else => unreachable,
8525	8802	};
8526	8803
...	...	@@ -8546,6 +8823,9 @@ fn genCall(self: *Self, info: union(enum) {
8546	8823	else => unreachable,
8547	8824	};
8548	8825
	8826	if (fn_info.is_var_args)
	8827	try self.asmRegisterImmediate(.{ ._, .mov }, .al, Immediate.u(call_info.fp_count));
	8828
8549	8829	// Due to incremental compilation, how function calls are generated depends
8550	8830	// on linking.
8551	8831	switch (info) {
...	...	@@ -8611,6 +8891,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
8611	8891	const mod = self.bin_file.options.module.?;
8612	8892	const un_op = self.air.instructions.items(.data)[inst].un_op;
8613	8893	const operand = try self.resolveInst(un_op);
	8894
8614	8895	const ret_ty = self.fn_type.fnReturnType(mod);
8615	8896	switch (self.ret_mcv.short) {
8616	8897	.none => {},
...	...	@@ -8625,6 +8906,8 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
8625	8906	},
8626	8907	else => unreachable,
8627	8908	}
	8909	self.ret_mcv.liveOut(self, inst);
	8910
8628	8911	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
8629	8912	// which is available if the jump is 127 bytes or less forward.
8630	8913	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -8635,6 +8918,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
8635	8918	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
8636	8919	const un_op = self.air.instructions.items(.data)[inst].un_op;
8637	8920	const ptr = try self.resolveInst(un_op);
	8921
8638	8922	const ptr_ty = self.typeOf(un_op);
8639	8923	switch (self.ret_mcv.short) {
8640	8924	.none => {},
...	...	@@ -8642,6 +8926,8 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
8642	8926	.indirect => \|reg_off\| try self.genSetReg(reg_off.reg, ptr_ty, ptr),
8643	8927	else => unreachable,
8644	8928	}
	8929	self.ret_mcv.liveOut(self, inst);
	8930
8645	8931	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
8646	8932	// which is available if the jump is 127 bytes or less forward.
8647	8933	const jmp_reloc = try self.asmJmpReloc(undefined);
...	...	@@ -8792,8 +9078,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
8792	9078	else => unreachable,
8793	9079	}
8794	9080
8795		const dst_addr_reg =
8796		(try self.register_manager.allocReg(null, gp)).to64();
	9081	const dst_addr_reg = (try self.register_manager.allocReg(
	9082	null,
	9083	abi.RegisterClass.gp,
	9084	)).to64();
8797	9085	const dst_addr_lock =
8798	9086	self.register_manager.lockRegAssumeUnused(dst_addr_reg);
8799	9087	errdefer self.register_manager.unlockReg(dst_addr_lock);
...	...	@@ -8848,8 +9136,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
8848	9136	else => unreachable,
8849	9137	}
8850	9138
8851		const src_addr_reg =
8852		(try self.register_manager.allocReg(null, gp)).to64();
	9139	const src_addr_reg = (try self.register_manager.allocReg(
	9140	null,
	9141	abi.RegisterClass.gp,
	9142	)).to64();
8853	9143	const src_addr_lock =
8854	9144	self.register_manager.lockRegAssumeUnused(src_addr_reg);
8855	9145	errdefer self.register_manager.unlockReg(src_addr_lock);
...	...	@@ -8868,8 +9158,11 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
8868	9158	defer if (src_info) \|info\|
8869	9159	self.register_manager.unlockReg(info.addr_lock);
8870	9160
8871		const regs =
8872		try self.register_manager.allocRegs(2, .{ null, null }, gp);
	9161	const regs = try self.register_manager.allocRegs(
	9162	2,
	9163	.{ null, null },
	9164	abi.RegisterClass.gp,
	9165	);
8873	9166	const acc_reg = regs[0].to64();
8874	9167	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
8875	9168	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -8956,12 +9249,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
8956	9249	switch (ty.floatBits(self.target.*)) {
8957	9250	16 => {
8958	9251	assert(self.hasFeature(.f16c));
8959		const tmp1_reg = (try self.register_manager.allocReg(null, sse)).to128();
	9252	const tmp1_reg =
	9253	(try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
8960	9254	const tmp1_mcv = MCValue{ .register = tmp1_reg };
8961	9255	const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg);
8962	9256	defer self.register_manager.unlockReg(tmp1_lock);
8963	9257
8964		const tmp2_reg = (try self.register_manager.allocReg(null, sse)).to128();
	9258	const tmp2_reg =
	9259	(try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
8965	9260	const tmp2_mcv = MCValue{ .register = tmp2_reg };
8966	9261	const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
8967	9262	defer self.register_manager.unlockReg(tmp2_lock);
...	...	@@ -9024,7 +9319,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
9024	9319	const mod = self.bin_file.options.module.?;
9025	9320	const un_op = self.air.instructions.items(.data)[inst].un_op;
9026	9321
9027		const addr_reg = try self.register_manager.allocReg(null, gp);
	9322	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
9028	9323	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
9029	9324	defer self.register_manager.unlockReg(addr_lock);
9030	9325	try self.genLazySymbolRef(.lea, addr_reg, link.File.LazySymbol.initDecl(.const_data, null, mod));
...	...	@@ -9086,13 +9381,13 @@ fn genTry(
9086	9381	const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv);
9087	9382
9088	9383	if (self.liveness.operandDies(inst, 0)) {
9089		if (Air.refToIndex(err_union)) \|err_union_inst\| self.processDeath(err_union_inst);
	9384	if (Air.refToIndex(err_union)) \|err_union_inst\| try self.processDeath(err_union_inst);
9090	9385	}
9091	9386
9092	9387	self.scope_generation += 1;
9093	9388	const state = try self.saveState();
9094	9389
9095		for (liveness_cond_br.else_deaths) \|operand\| self.processDeath(operand);
	9390	for (liveness_cond_br.else_deaths) \|operand\| try self.processDeath(operand);
9096	9391	try self.genBody(body);
9097	9392	try self.restoreState(state, &.{}, .{
9098	9393	.emit_instructions = false,
...	...	@@ -9103,7 +9398,7 @@ fn genTry(
9103	9398
9104	9399	try self.performReloc(reloc);
9105	9400
9106		for (liveness_cond_br.then_deaths) \|operand\| self.processDeath(operand);
	9401	for (liveness_cond_br.then_deaths) \|operand\| try self.processDeath(operand);
9107	9402
9108	9403	const result = if (self.liveness.isUnused(inst))
9109	9404	.unreach
...	...	@@ -9126,8 +9421,8 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
9126	9421	}
9127	9422
9128	9423	fn airDbgInline(self: *Self, inst: Air.Inst.Index) !void {
9129		const ty_fn = self.air.instructions.items(.data)[inst].ty_fn;
9130	9424	const mod = self.bin_file.options.module.?;
	9425	const ty_fn = self.air.instructions.items(.data)[inst].ty_fn;
9131	9426	const func = mod.funcInfo(ty_fn.func);
9132	9427	// TODO emit debug info for function change
9133	9428	_ = func;
...	...	@@ -9153,7 +9448,7 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
9153	9448	return self.finishAir(inst, .unreach, .{ operand, .none, .none });
9154	9449	}
9155	9450
9156		fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
	9451	fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
9157	9452	const mod = self.bin_file.options.module.?;
9158	9453	const abi_size = ty.abiSize(mod);
9159	9454	switch (mcv) {
...	...	@@ -9196,13 +9491,13 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
9196	9491	// that death now instead of later as this has an effect on
9197	9492	// whether it needs to be spilled in the branches
9198	9493	if (self.liveness.operandDies(inst, 0)) {
9199		if (Air.refToIndex(pl_op.operand)) \|op_inst\| self.processDeath(op_inst);
	9494	if (Air.refToIndex(pl_op.operand)) \|op_inst\| try self.processDeath(op_inst);
9200	9495	}
9201	9496
9202	9497	self.scope_generation += 1;
9203	9498	const state = try self.saveState();
9204	9499
9205		for (liveness_cond_br.then_deaths) \|operand\| self.processDeath(operand);
	9500	for (liveness_cond_br.then_deaths) \|operand\| try self.processDeath(operand);
9206	9501	try self.genBody(then_body);
9207	9502	try self.restoreState(state, &.{}, .{
9208	9503	.emit_instructions = false,
...	...	@@ -9213,7 +9508,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
9213	9508
9214	9509	try self.performReloc(reloc);
9215	9510
9216		for (liveness_cond_br.else_deaths) \|operand\| self.processDeath(operand);
	9511	for (liveness_cond_br.else_deaths) \|operand\| try self.processDeath(operand);
9217	9512	try self.genBody(else_body);
9218	9513	try self.restoreState(state, &.{}, .{
9219	9514	.emit_instructions = false,
...	...	@@ -9285,7 +9580,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
9285	9580	.load_direct,
9286	9581	.load_tlv,
9287	9582	=> {
9288		const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	9583	const addr_reg = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64();
9289	9584	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
9290	9585	defer self.register_manager.unlockReg(addr_reg_lock);
9291	9586
...	...	@@ -9535,7 +9830,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
9535	9830	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
9536	9831	const loop = self.air.extraData(Air.Block, ty_pl.payload);
9537	9832	const body = self.air.extra[loop.end..][0..loop.data.body_len];
9538		const jmp_target: u32 = @intCast(self.mir_instructions.len);
	9833	const jmp_target: Mir.Inst.Index = @intCast(self.mir_instructions.len);
9539	9834
9540	9835	self.scope_generation += 1;
9541	9836	const state = try self.saveState();
...	...	@@ -9580,7 +9875,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
9580	9875
9581	9876	if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i);
9582	9877	const tracking = &self.inst_tracking.values()[inst_tracking_i];
9583		if (self.liveness.isUnused(inst)) tracking.die(self, inst);
	9878	if (self.liveness.isUnused(inst)) try tracking.die(self, inst);
9584	9879	self.getValue(tracking.short, inst);
9585	9880	self.finishAirBookkeeping();
9586	9881	}
...	...	@@ -9599,7 +9894,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
9599	9894	// that death now instead of later as this has an effect on
9600	9895	// whether it needs to be spilled in the branches
9601	9896	if (self.liveness.operandDies(inst, 0)) {
9602		if (Air.refToIndex(pl_op.operand)) \|op_inst\| self.processDeath(op_inst);
	9897	if (Air.refToIndex(pl_op.operand)) \|op_inst\| try self.processDeath(op_inst);
9603	9898	}
9604	9899
9605	9900	self.scope_generation += 1;
...	...	@@ -9612,7 +9907,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
9612	9907	const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
9613	9908	extra_index = case.end + items.len + case_body.len;
9614	9909
9615		var relocs = try self.gpa.alloc(u32, items.len);
	9910	var relocs = try self.gpa.alloc(Mir.Inst.Index, items.len);
9616	9911	defer self.gpa.free(relocs);
9617	9912
9618	9913	try self.spillEflagsIfOccupied();
...	...	@@ -9622,7 +9917,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
9622	9917	reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
9623	9918	}
9624	9919
9625		for (liveness.deaths[case_i]) \|operand\| self.processDeath(operand);
	9920	for (liveness.deaths[case_i]) \|operand\| try self.processDeath(operand);
9626	9921
9627	9922	for (relocs[0 .. relocs.len - 1]) \|reloc\| try self.performReloc(reloc);
9628	9923	try self.genBody(case_body);
...	...	@@ -9640,7 +9935,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
9640	9935	const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
9641	9936
9642	9937	const else_deaths = liveness.deaths.len - 1;
9643		for (liveness.deaths[else_deaths]) \|operand\| self.processDeath(operand);
	9938	for (liveness.deaths[else_deaths]) \|operand\| try self.processDeath(operand);
9644	9939
9645	9940	try self.genBody(else_body);
9646	9941	try self.restoreState(state, &.{}, .{
...	...	@@ -9704,7 +9999,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
9704	9999
9705	10000	// Process operand death so that it is properly accounted for in the State below.
9706	10001	if (self.liveness.operandDies(inst, 0)) {
9707		if (Air.refToIndex(br.operand)) \|op_inst\| self.processDeath(op_inst);
	10002	if (Air.refToIndex(br.operand)) \|op_inst\| try self.processDeath(op_inst);
9708	10003	}
9709	10004
9710	10005	if (first_br) {
...	...	@@ -9718,7 +10013,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
9718	10013	});
9719	10014
9720	10015	// Stop tracking block result without forgetting tracking info
9721		self.freeValue(block_tracking.short);
	10016	try self.freeValue(block_tracking.short);
9722	10017
9723	10018	// Emit a jump with a relocation. It will be patched up after the block ends.
9724	10019	// Leave the jump offset undefined
...	...	@@ -9754,9 +10049,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9754	10049
9755	10050	var outputs_extra_i = extra_i;
9756	10051	for (outputs) \|output\| {
9757		const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
9758		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
9759		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
	10052	const extra_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);
	10053	const constraint = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
	10054	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
9760	10055	// This equation accounts for the fact that even if we have exactly 4 bytes
9761	10056	// for the string, we still use the next u32 for the null terminator.
9762	10057	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	...	@@ -9769,22 +10064,35 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9769	10064	.none => self.typeOfIndex(inst),
9770	10065	else => self.typeOf(output).childType(mod),
9771	10066	};
9772		const arg_maybe_reg: ?Register = if (mem.eql(u8, constraint, "=r"))
9773		self.register_manager.tryAllocReg(maybe_inst, regClassForType(ty, mod)) orelse
	10067	const is_read = switch (constraint[0]) {
	10068	'=' => false,
	10069	'+' => read: {
	10070	if (output == .none) return self.fail(
	10071	"read-write constraint unsupported for asm result: '{s}'",
	10072	.{constraint},
	10073	);
	10074	break :read true;
	10075	},
	10076	else => return self.fail("invalid constraint: '{s}'", .{constraint}),
	10077	};
	10078	const arg_maybe_reg: ?Register = if (mem.eql(u8, constraint[1..], "r"))
	10079	self.register_manager.tryAllocReg(maybe_inst, self.regClassForType(ty)) orelse
9774	10080	return self.fail("ran out of registers lowering inline asm", .{})
9775		else if (mem.eql(u8, constraint, "=m"))
	10081	else if (mem.eql(u8, constraint[1..], "m"))
9776	10082	if (output != .none) null else return self.fail(
9777		"memory constraint unsupported for asm result",
9778		.{},
9779		)
9780		else if (mem.eql(u8, constraint, "=g"))
9781		self.register_manager.tryAllocReg(maybe_inst, regClassForType(ty, mod)) orelse
9782		if (output != .none) null else return self.fail(
9783		"ran out of register lowering inline asm",
9784		.{},
	10083	"memory constraint unsupported for asm result: '{s}'",
	10084	.{constraint},
9785	10085	)
9786		else if (mem.startsWith(u8, constraint, "={") and mem.endsWith(u8, constraint, "}"))
9787		parseRegName(constraint["={".len .. constraint.len - "}".len]) orelse
	10086	else if (mem.eql(u8, constraint[1..], "g") or
	10087	mem.eql(u8, constraint[1..], "rm") or mem.eql(u8, constraint[1..], "mr") or
	10088	mem.eql(u8, constraint[1..], "r,m") or mem.eql(u8, constraint[1..], "m,r"))
	10089	self.register_manager.tryAllocReg(maybe_inst, self.regClassForType(ty)) orelse
	10090	if (output != .none)
	10091	null
	10092	else
	10093	return self.fail("ran out of registers lowering inline asm", .{})
	10094	else if (mem.startsWith(u8, constraint[1..], "{") and mem.endsWith(u8, constraint[1..], "}"))
	10095	parseRegName(constraint[1 + "{".len .. constraint.len - "}".len]) orelse
9788	10096	return self.fail("invalid register constraint: '{s}'", .{constraint})
9789	10097	else
9790	10098	return self.fail("invalid constraint: '{s}'", .{constraint});
...	...	@@ -9801,16 +10109,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9801	10109	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
9802	10110	_ = self.register_manager.lockRegAssumeUnused(reg);
9803	10111	};
9804		if (!std.mem.eql(u8, name, "_"))
	10112	if (!mem.eql(u8, name, "_"))
9805	10113	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
9806	10114	args.appendAssumeCapacity(arg_mcv);
9807	10115	if (output == .none) result = arg_mcv;
	10116	if (is_read) try self.load(arg_mcv, self.typeOf(output), .{ .air_ref = output });
9808	10117	}
9809	10118
9810	10119	for (inputs) \|input\| {
9811		const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
9812		const constraint = std.mem.sliceTo(input_bytes, 0);
9813		const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
	10120	const input_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);
	10121	const constraint = mem.sliceTo(input_bytes, 0);
	10122	const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
9814	10123	// This equation accounts for the fact that even if we have exactly 4 bytes
9815	10124	// for the string, we still use the next u32 for the null terminator.
9816	10125	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	...	@@ -9832,14 +10141,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9832	10141	break :arg temp_mcv;
9833	10142	},
9834	10143	}
9835		const addr_reg = self.register_manager.tryAllocReg(null, gp) orelse {
	10144	const addr_reg = self.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse {
9836	10145	const temp_mcv = try self.allocTempRegOrMem(ty, false);
9837	10146	try self.genCopy(ty, temp_mcv, input_mcv);
9838	10147	break :arg temp_mcv;
9839	10148	};
9840	10149	try self.genSetReg(addr_reg, Type.usize, input_mcv.address());
9841	10150	break :arg .{ .indirect = .{ .reg = addr_reg } };
9842		} else if (mem.eql(u8, constraint, "g")) arg: {
	10151	} else if (mem.eql(u8, constraint, "g") or
	10152	mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or
	10153	mem.eql(u8, constraint, "r,m") or mem.eql(u8, constraint, "m,r"))
	10154	arg: {
9843	10155	switch (input_mcv) {
9844	10156	.register, .indirect, .load_frame => break :arg input_mcv,
9845	10157	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
...	...	@@ -9861,7 +10173,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9861	10173	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
9862	10174	_ = self.register_manager.lockReg(reg);
9863	10175	};
9864		if (!std.mem.eql(u8, name, "_"))
	10176	if (!mem.eql(u8, name, "_"))
9865	10177	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
9866	10178	args.appendAssumeCapacity(arg_mcv);
9867	10179	}
...	...	@@ -9869,7 +10181,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9869	10181	{
9870	10182	var clobber_i: u32 = 0;
9871	10183	while (clobber_i < clobbers_len) : (clobber_i += 1) {
9872		const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
	10184	const clobber = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
9873	10185	// This equation accounts for the fact that even if we have exactly 4 bytes
9874	10186	// for the string, we still use the next u32 for the null terminator.
9875	10187	extra_i += clobber.len / 4 + 1;
...	...	@@ -9878,44 +10190,141 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9878	10190	}
9879	10191	}
9880	10192
	10193	const Label = struct {
	10194	target: Mir.Inst.Index = undefined,
	10195	pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
	10196
	10197	const Kind = enum { definition, reference };
	10198
	10199	fn isValid(kind: Kind, name: []const u8) bool {
	10200	for (name, 0..) \|c, i\| switch (c) {
	10201	else => return false,
	10202	'$' => if (i == 0) return false,
	10203	'.' => {},
	10204	'0'...'9' => if (i == 0) switch (kind) {
	10205	.definition => if (name.len != 1) return false,
	10206	.reference => {
	10207	if (name.len != 2) return false;
	10208	switch (name[1]) {
	10209	else => return false,
	10210	'B', 'F', 'b', 'f' => {},
	10211	}
	10212	},
	10213	},
	10214	'@', 'A'...'Z', '_', 'a'...'z' => {},
	10215	};
	10216	return name.len > 0;
	10217	}
	10218	};
	10219	var labels: std.StringHashMapUnmanaged(Label) = .{};
	10220	defer {
	10221	var label_it = labels.valueIterator();
	10222	while (label_it.next()) \|label\| label.pending_relocs.deinit(self.gpa);
	10223	labels.deinit(self.gpa);
	10224	}
	10225
9881	10226	const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
9882	10227	var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
9883		while (line_it.next()) \|line\| {
	10228	next_line: while (line_it.next()) \|line\| {
9884	10229	var mnem_it = mem.tokenizeAny(u8, line, " \t");
9885		const mnem_str = mnem_it.next() orelse continue;
9886		if (mem.startsWith(u8, mnem_str, "#")) continue;
9887
9888		const mnem_size: ?Memory.PtrSize = if (mem.endsWith(u8, mnem_str, "b"))
9889		.byte
9890		else if (mem.endsWith(u8, mnem_str, "w"))
9891		.word
9892		else if (mem.endsWith(u8, mnem_str, "l"))
9893		.dword
9894		else if (mem.endsWith(u8, mnem_str, "q"))
9895		.qword
9896		else
9897		null;
9898		const mnem_tag = Mir.Inst.FixedTag{ ._, mnem: {
9899		if (mnem_size) \|_\| {
9900		if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) \|mnem\| {
9901		break :mnem mnem;
9902		}
	10230	var prefix: Instruction.Prefix = .none;
	10231	const mnem_str = while (mnem_it.next()) \|mnem_str\| {
	10232	if (mem.startsWith(u8, mnem_str, "#")) continue :next_line;
	10233	if (std.meta.stringToEnum(Instruction.Prefix, mnem_str)) \|pre\| {
	10234	if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});
	10235	prefix = pre;
	10236	continue;
	10237	}
	10238	if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;
	10239	const label_name = mnem_str[0 .. mnem_str.len - ":".len];
	10240	if (!Label.isValid(.definition, label_name))
	10241	return self.fail("invalid label: '{s}'", .{label_name});
	10242	const label_gop = try labels.getOrPut(self.gpa, label_name);
	10243	if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else {
	10244	const anon = std.ascii.isDigit(label_name[0]);
	10245	if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0)
	10246	return self.fail("redefined label: '{s}'", .{label_name});
	10247	for (label_gop.value_ptr.pending_relocs.items) \|pending_reloc\|
	10248	try self.performReloc(pending_reloc);
	10249	if (anon)
	10250	label_gop.value_ptr.pending_relocs.clearRetainingCapacity()
	10251	else
	10252	label_gop.value_ptr.pending_relocs.clearAndFree(self.gpa);
9903	10253	}
9904		break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
9905		return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
9906		} };
9907
9908		var op_it = mem.tokenizeScalar(u8, mnem_it.rest(), ',');
9909		var ops = [1]encoder.Instruction.Operand{.none} ** 4;
9910		for (&ops) \|*op\| {
9911		const op_str = mem.trim(u8, op_it.next() orelse break, " \t");
9912		if (mem.startsWith(u8, op_str, "#")) break;
	10254	label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
	10255	} else continue;
	10256
	10257	var mnem_size: ?Memory.PtrSize = null;
	10258	const mnem_tag = mnem: {
	10259	mnem_size = if (mem.endsWith(u8, mnem_str, "b"))
	10260	.byte
	10261	else if (mem.endsWith(u8, mnem_str, "w"))
	10262	.word
	10263	else if (mem.endsWith(u8, mnem_str, "l"))
	10264	.dword
	10265	else if (mem.endsWith(u8, mnem_str, "q"))
	10266	.qword
	10267	else if (mem.endsWith(u8, mnem_str, "t"))
	10268	.tbyte
	10269	else
	10270	break :mnem null;
	10271	break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str[0 .. mnem_str.len - 1]);
	10272	} orelse mnem: {
	10273	mnem_size = null;
	10274	break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);
	10275	} orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
	10276	const mnem_name = @tagName(mnem_tag);
	10277	const mnem_fixed_tag: Mir.Inst.FixedTag = for (std.enums.values(Mir.Inst.Fixes)) \|fixes\| {
	10278	const fixes_name = @tagName(fixes);
	10279	const space_i = mem.indexOfScalar(u8, fixes_name, ' ');
	10280	const fixes_prefix = if (space_i) \|i\|
	10281	std.meta.stringToEnum(Instruction.Prefix, fixes_name[0..i]).?
	10282	else
	10283	.none;
	10284	if (fixes_prefix != prefix) continue;
	10285	const pattern = fixes_name[if (space_i) \|i\| i + " ".len else 0..];
	10286	const wildcard_i = mem.indexOfScalar(u8, pattern, '_').?;
	10287	const mnem_prefix = pattern[0..wildcard_i];
	10288	const mnem_suffix = pattern[wildcard_i + "_".len ..];
	10289	if (!mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
	10290	if (!mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
	10291	break .{ fixes, std.meta.stringToEnum(
	10292	Mir.Inst.Tag,
	10293	mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],
	10294	) orelse continue };
	10295	} else {
	10296	assert(prefix != .none);
	10297	return self.fail("invalid prefix for mnemonic: '{s} {s}'", .{
	10298	@tagName(prefix), mnem_str,
	10299	});
	10300	};
	10301
	10302	const Operand = union(enum) {
	10303	none,
	10304	reg: Register,
	10305	mem: Memory,
	10306	imm: Immediate,
	10307	inst: Mir.Inst.Index,
	10308	};
	10309	var ops: [4]Operand = .{.none} ** 4;
	10310
	10311	var last_op = false;
	10312	var op_it = mem.splitScalar(u8, mnem_it.rest(), ',');
	10313	next_op: for (&ops) \|*op\| {
	10314	const op_str = while (!last_op) {
	10315	const full_str = op_it.next() orelse break :next_op;
	10316	const trim_str = mem.trim(u8, if (mem.indexOfScalar(u8, full_str, '#')) \|hash\| hash: {
	10317	last_op = true;
	10318	break :hash full_str[0..hash];
	10319	} else full_str, " \t");
	10320	if (trim_str.len > 0) break trim_str;
	10321	} else break;
9913	10322	if (mem.startsWith(u8, op_str, "%%")) {
9914	10323	const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
9915	10324	const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
9916	10325	return self.fail("invalid register: '{s}'", .{op_str});
9917	10326	if (colon) \|colon_pos\| {
9918		const disp = std.fmt.parseInt(i32, op_str[colon_pos + 1 ..], 0) catch
	10327	const disp = std.fmt.parseInt(i32, op_str[colon_pos + ":".len ..], 0) catch
9919	10328	return self.fail("invalid displacement: '{s}'", .{op_str});
9920	10329	op.* = .{ .mem = Memory.sib(
9921	10330	mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
...	...	@@ -9929,33 +10338,33 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9929	10338	} else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {
9930	10339	const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
9931	10340	const modifier = if (colon) \|colon_pos\|
9932		op_str[colon_pos + 1 .. op_str.len - "]".len]
	10341	op_str[colon_pos + ":".len .. op_str.len - "]".len]
9933	10342	else
9934	10343	"";
9935	10344	op.* = switch (args.items[
9936	10345	arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse
9937	10346	return self.fail("no matching constraint: '{s}'", .{op_str})
9938	10347	]) {
9939		.register => \|reg\| if (std.mem.eql(u8, modifier, ""))
	10348	.register => \|reg\| if (mem.eql(u8, modifier, ""))
9940	10349	.{ .reg = reg }
9941	10350	else
9942	10351	return self.fail("invalid modifier: '{s}'", .{modifier}),
9943		.memory => \|addr\| if (std.mem.eql(u8, modifier, "") or
9944		std.mem.eql(u8, modifier, "P"))
	10352	.memory => \|addr\| if (mem.eql(u8, modifier, "") or
	10353	mem.eql(u8, modifier, "P"))
9945	10354	.{ .mem = Memory.sib(
9946	10355	mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
9947	10356	.{ .base = .{ .reg = .ds }, .disp = @intCast(@as(i64, @bitCast(addr))) },
9948	10357	) }
9949	10358	else
9950	10359	return self.fail("invalid modifier: '{s}'", .{modifier}),
9951		.indirect => \|reg_off\| if (std.mem.eql(u8, modifier, ""))
	10360	.indirect => \|reg_off\| if (mem.eql(u8, modifier, ""))
9952	10361	.{ .mem = Memory.sib(
9953	10362	mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
9954	10363	.{ .base = .{ .reg = reg_off.reg }, .disp = reg_off.off },
9955	10364	) }
9956	10365	else
9957	10366	return self.fail("invalid modifier: '{s}'", .{modifier}),
9958		.load_frame => \|frame_addr\| if (std.mem.eql(u8, modifier, ""))
	10367	.load_frame => \|frame_addr\| if (mem.eql(u8, modifier, ""))
9959	10368	.{ .mem = Memory.sib(
9960	10369	mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
9961	10370	.{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off },
...	...	@@ -9980,64 +10389,113 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9980	10389	}
9981	10390	op.* = .{ .imm = Immediate.u(u) };
9982	10391	} else \|_\| return self.fail("invalid immediate: '{s}'", .{op_str});
	10392	} else if (mem.endsWith(u8, op_str, ")")) {
	10393	const open = mem.indexOfScalar(u8, op_str, '(') orelse
	10394	return self.fail("invalid operand: '{s}'", .{op_str});
	10395	var sib_it = mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
	10396	const base_str = sib_it.next() orelse
	10397	return self.fail("invalid memory operand: '{s}'", .{op_str});
	10398	if (base_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
	10399	return self.fail("invalid memory operand: '{s}'", .{op_str});
	10400	const index_str = sib_it.next() orelse "";
	10401	if (index_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
	10402	return self.fail("invalid memory operand: '{s}'", .{op_str});
	10403	const scale_str = sib_it.next() orelse "";
	10404	if (index_str.len == 0 and scale_str.len > 0)
	10405	return self.fail("invalid memory operand: '{s}'", .{op_str});
	10406	const scale = if (scale_str.len > 0) switch (std.fmt.parseInt(u4, scale_str, 10) catch
	10407	return self.fail("invalid scale: '{s}'", .{op_str})) {
	10408	1, 2, 4, 8 => \|scale\| scale,
	10409	else => return self.fail("invalid scale: '{s}'", .{op_str}),
	10410	} else 1;
	10411	if (sib_it.next()) \|_\| return self.fail("invalid memory operand: '{s}'", .{op_str});
	10412	op.* = .{ .mem = Memory.sib(mnem_size orelse
	10413	return self.fail("unknown size: '{s}'", .{op_str}), .{
	10414	.disp = if (open > 0) std.fmt.parseInt(i32, op_str[0..open], 0) catch
	10415	return self.fail("invalid displacement: '{s}'", .{op_str}) else 0,
	10416	.base = if (base_str.len > 0) .{ .reg = parseRegName(base_str["%%".len..]) orelse
	10417	return self.fail("invalid base register: '{s}'", .{base_str}) } else .none,
	10418	.scale_index = if (index_str.len > 0) .{
	10419	.index = parseRegName(index_str["%%".len..]) orelse
	10420	return self.fail("invalid index register: '{s}'", .{op_str}),
	10421	.scale = scale,
	10422	} else null,
	10423	}) };
	10424	} else if (Label.isValid(.reference, op_str)) {
	10425	const anon = std.ascii.isDigit(op_str[0]);
	10426	const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]);
	10427	if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
	10428	if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing)
	10429	return self.fail("undefined label: '{s}'", .{op_str});
	10430	const pending_relocs = &label_gop.value_ptr.pending_relocs;
	10431	if (if (anon)
	10432	op_str[1] == 'f' or op_str[1] == 'F'
	10433	else
	10434	!label_gop.found_existing or pending_relocs.items.len > 0)
	10435	try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));
	10436	op.* = .{ .inst = label_gop.value_ptr.target };
9983	10437	} else return self.fail("invalid operand: '{s}'", .{op_str});
9984	10438	} else if (op_it.next()) \|op_str\| return self.fail("extra operand: '{s}'", .{op_str});
9985	10439
9986	10440	(switch (ops[0]) {
9987		.none => self.asmOpOnly(mnem_tag),
	10441	.none => self.asmOpOnly(mnem_fixed_tag),
9988	10442	.reg => \|reg0\| switch (ops[1]) {
9989		.none => self.asmRegister(mnem_tag, reg0),
	10443	.none => self.asmRegister(mnem_fixed_tag, reg0),
9990	10444	.reg => \|reg1\| switch (ops[2]) {
9991		.none => self.asmRegisterRegister(mnem_tag, reg1, reg0),
	10445	.none => self.asmRegisterRegister(mnem_fixed_tag, reg1, reg0),
9992	10446	.reg => \|reg2\| switch (ops[3]) {
9993		.none => self.asmRegisterRegisterRegister(mnem_tag, reg2, reg1, reg0),
	10447	.none => self.asmRegisterRegisterRegister(mnem_fixed_tag, reg2, reg1, reg0),
9994	10448	else => error.InvalidInstruction,
9995	10449	},
9996	10450	.mem => \|mem2\| switch (ops[3]) {
9997		.none => self.asmMemoryRegisterRegister(mnem_tag, mem2, reg1, reg0),
	10451	.none => self.asmMemoryRegisterRegister(mnem_fixed_tag, mem2, reg1, reg0),
9998	10452	else => error.InvalidInstruction,
9999	10453	},
10000	10454	else => error.InvalidInstruction,
10001	10455	},
10002	10456	.mem => \|mem1\| switch (ops[2]) {
10003		.none => self.asmMemoryRegister(mnem_tag, mem1, reg0),
	10457	.none => self.asmMemoryRegister(mnem_fixed_tag, mem1, reg0),
10004	10458	else => error.InvalidInstruction,
10005	10459	},
10006	10460	else => error.InvalidInstruction,
10007	10461	},
10008	10462	.mem => \|mem0\| switch (ops[1]) {
10009		.none => self.asmMemory(mnem_tag, mem0),
	10463	.none => self.asmMemory(mnem_fixed_tag, mem0),
10010	10464	.reg => \|reg1\| switch (ops[2]) {
10011		.none => self.asmRegisterMemory(mnem_tag, reg1, mem0),
	10465	.none => self.asmRegisterMemory(mnem_fixed_tag, reg1, mem0),
10012	10466	else => error.InvalidInstruction,
10013	10467	},
10014	10468	else => error.InvalidInstruction,
10015	10469	},
10016	10470	.imm => \|imm0\| switch (ops[1]) {
10017		.none => self.asmImmediate(mnem_tag, imm0),
	10471	.none => self.asmImmediate(mnem_fixed_tag, imm0),
10018	10472	.reg => \|reg1\| switch (ops[2]) {
10019		.none => self.asmRegisterImmediate(mnem_tag, reg1, imm0),
	10473	.none => self.asmRegisterImmediate(mnem_fixed_tag, reg1, imm0),
10020	10474	.reg => \|reg2\| switch (ops[3]) {
10021		.none => self.asmRegisterRegisterImmediate(mnem_tag, reg2, reg1, imm0),
	10475	.none => self.asmRegisterRegisterImmediate(mnem_fixed_tag, reg2, reg1, imm0),
10022	10476	else => error.InvalidInstruction,
10023	10477	},
10024	10478	.mem => \|mem2\| switch (ops[3]) {
10025		.none => self.asmMemoryRegisterImmediate(mnem_tag, mem2, reg1, imm0),
	10479	.none => self.asmMemoryRegisterImmediate(mnem_fixed_tag, mem2, reg1, imm0),
10026	10480	else => error.InvalidInstruction,
10027	10481	},
10028	10482	else => error.InvalidInstruction,
10029	10483	},
10030	10484	.mem => \|mem1\| switch (ops[2]) {
10031		.none => self.asmMemoryImmediate(mnem_tag, mem1, imm0),
	10485	.none => self.asmMemoryImmediate(mnem_fixed_tag, mem1, imm0),
10032	10486	else => error.InvalidInstruction,
10033	10487	},
10034	10488	else => error.InvalidInstruction,
10035	10489	},
	10490	.inst => \|inst0\| switch (ops[1]) {
	10491	.none => self.asmReloc(mnem_fixed_tag, inst0),
	10492	else => error.InvalidInstruction,
	10493	},
10036	10494	}) catch \|err\| switch (err) {
10037	10495	error.InvalidInstruction => return self.fail(
10038		"Invalid instruction: '{s} {s} {s} {s} {s}'",
	10496	"invalid instruction: '{s} {s} {s} {s} {s}'",
10039	10497	.{
10040		@tagName(mnem_tag[1]),
	10498	mnem_str,
10041	10499	@tagName(ops[0]),
10042	10500	@tagName(ops[1]),
10043	10501	@tagName(ops[2]),
...	...	@@ -10048,18 +10506,22 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
10048	10506	};
10049	10507	}
10050	10508
10051		for (outputs, args.items[0..outputs.len]) \|output, mcv\| {
10052		const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
	10509	var label_it = labels.iterator();
	10510	while (label_it.next()) \|label\| if (label.value_ptr.pending_relocs.items.len > 0)
	10511	return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
	10512
	10513	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {
	10514	const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
10053	10515	const constraint =
10054		std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
10055		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
	10516	mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
	10517	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
10056	10518	// This equation accounts for the fact that even if we have exactly 4 bytes
10057	10519	// for the string, we still use the next u32 for the null terminator.
10058	10520	outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
10059	10521
10060	10522	if (output == .none) continue;
10061		if (mcv != .register) continue;
10062		try self.store(self.typeOf(output), try self.resolveInst(output), mcv);
	10523	if (arg_mcv != .register) continue;
	10524	try self.store(self.typeOf(output), .{ .air_ref = output }, arg_mcv);
10063	10525	}
10064	10526
10065	10527	simple: {
...	...	@@ -10077,13 +10539,14 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
10077	10539	return self.finishAir(inst, result, buf);
10078	10540	}
10079	10541	var bt = self.liveness.iterateBigTomb(inst);
10080		for (outputs) \|output\| if (output != .none) self.feed(&bt, output);
10081		for (inputs) \|input\| self.feed(&bt, input);
	10542	for (outputs) \|output\| if (output != .none) try self.feed(&bt, output);
	10543	for (inputs) \|input\| try self.feed(&bt, input);
10082	10544	return self.finishAirResult(inst, result);
10083	10545	}
10084	10546
10085	10547	const MoveStrategy = union(enum) {
10086	10548	move: Mir.Inst.FixedTag,
	10549	x87_load_store,
10087	10550	insert_extract: InsertExtract,
10088	10551	vex_insert_extract: InsertExtract,
10089	10552
...	...	@@ -10091,6 +10554,44 @@ const MoveStrategy = union(enum) {
10091	10554	insert: Mir.Inst.FixedTag,
10092	10555	extract: Mir.Inst.FixedTag,
10093	10556	};
	10557
	10558	pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {
	10559	switch (strat) {
	10560	.move => \|tag\| try self.asmRegisterMemory(tag, dst_reg, src_mem),
	10561	.x87_load_store => {
	10562	try self.asmMemory(.{ .f_, .ld }, src_mem);
	10563	try self.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1));
	10564	},
	10565	.insert_extract => \|ie\| try self.asmRegisterMemoryImmediate(
	10566	ie.insert,
	10567	dst_reg,
	10568	src_mem,
	10569	Immediate.u(0),
	10570	),
	10571	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
	10572	ie.insert,
	10573	dst_reg,
	10574	dst_reg,
	10575	src_mem,
	10576	Immediate.u(0),
	10577	),
	10578	}
	10579	}
	10580	pub fn write(strat: MoveStrategy, self: *Self, dst_mem: Memory, src_reg: Register) !void {
	10581	switch (strat) {
	10582	.move => \|tag\| try self.asmMemoryRegister(tag, dst_mem, src_reg),
	10583	.x87_load_store => {
	10584	try self.asmRegister(.{ .f_, .ld }, src_reg);
	10585	try self.asmMemory(.{ .f_p, .st }, dst_mem);
	10586	},
	10587	.insert_extract, .vex_insert_extract => \|ie\| try self.asmMemoryRegisterImmediate(
	10588	ie.extract,
	10589	dst_mem,
	10590	src_reg,
	10591	Immediate.u(0),
	10592	),
	10593	}
	10594	}
10094	10595	};
10095	10596	fn moveStrategy(self: *Self, ty: Type, aligned: bool) !MoveStrategy {
10096	10597	const mod = self.bin_file.options.module.?;
...	...	@@ -10106,6 +10607,7 @@ fn moveStrategy(self: *Self, ty: Type, aligned: bool) !MoveStrategy {
10106	10607	} },
10107	10608	32 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov } },
10108	10609	64 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov } },
	10610	80 => return .x87_load_store,
10109	10611	128 => return .{ .move = if (self.hasFeature(.avx))
10110	10612	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
10111	10613	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
...	...	@@ -10274,10 +10776,12 @@ fn moveStrategy(self: *Self, ty: Type, aligned: bool) !MoveStrategy {
10274	10776	else => {},
10275	10777	},
10276	10778	}
10277		return self.fail("TODO moveStrategy for {}", .{ty.fmt(self.bin_file.options.module.?)});
	10779	return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});
10278	10780	}
10279	10781
10280	10782	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
	10783	const mod = self.bin_file.options.module.?;
	10784
10281	10785	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
10282	10786	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
10283	10787
...	...	@@ -10315,18 +10819,31 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
10315	10819	.off = -dst_reg_off.off,
10316	10820	} },
10317	10821	}),
10318		.register_pair => \|dst_regs\| for (dst_regs, 0..) \|dst_reg, dst_reg_i\| switch (src_mcv) {
10319		.register_pair => \|src_regs\| try self.genSetReg(
10320		dst_reg,
10321		Type.usize,
10322		.{ .register = src_regs[dst_reg_i] },
10323		),
10324		else => try self.genSetReg(
10325		dst_reg,
10326		Type.usize,
10327		src_mcv.address().offset(@intCast(dst_reg_i * 8)).deref(),
10328		),
10329		.air_ref => \|src_ref\| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref)),
	10822	.register_pair => \|dst_regs\| {
	10823	switch (src_mcv) {
	10824	.air_ref => \|src_ref\| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref)),
	10825	else => {},
	10826	}
	10827	const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
	10828	for (dst_regs, classes, 0..) \|dst_reg, class, dst_reg_i\| {
	10829	const class_ty = switch (class) {
	10830	.integer => Type.usize,
	10831	.sse => Type.f64,
	10832	else => unreachable,
	10833	};
	10834	switch (src_mcv) {
	10835	.register_pair => \|src_regs\| try self.genSetReg(
	10836	dst_reg,
	10837	class_ty,
	10838	.{ .register = src_regs[dst_reg_i] },
	10839	),
	10840	else => try self.genSetReg(
	10841	dst_reg,
	10842	class_ty,
	10843	src_mcv.address().offset(@intCast(dst_reg_i * 8)).deref(),
	10844	),
	10845	}
	10846	}
10330	10847	},
10331	10848	.indirect => \|reg_off\| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
10332	10849	.memory, .load_direct, .load_got, .load_tlv => {
...	...	@@ -10355,7 +10872,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
10355	10872	fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
10356	10873	const mod = self.bin_file.options.module.?;
10357	10874	const abi_size: u32 = @intCast(ty.abiSize(mod));
10358		if (abi_size * 8 > dst_reg.bitSize())
	10875	if (ty.bitSize(mod) > dst_reg.bitSize())
10359	10876	return self.fail("genSetReg called with a value larger than dst_reg", .{});
10360	10877	switch (src_mcv) {
10361	10878	.none,
...	...	@@ -10457,9 +10974,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
10457	10974	80 => null,
10458	10975	else => unreachable,
10459	10976	},
10460		}) orelse return self.fail("TODO implement genSetReg for {}", .{
10461		ty.fmt(self.bin_file.options.module.?),
10462		}),
	10977	}) orelse return self.fail("TODO implement genSetReg for {}", .{ty.fmt(mod)}),
10463	10978	registerAlias(dst_reg, abi_size),
10464	10979	registerAlias(src_reg, abi_size),
10465	10980	),
...	...	@@ -10471,9 +10986,21 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
10471	10986	.indirect,
10472	10987	.load_frame,
10473	10988	.lea_frame,
10474		=> {
10475		const dst_alias = registerAlias(dst_reg, abi_size);
10476		const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
	10989	=> try @as(MoveStrategy, switch (src_mcv) {
	10990	.register_offset => \|reg_off\| switch (reg_off.off) {
	10991	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
	10992	else => .{ .move = .{ ._, .lea } },
	10993	},
	10994	.indirect => try self.moveStrategy(ty, false),
	10995	.load_frame => \|frame_addr\| try self.moveStrategy(
	10996	ty,
	10997	self.getFrameAddrAlignment(frame_addr).compare(.gte, ty.abiAlignment(mod)),
	10998	),
	10999	.lea_frame => .{ .move = .{ ._, .lea } },
	11000	else => unreachable,
	11001	}).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
	11002	self.memPtrSize(ty),
	11003	switch (src_mcv) {
10477	11004	.register_offset, .indirect => \|reg_off\| .{
10478	11005	.base = .{ .reg = reg_off.reg },
10479	11006	.disp = reg_off.off,
...	...	@@ -10483,64 +11010,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
10483	11010	.disp = frame_addr.off,
10484	11011	},
10485	11012	else => unreachable,
10486		});
10487		switch (@as(MoveStrategy, switch (src_mcv) {
10488		.register_offset => \|reg_off\| switch (reg_off.off) {
10489		0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
10490		else => .{ .move = .{ ._, .lea } },
10491		},
10492		.indirect => try self.moveStrategy(ty, false),
10493		.load_frame => \|frame_addr\| try self.moveStrategy(
10494		ty,
10495		self.getFrameAddrAlignment(frame_addr).compare(.gte, ty.abiAlignment(mod)),
10496		),
10497		.lea_frame => .{ .move = .{ ._, .lea } },
10498		else => unreachable,
10499		})) {
10500		.move => \|tag\| try self.asmRegisterMemory(tag, dst_alias, src_mem),
10501		.insert_extract => \|ie\| try self.asmRegisterMemoryImmediate(
10502		ie.insert,
10503		dst_alias,
10504		src_mem,
10505		Immediate.u(0),
10506		),
10507		.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
10508		ie.insert,
10509		dst_alias,
10510		dst_alias,
10511		src_mem,
10512		Immediate.u(0),
10513		),
10514		}
10515		},
	11013	},
	11014	)),
10516	11015	.memory, .load_direct, .load_got, .load_tlv => {
10517	11016	switch (src_mcv) {
10518		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| {
10519		const dst_alias = registerAlias(dst_reg, abi_size);
10520		const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
10521		.base = .{ .reg = .ds },
10522		.disp = small_addr,
10523		});
10524		switch (try self.moveStrategy(ty, ty.abiAlignment(mod).check(
10525		@as(u32, @bitCast(small_addr)),
10526		))) {
10527		.move => \|tag\| try self.asmRegisterMemory(tag, dst_alias, src_mem),
10528		.insert_extract => \|ie\| try self.asmRegisterMemoryImmediate(
10529		ie.insert,
10530		dst_alias,
10531		src_mem,
10532		Immediate.u(0),
10533		),
10534		.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
10535		ie.insert,
10536		dst_alias,
10537		dst_alias,
10538		src_mem,
10539		Immediate.u(0),
10540		),
10541		}
10542		return;
10543		},
	11017	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
	11018	return (try self.moveStrategy(
	11019	ty,
	11020	ty.abiAlignment(mod).check(@as(u32, @bitCast(small_addr))),
	11021	)).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
	11022	self.memPtrSize(ty),
	11023	.{ .base = .{ .reg = .ds }, .disp = small_addr },
	11024	)),
10544	11025	.load_direct => \|sym_index\| switch (ty.zigTypeTag(mod)) {
10545	11026	else => {
10546	11027	const atom_index = try self.owner.getSymbolIndex(self);
...	...	@@ -10567,26 +11048,11 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
10567	11048	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
10568	11049	defer self.register_manager.unlockReg(addr_lock);
10569	11050
10570		const dst_alias = registerAlias(dst_reg, abi_size);
10571		const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
10572		.base = .{ .reg = addr_reg },
10573		});
10574		switch (try self.moveStrategy(ty, false)) {
10575		.move => \|tag\| try self.asmRegisterMemory(tag, dst_alias, src_mem),
10576		.insert_extract => \|ie\| try self.asmRegisterMemoryImmediate(
10577		ie.insert,
10578		dst_alias,
10579		src_mem,
10580		Immediate.u(0),
10581		),
10582		.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
10583		ie.insert,
10584		dst_alias,
10585		dst_alias,
10586		src_mem,
10587		Immediate.u(0),
10588		),
10589		}
	11051	try (try self.moveStrategy(ty, false)).read(
	11052	self,
	11053	registerAlias(dst_reg, abi_size),
	11054	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
	11055	);
10590	11056	},
10591	11057	.lea_direct, .lea_got => \|sym_index\| {
10592	11058	const atom_index = try self.owner.getSymbolIndex(self);
...	...	@@ -10683,39 +11149,23 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
10683	11149	},
10684	11150	},
10685	11151	.eflags => \|cc\| try self.asmSetccMemory(Memory.sib(.byte, .{ .base = base, .disp = disp }), cc),
10686		.register => \|src_reg\| {
10687		const dst_mem = Memory.sib(
10688		Memory.PtrSize.fromSize(abi_size),
10689		.{ .base = base, .disp = disp },
10690		);
10691		const src_alias = registerAlias(src_reg, abi_size);
10692		switch (try self.moveStrategy(ty, switch (base) {
10693		.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
10694		.reg => \|reg\| switch (reg) {
10695		.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
10696		else => false,
10697		},
10698		.frame => \|frame_index\| self.getFrameAddrAlignment(
10699		.{ .index = frame_index, .off = disp },
10700		).compare(.gte, ty.abiAlignment(mod)),
10701		})) {
10702		.move => \|tag\| try self.asmMemoryRegister(tag, dst_mem, src_alias),
10703		.insert_extract, .vex_insert_extract => \|ie\| try self.asmMemoryRegisterImmediate(
10704		ie.extract,
10705		dst_mem,
10706		src_alias,
10707		Immediate.u(0),
10708		),
10709		}
10710		},
	11152	.register => \|src_reg\| try (try self.moveStrategy(ty, switch (base) {
	11153	.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
	11154	.reg => \|reg\| switch (reg) {
	11155	.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
	11156	else => false,
	11157	},
	11158	.frame => \|frame_index\| self.getFrameAddrAlignment(
	11159	.{ .index = frame_index, .off = disp },
	11160	).compare(.gte, ty.abiAlignment(mod)),
	11161	})).write(
	11162	self,
	11163	Memory.sib(self.memPtrSize(ty), .{ .base = base, .disp = disp }),
	11164	registerAlias(src_reg, abi_size),
	11165	),
10711	11166	.register_pair => \|src_regs\| for (src_regs, 0..) \|src_reg, src_reg_i\| {
10712	11167	const part_size = @min(abi_size - src_reg_i * 8, 8);
10713		const dst_mem = Memory.sib(
10714		Memory.PtrSize.fromSize(part_size),
10715		.{ .base = base, .disp = disp + @as(i32, @intCast(src_reg_i * 8)) },
10716		);
10717		const src_alias = registerAlias(src_reg, part_size);
10718		switch (try self.moveStrategy(ty, switch (base) {
	11168	try (try self.moveStrategy(ty, switch (base) {
10719	11169	.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
10720	11170	.reg => \|reg\| switch (reg) {
10721	11171	.es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
...	...	@@ -10724,15 +11174,10 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
10724	11174	.frame => \|frame_index\| self.getFrameAddrAlignment(
10725	11175	.{ .index = frame_index, .off = disp },
10726	11176	).compare(.gte, ty.abiAlignment(mod)),
10727		})) {
10728		.move => \|tag\| try self.asmMemoryRegister(tag, dst_mem, src_alias),
10729		.insert_extract, .vex_insert_extract => \|ie\| try self.asmMemoryRegisterImmediate(
10730		ie.extract,
10731		dst_mem,
10732		src_alias,
10733		Immediate.u(0),
10734		),
10735		}
	11177	})).write(self, Memory.sib(
	11178	Memory.PtrSize.fromSize(part_size),
	11179	.{ .base = base, .disp = disp + @as(i32, @intCast(src_reg_i * 8)) },
	11180	), registerAlias(src_reg, part_size));
10736	11181	},
10737	11182	.register_overflow => \|ro\| {
10738	11183	try self.genSetMem(
...	...	@@ -10953,8 +11398,8 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
10953	11398	const src_ty = self.typeOf(ty_op.operand);
10954	11399
10955	11400	const result = result: {
10956		const dst_rc = regClassForType(dst_ty, mod);
10957		const src_rc = regClassForType(src_ty, mod);
	11401	const dst_rc = self.regClassForType(dst_ty);
	11402	const src_rc = self.regClassForType(src_ty);
10958	11403	const src_mcv = try self.resolveInst(ty_op.operand);
10959	11404
10960	11405	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
...	...	@@ -10973,11 +11418,15 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
10973	11418	break :dst dst_mcv;
10974	11419	};
10975	11420
	11421	if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
	11422
10976	11423	const dst_signedness =
10977	11424	if (dst_ty.isAbiInt(mod)) dst_ty.intInfo(mod).signedness else .unsigned;
10978		const src_signedness =
10979		if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
10980		if (dst_signedness == src_signedness) break :result dst_mcv;
	11425	if (!src_ty.isRuntimeFloat() or src_ty.floatBits(self.target.*) != 80) {
	11426	const src_signedness =
	11427	if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
	11428	if (dst_signedness == src_signedness) break :result dst_mcv;
	11429	}
10981	11430
10982	11431	const abi_size: u16 = @intCast(dst_ty.abiSize(mod));
10983	11432	const bit_size: u16 = @intCast(dst_ty.bitSize(mod));
...	...	@@ -11049,9 +11498,13 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
11049	11498	const result = result: {
11050	11499	if (switch (dst_bits) {
11051	11500	16, 80, 128 => true,
11052		32, 64 => src_size > 8 and src_size < 16,
	11501	32, 64 => src_size > 8,
11053	11502	else => unreachable,
11054	11503	}) {
	11504	if (src_bits > 128) return self.fail("TODO implement airFloatFromInt from {} to {}", .{
	11505	src_ty.fmt(mod), dst_ty.fmt(mod),
	11506	});
	11507
11055	11508	var callee: ["__floatun?i?f".len]u8 = undefined;
11056	11509	break :result try self.genCall(.{ .lib = .{
11057	11510	.return_type = dst_ty.toIntern(),
...	...	@@ -11067,10 +11520,6 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
11067	11520	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
11068	11521	}
11069	11522
11070		if (src_size > 8) return self.fail("TODO implement airFloatFromInt from {} to {}", .{
11071		src_ty.fmt(mod), dst_ty.fmt(mod),
11072		});
11073
11074	11523	const src_mcv = try self.resolveInst(ty_op.operand);
11075	11524	const src_reg = if (src_mcv.isRegister())
11076	11525	src_mcv.getReg().?
...	...	@@ -11081,7 +11530,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
11081	11530
11082	11531	if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
11083	11532
11084		const dst_reg = try self.register_manager.allocReg(inst, regClassForType(dst_ty, mod));
	11533	const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));
11085	11534	const dst_mcv = MCValue{ .register = dst_reg };
11086	11535	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
11087	11536	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -11128,9 +11577,13 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
11128	11577	const result = result: {
11129	11578	if (switch (src_bits) {
11130	11579	16, 80, 128 => true,
11131		32, 64 => dst_size > 8 and dst_size < 16,
	11580	32, 64 => dst_size > 8,
11132	11581	else => unreachable,
11133	11582	}) {
	11583	if (dst_bits > 128) return self.fail("TODO implement airIntFromFloat from {} to {}", .{
	11584	src_ty.fmt(mod), dst_ty.fmt(mod),
	11585	});
	11586
11134	11587	var callee: ["__fixuns?f?i".len]u8 = undefined;
11135	11588	break :result try self.genCall(.{ .lib = .{
11136	11589	.return_type = dst_ty.toIntern(),
...	...	@@ -11146,10 +11599,6 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
11146	11599	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
11147	11600	}
11148	11601
11149		if (dst_size > 8) return self.fail("TODO implement airIntFromFloat from {} to {}", .{
11150		src_ty.fmt(mod), dst_ty.fmt(mod),
11151		});
11152
11153	11602	const src_mcv = try self.resolveInst(ty_op.operand);
11154	11603	const src_reg = if (src_mcv.isRegister())
11155	11604	src_mcv.getReg().?
...	...	@@ -11158,7 +11607,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
11158	11607	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
11159	11608	defer self.register_manager.unlockReg(src_lock);
11160	11609
11161		const dst_reg = try self.register_manager.allocReg(inst, regClassForType(dst_ty, mod));
	11610	const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));
11162	11611	const dst_mcv = MCValue{ .register = dst_reg };
11163	11612	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
11164	11613	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -11339,7 +11788,7 @@ fn atomicOp(
11339	11788	.SeqCst => .xchg,
11340	11789	};
11341	11790
11342		const dst_reg = try self.register_manager.allocReg(null, gp);
	11791	const dst_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11343	11792	const dst_mcv = MCValue{ .register = dst_reg };
11344	11793	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
11345	11794	defer self.register_manager.unlockReg(dst_lock);
...	...	@@ -11361,13 +11810,13 @@ fn atomicOp(
11361	11810	return if (unused) .unreach else dst_mcv;
11362	11811	},
11363	11812	.loop => _ = if (val_abi_size <= 8) {
11364		const tmp_reg = try self.register_manager.allocReg(null, gp);
	11813	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11365	11814	const tmp_mcv = MCValue{ .register = tmp_reg };
11366	11815	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
11367	11816	defer self.register_manager.unlockReg(tmp_lock);
11368	11817
11369	11818	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
11370		const loop: u32 = @intCast(self.mir_instructions.len);
	11819	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
11371	11820	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
11372	11821	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
11373	11822	}
...	...	@@ -11441,7 +11890,7 @@ fn atomicOp(
11441	11890	.scale_index = ptr_mem.scaleIndex(),
11442	11891	.disp = ptr_mem.sib.disp + 8,
11443	11892	}));
11444		const loop: u32 = @intCast(self.mir_instructions.len);
	11893	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
11445	11894	const val_mem_mcv: MCValue = switch (val_mcv) {
11446	11895	.memory, .indirect, .load_frame => val_mcv,
11447	11896	else => .{ .indirect = .{
...	...	@@ -11486,7 +11935,7 @@ fn atomicOp(
11486	11935	try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
11487	11936	},
11488	11937	else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
11489		val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
	11938	val_ty.fmt(mod), @tagName(op),
11490	11939	}),
11491	11940	};
11492	11941	try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
...	...	@@ -11635,7 +12084,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
11635	12084
11636	12085	// Used to store the number of elements for comparison.
11637	12086	// After comparison, updated to store number of bytes needed to copy.
11638		const len_reg = try self.register_manager.allocReg(null, gp);
	12087	const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11639	12088	const len_mcv: MCValue = .{ .register = len_reg };
11640	12089	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
11641	12090	defer self.register_manager.unlockReg(len_lock);
...	...	@@ -11645,7 +12094,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
11645	12094	const skip_reloc = try self.asmJccReloc(undefined, .z);
11646	12095	try self.store(slice_ptr_ty, ptr, src_val);
11647	12096
11648		const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
	12097	const second_elem_ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11649	12098	const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
11650	12099	const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
11651	12100	defer self.register_manager.unlockReg(second_elem_ptr_lock);
...	...	@@ -11674,7 +12123,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
11674	12123	assert(len != 0); // prevented by Sema
11675	12124	try self.store(elem_ptr_ty, dst_ptr, src_val);
11676	12125
11677		const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
	12126	const second_elem_ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11678	12127	const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
11679	12128	const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
11680	12129	defer self.register_manager.unlockReg(second_elem_ptr_lock);
...	...	@@ -11781,16 +12230,16 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
11781	12230	const err_lock = self.register_manager.lockRegAssumeUnused(err_reg);
11782	12231	defer self.register_manager.unlockReg(err_lock);
11783	12232
11784		const addr_reg = try self.register_manager.allocReg(null, gp);
	12233	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11785	12234	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
11786	12235	defer self.register_manager.unlockReg(addr_lock);
11787	12236	try self.genLazySymbolRef(.lea, addr_reg, link.File.LazySymbol.initDecl(.const_data, null, mod));
11788	12237
11789		const start_reg = try self.register_manager.allocReg(null, gp);
	12238	const start_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11790	12239	const start_lock = self.register_manager.lockRegAssumeUnused(start_reg);
11791	12240	defer self.register_manager.unlockReg(start_lock);
11792	12241
11793		const end_reg = try self.register_manager.allocReg(null, gp);
	12242	const end_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
11794	12243	const end_lock = self.register_manager.lockRegAssumeUnused(end_reg);
11795	12244	defer self.register_manager.unlockReg(end_lock);
11796	12245
...	...	@@ -11858,7 +12307,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
11858	12307	const mod = self.bin_file.options.module.?;
11859	12308	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
11860	12309	const vector_ty = self.typeOfIndex(inst);
11861		const dst_rc = regClassForType(vector_ty, mod);
	12310	const dst_rc = self.regClassForType(vector_ty);
11862	12311	const scalar_ty = vector_ty.scalarType(mod);
11863	12312
11864	12313	const src_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -12046,9 +12495,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
12046	12495	else => unreachable,
12047	12496	},
12048	12497	}
12049		return self.fail("TODO implement airSplat for {}", .{
12050		vector_ty.fmt(self.bin_file.options.module.?),
12051		});
	12498	return self.fail("TODO implement airSplat for {}", .{vector_ty.fmt(mod)});
12052	12499	};
12053	12500	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
12054	12501	}
...	...	@@ -12214,7 +12661,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
12214	12661	return self.finishAir(inst, result, buf);
12215	12662	}
12216	12663	var bt = self.liveness.iterateBigTomb(inst);
12217		for (elements) \|elem\| self.feed(&bt, elem);
	12664	for (elements) \|elem\| try self.feed(&bt, elem);
12218	12665	return self.finishAirResult(inst, result);
12219	12666	}
12220	12667
...	...	@@ -12274,146 +12721,411 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
12274	12721	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
12275	12722	const ty = self.typeOfIndex(inst);
12276	12723
12277		if (!self.hasFeature(.fma)) return self.fail("TODO implement airMulAdd for {}", .{
12278		ty.fmt(self.bin_file.options.module.?),
12279		});
12280
12281	12724	const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
12282		var mcvs: [3]MCValue = undefined;
12283		var locks = [1]?RegisterManager.RegisterLock{null} ** 3;
12284		defer for (locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
12285		var order = [1]u2{0} ** 3;
12286		var unused = std.StaticBitSet(3).initFull();
12287		for (ops, &mcvs, &locks, 0..) \|op, mcv, lock, op_i\| {
12288		const op_index: u2 = @intCast(op_i);
12289		mcv.* = try self.resolveInst(op);
12290		if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
12291		order[op_index] = 1;
12292		unused.unset(0);
12293		} else if (unused.isSet(2) and mcv.isMemory()) {
12294		order[op_index] = 3;
12295		unused.unset(2);
	12725	const result = result: {
	12726	if (switch (ty.scalarType(mod).floatBits(self.target.*)) {
	12727	16, 80, 128 => true,
	12728	32, 64 => !self.hasFeature(.fma),
	12729	else => unreachable,
	12730	}) {
	12731	if (ty.zigTypeTag(mod) != .Float) return self.fail("TODO implement airMulAdd for {}", .{
	12732	ty.fmt(mod),
	12733	});
	12734
	12735	var callee: ["__fma?".len]u8 = undefined;
	12736	break :result try self.genCall(.{ .lib = .{
	12737	.return_type = ty.toIntern(),
	12738	.param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() },
	12739	.callee = std.fmt.bufPrint(&callee, "{s}fma{s}", .{
	12740	floatLibcAbiPrefix(ty),
	12741	floatLibcAbiSuffix(ty),
	12742	}) catch unreachable,
	12743	} }, &.{ ty, ty, ty }, &.{
	12744	.{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand },
	12745	});
12296	12746	}
12297		switch (mcv.*) {
12298		.register => \|reg\| lock.* = self.register_manager.lockReg(reg),
12299		else => {},
	12747
	12748	var mcvs: [3]MCValue = undefined;
	12749	var locks = [1]?RegisterManager.RegisterLock{null} ** 3;
	12750	defer for (locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	12751	var order = [1]u2{0} ** 3;
	12752	var unused = std.StaticBitSet(3).initFull();
	12753	for (ops, &mcvs, &locks, 0..) \|op, mcv, lock, op_i\| {
	12754	const op_index: u2 = @intCast(op_i);
	12755	mcv.* = try self.resolveInst(op);
	12756	if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
	12757	order[op_index] = 1;
	12758	unused.unset(0);
	12759	} else if (unused.isSet(2) and mcv.isMemory()) {
	12760	order[op_index] = 3;
	12761	unused.unset(2);
	12762	}
	12763	switch (mcv.*) {
	12764	.register => \|reg\| lock.* = self.register_manager.lockReg(reg),
	12765	else => {},
	12766	}
	12767	}
	12768	for (&order, &mcvs, &locks) \|mop_index, mcv, *lock\| {
	12769	if (mop_index.* != 0) continue;
	12770	mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?));
	12771	if (mop_index.* > 1 and mcv.isRegister()) continue;
	12772	const reg = try self.copyToTmpRegister(ty, mcv.*);
	12773	mcv.* = .{ .register = reg };
	12774	if (lock.*) \|old_lock\| self.register_manager.unlockReg(old_lock);
	12775	lock.* = self.register_manager.lockRegAssumeUnused(reg);
12300	12776	}
12301		}
12302		for (&order, &mcvs, &locks) \|mop_index, mcv, *lock\| {
12303		if (mop_index.* != 0) continue;
12304		mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?));
12305		if (mop_index.* > 1 and mcv.isRegister()) continue;
12306		const reg = try self.copyToTmpRegister(ty, mcv.*);
12307		mcv.* = .{ .register = reg };
12308		if (lock.*) \|old_lock\| self.register_manager.unlockReg(old_lock);
12309		lock.* = self.register_manager.lockRegAssumeUnused(reg);
12310		}
12311	12777
12312		const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or
12313		mem.eql(u2, &order, &.{ 3, 1, 2 }))
12314		switch (ty.zigTypeTag(mod)) {
12315		.Float => switch (ty.floatBits(self.target.*)) {
12316		32 => .{ .v_ss, .fmadd132 },
12317		64 => .{ .v_sd, .fmadd132 },
12318		16, 80, 128 => null,
12319		else => unreachable,
12320		},
12321		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12322		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
12323		32 => switch (ty.vectorLen(mod)) {
12324		1 => .{ .v_ss, .fmadd132 },
12325		2...8 => .{ .v_ps, .fmadd132 },
12326		else => null,
12327		},
12328		64 => switch (ty.vectorLen(mod)) {
12329		1 => .{ .v_sd, .fmadd132 },
12330		2...4 => .{ .v_pd, .fmadd132 },
12331		else => null,
12332		},
	12778	const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or
	12779	mem.eql(u2, &order, &.{ 3, 1, 2 }))
	12780	switch (ty.zigTypeTag(mod)) {
	12781	.Float => switch (ty.floatBits(self.target.*)) {
	12782	32 => .{ .v_ss, .fmadd132 },
	12783	64 => .{ .v_sd, .fmadd132 },
12333	12784	16, 80, 128 => null,
12334	12785	else => unreachable,
12335	12786	},
12336		else => unreachable,
12337		},
12338		else => unreachable,
12339		}
12340		else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
12341		switch (ty.zigTypeTag(mod)) {
12342		.Float => switch (ty.floatBits(self.target.*)) {
12343		32 => .{ .v_ss, .fmadd213 },
12344		64 => .{ .v_sd, .fmadd213 },
12345		16, 80, 128 => null,
12346		else => unreachable,
12347		},
12348		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12349		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
12350		32 => switch (ty.vectorLen(mod)) {
12351		1 => .{ .v_ss, .fmadd213 },
12352		2...8 => .{ .v_ps, .fmadd213 },
12353		else => null,
12354		},
12355		64 => switch (ty.vectorLen(mod)) {
12356		1 => .{ .v_sd, .fmadd213 },
12357		2...4 => .{ .v_pd, .fmadd213 },
12358		else => null,
	12787	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	12788	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	12789	32 => switch (ty.vectorLen(mod)) {
	12790	1 => .{ .v_ss, .fmadd132 },
	12791	2...8 => .{ .v_ps, .fmadd132 },
	12792	else => null,
	12793	},
	12794	64 => switch (ty.vectorLen(mod)) {
	12795	1 => .{ .v_sd, .fmadd132 },
	12796	2...4 => .{ .v_pd, .fmadd132 },
	12797	else => null,
	12798	},
	12799	16, 80, 128 => null,
	12800	else => unreachable,
12359	12801	},
12360		16, 80, 128 => null,
12361	12802	else => unreachable,
12362	12803	},
12363	12804	else => unreachable,
12364		},
12365		else => unreachable,
12366		}
12367		else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
12368		switch (ty.zigTypeTag(mod)) {
12369		.Float => switch (ty.floatBits(self.target.*)) {
12370		32 => .{ .v_ss, .fmadd231 },
12371		64 => .{ .v_sd, .fmadd231 },
12372		16, 80, 128 => null,
12373		else => unreachable,
12374		},
12375		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12376		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
12377		32 => switch (ty.vectorLen(mod)) {
12378		1 => .{ .v_ss, .fmadd231 },
12379		2...8 => .{ .v_ps, .fmadd231 },
12380		else => null,
12381		},
12382		64 => switch (ty.vectorLen(mod)) {
12383		1 => .{ .v_sd, .fmadd231 },
12384		2...4 => .{ .v_pd, .fmadd231 },
12385		else => null,
	12805	}
	12806	else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
	12807	switch (ty.zigTypeTag(mod)) {
	12808	.Float => switch (ty.floatBits(self.target.*)) {
	12809	32 => .{ .v_ss, .fmadd213 },
	12810	64 => .{ .v_sd, .fmadd213 },
	12811	16, 80, 128 => null,
	12812	else => unreachable,
	12813	},
	12814	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	12815	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	12816	32 => switch (ty.vectorLen(mod)) {
	12817	1 => .{ .v_ss, .fmadd213 },
	12818	2...8 => .{ .v_ps, .fmadd213 },
	12819	else => null,
	12820	},
	12821	64 => switch (ty.vectorLen(mod)) {
	12822	1 => .{ .v_sd, .fmadd213 },
	12823	2...4 => .{ .v_pd, .fmadd213 },
	12824	else => null,
	12825	},
	12826	16, 80, 128 => null,
	12827	else => unreachable,
12386	12828	},
	12829	else => unreachable,
	12830	},
	12831	else => unreachable,
	12832	}
	12833	else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
	12834	switch (ty.zigTypeTag(mod)) {
	12835	.Float => switch (ty.floatBits(self.target.*)) {
	12836	32 => .{ .v_ss, .fmadd231 },
	12837	64 => .{ .v_sd, .fmadd231 },
12387	12838	16, 80, 128 => null,
12388	12839	else => unreachable,
12389	12840	},
	12841	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	12842	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	12843	32 => switch (ty.vectorLen(mod)) {
	12844	1 => .{ .v_ss, .fmadd231 },
	12845	2...8 => .{ .v_ps, .fmadd231 },
	12846	else => null,
	12847	},
	12848	64 => switch (ty.vectorLen(mod)) {
	12849	1 => .{ .v_sd, .fmadd231 },
	12850	2...4 => .{ .v_pd, .fmadd231 },
	12851	else => null,
	12852	},
	12853	16, 80, 128 => null,
	12854	else => unreachable,
	12855	},
	12856	else => unreachable,
	12857	},
12390	12858	else => unreachable,
12391		},
12392		else => unreachable,
12393		}
12394		else
12395		unreachable) orelse return self.fail("TODO implement airMulAdd for {}", .{
12396		ty.fmt(self.bin_file.options.module.?),
12397		});
	12859	}
	12860	else
	12861	unreachable) orelse return self.fail("TODO implement airMulAdd for {}", .{ty.fmt(mod)});
12398	12862
12399		var mops: [3]MCValue = undefined;
12400		for (order, mcvs) \|mop_index, mcv\| mops[mop_index - 1] = mcv;
	12863	var mops: [3]MCValue = undefined;
	12864	for (order, mcvs) \|mop_index, mcv\| mops[mop_index - 1] = mcv;
12401	12865
12402		const abi_size: u32 = @intCast(ty.abiSize(mod));
12403		const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
12404		const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
12405		if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
12406		mir_tag,
12407		mop1_reg,
12408		mop2_reg,
12409		registerAlias(mops[2].getReg().?, abi_size),
12410		) else try self.asmRegisterRegisterMemory(
12411		mir_tag,
12412		mop1_reg,
12413		mop2_reg,
12414		mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
12415		);
12416		return self.finishAir(inst, mops[0], ops);
	12866	const abi_size: u32 = @intCast(ty.abiSize(mod));
	12867	const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
	12868	const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
	12869	if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
	12870	mir_tag,
	12871	mop1_reg,
	12872	mop2_reg,
	12873	registerAlias(mops[2].getReg().?, abi_size),
	12874	) else try self.asmRegisterRegisterMemory(
	12875	mir_tag,
	12876	mop1_reg,
	12877	mop2_reg,
	12878	mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
	12879	);
	12880	break :result mops[0];
	12881	};
	12882	return self.finishAir(inst, result, ops);
	12883	}
	12884
	12885	fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
	12886	const mod = self.bin_file.options.module.?;
	12887	const va_list_ty = self.air.instructions.items(.data)[inst].ty;
	12888	const ptr_anyopaque_ty = try mod.singleMutPtrType(Type.anyopaque);
	12889
	12890	const result: MCValue = switch (abi.resolveCallingConvention(
	12891	self.fn_type.fnCallingConvention(mod),
	12892	self.target.*,
	12893	)) {
	12894	.SysV => result: {
	12895	const info = self.va_info.sysv;
	12896	const dst_fi = try self.allocFrameIndex(FrameAlloc.initType(va_list_ty, mod));
	12897	var field_off: u31 = 0;
	12898	// gp_offset: c_uint,
	12899	try self.genSetMem(
	12900	.{ .frame = dst_fi },
	12901	field_off,
	12902	Type.c_uint,
	12903	.{ .immediate = info.gp_count * 8 },
	12904	);
	12905	field_off += @intCast(Type.c_uint.abiSize(mod));
	12906	// fp_offset: c_uint,
	12907	try self.genSetMem(
	12908	.{ .frame = dst_fi },
	12909	field_off,
	12910	Type.c_uint,
	12911	.{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 },
	12912	);
	12913	field_off += @intCast(Type.c_uint.abiSize(mod));
	12914	// overflow_arg_area: *anyopaque,
	12915	try self.genSetMem(
	12916	.{ .frame = dst_fi },
	12917	field_off,
	12918	ptr_anyopaque_ty,
	12919	.{ .lea_frame = info.overflow_arg_area },
	12920	);
	12921	field_off += @intCast(ptr_anyopaque_ty.abiSize(mod));
	12922	// reg_save_area: *anyopaque,
	12923	try self.genSetMem(
	12924	.{ .frame = dst_fi },
	12925	field_off,
	12926	ptr_anyopaque_ty,
	12927	.{ .lea_frame = info.reg_save_area },
	12928	);
	12929	field_off += @intCast(ptr_anyopaque_ty.abiSize(mod));
	12930	break :result .{ .load_frame = .{ .index = dst_fi } };
	12931	},
	12932	.Win64 => return self.fail("TODO implement c_va_start for Win64", .{}),
	12933	else => unreachable,
	12934	};
	12935	return self.finishAir(inst, result, .{ .none, .none, .none });
	12936	}
	12937
	12938	fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
	12939	const mod = self.bin_file.options.module.?;
	12940	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	12941	const ty = self.typeOfIndex(inst);
	12942	const promote_ty = self.promoteVarArg(ty);
	12943	const ptr_anyopaque_ty = try mod.singleMutPtrType(Type.anyopaque);
	12944	const unused = self.liveness.isUnused(inst);
	12945
	12946	const result: MCValue = switch (abi.resolveCallingConvention(
	12947	self.fn_type.fnCallingConvention(mod),
	12948	self.target.*,
	12949	)) {
	12950	.SysV => result: {
	12951	try self.spillEflagsIfOccupied();
	12952
	12953	const tmp_regs =
	12954	try self.register_manager.allocRegs(2, .{ null, null }, abi.RegisterClass.gp);
	12955	const offset_reg = tmp_regs[0].to32();
	12956	const addr_reg = tmp_regs[1].to64();
	12957	const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
	12958	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);
	12959
	12960	const promote_mcv = try self.allocTempRegOrMem(promote_ty, true);
	12961	const promote_lock = switch (promote_mcv) {
	12962	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	12963	else => null,
	12964	};
	12965	defer if (promote_lock) \|lock\| self.register_manager.unlockReg(lock);
	12966
	12967	const ptr_arg_list_reg =
	12968	try self.copyToTmpRegister(self.typeOf(ty_op.operand), .{ .air_ref = ty_op.operand });
	12969	const ptr_arg_list_lock = self.register_manager.lockRegAssumeUnused(ptr_arg_list_reg);
	12970	defer self.register_manager.unlockReg(ptr_arg_list_lock);
	12971
	12972	const gp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 0 } };
	12973	const fp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 4 } };
	12974	const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
	12975	const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
	12976
	12977	const classes = mem.sliceTo(&abi.classifySystemV(promote_ty, mod, .arg), .none);
	12978	switch (classes[0]) {
	12979	.integer => {
	12980	assert(classes.len == 1);
	12981
	12982	try self.genSetReg(offset_reg, Type.c_uint, gp_offset);
	12983	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
	12984	abi.SysV.c_abi_int_param_regs.len * 8,
	12985	));
	12986	const mem_reloc = try self.asmJccReloc(undefined, .ae);
	12987
	12988	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
	12989	if (!unused)
	12990	try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, Memory.sib(.qword, .{
	12991	.base = .{ .reg = addr_reg },
	12992	.scale_index = .{ .scale = 1, .index = offset_reg.to64() },
	12993	}));
	12994	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, Memory.sib(.qword, .{
	12995	.base = .{ .reg = offset_reg.to64() },
	12996	.disp = 8,
	12997	}));
	12998	try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg });
	12999	const done_reloc = try self.asmJmpReloc(undefined);
	13000
	13001	try self.performReloc(mem_reloc);
	13002	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
	13003	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), Memory.sib(.qword, .{
	13004	.base = .{ .reg = addr_reg },
	13005	.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
	13006	}));
	13007	try self.genCopy(
	13008	ptr_anyopaque_ty,
	13009	overflow_arg_area,
	13010	.{ .register = offset_reg.to64() },
	13011	);
	13012
	13013	try self.performReloc(done_reloc);
	13014	if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
	13015	.indirect = .{ .reg = addr_reg },
	13016	});
	13017	},
	13018	.sse => {
	13019	assert(classes.len == 1);
	13020
	13021	try self.genSetReg(offset_reg, Type.c_uint, fp_offset);
	13022	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
	13023	abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
	13024	));
	13025	const mem_reloc = try self.asmJccReloc(undefined, .ae);
	13026
	13027	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
	13028	if (!unused)
	13029	try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, Memory.sib(.qword, .{
	13030	.base = .{ .reg = addr_reg },
	13031	.scale_index = .{ .scale = 1, .index = offset_reg.to64() },
	13032	}));
	13033	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, Memory.sib(.qword, .{
	13034	.base = .{ .reg = offset_reg.to64() },
	13035	.disp = 16,
	13036	}));
	13037	try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg });
	13038	const done_reloc = try self.asmJmpReloc(undefined);
	13039
	13040	try self.performReloc(mem_reloc);
	13041	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
	13042	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), Memory.sib(.qword, .{
	13043	.base = .{ .reg = addr_reg },
	13044	.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
	13045	}));
	13046	try self.genCopy(
	13047	ptr_anyopaque_ty,
	13048	overflow_arg_area,
	13049	.{ .register = offset_reg.to64() },
	13050	);
	13051
	13052	try self.performReloc(done_reloc);
	13053	if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
	13054	.indirect = .{ .reg = addr_reg },
	13055	});
	13056	},
	13057	.memory => {
	13058	assert(classes.len == 1);
	13059	unreachable;
	13060	},
	13061	else => return self.fail("TODO implement c_va_arg for {} on SysV", .{
	13062	promote_ty.fmt(mod),
	13063	}),
	13064	}
	13065
	13066	if (unused) break :result .unreach;
	13067	if (ty.toIntern() == promote_ty.toIntern()) break :result promote_mcv;
	13068
	13069	if (!promote_ty.isRuntimeFloat()) {
	13070	const dst_mcv = try self.allocRegOrMem(inst, true);
	13071	try self.genCopy(ty, dst_mcv, promote_mcv);
	13072	break :result dst_mcv;
	13073	}
	13074
	13075	assert(ty.toIntern() == .f32_type and promote_ty.toIntern() == .f64_type);
	13076	const dst_mcv = if (promote_mcv.isRegister())
	13077	promote_mcv
	13078	else
	13079	try self.copyToRegisterWithInstTracking(inst, ty, promote_mcv);
	13080	const dst_reg = dst_mcv.getReg().?.to128();
	13081	const dst_lock = self.register_manager.lockReg(dst_reg);
	13082	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	13083
	13084	if (self.hasFeature(.avx)) if (promote_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	13085	.{ .v_ss, .cvtsd2 },
	13086	dst_reg,
	13087	dst_reg,
	13088	promote_mcv.mem(.qword),
	13089	) else try self.asmRegisterRegisterRegister(
	13090	.{ .v_ss, .cvtsd2 },
	13091	dst_reg,
	13092	dst_reg,
	13093	(if (promote_mcv.isRegister())
	13094	promote_mcv.getReg().?
	13095	else
	13096	try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
	13097	) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
	13098	.{ ._ss, .cvtsd2 },
	13099	dst_reg,
	13100	promote_mcv.mem(.qword),
	13101	) else try self.asmRegisterRegister(
	13102	.{ ._ss, .cvtsd2 },
	13103	dst_reg,
	13104	(if (promote_mcv.isRegister())
	13105	promote_mcv.getReg().?
	13106	else
	13107	try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
	13108	);
	13109	break :result promote_mcv;
	13110	},
	13111	.Win64 => return self.fail("TODO implement c_va_arg for Win64", .{}),
	13112	else => unreachable,
	13113	};
	13114	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	13115	}
	13116
	13117	fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {
	13118	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	13119	const ptr_va_list_ty = self.typeOf(ty_op.operand);
	13120
	13121	const dst_mcv = try self.allocRegOrMem(inst, true);
	13122	try self.load(dst_mcv, ptr_va_list_ty, .{ .air_ref = ty_op.operand });
	13123	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
	13124	}
	13125
	13126	fn airVaEnd(self: *Self, inst: Air.Inst.Index) !void {
	13127	const un_op = self.air.instructions.items(.data)[inst].un_op;
	13128	return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
12417	13129	}
12418	13130
12419	13131	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
...	...	@@ -12494,6 +13206,8 @@ const CallMCValues = struct {
12494	13206	return_value: InstTracking,
12495	13207	stack_byte_count: u31,
12496	13208	stack_align: Alignment,
	13209	gp_count: u32,
	13210	fp_count: u32,
12497	13211
12498	13212	fn deinit(self: CallMCValues, func: Self) void {
12499	13213	func.gpa.free(self.args);
...	...	@@ -12517,8 +13231,8 @@ fn resolveCallingConventionValues(
12517	13231	for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) \|*dest, src\| {
12518	13232	dest.* = src.toType();
12519	13233	}
12520		// TODO: promote var arg types
12521		for (param_types[fn_info.param_types.len..], var_args) \|param_ty, arg_ty\| param_ty. = arg_ty;
	13234	for (param_types[fn_info.param_types.len..], var_args) \|*param_ty, arg_ty\|
	13235	param_ty.* = self.promoteVarArg(arg_ty);
12522	13236
12523	13237	var result: CallMCValues = .{
12524	13238	.args = try self.gpa.alloc(MCValue, param_types.len),
...	...	@@ -12526,6 +13240,8 @@ fn resolveCallingConventionValues(
12526	13240	.return_value = undefined,
12527	13241	.stack_byte_count = 0,
12528	13242	.stack_align = undefined,
	13243	.gp_count = 0,
	13244	.fp_count = 0,
12529	13245	};
12530	13246	errdefer self.gpa.free(result.args);
12531	13247
...	...	@@ -12539,10 +13255,10 @@ fn resolveCallingConventionValues(
12539	13255	result.stack_align = .@"8";
12540	13256	},
12541	13257	.C, .SysV, .Win64 => {
12542		var ret_int_reg_i: usize = 0;
12543		var ret_sse_reg_i: usize = 0;
12544		var param_int_reg_i: usize = 0;
12545		var param_sse_reg_i: usize = 0;
	13258	var ret_int_reg_i: u32 = 0;
	13259	var ret_sse_reg_i: u32 = 0;
	13260	var param_int_reg_i: u32 = 0;
	13261	var param_sse_reg_i: u32 = 0;
12546	13262	result.stack_align = .@"16";
12547	13263
12548	13264	switch (resolved_cc) {
...	...	@@ -12566,7 +13282,7 @@ fn resolveCallingConventionValues(
12566	13282
12567	13283	const classes = switch (resolved_cc) {
12568	13284	.SysV => mem.sliceTo(&abi.classifySystemV(ret_ty, mod, .ret), .none),
12569		.Win64 => &[1]abi.Class{abi.classifyWindows(ret_ty, mod)},
	13285	.Win64 => &.{abi.classifyWindows(ret_ty, mod)},
12570	13286	else => unreachable,
12571	13287	};
12572	13288	for (classes) \|class\| switch (class) {
...	...	@@ -12639,9 +13355,10 @@ fn resolveCallingConventionValues(
12639	13355	var arg_mcv: [2]MCValue = undefined;
12640	13356	var arg_mcv_i: usize = 0;
12641	13357
12642		const classes = switch (self.target.os.tag) {
12643		.windows => &[1]abi.Class{abi.classifyWindows(ty, mod)},
12644		else => mem.sliceTo(&abi.classifySystemV(ty, mod, .arg), .none),
	13358	const classes = switch (resolved_cc) {
	13359	.SysV => mem.sliceTo(&abi.classifySystemV(ty, mod, .arg), .none),
	13360	.Win64 => &.{abi.classifyWindows(ty, mod)},
	13361	else => unreachable,
12645	13362	};
12646	13363	for (classes) \|class\| switch (class) {
12647	13364	.integer => {
...	...	@@ -12700,6 +13417,10 @@ fn resolveCallingConventionValues(
12700	13417	} };
12701	13418	result.stack_byte_count += param_size;
12702	13419	}
	13420	assert(param_int_reg_i <= 6);
	13421	result.gp_count = param_int_reg_i;
	13422	assert(param_sse_reg_i <= 16);
	13423	result.fp_count = param_sse_reg_i;
12703	13424	},
12704	13425	.Unspecified => {
12705	13426	result.stack_align = .@"16";
...	...	@@ -12805,6 +13526,14 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
12805	13526	};
12806	13527	}
12807	13528
	13529	fn memPtrSize(self: *Self, ty: Type) Memory.PtrSize {
	13530	const mod = self.bin_file.options.module.?;
	13531	return switch (ty.zigTypeTag(mod)) {
	13532	.Float => Memory.PtrSize.fromBitSize(ty.floatBits(self.target.*)),
	13533	else => Memory.PtrSize.fromSize(@intCast(ty.abiSize(mod))),
	13534	};
	13535	}
	13536
12808	13537	/// Truncates the value in the register in place.
12809	13538	/// Clobbers any remaining bits.
12810	13539	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
...	...	@@ -12943,3 +13672,35 @@ fn floatLibcAbiSuffix(ty: Type) []const u8 {
12943	13672	else => unreachable,
12944	13673	};
12945	13674	}
	13675
	13676	fn promoteVarArg(self: *Self, ty: Type) Type {
	13677	const mod = self.bin_file.options.module.?;
	13678	switch (ty.zigTypeTag(mod)) {
	13679	.Bool => return Type.c_int,
	13680	else => {
	13681	const int_info = ty.intInfo(mod);
	13682	for ([_]Type{
	13683	Type.c_int, Type.c_uint,
	13684	Type.c_long, Type.c_ulong,
	13685	Type.c_longlong, Type.c_ulonglong,
	13686	}) \|promote_ty\| {
	13687	const promote_info = promote_ty.intInfo(mod);
	13688	if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
	13689	if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
	13690	promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
	13691	}
	13692	unreachable;
	13693	},
	13694	.Float => switch (ty.floatBits(self.target.*)) {
	13695	32, 64 => return Type.f64,
	13696	else => \|float_bits\| {
	13697	assert(float_bits == self.target.c_type_bit_size(.longdouble));
	13698	return Type.c_longdouble;
	13699	},
	13700	},
	13701	.Pointer => {
	13702	assert(!ty.isSlice(mod));
	13703	return ty;
	13704	},
	13705	}
	13706	}

src/arch/x86_64/Encoding.zig+5-1

...	...	@@ -260,7 +260,7 @@ pub const Mnemonic = enum {
260	260	ud2,
261	261	xadd, xchg, xor,
262	262	// X87
263		fisttp, fld,
	263	fabs, fchs, ffree, fisttp, fld, fst, fstp,
264	264	// MMX
265	265	movd, movq,
266	266	packssdw, packsswb, packuswb,
...	...	@@ -316,6 +316,8 @@ pub const Mnemonic = enum {
316	316	xorpd,
317	317	// SSE3
318	318	movddup, movshdup, movsldup,
	319	// SSSE3
	320	pabsb, pabsd, pabsw,
319	321	// SSE4.1
320	322	blendpd, blendps, blendvpd, blendvps,
321	323	extractps,
...	...	@@ -353,6 +355,7 @@ pub const Mnemonic = enum {
353	355	vmovupd, vmovups,
354	356	vmulpd, vmulps, vmulsd, vmulss,
355	357	vorpd, vorps,
	358	vpabsb, vpabsd, vpabsw,
356	359	vpackssdw, vpacksswb, vpackusdw, vpackuswb,
357	360	vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
358	361	vpand, vpandn,
...	...	@@ -750,6 +753,7 @@ pub const Feature = enum {
750	753	sse2,
751	754	sse3,
752	755	sse4_1,
	756	ssse3,
753	757	x87,
754	758	};
755	759

src/arch/x86_64/Mir.zig+13-4

...	...	@@ -342,14 +342,10 @@ pub const Inst = struct {
342	342	div,
343	343	///
344	344	int3,
345		/// Store integer with truncation
346		istt,
347	345	/// Conditional jump
348	346	j,
349	347	/// Jump
350	348	jmp,
351		/// Load floating-point value
352		ld,
353	349	/// Load effective address
354	350	lea,
355	351	/// Load string
...	...	@@ -446,6 +442,19 @@ pub const Inst = struct {
446	442	/// Bitwise logical xor of packed double-precision floating-point values
447	443	xor,
448	444
	445	/// Absolute value
	446	abs,
	447	/// Change sign
	448	chs,
	449	/// Free floating-point register
	450	free,
	451	/// Store integer with truncation
	452	istt,
	453	/// Load floating-point value
	454	ld,
	455	/// Store floating-point value
	456	st,
	457
449	458	/// Pack with signed saturation
450	459	ackssw,
451	460	/// Pack with signed saturation

src/arch/x86_64/abi.zig+19-17

...	...	@@ -64,7 +64,7 @@ pub fn classifyWindows(ty: Type, mod: *Module) Class {
64	64	}
65	65	}
66	66
67		pub const Context = enum { ret, arg, other };
	67	pub const Context = enum { ret, arg, field, other };
68	68
69	69	/// There are a maximum of 8 possible return slots. Returned values are in
70	70	/// the beginning of the array; unused slots are filled with .none.
...	...	@@ -120,7 +120,7 @@ pub fn classifySystemV(ty: Type, mod: *Module, ctx: Context) [8]Class {
120	120	},
121	121	.Float => switch (ty.floatBits(target)) {
122	122	16 => {
123		if (ctx == .other) {
	123	if (ctx == .field) {
124	124	result[0] = .memory;
125	125	} else {
126	126	// TODO clang doesn't allow __fp16 as .ret or .arg
...	...	@@ -140,7 +140,7 @@ pub fn classifySystemV(ty: Type, mod: *Module, ctx: Context) [8]Class {
140	140	// "Arguments of types __float128, _Decimal128 and __m128 are
141	141	// split into two halves. The least significant ones belong
142	142	// to class SSE, the most significant one to class SSEUP."
143		if (ctx == .other) {
	143	if (ctx == .field) {
144	144	result[0] = .memory;
145	145	return result;
146	146	}
...	...	@@ -229,7 +229,7 @@ pub fn classifySystemV(ty: Type, mod: *Module, ctx: Context) [8]Class {
229	229	if (field_align != .none and field_align.compare(.lt, field_ty.abiAlignment(mod)))
230	230	return memory_class;
231	231	const field_size = field_ty.abiSize(mod);
232		const field_class_array = classifySystemV(field_ty, mod, .other);
	232	const field_class_array = classifySystemV(field_ty, mod, .field);
233	233	const field_class = std.mem.sliceTo(&field_class_array, .none);
234	234	if (byte_i + field_size <= 8) {
235	235	// Combine this field with the previous one.
...	...	@@ -347,7 +347,7 @@ pub fn classifySystemV(ty: Type, mod: *Module, ctx: Context) [8]Class {
347	347	return memory_class;
348	348	}
349	349	// Combine this field with the previous one.
350		const field_class = classifySystemV(field_ty.toType(), mod, .other);
	350	const field_class = classifySystemV(field_ty.toType(), mod, .field);
351	351	for (&result, 0..) \|*result_item, i\| {
352	352	const field_item = field_class[i];
353	353	// "If both classes are equal, this is the resulting class."
...	...	@@ -444,7 +444,7 @@ pub const SysV = struct {
444	444	/// These registers need to be preserved (saved on the stack) and restored by the caller before
445	445	/// the caller relinquishes control to a subroutine via call instruction (or similar).
446	446	/// In other words, these registers are free to use by the callee.
447		pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ sse_avx_regs;
	447	pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
448	448
449	449	pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
450	450	pub const c_abi_sse_param_regs = sse_avx_regs[0..8].*;
...	...	@@ -459,7 +459,7 @@ pub const Win64 = struct {
459	459	/// These registers need to be preserved (saved on the stack) and restored by the caller before
460	460	/// the caller relinquishes control to a subroutine via call instruction (or similar).
461	461	/// In other words, these registers are free to use by the callee.
462		pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ sse_avx_regs;
	462	pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
463	463
464	464	pub const c_abi_int_param_regs = [_]Register{ .rcx, .rdx, .r8, .r9 };
465	465	pub const c_abi_sse_param_regs = sse_avx_regs[0..4].*;
...	...	@@ -531,30 +531,32 @@ pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention) []const Register
531	531	const gp_regs = [_]Register{
532	532	.rax, .rcx, .rdx, .rbx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15,
533	533	};
	534	const x87_regs = [_]Register{
	535	.st0, .st1, .st2, .st3, .st4, .st5, .st6, .st7,
	536	};
534	537	const sse_avx_regs = [_]Register{
535	538	.ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7,
536	539	.ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15,
537	540	};
538		const allocatable_regs = gp_regs ++ sse_avx_regs;
539		pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_regs);
	541	const allocatable_regs = gp_regs ++ x87_regs[0 .. x87_regs.len - 1] ++ sse_avx_regs;
	542	pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, allocatable_regs);
540	543
541	544	// Register classes
542	545	const RegisterBitSet = RegisterManager.RegisterBitSet;
543	546	pub const RegisterClass = struct {
544	547	pub const gp: RegisterBitSet = blk: {
545	548	var set = RegisterBitSet.initEmpty();
546		set.setRangeValue(.{
547		.start = 0,
548		.end = gp_regs.len,
549		}, true);
	549	for (allocatable_regs, 0..) \|reg, index\| if (reg.class() == .general_purpose) set.set(index);
	550	break :blk set;
	551	};
	552	pub const x87: RegisterBitSet = blk: {
	553	var set = RegisterBitSet.initEmpty();
	554	for (allocatable_regs, 0..) \|reg, index\| if (reg.class() == .x87) set.set(index);
550	555	break :blk set;
551	556	};
552	557	pub const sse: RegisterBitSet = blk: {
553	558	var set = RegisterBitSet.initEmpty();
554		set.setRangeValue(.{
555		.start = gp_regs.len,
556		.end = allocatable_regs.len,
557		}, true);
	559	for (allocatable_regs, 0..) \|reg, index\| if (reg.class() == .sse) set.set(index);
558	560	break :blk set;
559	561	};
560	562	};

src/arch/x86_64/encodings.zig+34-3

...	...	@@ -829,13 +829,28 @@ pub const table = [_]Entry{
829	829	.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
830	830
831	831	// X87
	832	.{ .fabs, .np, &.{}, &.{ 0xd9, 0xe1 }, 0, .none, .x87 },
	833
	834	.{ .fchs, .np, &.{}, &.{ 0xd9, 0xe0 }, 0, .none, .x87 },
	835
	836	.{ .ffree, .o, &.{ .st }, &.{ 0xdd, 0xc0 }, 0, .none, .x87 },
	837
832	838	.{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
833	839	.{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
834	840	.{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .none, .x87 },
835	841
836		.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
837		.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
838		.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
	842	.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
	843	.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
	844	.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
	845	.{ .fld, .o, &.{ .st }, &.{ 0xd9, 0xc0 }, 0, .none, .x87 },
	846
	847	.{ .fst, .m, &.{ .m32 }, &.{ 0xd9 }, 2, .none, .x87 },
	848	.{ .fst, .m, &.{ .m64 }, &.{ 0xdd }, 2, .none, .x87 },
	849	.{ .fst, .o, &.{ .st }, &.{ 0xdd, 0xd0 }, 0, .none, .x87 },
	850	.{ .fstp, .m, &.{ .m32 }, &.{ 0xd9 }, 3, .none, .x87 },
	851	.{ .fstp, .m, &.{ .m64 }, &.{ 0xdd }, 3, .none, .x87 },
	852	.{ .fstp, .m, &.{ .m80 }, &.{ 0xdb }, 7, .none, .x87 },
	853	.{ .fstp, .o, &.{ .st }, &.{ 0xdd, 0xd8 }, 0, .none, .x87 },
839	854
840	855	// SSE
841	856	.{ .addps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .none, .sse },
...	...	@@ -1093,6 +1108,14 @@ pub const table = [_]Entry{
1093	1108
1094	1109	.{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
1095	1110
	1111	// SSSE3
	1112	.{ .pabsb, .rm, &.{ .mm, .mm_m64 }, &.{ 0x0f, 0x38, 0x1c }, 0, .none, .ssse3 },
	1113	.{ .pabsb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .none, .ssse3 },
	1114	.{ .pabsd, .rm, &.{ .mm, .mm_m64 }, &.{ 0x0f, 0x38, 0x1e }, 0, .none, .ssse3 },
	1115	.{ .pabsd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .none, .ssse3 },
	1116	.{ .pabsw, .rm, &.{ .mm, .mm_m64 }, &.{ 0x0f, 0x38, 0x1d }, 0, .none, .ssse3 },
	1117	.{ .pabsw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .none, .ssse3 },
	1118
1096	1119	// SSE4.1
1097	1120	.{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },
1098	1121
...	...	@@ -1353,6 +1376,10 @@ pub const table = [_]Entry{
1353	1376	.{ .vorps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .vex_128_wig, .avx },
1354	1377	.{ .vorps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x56 }, 0, .vex_256_wig, .avx },
1355	1378
	1379	.{ .vpabsb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_128_wig, .avx },
	1380	.{ .vpabsd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_128_wig, .avx },
	1381	.{ .vpabsw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_128_wig, .avx },
	1382
1356	1383	.{ .vpacksswb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x63 }, 0, .vex_128_wig, .avx },
1357	1384	.{ .vpackssdw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6b }, 0, .vex_128_wig, .avx },
1358	1385
...	...	@@ -1522,6 +1549,10 @@ pub const table = [_]Entry{
1522	1549	.{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
1523	1550	.{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
1524	1551
	1552	.{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },
	1553	.{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },
	1554	.{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },
	1555
1525	1556	.{ .vpacksswb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x63 }, 0, .vex_256_wig, .avx2 },
1526	1557	.{ .vpackssdw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6b }, 0, .vex_256_wig, .avx2 },
1527	1558

src/register_manager.zig+4-2

...	...	@@ -91,11 +91,13 @@ pub fn RegisterManager(
91	91	return null;
92	92	}
93	93
94		pub fn indexOfRegIntoTracked(reg: Register) ?RegisterBitSet.ShiftInt {
	94	pub fn indexOfRegIntoTracked(
	95	reg: Register,
	96	) ?std.math.IntFittingRange(0, tracked_registers.len) {
95	97	return indexOfReg(tracked_registers, reg);
96	98	}
97	99
98		pub fn regAtTrackedIndex(index: RegisterBitSet.ShiftInt) Register {
	100	pub fn regAtTrackedIndex(index: std.math.IntFittingRange(0, tracked_registers.len)) Register {
99	101	return tracked_registers[index];
100	102	}
101	103

test/behavior/abs.zig+2-2

...	...	@@ -95,7 +95,7 @@ test "@abs floats" {
95	95	try comptime testAbsFloats(f64);
96	96	try testAbsFloats(f64);
97	97	try comptime testAbsFloats(f80);
98		if (builtin.zig_backend != .stage2_x86_64 and builtin.zig_backend != .stage2_wasm) try testAbsFloats(f80);
	98	if (builtin.zig_backend != .stage2_wasm) try testAbsFloats(f80);
99	99	try comptime testAbsFloats(f128);
100	100	if (builtin.zig_backend != .stage2_wasm) try testAbsFloats(f128);
101	101	}
...	...	@@ -280,7 +280,7 @@ test "@abs float vectors" {
280	280	try testAbsFloatVectors(f16, 16);
281	281	try comptime testAbsFloatVectors(f16, 17);
282	282
283		try testAbsFloatVectors(f32, 17);
	283	try testAbsFloatVectors(f32, 1);
284	284	try comptime testAbsFloatVectors(f32, 1);
285	285	try testAbsFloatVectors(f32, 1);
286	286	try comptime testAbsFloatVectors(f32, 2);

test/behavior/asm.zig+2-4

...	...	@@ -42,7 +42,6 @@ test "module level assembly" {
42	42
43	43	test "output constraint modifiers" {
44	44	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
45		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
46	45	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
47	46	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
48	47	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -66,7 +65,6 @@ test "output constraint modifiers" {
66	65
67	66	test "alternative constraints" {
68	67	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
69		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
70	68	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
71	69	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
72	70	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -163,8 +161,8 @@ export fn derp() i32 {
163	161	}
164	162
165	163	test "rw constraint (x86_64)" {
166		if (builtin.target.cpu.arch != .x86_64 or builtin.zig_backend != .stage2_llvm)
167		return error.SkipZigTest;
	164	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
	165	if (builtin.target.cpu.arch != .x86_64) return error.SkipZigTest;
168	166
169	167	var res: i32 = 5;
170	168	asm ("addl %[b], %[a]"

test/behavior/cast.zig+2-2

...	...	@@ -118,11 +118,11 @@ test "@floatFromInt" {
118	118
119	119	test "@floatFromInt(f80)" {
120	120	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
121		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
122	121	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
123	122	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
124	123	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
125	124	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
	125	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
126	126
127	127	const S = struct {
128	128	fn doTheTest(comptime Int: type) !void {
...	...	@@ -1476,9 +1476,9 @@ test "pointer to empty struct literal to mutable slice" {
1476	1476	test "coerce between pointers of compatible differently-named floats" {
1477	1477	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1478	1478	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1479		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1480	1479	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1481	1480	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
	1481	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
1482	1482
1483	1483	if (builtin.os.tag == .windows) {
1484	1484	// https://github.com/ziglang/zig/issues/12396

test/behavior/floatop.zig+17-17

...	...	@@ -28,7 +28,7 @@ test "add f32/f64" {
28	28	}
29	29
30	30	test "add f80/f128/c_longdouble" {
31		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	31	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
32	32
33	33	try testAdd(f80);
34	34	try comptime testAdd(f80);
...	...	@@ -59,7 +59,7 @@ test "sub f32/f64" {
59	59	}
60	60
61	61	test "sub f80/f128/c_longdouble" {
62		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	62	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
63	63
64	64	try testSub(f80);
65	65	try comptime testSub(f80);
...	...	@@ -90,7 +90,7 @@ test "mul f32/f64" {
90	90	}
91	91
92	92	test "mul f80/f128/c_longdouble" {
93		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	93	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
94	94
95	95	try testMul(f80);
96	96	try comptime testMul(f80);
...	...	@@ -270,7 +270,7 @@ test "@sqrt f80/f128/c_longdouble" {
270	270	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
271	271	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
272	272	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
273		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	273	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
274	274
275	275	if (builtin.os.tag == .freebsd) {
276	276	// TODO https://github.com/ziglang/zig/issues/10875
...	...	@@ -378,7 +378,7 @@ test "@sin f80/f128/c_longdouble" {
378	378	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
379	379	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
380	380	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
381		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	381	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
382	382
383	383	try testSin(f80);
384	384	comptime try testSin(f80);
...	...	@@ -447,7 +447,7 @@ test "@cos f80/f128/c_longdouble" {
447	447	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
448	448	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
449	449	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
450		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	450	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
451	451
452	452	try testCos(f80);
453	453	try comptime testCos(f80);
...	...	@@ -516,7 +516,7 @@ test "@tan f80/f128/c_longdouble" {
516	516	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
517	517	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
518	518	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
519		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	519	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
520	520
521	521	try testTan(f80);
522	522	try comptime testTan(f80);
...	...	@@ -585,7 +585,7 @@ test "@exp f80/f128/c_longdouble" {
585	585	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
586	586	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
587	587	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
588		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	588	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
589	589
590	590	try testExp(f80);
591	591	try comptime testExp(f80);
...	...	@@ -654,7 +654,7 @@ test "@exp2 f80/f128/c_longdouble" {
654	654	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
655	655	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
656	656	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
657		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	657	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
658	658
659	659	try testExp2(f80);
660	660	try comptime testExp2(f80);
...	...	@@ -723,7 +723,7 @@ test "@log f80/f128/c_longdouble" {
723	723	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
724	724	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
725	725	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
726		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	726	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
727	727
728	728	try testLog(f80);
729	729	try comptime testLog(f80);
...	...	@@ -790,7 +790,7 @@ test "@log2 f80/f128/c_longdouble" {
790	790	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
791	791	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
792	792	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
793		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	793	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
794	794
795	795	try testLog2(f80);
796	796	try comptime testLog2(f80);
...	...	@@ -863,7 +863,7 @@ test "@log10 f80/f128/c_longdouble" {
863	863	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
864	864	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
865	865	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
866		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	866	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
867	867
868	868	try testLog10(f80);
869	869	try comptime testLog10(f80);
...	...	@@ -929,7 +929,7 @@ test "@abs f80/f128/c_longdouble" {
929	929	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
930	930	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
931	931	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
932		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	932	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
933	933
934	934	try testFabs(f80);
935	935	try comptime testFabs(f80);
...	...	@@ -1029,7 +1029,7 @@ test "@floor f80/f128/c_longdouble" {
1029	1029	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1030	1030	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
1031	1031	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1032		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1032	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
1033	1033
1034	1034	if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {
1035	1035	// https://github.com/ziglang/zig/issues/12602
...	...	@@ -1111,7 +1111,7 @@ test "@ceil f80/f128/c_longdouble" {
1111	1111	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1112	1112	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
1113	1113	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1114		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1114	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
1115	1115
1116	1116	if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {
1117	1117	// https://github.com/ziglang/zig/issues/12602
...	...	@@ -1203,7 +1203,7 @@ test "@trunc f80/f128/c_longdouble" {
1203	1203	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1204	1204	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
1205	1205	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1206		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1206	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
1207	1207
1208	1208	if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {
1209	1209	// https://github.com/ziglang/zig/issues/12602
...	...	@@ -1294,7 +1294,7 @@ test "neg f80/f128/c_longdouble" {
1294	1294	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1295	1295	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1296	1296	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
1297		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	1297	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
1298	1298
1299	1299	try testNeg(f80);
1300	1300	try comptime testNeg(f80);

test/behavior/math.zig+3-3

...	...	@@ -495,11 +495,11 @@ fn testDivision() !void {
495	495	}
496	496
497	497	test "division half-precision floats" {
498		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
499	498	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
500	499	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
501	500	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
502	501	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
	502	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
503	503
504	504	try testDivisionFP16();
505	505	try comptime testDivisionFP16();
...	...	@@ -1332,6 +1332,7 @@ test "remainder division" {
1332	1332	try comptime remdiv(f80);
1333	1333	try comptime remdiv(f128);
1334	1334	try remdiv(f16);
	1335	try remdiv(f32);
1335	1336	try remdiv(f64);
1336	1337	try remdiv(f80);
1337	1338	try remdiv(f128);
...	...	@@ -1356,7 +1357,7 @@ test "float remainder division using @rem" {
1356	1357	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1357	1358	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1358	1359	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1359		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	1360	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
1360	1361
1361	1362	try comptime frem(f16);
1362	1363	try comptime frem(f32);
...	...	@@ -1468,7 +1469,6 @@ test "@round f32/f64" {
1468	1469	test "@round f80" {
1469	1470	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1470	1471	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1471		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1472	1472	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1473	1473	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1474	1474	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;

test/behavior/maximum_minimum.zig+2-1

...	...	@@ -276,7 +276,8 @@ test "@min/@max notices bounds from vector types when element of comptime-known
276	276	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
277	277	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
278	278	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
279		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
	279	if (builtin.zig_backend == .stage2_x86_64 and
	280	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest;
280	281
281	282	var x: @Vector(2, u32) = .{ 1_000_000, 12345 };
282	283	const y: @Vector(2, u16) = .{ 10, undefined };

test/behavior/muladd.zig+3-3

...	...	@@ -32,11 +32,11 @@ fn testMulAdd() !void {
32	32	}
33	33
34	34	test "@mulAdd f16" {
35		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
36	35	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
37	36	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
38	37	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
39	38	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
	39	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
40	40
41	41	try comptime testMulAdd16();
42	42	try testMulAdd16();
...	...	@@ -50,12 +50,12 @@ fn testMulAdd16() !void {
50	50	}
51	51
52	52	test "@mulAdd f80" {
53		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
54	53	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
55	54	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
56	55	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
57	56	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
58	57	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
	58	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
59	59
60	60	try comptime testMulAdd80();
61	61	try testMulAdd80();
...	...	@@ -69,12 +69,12 @@ fn testMulAdd80() !void {
69	69	}
70	70
71	71	test "@mulAdd f128" {
72		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
73	72	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
74	73	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
75	74	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
76	75	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
77	76	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
	77	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
78	78
79	79	try comptime testMulAdd128();
80	80	try testMulAdd128();

test/behavior/var_args.zig-4

...	...	@@ -108,7 +108,6 @@ test "simple variadic function" {
108	108	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
109	109	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
110	110	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
111		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
112	111	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
113	112	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
114	113	// https://github.com/ziglang/zig/issues/14096
...	...	@@ -158,7 +157,6 @@ test "variadic functions" {
158	157	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
159	158	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
160	159	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
161		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
162	160	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
163	161	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
164	162	// https://github.com/ziglang/zig/issues/14096
...	...	@@ -202,7 +200,6 @@ test "copy VaList" {
202	200	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
203	201	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
204	202	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
205		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
206	203	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
207	204	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
208	205	// https://github.com/ziglang/zig/issues/14096
...	...	@@ -235,7 +232,6 @@ test "unused VaList arg" {
235	232	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
236	233	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
237	234	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
238		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
239	235	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
240	236	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {
241	237	// https://github.com/ziglang/zig/issues/14096

test/behavior/widening.zig+3-6

...	...	@@ -2,7 +2,6 @@ const std = @import("std");
2	2	const expect = std.testing.expect;
3	3	const mem = std.mem;
4	4	const builtin = @import("builtin");
5		const has_f80_rt = @import("builtin").cpu.arch == .x86_64;
6	5
7	6	test "integer widening" {
8	7	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
...	...	@@ -40,11 +39,11 @@ test "implicit unsigned integer to signed integer" {
40	39	}
41	40
42	41	test "float widening" {
43		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
44	42	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
45	43	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
46	44	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
47	45	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
	46	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
48	47
49	48	var a: f16 = 12.34;
50	49	var b: f32 = a;
...	...	@@ -53,10 +52,8 @@ test "float widening" {
53	52	try expect(a == b);
54	53	try expect(b == c);
55	54	try expect(c == d);
56		if (has_f80_rt) {
57		var e: f80 = c;
58		try expect(c == e);
59		}
	55	var e: f80 = c;
	56	try expect(c == e);
60	57	}
61	58
62	59	test "float widening f16 to f128" {