Commit 85902115d4fde6120b6cea610cd6f79baede5aaa | zig

author	Jacob Young <jacobly@ziglang.org> 2023-06-25 19:13:47-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-06-25 19:14:26-04:00
log	85902115d4fde6120b6cea610cd6f79baede5aaa
tree	50bac6719a89a52f51efb1b0ec4b42b905868024
parent	7013567f8abde19e5176d902ce88abc754fc9a3e

x86_64: cleanup `@as` invasion

1 files changed, 287 insertions(+), 309 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig

src/arch/x86_64/CodeGen.zig+287-309

...	...	@@ -329,7 +329,7 @@ pub const MCValue = union(enum) {
329	329	.load_frame,
330	330	.reserved_frame,
331	331	=> unreachable, // not offsettable
332		.immediate => \|imm\| .{ .immediate = @as(u64, @bitCast(@as(i64, @bitCast(imm)) +% off)) },
	332	.immediate => \|imm\| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) },
333	333	.register => \|reg\| .{ .register_offset = .{ .reg = reg, .off = off } },
334	334	.register_offset => \|reg_off\| .{
335	335	.register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off },
...	...	@@ -606,7 +606,7 @@ const FrameAlloc = struct {
606	606	fn init(alloc_abi: struct { size: u64, alignment: u32 }) FrameAlloc {
607	607	assert(math.isPowerOfTwo(alloc_abi.alignment));
608	608	return .{
609		.abi_size = @as(u31, @intCast(alloc_abi.size)),
	609	.abi_size = @intCast(alloc_abi.size),
610	610	.abi_align = math.log2_int(u32, alloc_abi.alignment),
611	611	.ref_count = 0,
612	612	};
...	...	@@ -694,7 +694,7 @@ pub fn generate(
694	694	FrameAlloc.init(.{
695	695	.size = 0,
696	696	.alignment = if (mod.align_stack_fns.get(module_fn_index)) \|set_align_stack\|
697		@as(u32, @intCast(set_align_stack.alignment.toByteUnitsOptional().?))
	697	@intCast(set_align_stack.alignment.toByteUnitsOptional().?)
698	698	else
699	699	1,
700	700	}),
...	...	@@ -979,7 +979,7 @@ fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {
979	979	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
980	980	const gpa = self.gpa;
981	981	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
982		const result_index = @as(Mir.Inst.Index, @intCast(self.mir_instructions.len));
	982	const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
983	983	self.mir_instructions.appendAssumeCapacity(inst);
984	984	if (inst.tag != .pseudo or switch (inst.ops) {
985	985	else => true,
...	...	@@ -1000,11 +1000,11 @@ fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
1000	1000
1001	1001	fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1002	1002	const fields = std.meta.fields(@TypeOf(extra));
1003		const result = @as(u32, @intCast(self.mir_extra.items.len));
	1003	const result: u32 = @intCast(self.mir_extra.items.len);
1004	1004	inline for (fields) \|field\| {
1005	1005	self.mir_extra.appendAssumeCapacity(switch (field.type) {
1006	1006	u32 => @field(extra, field.name),
1007		i32 => @as(u32, @bitCast(@field(extra, field.name))),
	1007	i32 => @bitCast(@field(extra, field.name)),
1008	1008	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
1009	1009	});
1010	1010	}
...	...	@@ -1214,8 +1214,8 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1214	1214	.data = .{ .i = .{
1215	1215	.fixes = tag[0],
1216	1216	.i = switch (imm) {
1217		.signed => \|s\| @as(u32, @bitCast(s)),
1218		.unsigned => \|u\| @as(u32, @intCast(u)),
	1217	.signed => \|s\| @bitCast(s),
	1218	.unsigned => \|u\| @intCast(u),
1219	1219	},
1220	1220	} },
1221	1221	});
...	...	@@ -1246,8 +1246,8 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm:
1246	1246	.fixes = tag[0],
1247	1247	.r1 = reg,
1248	1248	.i = switch (imm) {
1249		.signed => \|s\| @as(u32, @bitCast(s)),
1250		.unsigned => \|u\| @as(u32, @intCast(u)),
	1249	.signed => \|s\| @bitCast(s),
	1250	.unsigned => \|u\| @intCast(u),
1251	1251	},
1252	1252	} },
1253	1253	.ri64 => .{ .rx = .{
...	...	@@ -1339,8 +1339,8 @@ fn asmRegisterRegisterImmediate(
1339	1339	.r1 = reg1,
1340	1340	.r2 = reg2,
1341	1341	.i = switch (imm) {
1342		.signed => \|s\| @as(u32, @bitCast(s)),
1343		.unsigned => \|u\| @as(u32, @intCast(u)),
	1342	.signed => \|s\| @bitCast(s),
	1343	.unsigned => \|u\| @intCast(u),
1344	1344	},
1345	1345	} },
1346	1346	});
...	...	@@ -1458,7 +1458,7 @@ fn asmRegisterRegisterMemoryImmediate(
1458	1458	.fixes = tag[0],
1459	1459	.r1 = reg1,
1460	1460	.r2 = reg2,
1461		.i = @as(u8, @intCast(imm.unsigned)),
	1461	.i = @intCast(imm.unsigned),
1462	1462	.payload = switch (m) {
1463	1463	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1464	1464	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...	...	@@ -1490,8 +1490,8 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist
1490	1490
1491	1491	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1492	1492	const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
1493		.signed => \|s\| @as(u32, @bitCast(s)),
1494		.unsigned => \|u\| @as(u32, @intCast(u)),
	1493	.signed => \|s\| @bitCast(s),
	1494	.unsigned => \|u\| @intCast(u),
1495	1495	} });
1496	1496	assert(payload + 1 == switch (m) {
1497	1497	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
...	...	@@ -1562,7 +1562,7 @@ fn asmMemoryRegisterImmediate(
1562	1562	.data = .{ .rix = .{
1563	1563	.fixes = tag[0],
1564	1564	.r1 = reg,
1565		.i = @as(u8, @intCast(imm.unsigned)),
	1565	.i = @intCast(imm.unsigned),
1566	1566	.payload = switch (m) {
1567	1567	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1568	1568	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...	...	@@ -1617,7 +1617,7 @@ fn gen(self: *Self) InnerError!void {
1617	1617	// Eliding the reloc will cause a miscompilation in this case.
1618	1618	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
1619	1619	self.mir_instructions.items(.data)[jmp_reloc].inst.inst =
1620		@as(u32, @intCast(self.mir_instructions.len));
	1620	@intCast(self.mir_instructions.len);
1621	1621	}
1622	1622
1623	1623	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
...	...	@@ -1739,7 +1739,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1739	1739
1740	1740	for (body) \|inst\| {
1741	1741	if (builtin.mode == .Debug) {
1742		const mir_inst = @as(Mir.Inst.Index, @intCast(self.mir_instructions.len));
	1742	const mir_inst: Mir.Inst.Index = @intCast(self.mir_instructions.len);
1743	1743	try self.mir_to_air_map.put(self.gpa, mir_inst, inst);
1744	1744	}
1745	1745
...	...	@@ -2034,7 +2034,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2034	2034
2035	2035	var data_off: i32 = 0;
2036	2036	for (exitlude_jump_relocs, 0..) \|*exitlude_jump_reloc, index_usize\| {
2037		const index = @as(u32, @intCast(index_usize));
	2037	const index: u32 = @intCast(index_usize);
2038	2038	const tag_name = mod.intern_pool.stringToSlice(enum_ty.enumFields(mod)[index_usize]);
2039	2039	const tag_val = try mod.enumValueFieldIndex(enum_ty, index);
2040	2040	const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });
...	...	@@ -2052,7 +2052,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2052	2052	exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);
2053	2053	try self.performReloc(skip_reloc);
2054	2054
2055		data_off += @as(i32, @intCast(tag_name.len + 1));
	2055	data_off += @intCast(tag_name.len + 1);
2056	2056	}
2057	2057
2058	2058	try self.airTrap();
...	...	@@ -2169,7 +2169,7 @@ fn computeFrameLayout(self: *Self) !FrameLayout {
2169	2169	const frame_offset = self.frame_locs.items(.disp);
2170	2170
2171	2171	for (stack_frame_order, FrameIndex.named_count..) \|*frame_order, frame_index\|
2172		frame_order.* = @as(FrameIndex, @enumFromInt(frame_index));
	2172	frame_order.* = @enumFromInt(frame_index);
2173	2173	{
2174	2174	const SortContext = struct {
2175	2175	frame_align: @TypeOf(frame_align),
...	...	@@ -2197,7 +2197,7 @@ fn computeFrameLayout(self: *Self) !FrameLayout {
2197	2197	}
2198	2198	}
2199	2199
2200		var rbp_offset = @as(i32, @intCast(save_reg_list.count() * 8));
	2200	var rbp_offset: i32 = @intCast(save_reg_list.count() * 8);
2201	2201	self.setFrameLoc(.base_ptr, .rbp, &rbp_offset, false);
2202	2202	self.setFrameLoc(.ret_addr, .rbp, &rbp_offset, false);
2203	2203	self.setFrameLoc(.args_frame, .rbp, &rbp_offset, false);
...	...	@@ -2212,11 +2212,11 @@ fn computeFrameLayout(self: *Self) !FrameLayout {
2212	2212	rsp_offset = mem.alignForward(i32, rsp_offset, @as(i32, 1) << needed_align);
2213	2213	rsp_offset -= stack_frame_align_offset;
2214	2214	frame_size[@intFromEnum(FrameIndex.call_frame)] =
2215		@as(u31, @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]));
	2215	@intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]);
2216	2216
2217	2217	return .{
2218	2218	.stack_mask = @as(u32, math.maxInt(u32)) << (if (need_align_stack) needed_align else 0),
2219		.stack_adjust = @as(u32, @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)])),
	2219	.stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]),
2220	2220	.save_reg_list = save_reg_list,
2221	2221	};
2222	2222	}
...	...	@@ -2247,7 +2247,7 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
2247	2247	_ = self.free_frame_indices.swapRemoveAt(free_i);
2248	2248	return frame_index;
2249	2249	}
2250		const frame_index = @as(FrameIndex, @enumFromInt(self.frame_allocs.len));
	2250	const frame_index: FrameIndex = @enumFromInt(self.frame_allocs.len);
2251	2251	try self.frame_allocs.append(self.gpa, alloc);
2252	2252	return frame_index;
2253	2253	}
...	...	@@ -2323,7 +2323,7 @@ const State = struct {
2323	2323
2324	2324	fn initRetroactiveState(self: *Self) State {
2325	2325	var state: State = undefined;
2326		state.inst_tracking_len = @as(u32, @intCast(self.inst_tracking.count()));
	2326	state.inst_tracking_len = @intCast(self.inst_tracking.count());
2327	2327	state.scope_generation = self.scope_generation;
2328	2328	return state;
2329	2329	}
...	...	@@ -2394,9 +2394,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2394	2394	self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
2395	2395	}
2396	2396	{
2397		const reg = RegisterManager.regAtTrackedIndex(
2398		@as(RegisterManager.RegisterBitSet.ShiftInt, @intCast(index)),
2399		);
	2397	const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
2400	2398	self.register_manager.freeReg(reg);
2401	2399	self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
2402	2400	}
...	...	@@ -2630,7 +2628,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2630	2628
2631	2629	const dst_ty = self.typeOfIndex(inst);
2632	2630	const dst_int_info = dst_ty.intInfo(mod);
2633		const abi_size = @as(u32, @intCast(dst_ty.abiSize(mod)));
	2631	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
2634	2632
2635	2633	const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
2636	2634	const extend = switch (src_int_info.signedness) {
...	...	@@ -2708,9 +2706,9 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2708	2706	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2709	2707
2710	2708	const dst_ty = self.typeOfIndex(inst);
2711		const dst_abi_size = @as(u32, @intCast(dst_ty.abiSize(mod)));
	2709	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
2712	2710	const src_ty = self.typeOf(ty_op.operand);
2713		const src_abi_size = @as(u32, @intCast(src_ty.abiSize(mod)));
	2711	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
2714	2712
2715	2713	const result = result: {
2716	2714	const src_mcv = try self.resolveInst(ty_op.operand);
...	...	@@ -2727,7 +2725,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2727	2725	assert(src_ty.zigTypeTag(mod) == .Vector and dst_ty.vectorLen(mod) == src_ty.vectorLen(mod));
2728	2726	const dst_info = dst_ty.childType(mod).intInfo(mod);
2729	2727	const src_info = src_ty.childType(mod).intInfo(mod);
2730		const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (dst_info.bits) {
	2728	const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_info.bits) {
2731	2729	8 => switch (src_info.bits) {
2732	2730	16 => switch (dst_ty.vectorLen(mod)) {
2733	2731	1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
...	...	@@ -2750,18 +2748,18 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2750	2748	else => null,
2751	2749	},
2752	2750	else => null,
2753		})) \|tag\| tag else return self.fail("TODO implement airTrunc for {}", .{
	2751	}) orelse return self.fail("TODO implement airTrunc for {}", .{
2754	2752	dst_ty.fmt(self.bin_file.options.module.?),
2755	2753	});
2756	2754
2757	2755	const elem_ty = src_ty.childType(mod);
2758		const mask_val = try mod.intValue(elem_ty, @as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - dst_info.bits)));
	2756	const mask_val = try mod.intValue(elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
2759	2757
2760	2758	const splat_ty = try mod.vectorType(.{
2761		.len = @as(u32, @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits))),
	2759	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
2762	2760	.child = elem_ty.ip_index,
2763	2761	});
2764		const splat_abi_size = @as(u32, @intCast(splat_ty.abiSize(mod)));
	2762	const splat_abi_size: u32 = @intCast(splat_ty.abiSize(mod));
2765	2763
2766	2764	const splat_val = try mod.intern(.{ .aggregate = .{
2767	2765	.ty = splat_ty.ip_index,
...	...	@@ -2836,7 +2834,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
2836	2834	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
2837	2835	try self.genSetMem(
2838	2836	.{ .frame = frame_index },
2839		@as(i32, @intCast(ptr_ty.abiSize(mod))),
	2837	@intCast(ptr_ty.abiSize(mod)),
2840	2838	len_ty,
2841	2839	len,
2842	2840	);
...	...	@@ -2970,7 +2968,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2970	2968	try self.genSetReg(limit_reg, ty, dst_mcv);
2971	2969	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2972	2970	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2973		.immediate = (@as(u64, 1) << @as(u6, @intCast(reg_bits - 1))) - 1,
	2971	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
2974	2972	});
2975	2973	if (reg_extra_bits > 0) {
2976	2974	const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv);
...	...	@@ -2989,7 +2987,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2989	2987	break :cc .o;
2990	2988	} else cc: {
2991	2989	try self.genSetReg(limit_reg, ty, .{
2992		.immediate = @as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - ty.bitSize(mod))),
	2990	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - ty.bitSize(mod)),
2993	2991	});
2994	2992
2995	2993	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
...	...	@@ -3053,7 +3051,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
3053	3051	try self.genSetReg(limit_reg, ty, dst_mcv);
3054	3052	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
3055	3053	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
3056		.immediate = (@as(u64, 1) << @as(u6, @intCast(reg_bits - 1))) - 1,
	3054	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
3057	3055	});
3058	3056	if (reg_extra_bits > 0) {
3059	3057	const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv);
...	...	@@ -3128,12 +3126,12 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3128	3126	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
3129	3127	try self.genShiftBinOpMir(.{ ._, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
3130	3128	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
3131		.immediate = (@as(u64, 1) << @as(u6, @intCast(reg_bits - 1))) - 1,
	3129	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
3132	3130	});
3133	3131	break :cc .o;
3134	3132	} else cc: {
3135	3133	try self.genSetReg(limit_reg, ty, .{
3136		.immediate = @as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - reg_bits)),
	3134	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),
3137	3135	});
3138	3136	break :cc .c;
3139	3137	};
...	...	@@ -3186,13 +3184,13 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3186	3184	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
3187	3185	try self.genSetMem(
3188	3186	.{ .frame = frame_index },
3189		@as(i32, @intCast(tuple_ty.structFieldOffset(1, mod))),
	3187	@intCast(tuple_ty.structFieldOffset(1, mod)),
3190	3188	Type.u1,
3191	3189	.{ .eflags = cc },
3192	3190	);
3193	3191	try self.genSetMem(
3194	3192	.{ .frame = frame_index },
3195		@as(i32, @intCast(tuple_ty.structFieldOffset(0, mod))),
	3193	@intCast(tuple_ty.structFieldOffset(0, mod)),
3196	3194	ty,
3197	3195	partial_mcv,
3198	3196	);
...	...	@@ -3259,13 +3257,13 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3259	3257	try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
3260	3258	try self.genSetMem(
3261	3259	.{ .frame = frame_index },
3262		@as(i32, @intCast(tuple_ty.structFieldOffset(1, mod))),
	3260	@intCast(tuple_ty.structFieldOffset(1, mod)),
3263	3261	tuple_ty.structFieldType(1, mod),
3264	3262	.{ .eflags = cc },
3265	3263	);
3266	3264	try self.genSetMem(
3267	3265	.{ .frame = frame_index },
3268		@as(i32, @intCast(tuple_ty.structFieldOffset(0, mod))),
	3266	@intCast(tuple_ty.structFieldOffset(0, mod)),
3269	3267	tuple_ty.structFieldType(0, mod),
3270	3268	partial_mcv,
3271	3269	);
...	...	@@ -3333,7 +3331,7 @@ fn genSetFrameTruncatedOverflowCompare(
3333	3331	);
3334	3332	}
3335	3333
3336		const payload_off = @as(i32, @intCast(tuple_ty.structFieldOffset(0, mod)));
	3334	const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, mod));
3337	3335	if (hi_limb_off > 0) try self.genSetMem(.{ .frame = frame_index }, payload_off, rest_ty, src_mcv);
3338	3336	try self.genSetMem(
3339	3337	.{ .frame = frame_index },
...	...	@@ -3343,7 +3341,7 @@ fn genSetFrameTruncatedOverflowCompare(
3343	3341	);
3344	3342	try self.genSetMem(
3345	3343	.{ .frame = frame_index },
3346		@as(i32, @intCast(tuple_ty.structFieldOffset(1, mod))),
	3344	@intCast(tuple_ty.structFieldOffset(1, mod)),
3347	3345	tuple_ty.structFieldType(1, mod),
3348	3346	if (overflow_cc) \|_\| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
3349	3347	);
...	...	@@ -3400,13 +3398,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3400	3398	if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
3401	3399	try self.genSetMem(
3402	3400	.{ .frame = frame_index },
3403		@as(i32, @intCast(tuple_ty.structFieldOffset(0, mod))),
	3401	@intCast(tuple_ty.structFieldOffset(0, mod)),
3404	3402	tuple_ty.structFieldType(0, mod),
3405	3403	partial_mcv,
3406	3404	);
3407	3405	try self.genSetMem(
3408	3406	.{ .frame = frame_index },
3409		@as(i32, @intCast(tuple_ty.structFieldOffset(1, mod))),
	3407	@intCast(tuple_ty.structFieldOffset(1, mod)),
3410	3408	tuple_ty.structFieldType(1, mod),
3411	3409	.{ .immediate = 0 }, // cc being set is impossible
3412	3410	);
...	...	@@ -3430,7 +3428,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3430	3428	/// Quotient is saved in .rax and remainder in .rdx.
3431	3429	fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
3432	3430	const mod = self.bin_file.options.module.?;
3433		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	3431	const abi_size: u32 = @intCast(ty.abiSize(mod));
3434	3432	if (abi_size > 8) {
3435	3433	return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
3436	3434	}
...	...	@@ -3470,7 +3468,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
3470	3468	/// Clobbers .rax and .rdx registers.
3471	3469	fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
3472	3470	const mod = self.bin_file.options.module.?;
3473		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	3471	const abi_size: u32 = @intCast(ty.abiSize(mod));
3474	3472	const int_info = ty.intInfo(mod);
3475	3473	const dividend: Register = switch (lhs) {
3476	3474	.register => \|reg\| reg,
...	...	@@ -3609,7 +3607,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3609	3607	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
3610	3608
3611	3609	const pl_ty = dst_ty.childType(mod);
3612		const pl_abi_size = @as(i32, @intCast(pl_ty.abiSize(mod)));
	3610	const pl_abi_size: i32 = @intCast(pl_ty.abiSize(mod));
3613	3611	try self.genSetMem(.{ .reg = dst_mcv.getReg().? }, pl_abi_size, Type.bool, .{ .immediate = 1 });
3614	3612	break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
3615	3613	};
...	...	@@ -3641,17 +3639,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
3641	3639	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
3642	3640
3643	3641	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
3644		if (err_off > 0) {
3645		const shift = @as(u6, @intCast(err_off * 8));
3646		try self.genShiftBinOpMir(
3647		.{ ._r, .sh },
3648		err_union_ty,
3649		result,
3650		.{ .immediate = shift },
3651		);
3652		} else {
3653		try self.truncateRegister(Type.anyerror, result.register);
3654		}
	3642	if (err_off > 0) try self.genShiftBinOpMir(
	3643	.{ ._r, .sh },
	3644	err_union_ty,
	3645	result,
	3646	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
	3647	) else try self.truncateRegister(Type.anyerror, result.register);
3655	3648	break :result result;
3656	3649	},
3657	3650	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
...	...	@@ -3699,17 +3692,12 @@ fn genUnwrapErrorUnionPayloadMir(
3699	3692	try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)
3700	3693	else
3701	3694	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
3702		if (payload_off > 0) {
3703		const shift = @as(u6, @intCast(payload_off * 8));
3704		try self.genShiftBinOpMir(
3705		.{ ._r, .sh },
3706		err_union_ty,
3707		result_mcv,
3708		.{ .immediate = shift },
3709		);
3710		} else {
3711		try self.truncateRegister(payload_ty, result_mcv.register);
3712		}
	3695	if (payload_off > 0) try self.genShiftBinOpMir(
	3696	.{ ._r, .sh },
	3697	err_union_ty,
	3698	result_mcv,
	3699	.{ .immediate = @as(u6, @intCast(payload_off * 8)) },
	3700	) else try self.truncateRegister(payload_ty, result_mcv.register);
3713	3701	break :result result_mcv;
3714	3702	},
3715	3703	else => return self.fail("TODO implement genUnwrapErrorUnionPayloadMir for {}", .{err_union}),
...	...	@@ -3741,8 +3729,8 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3741	3729	const eu_ty = src_ty.childType(mod);
3742	3730	const pl_ty = eu_ty.errorUnionPayload(mod);
3743	3731	const err_ty = eu_ty.errorUnionSet(mod);
3744		const err_off = @as(i32, @intCast(errUnionErrorOffset(pl_ty, mod)));
3745		const err_abi_size = @as(u32, @intCast(err_ty.abiSize(mod)));
	3732	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
	3733	const err_abi_size: u32 = @intCast(err_ty.abiSize(mod));
3746	3734	try self.asmRegisterMemory(
3747	3735	.{ ._, .mov },
3748	3736	registerAlias(dst_reg, err_abi_size),
...	...	@@ -3780,8 +3768,8 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
3780	3768
3781	3769	const eu_ty = src_ty.childType(mod);
3782	3770	const pl_ty = eu_ty.errorUnionPayload(mod);
3783		const pl_off = @as(i32, @intCast(errUnionPayloadOffset(pl_ty, mod)));
3784		const dst_abi_size = @as(u32, @intCast(dst_ty.abiSize(mod)));
	3771	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
	3772	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
3785	3773	try self.asmRegisterMemory(
3786	3774	.{ ._, .lea },
3787	3775	registerAlias(dst_reg, dst_abi_size),
...	...	@@ -3807,8 +3795,8 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3807	3795	const eu_ty = src_ty.childType(mod);
3808	3796	const pl_ty = eu_ty.errorUnionPayload(mod);
3809	3797	const err_ty = eu_ty.errorUnionSet(mod);
3810		const err_off = @as(i32, @intCast(errUnionErrorOffset(pl_ty, mod)));
3811		const err_abi_size = @as(u32, @intCast(err_ty.abiSize(mod)));
	3798	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
	3799	const err_abi_size: u32 = @intCast(err_ty.abiSize(mod));
3812	3800	try self.asmMemoryImmediate(
3813	3801	.{ ._, .mov },
3814	3802	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
...	...	@@ -3828,8 +3816,8 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3828	3816	const dst_lock = self.register_manager.lockReg(dst_reg);
3829	3817	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
3830	3818
3831		const pl_off = @as(i32, @intCast(errUnionPayloadOffset(pl_ty, mod)));
3832		const dst_abi_size = @as(u32, @intCast(dst_ty.abiSize(mod)));
	3819	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
	3820	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
3833	3821	try self.asmRegisterMemory(
3834	3822	.{ ._, .lea },
3835	3823	registerAlias(dst_reg, dst_abi_size),
...	...	@@ -3878,7 +3866,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
3878	3866	try self.genCopy(pl_ty, opt_mcv, pl_mcv);
3879	3867
3880	3868	if (!same_repr) {
3881		const pl_abi_size = @as(i32, @intCast(pl_ty.abiSize(mod)));
	3869	const pl_abi_size: i32 = @intCast(pl_ty.abiSize(mod));
3882	3870	switch (opt_mcv) {
3883	3871	else => unreachable,
3884	3872
...	...	@@ -3917,8 +3905,8 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
3917	3905	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .{ .immediate = 0 };
3918	3906
3919	3907	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));
3920		const pl_off = @as(i32, @intCast(errUnionPayloadOffset(pl_ty, mod)));
3921		const err_off = @as(i32, @intCast(errUnionErrorOffset(pl_ty, mod)));
	3908	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
	3909	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
3922	3910	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
3923	3911	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
3924	3912	break :result .{ .load_frame = .{ .index = frame_index } };
...	...	@@ -3939,8 +3927,8 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
3939	3927	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);
3940	3928
3941	3929	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));
3942		const pl_off = @as(i32, @intCast(errUnionPayloadOffset(pl_ty, mod)));
3943		const err_off = @as(i32, @intCast(errUnionErrorOffset(pl_ty, mod)));
	3930	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
	3931	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
3944	3932	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
3945	3933	const operand = try self.resolveInst(ty_op.operand);
3946	3934	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
...	...	@@ -4002,7 +3990,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
4002	3990	const dst_lock = self.register_manager.lockReg(dst_reg);
4003	3991	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
4004	3992
4005		const dst_abi_size = @as(u32, @intCast(dst_ty.abiSize(mod)));
	3993	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
4006	3994	try self.asmRegisterMemory(
4007	3995	.{ ._, .lea },
4008	3996	registerAlias(dst_reg, dst_abi_size),
...	...	@@ -4179,7 +4167,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
4179	4167	// additional `mov` is needed at the end to get the actual value
4180	4168
4181	4169	const elem_ty = ptr_ty.elemType2(mod);
4182		const elem_abi_size = @as(u32, @intCast(elem_ty.abiSize(mod)));
	4170	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
4183	4171	const index_ty = self.typeOf(bin_op.rhs);
4184	4172	const index_mcv = try self.resolveInst(bin_op.rhs);
4185	4173	const index_lock = switch (index_mcv) {
...	...	@@ -4319,7 +4307,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
4319	4307	.load_frame => \|frame_addr\| {
4320	4308	if (tag_abi_size <= 8) {
4321	4309	const off: i32 = if (layout.tag_align < layout.payload_align)
4322		@as(i32, @intCast(layout.payload_size))
	4310	@intCast(layout.payload_size)
4323	4311	else
4324	4312	0;
4325	4313	break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{
...	...	@@ -4331,13 +4319,13 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
4331	4319	},
4332	4320	.register => {
4333	4321	const shift: u6 = if (layout.tag_align < layout.payload_align)
4334		@as(u6, @intCast(layout.payload_size * 8))
	4322	@intCast(layout.payload_size * 8)
4335	4323	else
4336	4324	0;
4337	4325	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
4338	4326	try self.genShiftBinOpMir(.{ ._r, .sh }, Type.usize, result, .{ .immediate = shift });
4339	4327	break :blk MCValue{
4340		.register = registerAlias(result.register, @as(u32, @intCast(layout.tag_size))),
	4328	.register = registerAlias(result.register, @intCast(layout.tag_size)),
4341	4329	};
4342	4330	},
4343	4331	else => return self.fail("TODO implement get_union_tag for {}", .{operand}),
...	...	@@ -4447,7 +4435,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4447	4435	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
4448	4436	} else {
4449	4437	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
4450		.immediate = @as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - self.regBitSize(dst_ty))),
	4438	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
4451	4439	});
4452	4440	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
4453	4441	defer self.register_manager.unlockReg(imm_lock);
...	...	@@ -4518,8 +4506,8 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4518	4506	.{ ._, .@"or" },
4519	4507	wide_ty,
4520	4508	tmp_mcv,
4521		.{ .immediate = (@as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - extra_bits))) <<
4522		@as(u6, @intCast(src_bits)) },
	4509	.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
	4510	@intCast(src_bits) },
4523	4511	);
4524	4512	break :masked tmp_mcv;
4525	4513	} else mat_src_mcv;
...	...	@@ -4536,7 +4524,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4536	4524	.{ ._, .@"or" },
4537	4525	Type.u64,
4538	4526	dst_mcv,
4539		.{ .immediate = @as(u64, math.maxInt(u64)) << @as(u6, @intCast(src_bits - 64)) },
	4527	.{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(src_bits - 64) },
4540	4528	);
4541	4529	break :masked dst_mcv;
4542	4530	} else mat_src_mcv.address().offset(8).deref();
...	...	@@ -4607,7 +4595,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4607	4595	break :result dst_mcv;
4608	4596	}
4609	4597
4610		const mask = @as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - src_abi_size * 8));
	4598	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
4611	4599	const imm_0_1 = Immediate.u(mask / 0b1_1);
4612	4600	const imm_00_11 = Immediate.u(mask / 0b01_01);
4613	4601	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
...	...	@@ -4780,7 +4768,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4780	4768	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
4781	4769
4782	4770	const src_ty = self.typeOf(ty_op.operand);
4783		const src_abi_size = @as(u32, @intCast(src_ty.abiSize(mod)));
	4771	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
4784	4772	const src_mcv = try self.resolveInst(ty_op.operand);
4785	4773
4786	4774	const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, false);
...	...	@@ -4800,7 +4788,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4800	4788	else
4801	4789	undefined;
4802	4790
4803		const mask = @as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - src_abi_size * 8));
	4791	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
4804	4792	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
4805	4793	const imm_00_11 = Immediate.u(mask / 0b01_01);
4806	4794	const imm_0_1 = Immediate.u(mask / 0b1_1);
...	...	@@ -5016,7 +5004,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
5016	5004	if (!self.hasFeature(.sse4_1))
5017	5005	return self.fail("TODO implement genRound without sse4_1 feature", .{});
5018	5006
5019		const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) {
	5007	const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) {
5020	5008	.Float => switch (ty.floatBits(self.target.*)) {
5021	5009	32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
5022	5010	64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
...	...	@@ -5043,10 +5031,10 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
5043	5031	else => null,
5044	5032	},
5045	5033	else => unreachable,
5046		})) \|tag\| tag else return self.fail("TODO implement genRound for {}", .{
	5034	}) orelse return self.fail("TODO implement genRound for {}", .{
5047	5035	ty.fmt(self.bin_file.options.module.?),
5048	5036	});
5049		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	5037	const abi_size: u32 = @intCast(ty.abiSize(mod));
5050	5038	const dst_alias = registerAlias(dst_reg, abi_size);
5051	5039	switch (mir_tag[0]) {
5052	5040	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
...	...	@@ -5086,7 +5074,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
5086	5074	const mod = self.bin_file.options.module.?;
5087	5075	const un_op = self.air.instructions.items(.data)[inst].un_op;
5088	5076	const ty = self.typeOf(un_op);
5089		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	5077	const abi_size: u32 = @intCast(ty.abiSize(mod));
5090	5078
5091	5079	const src_mcv = try self.resolveInst(un_op);
5092	5080	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
...	...	@@ -5098,7 +5086,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
5098	5086	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5099	5087
5100	5088	const result: MCValue = result: {
5101		const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) {
	5089	const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) {
5102	5090	.Float => switch (ty.floatBits(self.target.*)) {
5103	5091	16 => if (self.hasFeature(.f16c)) {
5104	5092	const mat_src_reg = if (src_mcv.isRegister())
...	...	@@ -5152,7 +5140,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
5152	5140	.{ .v_ps, .cvtph2 },
5153	5141	wide_reg,
5154	5142	src_mcv.mem(Memory.PtrSize.fromSize(
5155		@as(u32, @intCast(@divExact(wide_reg.bitSize(), 16))),
	5143	@intCast(@divExact(wide_reg.bitSize(), 16)),
5156	5144	)),
5157	5145	) else try self.asmRegisterRegister(
5158	5146	.{ .v_ps, .cvtph2 },
...	...	@@ -5191,7 +5179,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
5191	5179	else => unreachable,
5192	5180	},
5193	5181	else => unreachable,
5194		})) \|tag\| tag else return self.fail("TODO implement airSqrt for {}", .{
	5182	}) orelse return self.fail("TODO implement airSqrt for {}", .{
5195	5183	ty.fmt(mod),
5196	5184	});
5197	5185	switch (mir_tag[0]) {
...	...	@@ -5284,10 +5272,10 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
5284	5272	const ptr_info = ptr_ty.ptrInfo(mod);
5285	5273
5286	5274	const val_ty = ptr_info.child.toType();
5287		const val_abi_size = @as(u32, @intCast(val_ty.abiSize(mod)));
	5275	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
5288	5276	const limb_abi_size: u32 = @min(val_abi_size, 8);
5289	5277	const limb_abi_bits = limb_abi_size * 8;
5290		const val_byte_off = @as(i32, @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size));
	5278	const val_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
5291	5279	const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;
5292	5280	const val_extra_bits = self.regExtraBits(val_ty);
5293	5281
...	...	@@ -5434,7 +5422,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
5434	5422	const limb_ty = try mod.intType(.unsigned, limb_abi_bits);
5435	5423
5436	5424	const src_bit_size = src_ty.bitSize(mod);
5437		const src_byte_off = @as(i32, @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size));
	5425	const src_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
5438	5426	const src_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;
5439	5427
5440	5428	const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
...	...	@@ -5451,10 +5439,9 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
5451	5439	.disp = src_byte_off + limb_i * limb_abi_bits,
5452	5440	});
5453	5441
5454		const part_mask = (@as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - part_bit_size))) <<
5455		@as(u6, @intCast(part_bit_off));
5456		const part_mask_not = part_mask ^
5457		(@as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - limb_abi_bits)));
	5442	const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
	5443	@intCast(part_bit_off);
	5444	const part_mask_not = part_mask ^ (@as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));
5458	5445	if (limb_abi_size <= 4) {
5459	5446	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
5460	5447	} else if (math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {
...	...	@@ -5572,14 +5559,14 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
5572	5559	const ptr_field_ty = self.typeOfIndex(inst);
5573	5560	const ptr_container_ty = self.typeOf(operand);
5574	5561	const container_ty = ptr_container_ty.childType(mod);
5575		const field_offset = @as(i32, @intCast(switch (container_ty.containerLayout(mod)) {
	5562	const field_offset: i32 = @intCast(switch (container_ty.containerLayout(mod)) {
5576	5563	.Auto, .Extern => container_ty.structFieldOffset(index, mod),
5577	5564	.Packed => if (container_ty.zigTypeTag(mod) == .Struct and
5578	5565	ptr_field_ty.ptrInfo(mod).packed_offset.host_size == 0)
5579	5566	container_ty.packedStructFieldByteOffset(index, mod)
5580	5567	else
5581	5568	0,
5582		}));
	5569	});
5583	5570
5584	5571	const src_mcv = try self.resolveInst(operand);
5585	5572	const dst_mcv = if (switch (src_mcv) {
...	...	@@ -5606,8 +5593,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5606	5593	const field_is_gp = field_rc.supersetOf(gp);
5607	5594
5608	5595	const src_mcv = try self.resolveInst(operand);
5609		const field_off = switch (container_ty.containerLayout(mod)) {
5610		.Auto, .Extern => @as(u32, @intCast(container_ty.structFieldOffset(index, mod) * 8)),
	5596	const field_off: u32 = switch (container_ty.containerLayout(mod)) {
	5597	.Auto, .Extern => @intCast(container_ty.structFieldOffset(index, mod) * 8),
5611	5598	.Packed => if (mod.typeToStruct(container_ty)) \|struct_obj\|
5612	5599	struct_obj.packedFieldBitOffset(mod, index)
5613	5600	else
...	...	@@ -5650,7 +5637,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5650	5637
5651	5638	const limb_abi_size: u32 = @min(field_abi_size, 8);
5652	5639	const limb_abi_bits = limb_abi_size * 8;
5653		const field_byte_off = @as(i32, @intCast(field_off / limb_abi_bits * limb_abi_size));
	5640	const field_byte_off: i32 = @intCast(field_off / limb_abi_bits * limb_abi_size);
5654	5641	const field_bit_off = field_off % limb_abi_bits;
5655	5642
5656	5643	if (field_abi_size > 8) {
...	...	@@ -5775,7 +5762,7 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
5775	5762
5776	5763	const inst_ty = self.typeOfIndex(inst);
5777	5764	const parent_ty = inst_ty.childType(mod);
5778		const field_offset = @as(i32, @intCast(parent_ty.structFieldOffset(extra.field_index, mod)));
	5765	const field_offset: i32 = @intCast(parent_ty.structFieldOffset(extra.field_index, mod));
5779	5766
5780	5767	const src_mcv = try self.resolveInst(extra.field_ptr);
5781	5768	const dst_mcv = if (src_mcv.isRegisterOffset() and
...	...	@@ -5824,14 +5811,14 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
5824	5811
5825	5812	switch (tag) {
5826	5813	.not => {
5827		const limb_abi_size = @as(u16, @intCast(@min(src_ty.abiSize(mod), 8)));
	5814	const limb_abi_size: u16 = @intCast(@min(src_ty.abiSize(mod), 8));
5828	5815	const int_info = if (src_ty.ip_index == .bool_type)
5829	5816	std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 }
5830	5817	else
5831	5818	src_ty.intInfo(mod);
5832	5819	var byte_off: i32 = 0;
5833	5820	while (byte_off * 8 < int_info.bits) : (byte_off += limb_abi_size) {
5834		const limb_bits = @as(u16, @intCast(@min(int_info.bits - byte_off * 8, limb_abi_size * 8)));
	5821	const limb_bits: u16 = @intCast(@min(int_info.bits - byte_off * 8, limb_abi_size * 8));
5835	5822	const limb_ty = try mod.intType(int_info.signedness, limb_bits);
5836	5823	const limb_mcv = switch (byte_off) {
5837	5824	0 => dst_mcv,
...	...	@@ -5839,7 +5826,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
5839	5826	};
5840	5827
5841	5828	if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {
5842		const mask = @as(u64, math.maxInt(u64)) >> @as(u6, @intCast(64 - limb_bits));
	5829	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_bits);
5843	5830	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
5844	5831	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
5845	5832	}
...	...	@@ -5852,7 +5839,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
5852	5839
5853	5840	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
5854	5841	const mod = self.bin_file.options.module.?;
5855		const abi_size = @as(u32, @intCast(dst_ty.abiSize(mod)));
	5842	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
5856	5843	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
5857	5844	mir_tag,
5858	5845	dst_ty.fmt(self.bin_file.options.module.?),
...	...	@@ -5914,7 +5901,7 @@ fn genShiftBinOpMir(
5914	5901	break :rhs .{ .register = .rcx };
5915	5902	};
5916	5903
5917		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	5904	const abi_size: u32 = @intCast(ty.abiSize(mod));
5918	5905	if (abi_size <= 8) {
5919	5906	switch (lhs_mcv) {
5920	5907	.register => \|lhs_reg\| switch (rhs_mcv) {
...	...	@@ -6602,7 +6589,7 @@ fn genBinOp(
6602	6589	Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
6603	6590	.memory => \|addr\| .{
6604	6591	.base = .{ .reg = .ds },
6605		.disp = @as(i32, @intCast(@as(i64, @bitCast(addr)))),
	6592	.disp = @intCast(@as(i64, @bitCast(addr))),
6606	6593	},
6607	6594	.indirect => \|reg_off\| .{
6608	6595	.base = .{ .reg = reg_off.reg },
...	...	@@ -6628,7 +6615,7 @@ fn genBinOp(
6628	6615	}
6629	6616
6630	6617	const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
6631		const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	6618	const mir_tag = @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
6632	6619	else => unreachable,
6633	6620	.Float => switch (lhs_ty.floatBits(self.target.*)) {
6634	6621	16 => if (self.hasFeature(.f16c)) {
...	...	@@ -7190,7 +7177,7 @@ fn genBinOp(
7190	7177	else => unreachable,
7191	7178	},
7192	7179	},
7193		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7180	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7194	7181	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7195	7182	});
7196	7183
...	...	@@ -7253,7 +7240,7 @@ fn genBinOp(
7253	7240	const rhs_copy_reg = registerAlias(src_mcv.getReg().?, abi_size);
7254	7241
7255	7242	try self.asmRegisterRegisterRegisterImmediate(
7256		if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	7243	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
7257	7244	.Float => switch (lhs_ty.floatBits(self.target.*)) {
7258	7245	32 => .{ .v_ss, .cmp },
7259	7246	64 => .{ .v_sd, .cmp },
...	...	@@ -7278,7 +7265,7 @@ fn genBinOp(
7278	7265	else => unreachable,
7279	7266	},
7280	7267	else => unreachable,
7281		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7268	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7282	7269	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7283	7270	}),
7284	7271	mask_reg,
...	...	@@ -7287,7 +7274,7 @@ fn genBinOp(
7287	7274	Immediate.u(3), // unord
7288	7275	);
7289	7276	try self.asmRegisterRegisterRegisterRegister(
7290		if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	7277	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
7291	7278	.Float => switch (lhs_ty.floatBits(self.target.*)) {
7292	7279	32 => .{ .v_ps, .blendv },
7293	7280	64 => .{ .v_pd, .blendv },
...	...	@@ -7310,7 +7297,7 @@ fn genBinOp(
7310	7297	else => unreachable,
7311	7298	},
7312	7299	else => unreachable,
7313		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7300	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7314	7301	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7315	7302	}),
7316	7303	dst_reg,
...	...	@@ -7321,7 +7308,7 @@ fn genBinOp(
7321	7308	} else {
7322	7309	const has_blend = self.hasFeature(.sse4_1);
7323	7310	try self.asmRegisterRegisterImmediate(
7324		if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	7311	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
7325	7312	.Float => switch (lhs_ty.floatBits(self.target.*)) {
7326	7313	32 => .{ ._ss, .cmp },
7327	7314	64 => .{ ._sd, .cmp },
...	...	@@ -7346,7 +7333,7 @@ fn genBinOp(
7346	7333	else => unreachable,
7347	7334	},
7348	7335	else => unreachable,
7349		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7336	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7350	7337	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7351	7338	}),
7352	7339	mask_reg,
...	...	@@ -7354,7 +7341,7 @@ fn genBinOp(
7354	7341	Immediate.u(if (has_blend) 3 else 7), // unord, ord
7355	7342	);
7356	7343	if (has_blend) try self.asmRegisterRegisterRegister(
7357		if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	7344	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
7358	7345	.Float => switch (lhs_ty.floatBits(self.target.*)) {
7359	7346	32 => .{ ._ps, .blendv },
7360	7347	64 => .{ ._pd, .blendv },
...	...	@@ -7377,7 +7364,7 @@ fn genBinOp(
7377	7364	else => unreachable,
7378	7365	},
7379	7366	else => unreachable,
7380		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7367	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7381	7368	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7382	7369	}),
7383	7370	dst_reg,
...	...	@@ -7385,7 +7372,7 @@ fn genBinOp(
7385	7372	mask_reg,
7386	7373	) else {
7387	7374	try self.asmRegisterRegister(
7388		if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	7375	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
7389	7376	.Float => switch (lhs_ty.floatBits(self.target.*)) {
7390	7377	32 => .{ ._ps, .@"and" },
7391	7378	64 => .{ ._pd, .@"and" },
...	...	@@ -7408,14 +7395,14 @@ fn genBinOp(
7408	7395	else => unreachable,
7409	7396	},
7410	7397	else => unreachable,
7411		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7398	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7412	7399	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7413	7400	}),
7414	7401	dst_reg,
7415	7402	mask_reg,
7416	7403	);
7417	7404	try self.asmRegisterRegister(
7418		if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	7405	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
7419	7406	.Float => switch (lhs_ty.floatBits(self.target.*)) {
7420	7407	32 => .{ ._ps, .andn },
7421	7408	64 => .{ ._pd, .andn },
...	...	@@ -7438,14 +7425,14 @@ fn genBinOp(
7438	7425	else => unreachable,
7439	7426	},
7440	7427	else => unreachable,
7441		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7428	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7442	7429	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7443	7430	}),
7444	7431	mask_reg,
7445	7432	lhs_copy_reg.?,
7446	7433	);
7447	7434	try self.asmRegisterRegister(
7448		if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
	7435	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
7449	7436	.Float => switch (lhs_ty.floatBits(self.target.*)) {
7450	7437	32 => .{ ._ps, .@"or" },
7451	7438	64 => .{ ._pd, .@"or" },
...	...	@@ -7468,7 +7455,7 @@ fn genBinOp(
7468	7455	else => unreachable,
7469	7456	},
7470	7457	else => unreachable,
7471		})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	7458	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
7472	7459	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7473	7460	}),
7474	7461	dst_reg,
...	...	@@ -7490,7 +7477,7 @@ fn genBinOpMir(
7490	7477	src_mcv: MCValue,
7491	7478	) !void {
7492	7479	const mod = self.bin_file.options.module.?;
7493		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	7480	const abi_size: u32 = @intCast(ty.abiSize(mod));
7494	7481	switch (dst_mcv) {
7495	7482	.none,
7496	7483	.unreach,
...	...	@@ -7738,10 +7725,10 @@ fn genBinOpMir(
7738	7725	else => unreachable,
7739	7726	},
7740	7727	.immediate => \|src_imm\| {
7741		const imm = switch (off) {
	7728	const imm: u64 = switch (off) {
7742	7729	0 => src_imm,
7743	7730	else => switch (ty_signedness) {
7744		.signed => @as(u64, @bitCast(@as(i64, @bitCast(src_imm)) >> 63)),
	7731	.signed => @bitCast(@as(i64, @bitCast(src_imm)) >> 63),
7745	7732	.unsigned => 0,
7746	7733	},
7747	7734	};
...	...	@@ -7814,7 +7801,7 @@ fn genBinOpMir(
7814	7801	0 => src_mcv,
7815	7802	else => .{ .immediate = 0 },
7816	7803	},
7817		.memory => \|addr\| .{ .memory = @as(u64, @bitCast(@as(i64, @bitCast(addr)) + off)) },
	7804	.memory => \|addr\| .{ .memory = @bitCast(@as(i64, @bitCast(addr)) + off) },
7818	7805	.indirect => \|reg_off\| .{ .indirect = .{
7819	7806	.reg = reg_off.reg,
7820	7807	.off = reg_off.off + off,
...	...	@@ -7841,7 +7828,7 @@ fn genBinOpMir(
7841	7828	/// Does not support byte-size operands.
7842	7829	fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
7843	7830	const mod = self.bin_file.options.module.?;
7844		const abi_size = @as(u32, @intCast(dst_ty.abiSize(mod)));
	7831	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
7845	7832	switch (dst_mcv) {
7846	7833	.none,
7847	7834	.unreach,
...	...	@@ -8075,7 +8062,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
8075	8062	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
8076	8063	const callee = pl_op.operand;
8077	8064	const extra = self.air.extraData(Air.Call, pl_op.payload);
8078		const args = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]));
	8065	const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]);
8079	8066	const ty = self.typeOf(callee);
8080	8067
8081	8068	const fn_ty = switch (ty.zigTypeTag(mod)) {
...	...	@@ -8168,7 +8155,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
8168	8155	const got_addr = atom.getOffsetTableAddress(elf_file);
8169	8156	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
8170	8157	.base = .{ .reg = .ds },
8171		.disp = @as(i32, @intCast(got_addr)),
	8158	.disp = @intCast(got_addr),
8172	8159	}));
8173	8160	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
8174	8161	const atom = try coff_file.getOrCreateAtomForDecl(owner_decl);
...	...	@@ -8185,7 +8172,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
8185	8172	const atom = p9.getAtom(atom_index);
8186	8173	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
8187	8174	.base = .{ .reg = .ds },
8188		.disp = @as(i32, @intCast(atom.getOffsetTableAddress(p9))),
	8175	.disp = @intCast(atom.getOffsetTableAddress(p9)),
8189	8176	}));
8190	8177	} else unreachable;
8191	8178	} else if (func_value.getExternFunc(mod)) \|extern_func\| {
...	...	@@ -8305,7 +8292,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
8305	8292	const result = MCValue{
8306	8293	.eflags = switch (ty.zigTypeTag(mod)) {
8307	8294	else => result: {
8308		const abi_size = @as(u16, @intCast(ty.abiSize(mod)));
	8295	const abi_size: u16 = @intCast(ty.abiSize(mod));
8309	8296	const may_flip: enum {
8310	8297	may_flip,
8311	8298	must_flip,
...	...	@@ -8502,7 +8489,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
8502	8489	self.eflags_inst = inst;
8503	8490
8504	8491	const op_ty = self.typeOf(un_op);
8505		const op_abi_size = @as(u32, @intCast(op_ty.abiSize(mod)));
	8492	const op_abi_size: u32 = @intCast(op_ty.abiSize(mod));
8506	8493	const op_mcv = try self.resolveInst(un_op);
8507	8494	const dst_reg = switch (op_mcv) {
8508	8495	.register => \|reg\| reg,
...	...	@@ -8711,7 +8698,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
8711	8698	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(mod))
8712	8699	.{ .off = 0, .ty = if (pl_ty.isSlice(mod)) pl_ty.slicePtrFieldType(mod) else pl_ty }
8713	8700	else
8714		.{ .off = @as(i32, @intCast(pl_ty.abiSize(mod))), .ty = Type.bool };
	8701	.{ .off = @intCast(pl_ty.abiSize(mod)), .ty = Type.bool };
8715	8702
8716	8703	switch (opt_mcv) {
8717	8704	.none,
...	...	@@ -8731,14 +8718,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
8731	8718
8732	8719	.register => \|opt_reg\| {
8733	8720	if (some_info.off == 0) {
8734		const some_abi_size = @as(u32, @intCast(some_info.ty.abiSize(mod)));
	8721	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
8735	8722	const alias_reg = registerAlias(opt_reg, some_abi_size);
8736	8723	assert(some_abi_size * 8 == alias_reg.bitSize());
8737	8724	try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
8738	8725	return .{ .eflags = .z };
8739	8726	}
8740	8727	assert(some_info.ty.ip_index == .bool_type);
8741		const opt_abi_size = @as(u32, @intCast(opt_ty.abiSize(mod)));
	8728	const opt_abi_size: u32 = @intCast(opt_ty.abiSize(mod));
8742	8729	try self.asmRegisterImmediate(
8743	8730	.{ ._, .bt },
8744	8731	registerAlias(opt_reg, opt_abi_size),
...	...	@@ -8757,7 +8744,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
8757	8744	defer self.register_manager.unlockReg(addr_reg_lock);
8758	8745
8759	8746	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
8760		const some_abi_size = @as(u32, @intCast(some_info.ty.abiSize(mod)));
	8747	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
8761	8748	try self.asmMemoryImmediate(
8762	8749	.{ ._, .cmp },
8763	8750	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
...	...	@@ -8770,7 +8757,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
8770	8757	},
8771	8758
8772	8759	.indirect, .load_frame => {
8773		const some_abi_size = @as(u32, @intCast(some_info.ty.abiSize(mod)));
	8760	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
8774	8761	try self.asmMemoryImmediate(
8775	8762	.{ ._, .cmp },
8776	8763	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
...	...	@@ -8802,7 +8789,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
8802	8789	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(mod))
8803	8790	.{ .off = 0, .ty = if (pl_ty.isSlice(mod)) pl_ty.slicePtrFieldType(mod) else pl_ty }
8804	8791	else
8805		.{ .off = @as(i32, @intCast(pl_ty.abiSize(mod))), .ty = Type.bool };
	8792	.{ .off = @intCast(pl_ty.abiSize(mod)), .ty = Type.bool };
8806	8793
8807	8794	const ptr_reg = switch (ptr_mcv) {
8808	8795	.register => \|reg\| reg,
...	...	@@ -8811,7 +8798,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
8811	8798	const ptr_lock = self.register_manager.lockReg(ptr_reg);
8812	8799	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
8813	8800
8814		const some_abi_size = @as(u32, @intCast(some_info.ty.abiSize(mod)));
	8801	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
8815	8802	try self.asmMemoryImmediate(
8816	8803	.{ ._, .cmp },
8817	8804	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
...	...	@@ -8844,12 +8831,11 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
8844	8831
8845	8832	const tmp_reg = try self.copyToTmpRegister(ty, operand);
8846	8833	if (err_off > 0) {
8847		const shift = @as(u6, @intCast(err_off * 8));
8848	8834	try self.genShiftBinOpMir(
8849	8835	.{ ._r, .sh },
8850	8836	ty,
8851	8837	.{ .register = tmp_reg },
8852		.{ .immediate = shift },
	8838	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
8853	8839	);
8854	8840	} else {
8855	8841	try self.truncateRegister(Type.anyerror, tmp_reg);
...	...	@@ -9004,7 +8990,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
9004	8990	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
9005	8991	const loop = self.air.extraData(Air.Block, ty_pl.payload);
9006	8992	const body = self.air.extra[loop.end..][0..loop.data.body_len];
9007		const jmp_target = @as(u32, @intCast(self.mir_instructions.len));
	8993	const jmp_target: u32 = @intCast(self.mir_instructions.len);
9008	8994
9009	8995	self.scope_generation += 1;
9010	8996	const state = try self.saveState();
...	...	@@ -9076,10 +9062,8 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
9076	9062
9077	9063	while (case_i < switch_br.data.cases_len) : (case_i += 1) {
9078	9064	const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
9079		const items = @as(
9080		[]const Air.Inst.Ref,
9081		@ptrCast(self.air.extra[case.end..][0..case.data.items_len]),
9082		);
	9065	const items: []const Air.Inst.Ref =
	9066	@ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
9083	9067	const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
9084	9068	extra_index = case.end + items.len + case_body.len;
9085	9069
...	...	@@ -9127,7 +9111,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
9127	9111	}
9128	9112
9129	9113	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
9130		const next_inst = @as(u32, @intCast(self.mir_instructions.len));
	9114	const next_inst: u32 = @intCast(self.mir_instructions.len);
9131	9115	switch (self.mir_instructions.items(.tag)[reloc]) {
9132	9116	.j, .jmp => {},
9133	9117	.pseudo => switch (self.mir_instructions.items(.ops)[reloc]) {
...	...	@@ -9202,11 +9186,12 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
9202	9186	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9203	9187	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
9204	9188	const extra = self.air.extraData(Air.Asm, ty_pl.payload);
9205		const clobbers_len = @as(u31, @truncate(extra.data.flags));
	9189	const clobbers_len: u31 = @truncate(extra.data.flags);
9206	9190	var extra_i: usize = extra.end;
9207		const outputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]));
	9191	const outputs: []const Air.Inst.Ref =
	9192	@ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
9208	9193	extra_i += outputs.len;
9209		const inputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]));
	9194	const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);
9210	9195	extra_i += inputs.len;
9211	9196
9212	9197	var result: MCValue = .none;
...	...	@@ -9341,16 +9326,14 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9341	9326	} else if (mem.startsWith(u8, op_str, "$")) {
9342	9327	if (std.fmt.parseInt(i32, op_str["$".len..], 0)) \|s\| {
9343	9328	if (mnem_size) \|size\| {
9344		const max = @as(u64, math.maxInt(u64)) >>
9345		@as(u6, @intCast(64 - (size.bitSize() - 1)));
	9329	const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));
9346	9330	if ((if (s < 0) ~s else s) > max)
9347	9331	return self.fail("Invalid immediate size: '{s}'", .{op_str});
9348	9332	}
9349	9333	op.* = .{ .imm = Immediate.s(s) };
9350	9334	} else \|_\| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) \|u\| {
9351	9335	if (mnem_size) \|size\| {
9352		const max = @as(u64, math.maxInt(u64)) >>
9353		@as(u6, @intCast(64 - size.bitSize()));
	9336	const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - size.bitSize());
9354	9337	if (u > max)
9355	9338	return self.fail("Invalid immediate size: '{s}'", .{op_str});
9356	9339	}
...	...	@@ -9702,7 +9685,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
9702	9685
9703	9686	fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
9704	9687	const mod = self.bin_file.options.module.?;
9705		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	9688	const abi_size: u32 = @intCast(ty.abiSize(mod));
9706	9689	if (abi_size * 8 > dst_reg.bitSize())
9707	9690	return self.fail("genSetReg called with a value larger than dst_reg", .{});
9708	9691	switch (src_mcv) {
...	...	@@ -9727,7 +9710,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
9727	9710	try self.asmRegisterImmediate(
9728	9711	.{ ._, .mov },
9729	9712	registerAlias(dst_reg, abi_size),
9730		Immediate.s(@as(i32, @intCast(@as(i64, @bitCast(imm))))),
	9713	Immediate.s(@intCast(@as(i64, @bitCast(imm)))),
9731	9714	);
9732	9715	} else {
9733	9716	try self.asmRegisterImmediate(
...	...	@@ -9785,7 +9768,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
9785	9768	.{ .register = try self.copyToTmpRegister(ty, src_mcv) },
9786	9769	),
9787	9770	.sse => try self.asmRegisterRegister(
9788		if (@as(?Mir.Inst.FixedTag, switch (ty.scalarType(mod).zigTypeTag(mod)) {
	9771	@as(?Mir.Inst.FixedTag, switch (ty.scalarType(mod).zigTypeTag(mod)) {
9789	9772	else => switch (abi_size) {
9790	9773	1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
9791	9774	5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
...	...	@@ -9809,7 +9792,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
9809	9792	80 => null,
9810	9793	else => unreachable,
9811	9794	},
9812		})) \|tag\| tag else return self.fail("TODO implement genSetReg for {}", .{
	9795	}) orelse return self.fail("TODO implement genSetReg for {}", .{
9813	9796	ty.fmt(self.bin_file.options.module.?),
9814	9797	}),
9815	9798	registerAlias(dst_reg, abi_size),
...	...	@@ -9989,9 +9972,9 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
9989	9972
9990	9973	fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCValue) InnerError!void {
9991	9974	const mod = self.bin_file.options.module.?;
9992		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	9975	const abi_size: u32 = @intCast(ty.abiSize(mod));
9993	9976	const dst_ptr_mcv: MCValue = switch (base) {
9994		.none => .{ .immediate = @as(u64, @bitCast(@as(i64, disp))) },
	9977	.none => .{ .immediate = @bitCast(@as(i64, disp)) },
9995	9978	.reg => \|base_reg\| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
9996	9979	.frame => \|base_frame_index\| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
9997	9980	};
...	...	@@ -10002,7 +9985,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
10002	9985	.immediate => \|imm\| switch (abi_size) {
10003	9986	1, 2, 4 => {
10004	9987	const immediate = if (ty.isSignedInt(mod))
10005		Immediate.s(@as(i32, @truncate(@as(i64, @bitCast(imm)))))
	9988	Immediate.s(@truncate(@as(i64, @bitCast(imm))))
10006	9989	else
10007	9990	Immediate.u(@as(u32, @intCast(imm)));
10008	9991	try self.asmMemoryImmediate(
...	...	@@ -10024,10 +10007,9 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
10024	10007	.{ ._, .mov },
10025	10008	Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
10026	10009	if (ty.isSignedInt(mod))
10027		Immediate.s(@as(
10028		i32,
	10010	Immediate.s(
10029	10011	@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),
10030		))
	10012	)
10031	10013	else
10032	10014	Immediate.u(@as(
10033	10015	u32,
...	...	@@ -10138,7 +10120,7 @@ fn genLazySymbolRef(
10138	10120	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
10139	10121	const got_addr = atom.getOffsetTableAddress(elf_file);
10140	10122	const got_mem =
10141		Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @as(i32, @intCast(got_addr)) });
	10123	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(got_addr) });
10142	10124	switch (tag) {
10143	10125	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
10144	10126	.call => try self.asmMemory(.{ ._, .call }, got_mem),
...	...	@@ -10160,7 +10142,7 @@ fn genLazySymbolRef(
10160	10142	_ = atom.getOrCreateOffsetTableEntry(p9_file);
10161	10143	const got_addr = atom.getOffsetTableAddress(p9_file);
10162	10144	const got_mem =
10163		Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @as(i32, @intCast(got_addr)) });
	10145	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(got_addr) });
10164	10146	switch (tag) {
10165	10147	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
10166	10148	.call => try self.asmMemory(.{ ._, .call }, got_mem),
...	...	@@ -10256,8 +10238,8 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
10256	10238	if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
10257	10239	if (dst_signedness == src_signedness) break :result dst_mcv;
10258	10240
10259		const abi_size = @as(u16, @intCast(dst_ty.abiSize(mod)));
10260		const bit_size = @as(u16, @intCast(dst_ty.bitSize(mod)));
	10241	const abi_size: u16 = @intCast(dst_ty.abiSize(mod));
	10242	const bit_size: u16 = @intCast(dst_ty.bitSize(mod));
10261	10243	if (abi_size * 8 <= bit_size) break :result dst_mcv;
10262	10244
10263	10245	const dst_limbs_len = math.divCeil(i32, bit_size, 64) catch unreachable;
...	...	@@ -10298,7 +10280,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
10298	10280	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
10299	10281	try self.genSetMem(
10300	10282	.{ .frame = frame_index },
10301		@as(i32, @intCast(ptr_ty.abiSize(mod))),
	10283	@intCast(ptr_ty.abiSize(mod)),
10302	10284	Type.usize,
10303	10285	.{ .immediate = array_len },
10304	10286	);
...	...	@@ -10312,7 +10294,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
10312	10294	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
10313	10295
10314	10296	const src_ty = self.typeOf(ty_op.operand);
10315		const src_bits = @as(u32, @intCast(src_ty.bitSize(mod)));
	10297	const src_bits: u32 = @intCast(src_ty.bitSize(mod));
10316	10298	const src_signedness =
10317	10299	if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
10318	10300	const dst_ty = self.typeOfIndex(inst);
...	...	@@ -10340,7 +10322,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
10340	10322	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
10341	10323	defer self.register_manager.unlockReg(dst_lock);
10342	10324
10343		const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (dst_ty.zigTypeTag(mod)) {
	10325	const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_ty.zigTypeTag(mod)) {
10344	10326	.Float => switch (dst_ty.floatBits(self.target.*)) {
10345	10327	32 => if (self.hasFeature(.avx)) .{ .v_ss, .cvtsi2 } else .{ ._ss, .cvtsi2 },
10346	10328	64 => if (self.hasFeature(.avx)) .{ .v_sd, .cvtsi2 } else .{ ._sd, .cvtsi2 },
...	...	@@ -10348,7 +10330,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
10348	10330	else => unreachable,
10349	10331	},
10350	10332	else => null,
10351		})) \|tag\| tag else return self.fail("TODO implement airFloatFromInt from {} to {}", .{
	10333	}) orelse return self.fail("TODO implement airFloatFromInt from {} to {}", .{
10352	10334	src_ty.fmt(mod), dst_ty.fmt(mod),
10353	10335	});
10354	10336	const dst_alias = dst_reg.to128();
...	...	@@ -10367,7 +10349,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
10367	10349
10368	10350	const src_ty = self.typeOf(ty_op.operand);
10369	10351	const dst_ty = self.typeOfIndex(inst);
10370		const dst_bits = @as(u32, @intCast(dst_ty.bitSize(mod)));
	10352	const dst_bits: u32 = @intCast(dst_ty.bitSize(mod));
10371	10353	const dst_signedness =
10372	10354	if (dst_ty.isAbiInt(mod)) dst_ty.intInfo(mod).signedness else .unsigned;
10373	10355
...	...	@@ -10393,7 +10375,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
10393	10375	defer self.register_manager.unlockReg(dst_lock);
10394	10376
10395	10377	try self.asmRegisterRegister(
10396		if (@as(?Mir.Inst.FixedTag, switch (src_ty.zigTypeTag(mod)) {
	10378	@as(?Mir.Inst.FixedTag, switch (src_ty.zigTypeTag(mod)) {
10397	10379	.Float => switch (src_ty.floatBits(self.target.*)) {
10398	10380	32 => if (self.hasFeature(.avx)) .{ .v_, .cvttss2si } else .{ ._, .cvttss2si },
10399	10381	64 => if (self.hasFeature(.avx)) .{ .v_, .cvttsd2si } else .{ ._, .cvttsd2si },
...	...	@@ -10401,7 +10383,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
10401	10383	else => unreachable,
10402	10384	},
10403	10385	else => null,
10404		})) \|tag\| tag else return self.fail("TODO implement airIntFromFloat from {} to {}", .{
	10386	}) orelse return self.fail("TODO implement airIntFromFloat from {} to {}", .{
10405	10387	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
10406	10388	}),
10407	10389	registerAlias(dst_reg, dst_size),
...	...	@@ -10420,7 +10402,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
10420	10402
10421	10403	const ptr_ty = self.typeOf(extra.ptr);
10422	10404	const val_ty = self.typeOf(extra.expected_value);
10423		const val_abi_size = @as(u32, @intCast(val_ty.abiSize(mod)));
	10405	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
10424	10406
10425	10407	try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx });
10426	10408	const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx });
...	...	@@ -10522,7 +10504,7 @@ fn atomicOp(
10522	10504	};
10523	10505	defer if (val_lock) \|lock\| self.register_manager.unlockReg(lock);
10524	10506
10525		const val_abi_size = @as(u32, @intCast(val_ty.abiSize(mod)));
	10507	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
10526	10508	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
10527	10509	const ptr_mem = switch (ptr_mcv) {
10528	10510	.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
...	...	@@ -10600,7 +10582,7 @@ fn atomicOp(
10600	10582	defer self.register_manager.unlockReg(tmp_lock);
10601	10583
10602	10584	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
10603		const loop = @as(u32, @intCast(self.mir_instructions.len));
	10585	const loop: u32 = @intCast(self.mir_instructions.len);
10604	10586	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
10605	10587	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
10606	10588	}
...	...	@@ -10674,7 +10656,7 @@ fn atomicOp(
10674	10656	.scale_index = ptr_mem.scaleIndex(),
10675	10657	.disp = ptr_mem.sib.disp + 8,
10676	10658	}));
10677		const loop = @as(u32, @intCast(self.mir_instructions.len));
	10659	const loop: u32 = @intCast(self.mir_instructions.len);
10678	10660	const val_mem_mcv: MCValue = switch (val_mcv) {
10679	10661	.memory, .indirect, .load_frame => val_mcv,
10680	10662	else => .{ .indirect = .{
...	...	@@ -10830,7 +10812,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
10830	10812	};
10831	10813	defer if (src_val_lock) \|lock\| self.register_manager.unlockReg(lock);
10832	10814
10833		const elem_abi_size = @as(u31, @intCast(elem_ty.abiSize(mod)));
	10815	const elem_abi_size: u31 = @intCast(elem_ty.abiSize(mod));
10834	10816
10835	10817	if (elem_abi_size == 1) {
10836	10818	const ptr: MCValue = switch (dst_ptr_ty.ptrSize(mod)) {
...	...	@@ -11310,9 +11292,9 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
11310	11292	fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
11311	11293	const mod = self.bin_file.options.module.?;
11312	11294	const result_ty = self.typeOfIndex(inst);
11313		const len = @as(usize, @intCast(result_ty.arrayLen(mod)));
	11295	const len: usize = @intCast(result_ty.arrayLen(mod));
11314	11296	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
11315		const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len]));
	11297	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
11316	11298	const result: MCValue = result: {
11317	11299	switch (result_ty.zigTypeTag(mod)) {
11318	11300	.Struct => {
...	...	@@ -11329,17 +11311,17 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
11329	11311	if ((try result_ty.structFieldValueComptime(mod, elem_i)) != null) continue;
11330	11312
11331	11313	const elem_ty = result_ty.structFieldType(elem_i, mod);
11332		const elem_bit_size = @as(u32, @intCast(elem_ty.bitSize(mod)));
	11314	const elem_bit_size: u32 = @intCast(elem_ty.bitSize(mod));
11333	11315	if (elem_bit_size > 64) {
11334	11316	return self.fail(
11335	11317	"TODO airAggregateInit implement packed structs with large fields",
11336	11318	.{},
11337	11319	);
11338	11320	}
11339		const elem_abi_size = @as(u32, @intCast(elem_ty.abiSize(mod)));
	11321	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
11340	11322	const elem_abi_bits = elem_abi_size * 8;
11341	11323	const elem_off = struct_obj.packedFieldBitOffset(mod, elem_i);
11342		const elem_byte_off = @as(i32, @intCast(elem_off / elem_abi_bits * elem_abi_size));
	11324	const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
11343	11325	const elem_bit_off = elem_off % elem_abi_bits;
11344	11326	const elem_mcv = try self.resolveInst(elem);
11345	11327	const mat_elem_mcv = switch (elem_mcv) {
...	...	@@ -11401,7 +11383,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
11401	11383	if ((try result_ty.structFieldValueComptime(mod, elem_i)) != null) continue;
11402	11384
11403	11385	const elem_ty = result_ty.structFieldType(elem_i, mod);
11404		const elem_off = @as(i32, @intCast(result_ty.structFieldOffset(elem_i, mod)));
	11386	const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, mod));
11405	11387	const elem_mcv = try self.resolveInst(elem);
11406	11388	const mat_elem_mcv = switch (elem_mcv) {
11407	11389	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
...	...	@@ -11415,7 +11397,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
11415	11397	const frame_index =
11416	11398	try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
11417	11399	const elem_ty = result_ty.childType(mod);
11418		const elem_size = @as(u32, @intCast(elem_ty.abiSize(mod)));
	11400	const elem_size: u32 = @intCast(elem_ty.abiSize(mod));
11419	11401
11420	11402	for (elements, 0..) \|elem, elem_i\| {
11421	11403	const elem_mcv = try self.resolveInst(elem);
...	...	@@ -11423,12 +11405,12 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
11423	11405	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
11424	11406	else => elem_mcv,
11425	11407	};
11426		const elem_off = @as(i32, @intCast(elem_size * elem_i));
	11408	const elem_off: i32 = @intCast(elem_size * elem_i);
11427	11409	try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
11428	11410	}
11429	11411	if (result_ty.sentinel(mod)) \|sentinel\| try self.genSetMem(
11430	11412	.{ .frame = frame_index },
11431		@as(i32, @intCast(elem_size * elements.len)),
	11413	@intCast(elem_size * elements.len),
11432	11414	elem_ty,
11433	11415	try self.genTypedValue(.{ .ty = elem_ty, .val = sentinel }),
11434	11416	);
...	...	@@ -11476,16 +11458,16 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
11476	11458	const tag_val = try mod.enumValueFieldIndex(tag_ty, field_index);
11477	11459	const tag_int_val = try tag_val.intFromEnum(tag_ty, mod);
11478	11460	const tag_int = tag_int_val.toUnsignedInt(mod);
11479		const tag_off = if (layout.tag_align < layout.payload_align)
11480		@as(i32, @intCast(layout.payload_size))
	11461	const tag_off: i32 = if (layout.tag_align < layout.payload_align)
	11462	@intCast(layout.payload_size)
11481	11463	else
11482	11464	0;
11483	11465	try self.genCopy(tag_ty, dst_mcv.address().offset(tag_off).deref(), .{ .immediate = tag_int });
11484	11466
11485		const pl_off = if (layout.tag_align < layout.payload_align)
	11467	const pl_off: i32 = if (layout.tag_align < layout.payload_align)
11486	11468	0
11487	11469	else
11488		@as(i32, @intCast(layout.tag_size));
	11470	@intCast(layout.tag_size);
11489	11471	try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv);
11490	11472
11491	11473	break :result dst_mcv;
...	...	@@ -11515,7 +11497,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
11515	11497	var order = [1]u2{0} ** 3;
11516	11498	var unused = std.StaticBitSet(3).initFull();
11517	11499	for (ops, &mcvs, &locks, 0..) \|op, mcv, lock, op_i\| {
11518		const op_index = @as(u2, @intCast(op_i));
	11500	const op_index: u2 = @intCast(op_i);
11519	11501	mcv.* = try self.resolveInst(op);
11520	11502	if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
11521	11503	order[op_index] = 1;
...	...	@@ -11539,99 +11521,97 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
11539	11521	lock.* = self.register_manager.lockRegAssumeUnused(reg);
11540	11522	}
11541	11523
11542		const mir_tag = if (@as(
11543		?Mir.Inst.FixedTag,
11544		if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
11545		switch (ty.zigTypeTag(mod)) {
11546		.Float => switch (ty.floatBits(self.target.*)) {
11547		32 => .{ .v_ss, .fmadd132 },
11548		64 => .{ .v_sd, .fmadd132 },
11549		16, 80, 128 => null,
11550		else => unreachable,
11551		},
11552		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
11553		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
11554		32 => switch (ty.vectorLen(mod)) {
11555		1 => .{ .v_ss, .fmadd132 },
11556		2...8 => .{ .v_ps, .fmadd132 },
11557		else => null,
11558		},
11559		64 => switch (ty.vectorLen(mod)) {
11560		1 => .{ .v_sd, .fmadd132 },
11561		2...4 => .{ .v_pd, .fmadd132 },
11562		else => null,
11563		},
11564		16, 80, 128 => null,
11565		else => unreachable,
11566		},
11567		else => unreachable,
11568		},
	11524	const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or
	11525	mem.eql(u2, &order, &.{ 3, 1, 2 }))
	11526	switch (ty.zigTypeTag(mod)) {
	11527	.Float => switch (ty.floatBits(self.target.*)) {
	11528	32 => .{ .v_ss, .fmadd132 },
	11529	64 => .{ .v_sd, .fmadd132 },
	11530	16, 80, 128 => null,
11569	11531	else => unreachable,
11570		}
11571		else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
11572		switch (ty.zigTypeTag(mod)) {
11573		.Float => switch (ty.floatBits(self.target.*)) {
11574		32 => .{ .v_ss, .fmadd213 },
11575		64 => .{ .v_sd, .fmadd213 },
11576		16, 80, 128 => null,
11577		else => unreachable,
11578		},
11579		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
11580		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
11581		32 => switch (ty.vectorLen(mod)) {
11582		1 => .{ .v_ss, .fmadd213 },
11583		2...8 => .{ .v_ps, .fmadd213 },
11584		else => null,
11585		},
11586		64 => switch (ty.vectorLen(mod)) {
11587		1 => .{ .v_sd, .fmadd213 },
11588		2...4 => .{ .v_pd, .fmadd213 },
11589		else => null,
11590		},
11591		16, 80, 128 => null,
11592		else => unreachable,
	11532	},
	11533	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	11534	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	11535	32 => switch (ty.vectorLen(mod)) {
	11536	1 => .{ .v_ss, .fmadd132 },
	11537	2...8 => .{ .v_ps, .fmadd132 },
	11538	else => null,
11593	11539	},
	11540	64 => switch (ty.vectorLen(mod)) {
	11541	1 => .{ .v_sd, .fmadd132 },
	11542	2...4 => .{ .v_pd, .fmadd132 },
	11543	else => null,
	11544	},
	11545	16, 80, 128 => null,
11594	11546	else => unreachable,
11595	11547	},
11596	11548	else => unreachable,
11597		}
11598		else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
11599		switch (ty.zigTypeTag(mod)) {
11600		.Float => switch (ty.floatBits(self.target.*)) {
11601		32 => .{ .v_ss, .fmadd231 },
11602		64 => .{ .v_sd, .fmadd231 },
	11549	},
	11550	else => unreachable,
	11551	}
	11552	else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
	11553	switch (ty.zigTypeTag(mod)) {
	11554	.Float => switch (ty.floatBits(self.target.*)) {
	11555	32 => .{ .v_ss, .fmadd213 },
	11556	64 => .{ .v_sd, .fmadd213 },
	11557	16, 80, 128 => null,
	11558	else => unreachable,
	11559	},
	11560	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	11561	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	11562	32 => switch (ty.vectorLen(mod)) {
	11563	1 => .{ .v_ss, .fmadd213 },
	11564	2...8 => .{ .v_ps, .fmadd213 },
	11565	else => null,
	11566	},
	11567	64 => switch (ty.vectorLen(mod)) {
	11568	1 => .{ .v_sd, .fmadd213 },
	11569	2...4 => .{ .v_pd, .fmadd213 },
	11570	else => null,
	11571	},
11603	11572	16, 80, 128 => null,
11604	11573	else => unreachable,
11605	11574	},
11606		.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
11607		.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
11608		32 => switch (ty.vectorLen(mod)) {
11609		1 => .{ .v_ss, .fmadd231 },
11610		2...8 => .{ .v_ps, .fmadd231 },
11611		else => null,
11612		},
11613		64 => switch (ty.vectorLen(mod)) {
11614		1 => .{ .v_sd, .fmadd231 },
11615		2...4 => .{ .v_pd, .fmadd231 },
11616		else => null,
11617		},
11618		16, 80, 128 => null,
11619		else => unreachable,
	11575	else => unreachable,
	11576	},
	11577	else => unreachable,
	11578	}
	11579	else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
	11580	switch (ty.zigTypeTag(mod)) {
	11581	.Float => switch (ty.floatBits(self.target.*)) {
	11582	32 => .{ .v_ss, .fmadd231 },
	11583	64 => .{ .v_sd, .fmadd231 },
	11584	16, 80, 128 => null,
	11585	else => unreachable,
	11586	},
	11587	.Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
	11588	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	11589	32 => switch (ty.vectorLen(mod)) {
	11590	1 => .{ .v_ss, .fmadd231 },
	11591	2...8 => .{ .v_ps, .fmadd231 },
	11592	else => null,
	11593	},
	11594	64 => switch (ty.vectorLen(mod)) {
	11595	1 => .{ .v_sd, .fmadd231 },
	11596	2...4 => .{ .v_pd, .fmadd231 },
	11597	else => null,
11620	11598	},
	11599	16, 80, 128 => null,
11621	11600	else => unreachable,
11622	11601	},
11623	11602	else => unreachable,
11624		}
11625		else
11626		unreachable,
11627		)) \|tag\| tag else return self.fail("TODO implement airMulAdd for {}", .{
	11603	},
	11604	else => unreachable,
	11605	}
	11606	else
	11607	unreachable) orelse return self.fail("TODO implement airMulAdd for {}", .{
11628	11608	ty.fmt(self.bin_file.options.module.?),
11629	11609	});
11630	11610
11631	11611	var mops: [3]MCValue = undefined;
11632	11612	for (order, mcvs) \|mop_index, mcv\| mops[mop_index - 1] = mcv;
11633	11613
11634		const abi_size = @as(u32, @intCast(ty.abiSize(mod)));
	11614	const abi_size: u32 = @intCast(ty.abiSize(mod));
11635	11615	const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
11636	11616	const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
11637	11617	if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
...	...	@@ -11784,7 +11764,7 @@ fn resolveCallingConventionValues(
11784	11764	switch (self.target.os.tag) {
11785	11765	.windows => {
11786	11766	// Align the stack to 16bytes before allocating shadow stack space (if any).
11787		result.stack_byte_count += @as(u31, @intCast(4 * Type.usize.abiSize(mod)));
	11767	result.stack_byte_count += @intCast(4 * Type.usize.abiSize(mod));
11788	11768	},
11789	11769	else => {},
11790	11770	}
...	...	@@ -11807,7 +11787,7 @@ fn resolveCallingConventionValues(
11807	11787	result.return_value = switch (classes[0]) {
11808	11788	.integer => InstTracking.init(.{ .register = registerAlias(
11809	11789	ret_reg,
11810		@as(u32, @intCast(ret_ty.abiSize(mod))),
	11790	@intCast(ret_ty.abiSize(mod)),
11811	11791	) }),
11812	11792	.float, .sse => InstTracking.init(.{ .register = .xmm0 }),
11813	11793	.memory => ret: {
...	...	@@ -11843,18 +11823,16 @@ fn resolveCallingConventionValues(
11843	11823	},
11844	11824	.float, .sse => switch (self.target.os.tag) {
11845	11825	.windows => if (param_reg_i < 4) {
11846		arg.* = .{ .register = @as(
11847		Register,
11848		@enumFromInt(@intFromEnum(Register.xmm0) + param_reg_i),
11849		) };
	11826	arg.* = .{
	11827	.register = @enumFromInt(@intFromEnum(Register.xmm0) + param_reg_i),
	11828	};
11850	11829	param_reg_i += 1;
11851	11830	continue;
11852	11831	},
11853	11832	else => if (param_sse_reg_i < 8) {
11854		arg.* = .{ .register = @as(
11855		Register,
11856		@enumFromInt(@intFromEnum(Register.xmm0) + param_sse_reg_i),
11857		) };
	11833	arg.* = .{
	11834	.register = @enumFromInt(@intFromEnum(Register.xmm0) + param_sse_reg_i),
	11835	};
11858	11836	param_sse_reg_i += 1;
11859	11837	continue;
11860	11838	},
...	...	@@ -11865,8 +11843,8 @@ fn resolveCallingConventionValues(
11865	11843	}),
11866	11844	}
11867	11845
11868		const param_size = @as(u31, @intCast(ty.abiSize(mod)));
11869		const param_align = @as(u31, @intCast(ty.abiAlignment(mod)));
	11846	const param_size: u31 = @intCast(ty.abiSize(mod));
	11847	const param_align: u31 = @intCast(ty.abiAlignment(mod));
11870	11848	result.stack_byte_count =
11871	11849	mem.alignForward(u31, result.stack_byte_count, param_align);
11872	11850	arg.* = .{ .load_frame = .{
...	...	@@ -11886,7 +11864,7 @@ fn resolveCallingConventionValues(
11886	11864	result.return_value = InstTracking.init(.none);
11887	11865	} else {
11888	11866	const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
11889		const ret_ty_size = @as(u31, @intCast(ret_ty.abiSize(mod)));
	11867	const ret_ty_size: u31 = @intCast(ret_ty.abiSize(mod));
11890	11868	if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
11891	11869	const aliased_reg = registerAlias(ret_reg, ret_ty_size);
11892	11870	result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
...	...	@@ -11905,8 +11883,8 @@ fn resolveCallingConventionValues(
11905	11883	arg.* = .none;
11906	11884	continue;
11907	11885	}
11908		const param_size = @as(u31, @intCast(ty.abiSize(mod)));
11909		const param_align = @as(u31, @intCast(ty.abiAlignment(mod)));
	11886	const param_size: u31 = @intCast(ty.abiSize(mod));
	11887	const param_align: u31 = @intCast(ty.abiAlignment(mod));
11910	11888	result.stack_byte_count =
11911	11889	mem.alignForward(u31, result.stack_byte_count, param_align);
11912	11890	arg.* = .{ .load_frame = .{
...	...	@@ -11993,12 +11971,12 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
11993	11971	const mod = self.bin_file.options.module.?;
11994	11972	const int_info = if (ty.isAbiInt(mod)) ty.intInfo(mod) else std.builtin.Type.Int{
11995	11973	.signedness = .unsigned,
11996		.bits = @as(u16, @intCast(ty.bitSize(mod))),
	11974	.bits = @intCast(ty.bitSize(mod)),
11997	11975	};
11998	11976	const max_reg_bit_width = Register.rax.bitSize();
11999	11977	switch (int_info.signedness) {
12000	11978	.signed => {
12001		const shift = @as(u6, @intCast(max_reg_bit_width - int_info.bits));
	11979	const shift: u6 = @intCast(max_reg_bit_width - int_info.bits);
12002	11980	try self.genShiftBinOpMir(
12003	11981	.{ ._l, .sa },
12004	11982	Type.isize,
...	...	@@ -12013,8 +11991,8 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
12013	11991	);
12014	11992	},
12015	11993	.unsigned => {
12016		const shift = @as(u6, @intCast(max_reg_bit_width - int_info.bits));
12017		const mask = (~@as(u64, 0)) >> shift;
	11994	const shift: u6 = @intCast(max_reg_bit_width - int_info.bits);
	11995	const mask = ~@as(u64, 0) >> shift;
12018	11996	if (int_info.bits <= 32) {
12019	11997	try self.genBinOpMir(
12020	11998	.{ ._, .@"and" },