Commit af1191ea8ba55a6eab53e0b561355bb116bdbf2d | zig

author	Jacob Young <jacobly@ziglang.org> 2024-11-25 02:58:30-05:00
committer	Jacob Young <jacobly@ziglang.org> 2025-01-16 20:42:07-05:00
log	af1191ea8ba55a6eab53e0b561355bb116bdbf2d
tree	0bd2977ad2f8e379dcac7fdec5ba58e294bb06f5
parent	257054a1467b2612725bd66852d84496024cf66c

x86_64: rewrite

12 files changed, 1885 insertions(+), 948 deletions(-)

src
- Air.zig
- Compilation.zig
- Liveness.zig
- arch
  - aarch64
    CodeGen.zig
  - arm
    CodeGen.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - x86_64
    CodeGen.zig
    abi.zig
    bits.zig
- print_air.zig
tools
- lldb_pretty_printers.py

src/Air.zig+30-6

...	@@ -893,14 +893,38 @@ pub const Inst = struct {	...	@@ -893,14 +893,38 @@ pub const Inst = struct {
893	pub const Index = enum(u32) {	893	pub const Index = enum(u32) {
894	_,	894	_,
895		895
896	pub fn toRef(i: Index) Inst.Ref {	896	pub fn unwrap(index: Index) union(enum) { ref: Inst.Ref, target: u31 } {
897	assert(@intFromEnum(i) >> 31 == 0);	897	const low_index: u31 = @truncate(@intFromEnum(index));
898	return @enumFromInt((1 << 31) \| @intFromEnum(i));	898	return switch (@as(u1, @intCast(@intFromEnum(index) >> 31))) {
		899	0 => .{ .ref = @enumFromInt(@as(u32, 1 << 31) \| low_index) },
		900	1 => .{ .target = low_index },
		901	};
		902	}
		903
		904	pub fn toRef(index: Index) Inst.Ref {
		905	return index.unwrap().ref;
		906	}
		907
		908	pub fn fromTargetIndex(index: u31) Index {
		909	return @enumFromInt((1 << 31) \| @as(u32, index));
		910	}
		911
		912	pub fn toTargetIndex(index: Index) u31 {
		913	return index.unwrap().target;
899	}	914	}
900		915
901	pub fn toTargetIndex(i: Index) u31 {	916	pub fn format(
902	assert(@intFromEnum(i) >> 31 == 1);	917	index: Index,
903	return @truncate(@intFromEnum(i));	918	comptime _: []const u8,
		919	_: std.fmt.FormatOptions,
		920	writer: anytype,
		921	) @TypeOf(writer).Error!void {
		922	try writer.writeByte('%');
		923	switch (index.unwrap()) {
		924	.ref => {},
		925	.target => try writer.writeByte('t'),
		926	}
		927	try writer.print("{d}", .{@as(u31, @truncate(@intFromEnum(index)))});
904	}	928	}
905	};	929	};
906		930

src/Compilation.zig+1

...	@@ -3067,6 +3067,7 @@ pub fn saveState(comp: *Compilation) !void {	...	@@ -3067,6 +3067,7 @@ pub fn saveState(comp: *Compilation) !void {
3067	// linker state	3067	// linker state
3068	switch (lf.tag) {	3068	switch (lf.tag) {
3069	.wasm => {	3069	.wasm => {
		3070	dev.check(link.File.Tag.wasm.devFeature());
3070	const wasm = lf.cast(.wasm).?;	3071	const wasm = lf.cast(.wasm).?;
3071	const is_obj = comp.config.output_mode == .Obj;	3072	const is_obj = comp.config.output_mode == .Obj;
3072	try bufs.ensureUnusedCapacity(85);	3073	try bufs.ensureUnusedCapacity(85);

src/Liveness.zig-14

...	@@ -202,14 +202,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool	...	@@ -202,14 +202,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool
202	return (l.tomb_bits[usize_index] & mask) != 0;	202	return (l.tomb_bits[usize_index] & mask) != 0;
203	}	203	}
204		204
205	pub fn clearOperandDeath(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) void {
206	assert(operand < bpi - 1);
207	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
208	const mask = @as(usize, 1) <<
209	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
210	l.tomb_bits[usize_index] &= ~mask;
211	}
212
213	const OperandCategory = enum {	205	const OperandCategory = enum {
214	/// The operand lives on, but this instruction cannot possibly mutate memory.	206	/// The operand lives on, but this instruction cannot possibly mutate memory.
215	none,	207	none,
...	@@ -844,12 +836,6 @@ const Analysis = struct {	...	@@ -844,12 +836,6 @@ const Analysis = struct {
844	special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),	836	special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
845	extra: std.ArrayListUnmanaged(u32),	837	extra: std.ArrayListUnmanaged(u32),
846		838
847	fn storeTombBits(a: *Analysis, inst: Air.Inst.Index, tomb_bits: Bpi) void {
848	const usize_index = (inst * bpi) / @bitSizeOf(usize);
849	a.tomb_bits[usize_index] \|= @as(usize, tomb_bits) <<
850	@as(Log2Int(usize), @intCast((inst % (@bitSizeOf(usize) / bpi)) * bpi));
851	}
852
853	fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {	839	fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
854	const fields = std.meta.fields(@TypeOf(extra));	840	const fields = std.meta.fields(@TypeOf(extra));
855	try a.extra.ensureUnusedCapacity(a.gpa, fields.len);	841	try a.extra.ensureUnusedCapacity(a.gpa, fields.len);

src/arch/aarch64/CodeGen.zig+10-10

 @@ -71,6 +71,8 @@ end_di_column: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// We postpone the creation of debug info for function args and locals
 /// until after all Mir instructions have been generated. Only then we
 /// will know saved_regs_stack_space which is necessary in order to
 @@ -646,6 +648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         const old_air_bookkeeping = self.air_bookkeeping;
         try self.ensureProcessDeathCapacity(Liveness.bpi);
+        self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
         switch (air_tags[@intFromEnum(inst)]) {
             // zig fmt: off
             .add             => try self.airBinOp(inst, .add),
 @@ -927,16 +930,13 @@ fn finishAirBookkeeping(self: *Self) void {
 }
 fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
-    var tomb_bits = self.liveness.getTombBits(inst);
+    const tomb_bits = self.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (self.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
+        self.processDeath(op.toIndexAllowNone() orelse continue);
-        const op_index = op.toIndex() orelse continue;
-        self.processDeath(op_index);
     }
-    const is_used = @as(u1, @truncate(tomb_bits)) == 0;
+    if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
-    if (is_used) {
         log.debug("%{d} => {}", .{ inst, result });
         const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
         branch.inst_table.putAssumeCapacityNoClobber(inst, result);
 @@ -3614,7 +3614,7 @@ fn reuseOperand(
     }
     // Prevent the operand deaths processing code from deallocating it.
-    self.liveness.clearOperandDeath(inst, op_index);
+    self.reused_operands.set(op_index);
     // That makes us responsible for doing the rest of the stuff that processDeath would have done.
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/arm/CodeGen.zig+10-10

 @@ -72,6 +72,8 @@ end_di_column: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// We postpone the creation of debug info for function args and locals
 /// until after all Mir instructions have been generated. Only then we
 /// will know saved_regs_stack_space which is necessary in order to
 @@ -635,6 +637,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         const old_air_bookkeeping = self.air_bookkeeping;
         try self.ensureProcessDeathCapacity(Liveness.bpi);
+        self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
         switch (air_tags[@intFromEnum(inst)]) {
             // zig fmt: off
             .add,            => try self.airBinOp(inst, .add),
 @@ -918,16 +921,13 @@ fn finishAirBookkeeping(self: *Self) void {
 }
 fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
-    var tomb_bits = self.liveness.getTombBits(inst);
+    const tomb_bits = self.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (self.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
+        self.processDeath(op.toIndexAllowNone() orelse continue);
-        const op_index = op.toIndex() orelse continue;
-        self.processDeath(op_index);
     }
-    const is_used = @as(u1, @truncate(tomb_bits)) == 0;
+    if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
-    if (is_used) {
         log.debug("%{d} => {}", .{ inst, result });
         const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
         branch.inst_table.putAssumeCapacityNoClobber(inst, result);
 @@ -2650,7 +2650,7 @@ fn reuseOperand(
     }
     // Prevent the operand deaths processing code from deallocating it.
-    self.liveness.clearOperandDeath(inst, op_index);
+    self.reused_operands.set(op_index);
     // That makes us responsible for doing the rest of the stuff that processDeath would have done.
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/riscv64/CodeGen.zig+11-7

 @@ -82,6 +82,8 @@ scope_generation: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// Whenever there is a runtime branch, we push a Branch onto this stack,
 /// and pop it off when the runtime branch joins. This provides an "overlay"
 /// of the table of mappings from instructions to `MCValue` from within the branch.
 @@ -1443,8 +1445,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
         verbose_tracking_log.debug("{}", .{func.fmtTracking()});
         const old_air_bookkeeping = func.air_bookkeeping;
+        try func.ensureProcessDeathCapacity(Liveness.bpi);
+        func.reused_operands = @TypeOf(func.reused_operands).initEmpty();
         try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
-        const tag: Air.Inst.Tag = air_tags[@intFromEnum(inst)];
+        const tag = air_tags[@intFromEnum(inst)];
         switch (tag) {
             // zig fmt: off
             .add,
 @@ -1783,11 +1788,10 @@ fn finishAir(
     result: MCValue,
     operands: [Liveness.bpi - 1]Air.Inst.Ref,
 ) !void {
-    var tomb_bits = func.liveness.getTombBits(inst);
+    const tomb_bits = func.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (func.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
         try func.processDeath(op.toIndexAllowNone() orelse continue);
     }
     func.finishAirResult(inst, result);
 @@ -4424,7 +4428,7 @@ fn reuseOperandAdvanced(
     }
     // Prevent the operand deaths processing code from deallocating it.
-    func.liveness.clearOperandDeath(inst, op_index);
+    func.reused_operands.set(op_index);
     const op_inst = operand.toIndex().?;
     func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst);

src/arch/sparc64/CodeGen.zig+10-10

 @@ -78,6 +78,8 @@ end_di_column: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// Whenever there is a runtime branch, we push a Branch onto this stack,
 /// and pop it off when the runtime branch joins. This provides an "overlay"
 /// of the table of mappings from instructions to `MCValue` from within the branch.
 @@ -493,6 +495,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         const old_air_bookkeeping = self.air_bookkeeping;
         try self.ensureProcessDeathCapacity(Liveness.bpi);
+        self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
         switch (air_tags[@intFromEnum(inst)]) {
             // zig fmt: off
             .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
 @@ -3523,16 +3526,13 @@ fn finishAirBookkeeping(self: *Self) void {
 }
 fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
-    var tomb_bits = self.liveness.getTombBits(inst);
+    const tomb_bits = self.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (self.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
+        self.processDeath(op.toIndexAllowNone() orelse continue);
-        const op_index = op.toIndex() orelse continue;
-        self.processDeath(op_index);
     }
-    const is_used = @as(u1, @truncate(tomb_bits)) == 0;
+    if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
-    if (is_used) {
         log.debug("%{d} => {}", .{ inst, result });
         const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
         branch.inst_table.putAssumeCapacityNoClobber(inst, result);
 @@ -4568,7 +4568,7 @@ fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_ind
     }
     // Prevent the operand deaths processing code from deallocating it.
-    self.liveness.clearOperandDeath(inst, op_index);
+    self.reused_operands.set(op_index);
     // That makes us responsible for doing the rest of the stuff that processDeath would have done.
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/x86_64/CodeGen.zig+1765-874

...	@@ -1,41 +1,26 @@	...	@@ -1,41 +1,26 @@
1	const std = @import("std");	1	const std = @import("std");
2	const build_options = @import("build_options");
3	const builtin = @import("builtin");
4	const assert = std.debug.assert;	2	const assert = std.debug.assert;
5	const codegen = @import("../../codegen.zig");	3	const codegen = @import("../../codegen.zig");
6	const leb128 = std.leb;
7	const link = @import("../../link.zig");	4	const link = @import("../../link.zig");
8	const log = std.log.scoped(.codegen);	5	const log = std.log.scoped(.codegen);
9	const tracking_log = std.log.scoped(.tracking);	6	const tracking_log = std.log.scoped(.tracking);
10	const verbose_tracking_log = std.log.scoped(.verbose_tracking);	7	const verbose_tracking_log = std.log.scoped(.verbose_tracking);
11	const wip_mir_log = std.log.scoped(.wip_mir);	8	const wip_mir_log = std.log.scoped(.wip_mir);
12	const math = std.math;
13	const mem = std.mem;
14	const target_util = @import("../../target.zig");
15	const trace = @import("../../tracy.zig").trace;
16		9
17	const Air = @import("../../Air.zig");	10	const Air = @import("../../Air.zig");
18	const Allocator = mem.Allocator;	11	const Allocator = std.mem.Allocator;
19	const CodeGenError = codegen.CodeGenError;
20	const Compilation = @import("../../Compilation.zig");
21	const ErrorMsg = Zcu.ErrorMsg;
22	const Emit = @import("Emit.zig");	12	const Emit = @import("Emit.zig");
23	const Liveness = @import("../../Liveness.zig");	13	const Liveness = @import("../../Liveness.zig");
24	const Lower = @import("Lower.zig");	14	const Lower = @import("Lower.zig");
25	const Mir = @import("Mir.zig");	15	const Mir = @import("Mir.zig");
26	const Package = @import("../../Package.zig");
27	const Zcu = @import("../../Zcu.zig");	16	const Zcu = @import("../../Zcu.zig");
		17	const Module = @import("../../Package/Module.zig");
28	const InternPool = @import("../../InternPool.zig");	18	const InternPool = @import("../../InternPool.zig");
29	const Alignment = InternPool.Alignment;
30	const Target = std.Target;
31	const Type = @import("../../Type.zig");	19	const Type = @import("../../Type.zig");
32	const Value = @import("../../Value.zig");	20	const Value = @import("../../Value.zig");
33	const Instruction = @import("encoder.zig").Instruction;
34		21
35	const abi = @import("abi.zig");	22	const abi = @import("abi.zig");
36	const bits = @import("bits.zig");	23	const bits = @import("bits.zig");
37	const errUnionErrorOffset = codegen.errUnionErrorOffset;
38	const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
39	const encoder = @import("encoder.zig");	24	const encoder = @import("encoder.zig");
40		25
41	const Condition = bits.Condition;	26	const Condition = bits.Condition;
...	@@ -46,7 +31,7 @@ const RegisterManager = abi.RegisterManager;	...	@@ -46,7 +31,7 @@ const RegisterManager = abi.RegisterManager;
46	const RegisterLock = RegisterManager.RegisterLock;	31	const RegisterLock = RegisterManager.RegisterLock;
47	const FrameIndex = bits.FrameIndex;	32	const FrameIndex = bits.FrameIndex;
48		33
49	const InnerError = CodeGenError \|\| error{OutOfRegisters};	34	const InnerError = codegen.CodeGenError \|\| error{OutOfRegisters};
50		35
51	gpa: Allocator,	36	gpa: Allocator,
52	pt: Zcu.PerThread,	37	pt: Zcu.PerThread,
...	@@ -57,7 +42,7 @@ debug_output: link.File.DebugInfoOutput,	...	@@ -57,7 +42,7 @@ debug_output: link.File.DebugInfoOutput,
57	target: *const std.Target,	42	target: *const std.Target,
58	owner: Owner,	43	owner: Owner,
59	inline_func: InternPool.Index,	44	inline_func: InternPool.Index,
60	mod: *Package.Module,	45	mod: *Module,
61	arg_index: u32,	46	arg_index: u32,
62	args: []MCValue,	47	args: []MCValue,
63	va_info: union {	48	va_info: union {
...	@@ -89,6 +74,7 @@ end_di_column: u32,	...	@@ -89,6 +74,7 @@ end_di_column: u32,
89	/// which is a relative jump, based on the address following the reloc.	74	/// which is a relative jump, based on the address following the reloc.
90	exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,	75	exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
91		76
		77	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
92	const_tracking: ConstTrackingMap = .{},	78	const_tracking: ConstTrackingMap = .{},
93	inst_tracking: InstTrackingMap = .{},	79	inst_tracking: InstTrackingMap = .{},
94		80
...	@@ -108,13 +94,11 @@ loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {	...	@@ -108,13 +94,11 @@ loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
108	/// The state to restore before branching.	94	/// The state to restore before branching.
109	state: State,	95	state: State,
110	/// The branch target.	96	/// The branch target.
111	jmp_target: Mir.Inst.Index,	97	target: Mir.Inst.Index,
112	}) = .{},	98	}) = .{},
113		99
114	/// Debug field, used to find bugs in the compiler.	100	next_temp_index: Temp.Index = @enumFromInt(0),
115	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,	101	temp_type: [Temp.Index.max]Type = undefined,
116
117	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
118		102
119	const Owner = union(enum) {	103	const Owner = union(enum) {
120	nav_index: InternPool.Nav.Index,	104	nav_index: InternPool.Nav.Index,
...	@@ -433,7 +417,7 @@ pub const MCValue = union(enum) {	...	@@ -433,7 +417,7 @@ pub const MCValue = union(enum) {
433	.reserved_frame,	417	.reserved_frame,
434	.lea_symbol,	418	.lea_symbol,
435	=> unreachable,	419	=> unreachable,
436	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{	420	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
437	.base = .{ .reg = .ds },	421	.base = .{ .reg = .ds },
438	.mod = .{ .rm = .{	422	.mod = .{ .rm = .{
439	.size = size,	423	.size = size,
...	@@ -484,8 +468,8 @@ pub const MCValue = union(enum) {	...	@@ -484,8 +468,8 @@ pub const MCValue = union(enum) {
484	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{	468	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{
485	@tagName(pl.eflags), @tagName(pl.reg),	469	@tagName(pl.eflags), @tagName(pl.reg),
486	}),	470	}),
487	.load_symbol => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.sym_index, pl.off }),	471	.load_symbol => \|pl\| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }),
488	.lea_symbol => \|pl\| try writer.print("{} + 0x{x}", .{ pl.sym_index, pl.off }),	472	.lea_symbol => \|pl\| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }),
489	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),	473	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
490	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),	474	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),
491	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),	475	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),
...	@@ -562,7 +546,7 @@ const InstTracking = struct {	...	@@ -562,7 +546,7 @@ const InstTracking = struct {
562	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },	546	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
563	else => unreachable,	547	else => unreachable,
564	}	548	}
565	tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });	549	tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long });
566	try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{});	550	try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{});
567	}	551	}
568		552
...	@@ -605,7 +589,7 @@ const InstTracking = struct {	...	@@ -605,7 +589,7 @@ const InstTracking = struct {
605	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {	589	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
606	try function.freeValue(self.short);	590	try function.freeValue(self.short);
607	self.reuseFrame();	591	self.reuseFrame();
608	tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });	592	tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });
609	}	593	}
610		594
611	fn verifyMaterialize(self: InstTracking, target: InstTracking) void {	595	fn verifyMaterialize(self: InstTracking, target: InstTracking) void {
...	@@ -678,14 +662,14 @@ const InstTracking = struct {	...	@@ -678,14 +662,14 @@ const InstTracking = struct {
678	else => target.long,	662	else => target.long,
679	} else target.long;	663	} else target.long;
680	self.short = target.short;	664	self.short = target.short;
681	tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });	665	tracking_log.debug("{} => {} (materialize)", .{ inst, self.* });
682	}	666	}
683		667
684	fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {	668	fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
685	switch (self.short) {	669	switch (self.short) {
686	.dead => \|die_generation\| if (die_generation >= scope_generation) {	670	.dead => \|die_generation\| if (die_generation >= scope_generation) {
687	self.reuseFrame();	671	self.reuseFrame();
688	tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });	672	tracking_log.debug("{} => {} (resurrect)", .{ inst, self.* });
689	},	673	},
690	else => {},	674	else => {},
691	}	675	}
...	@@ -695,7 +679,7 @@ const InstTracking = struct {	...	@@ -695,7 +679,7 @@ const InstTracking = struct {
695	if (self.short == .dead) return;	679	if (self.short == .dead) return;
696	try function.freeValue(self.short);	680	try function.freeValue(self.short);
697	self.short = .{ .dead = function.scope_generation };	681	self.short = .{ .dead = function.scope_generation };
698	tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });	682	tracking_log.debug("{} => {} (death)", .{ inst, self.* });
699	}	683	}
700		684
701	fn reuse(	685	fn reuse(
...	@@ -705,16 +689,13 @@ const InstTracking = struct {	...	@@ -705,16 +689,13 @@ const InstTracking = struct {
705	old_inst: Air.Inst.Index,	689	old_inst: Air.Inst.Index,
706	) void {	690	) void {
707	self.short = .{ .dead = function.scope_generation };	691	self.short = .{ .dead = function.scope_generation };
708	if (new_inst) \|inst\|	692	tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst });
709	tracking_log.debug("%{d} => {} (reuse %{d})", .{ inst, self.*, old_inst })
710	else
711	tracking_log.debug("tmp => {} (reuse %{d})", .{ self.*, old_inst });
712	}	693	}
713		694
714	fn liveOut(self: InstTracking, function: Self, inst: Air.Inst.Index) void {	695	fn liveOut(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
715	for (self.getRegs()) \|reg\| {	696	for (self.getRegs()) \|reg\| {
716	if (function.register_manager.isRegFree(reg)) {	697	if (function.register_manager.isRegFree(reg)) {
717	tracking_log.debug("%{d} => {} (live-out)", .{ inst, self.* });	698	tracking_log.debug("{} => {} (live-out)", .{ inst, self.* });
718	continue;	699	continue;
719	}	700	}
720		701
...	@@ -741,7 +722,7 @@ const InstTracking = struct {	...	@@ -741,7 +722,7 @@ const InstTracking = struct {
741	// Perform side-effects of freeValue manually.	722	// Perform side-effects of freeValue manually.
742	function.register_manager.freeReg(reg);	723	function.register_manager.freeReg(reg);
743		724
744	tracking_log.debug("%{d} => {} (live-out %{d})", .{ inst, self.*, tracked_inst });	725	tracking_log.debug("{} => {} (live-out {})", .{ inst, self.*, tracked_inst });
745	}	726	}
746	}	727	}
747		728
...	@@ -759,10 +740,10 @@ const InstTracking = struct {	...	@@ -759,10 +740,10 @@ const InstTracking = struct {
759	const FrameAlloc = struct {	740	const FrameAlloc = struct {
760	abi_size: u31,	741	abi_size: u31,
761	spill_pad: u3,	742	spill_pad: u3,
762	abi_align: Alignment,	743	abi_align: InternPool.Alignment,
763	ref_count: u16,	744	ref_count: u16,
764		745
765	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {	746	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc {
766	return .{	747	return .{
767	.abi_size = @intCast(alloc_abi.size),	748	.abi_size = @intCast(alloc_abi.size),
768	.spill_pad = alloc_abi.pad,	749	.spill_pad = alloc_abi.pad,
...	@@ -779,14 +760,14 @@ const FrameAlloc = struct {	...	@@ -779,14 +760,14 @@ const FrameAlloc = struct {
779	fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {	760	fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {
780	const abi_size = ty.abiSize(zcu);	761	const abi_size = ty.abiSize(zcu);
781	const spill_size = if (abi_size < 8)	762	const spill_size = if (abi_size < 8)
782	math.ceilPowerOfTwoAssert(u64, abi_size)	763	std.math.ceilPowerOfTwoAssert(u64, abi_size)
783	else	764	else
784	std.mem.alignForward(u64, abi_size, 8);	765	std.mem.alignForward(u64, abi_size, 8);
785	return init(.{	766	return init(.{
786	.size = spill_size,	767	.size = spill_size,
787	.pad = @intCast(spill_size - abi_size),	768	.pad = @intCast(spill_size - abi_size),
788	.alignment = ty.abiAlignment(zcu).maxStrict(	769	.alignment = ty.abiAlignment(zcu).maxStrict(
789	Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),	770	InternPool.Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
790	),	771	),
791	});	772	});
792	}	773	}
...	@@ -819,7 +800,7 @@ pub fn generate(	...	@@ -819,7 +800,7 @@ pub fn generate(
819	liveness: Liveness,	800	liveness: Liveness,
820	code: *std.ArrayListUnmanaged(u8),	801	code: *std.ArrayListUnmanaged(u8),
821	debug_output: link.File.DebugInfoOutput,	802	debug_output: link.File.DebugInfoOutput,
822	) CodeGenError!void {	803	) codegen.CodeGenError!void {
823	const zcu = pt.zcu;	804	const zcu = pt.zcu;
824	const comp = zcu.comp;	805	const comp = zcu.comp;
825	const gpa = zcu.gpa;	806	const gpa = zcu.gpa;
...	@@ -862,6 +843,11 @@ pub fn generate(	...	@@ -862,6 +843,11 @@ pub fn generate(
862	function.mir_instructions.deinit(gpa);	843	function.mir_instructions.deinit(gpa);
863	function.mir_extra.deinit(gpa);	844	function.mir_extra.deinit(gpa);
864	}	845	}
		846	try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);
		847	for (0..Temp.Index.max) \|temp_index\| {
		848	const temp: Temp.Index = @enumFromInt(temp_index);
		849	function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), InstTracking.init(.none));
		850	}
865		851
866	wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});	852	wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});
867		853
...	@@ -891,9 +877,9 @@ pub fn generate(	...	@@ -891,9 +877,9 @@ pub fn generate(
891	}));	877	}));
892	function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{	878	function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{
893	.size = Type.usize.abiSize(zcu),	879	.size = Type.usize.abiSize(zcu),
894	.alignment = Alignment.min(	880	.alignment = InternPool.Alignment.min(
895	call_info.stack_align,	881	call_info.stack_align,
896	Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),	882	InternPool.Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
897	),	883	),
898	}));	884	}));
899	function.frame_allocs.set(	885	function.frame_allocs.set(
...	@@ -972,7 +958,7 @@ pub fn generateLazy(	...	@@ -972,7 +958,7 @@ pub fn generateLazy(
972	lazy_sym: link.File.LazySymbol,	958	lazy_sym: link.File.LazySymbol,
973	code: *std.ArrayListUnmanaged(u8),	959	code: *std.ArrayListUnmanaged(u8),
974	debug_output: link.File.DebugInfoOutput,	960	debug_output: link.File.DebugInfoOutput,
975	) CodeGenError!void {	961	) codegen.CodeGenError!void {
976	const comp = bin_file.comp;	962	const comp = bin_file.comp;
977	const gpa = comp.gpa;	963	const gpa = comp.gpa;
978	// This function is for generating global code, so we use the root module.	964	// This function is for generating global code, so we use the root module.
...	@@ -1169,14 +1155,14 @@ fn formatWipMir(	...	@@ -1169,14 +1155,14 @@ fn formatWipMir(
1169	lower.mir.extraData(Mir.Imm64, mir_inst.data.ai.i).data.decode(),	1155	lower.mir.extraData(Mir.Imm64, mir_inst.data.ai.i).data.decode(),
1170	}),	1156	}),
1171	.pseudo_dbg_local_as => {	1157	.pseudo_dbg_local_as => {
1172	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1158	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1173	.base = .{ .reloc = mir_inst.data.as.sym_index },	1159	.base = .{ .reloc = mir_inst.data.as.sym_index },
1174	}) };	1160	}) };
1175	try writer.print(" {}, {}", .{ mir_inst.data.as.air_inst, mem_op.fmt(.m) });	1161	try writer.print(" {}, {}", .{ mir_inst.data.as.air_inst, mem_op.fmt(.m) });
1176	},	1162	},
1177	.pseudo_dbg_local_aso => {	1163	.pseudo_dbg_local_aso => {
1178	const sym_off = lower.mir.extraData(bits.SymbolOffset, mir_inst.data.ax.payload).data;	1164	const sym_off = lower.mir.extraData(bits.SymbolOffset, mir_inst.data.ax.payload).data;
1179	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1165	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1180	.base = .{ .reloc = sym_off.sym_index },	1166	.base = .{ .reloc = sym_off.sym_index },
1181	.disp = sym_off.off,	1167	.disp = sym_off.off,
1182	}) };	1168	}) };
...	@@ -1184,7 +1170,7 @@ fn formatWipMir(	...	@@ -1184,7 +1170,7 @@ fn formatWipMir(
1184	},	1170	},
1185	.pseudo_dbg_local_aro => {	1171	.pseudo_dbg_local_aro => {
1186	const air_off = lower.mir.extraData(Mir.AirOffset, mir_inst.data.rx.payload).data;	1172	const air_off = lower.mir.extraData(Mir.AirOffset, mir_inst.data.rx.payload).data;
1187	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1173	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1188	.base = .{ .reg = mir_inst.data.rx.r1 },	1174	.base = .{ .reg = mir_inst.data.rx.r1 },
1189	.disp = air_off.off,	1175	.disp = air_off.off,
1190	}) };	1176	}) };
...	@@ -1192,14 +1178,14 @@ fn formatWipMir(	...	@@ -1192,14 +1178,14 @@ fn formatWipMir(
1192	},	1178	},
1193	.pseudo_dbg_local_af => {	1179	.pseudo_dbg_local_af => {
1194	const frame_addr = lower.mir.extraData(bits.FrameAddr, mir_inst.data.ax.payload).data;	1180	const frame_addr = lower.mir.extraData(bits.FrameAddr, mir_inst.data.ax.payload).data;
1195	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1181	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1196	.base = .{ .frame = frame_addr.index },	1182	.base = .{ .frame = frame_addr.index },
1197	.disp = frame_addr.off,	1183	.disp = frame_addr.off,
1198	}) };	1184	}) };
1199	try writer.print(" {}, {d}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });	1185	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
1200	},	1186	},
1201	.pseudo_dbg_local_am => {	1187	.pseudo_dbg_local_am => {
1202	const mem_op: Instruction.Operand = .{	1188	const mem_op: encoder.Instruction.Operand = .{
1203	.mem = lower.mir.extraData(Mir.Memory, mir_inst.data.ax.payload).data.decode(),	1189	.mem = lower.mir.extraData(Mir.Memory, mir_inst.data.ax.payload).data.decode(),
1204	};	1190	};
1205	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });	1191	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
...	@@ -1221,7 +1207,7 @@ fn formatTracking(	...	@@ -1221,7 +1207,7 @@ fn formatTracking(
1221	writer: anytype,	1207	writer: anytype,
1222	) @TypeOf(writer).Error!void {	1208	) @TypeOf(writer).Error!void {
1223	var it = data.self.inst_tracking.iterator();	1209	var it = data.self.inst_tracking.iterator();
1224	while (it.next()) \|entry\| try writer.print("\n%{d} = {}", .{ entry.key_ptr., entry.value_ptr. });	1210	while (it.next()) \|entry\| try writer.print("\n{} = {}", .{ entry.key_ptr., entry.value_ptr. });
1225	}	1211	}
1226	fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {	1212	fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {
1227	return .{ .data = .{ .self = self } };	1213	return .{ .data = .{ .self = self } };
...	@@ -1427,7 +1413,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed	...	@@ -1427,7 +1413,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
1427	.i = @bitCast(s),	1413	.i = @bitCast(s),
1428	} },	1414	} },
1429	}),	1415	}),
1430	.unsigned => \|u\| _ = if (math.cast(u32, u)) \|small\| try self.addInst(.{	1416	.unsigned => \|u\| _ = if (std.math.cast(u32, u)) \|small\| try self.addInst(.{
1431	.tag = .pseudo,	1417	.tag = .pseudo,
1432	.ops = switch (tag) {	1418	.ops = switch (tag) {
1433	.dbg_local => .pseudo_dbg_local_ai_u,	1419	.dbg_local => .pseudo_dbg_local_ai_u,
...	@@ -1632,7 +1618,7 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2	...	@@ -1632,7 +1618,7 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2
1632	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {	1618	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1633	const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {	1619	const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {
1634	.signed => \|s\| .{ .ri_s, @bitCast(s) },	1620	.signed => \|s\| .{ .ri_s, @bitCast(s) },
1635	.unsigned => \|u\| if (math.cast(u32, u)) \|small\|	1621	.unsigned => \|u\| if (std.math.cast(u32, u)) \|small\|
1636	.{ .ri_u, small }	1622	.{ .ri_u, small }
1637	else	1623	else
1638	.{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) },	1624	.{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) },
...	@@ -1831,8 +1817,8 @@ fn asmRegisterMemoryImmediate(	...	@@ -1831,8 +1817,8 @@ fn asmRegisterMemoryImmediate(
1831	imm: Immediate,	1817	imm: Immediate,
1832	) !void {	1818	) !void {
1833	if (switch (imm) {	1819	if (switch (imm) {
1834	.signed => \|s\| if (math.cast(i16, s)) \|x\| @as(u16, @bitCast(x)) else null,	1820	.signed => \|s\| if (std.math.cast(i16, s)) \|x\| @as(u16, @bitCast(x)) else null,
1835	.unsigned => \|u\| math.cast(u16, u),	1821	.unsigned => \|u\| std.math.cast(u16, u),
1836	.reloc => unreachable,	1822	.reloc => unreachable,
1837	}) \|small_imm\| {	1823	}) \|small_imm\| {
1838	_ = try self.addInst(.{	1824	_ = try self.addInst(.{
...	@@ -1967,8 +1953,8 @@ fn gen(self: *Self) InnerError!void {	...	@@ -1967,8 +1953,8 @@ fn gen(self: *Self) InnerError!void {
1967	const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);	1953	const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
1968	if (cc != .naked) {	1954	if (cc != .naked) {
1969	try self.asmRegister(.{ ._, .push }, .rbp);	1955	try self.asmRegister(.{ ._, .push }, .rbp);
1970	try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, Immediate.s(8));	1956	try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, .s(8));
1971	try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, Immediate.s(0));	1957	try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, .s(0));
1972	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);	1958	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1973	try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp);	1959	try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp);
1974	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();	1960	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
...	@@ -2016,7 +2002,7 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2016,7 +2002,7 @@ fn gen(self: *Self) InnerError!void {
2016	.{},	2002	.{},
2017	);	2003	);
2018		2004
2019	try self.asmRegisterImmediate(.{ ._, .cmp }, .al, Immediate.u(info.fp_count));	2005	try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count));
2020	const skip_sse_reloc = try self.asmJccReloc(.na, undefined);	2006	const skip_sse_reloc = try self.asmJccReloc(.na, undefined);
2021		2007
2022	const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type });	2008	const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type });
...	@@ -2055,15 +2041,15 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2055,15 +2041,15 @@ fn gen(self: *Self) InnerError!void {
2055	const backpatch_stack_dealloc = try self.asmPlaceholder();	2041	const backpatch_stack_dealloc = try self.asmPlaceholder();
2056	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();	2042	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
2057	try self.asmRegister(.{ ._, .pop }, .rbp);	2043	try self.asmRegister(.{ ._, .pop }, .rbp);
2058	try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, Immediate.s(8));	2044	try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, .s(8));
2059	try self.asmOpOnly(.{ ._, .ret });	2045	try self.asmOpOnly(.{ ._, .ret });
2060		2046
2061	const frame_layout = try self.computeFrameLayout(cc);	2047	const frame_layout = try self.computeFrameLayout(cc);
2062	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);	2048	const need_frame_align = frame_layout.stack_mask != std.math.maxInt(u32);
2063	const need_stack_adjust = frame_layout.stack_adjust > 0;	2049	const need_stack_adjust = frame_layout.stack_adjust > 0;
2064	const need_save_reg = frame_layout.save_reg_list.count() > 0;	2050	const need_save_reg = frame_layout.save_reg_list.count() > 0;
2065	if (need_frame_align) {	2051	if (need_frame_align) {
2066	const page_align = @as(u32, math.maxInt(u32)) << 12;	2052	const page_align = @as(u32, std.math.maxInt(u32)) << 12;
2067	self.mir_instructions.set(backpatch_frame_align, .{	2053	self.mir_instructions.set(backpatch_frame_align, .{
2068	.tag = .@"and",	2054	.tag = .@"and",
2069	.ops = .ri_s,	2055	.ops = .ri_s,
...	@@ -2170,23 +2156,18 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2170,23 +2156,18 @@ fn gen(self: *Self) InnerError!void {
2170	});	2156	});
2171	}	2157	}
2172		2158
2173	fn checkInvariantsAfterAirInst(self: *Self, inst: Air.Inst.Index, old_air_bookkeeping: @TypeOf(air_bookkeeping_init)) void {	2159	fn checkInvariantsAfterAirInst(self: *Self) void {
2174	assert(!self.register_manager.lockedRegsExist());	2160	assert(!self.register_manager.lockedRegsExist());
2175		2161
2176	if (std.debug.runtime_safety) {	2162	if (std.debug.runtime_safety) {
2177	if (self.air_bookkeeping < old_air_bookkeeping + 1) {	2163	// check consistency of tracked registers
2178	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, self.air.instructions.items(.tag)[@intFromEnum(inst)] });	2164	var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
2179	}	2165	while (it.next()) \|index\| {
2180		2166	const tracked_inst = self.register_manager.registers[index];
2181	{ // check consistency of tracked registers	2167	const tracking = self.getResolvedInstValue(tracked_inst);
2182	var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });	2168	for (tracking.getRegs()) \|reg\| {
2183	while (it.next()) \|index\| {	2169	if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
2184	const tracked_inst = self.register_manager.registers[index];	2170	} else unreachable; // tracked register not in use
2185	const tracking = self.getResolvedInstValue(tracked_inst);
2186	for (tracking.getRegs()) \|reg\| {
2187	if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
2188	} else unreachable; // tracked register not in use
2189	}
2190	}	2171	}
2191	}	2172	}
2192	}	2173	}
...	@@ -2202,6 +2183,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2202,6 +2183,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2202	const zcu = pt.zcu;	2183	const zcu = pt.zcu;
2203	const ip = &zcu.intern_pool;	2184	const ip = &zcu.intern_pool;
2204	const air_tags = self.air.instructions.items(.tag);	2185	const air_tags = self.air.instructions.items(.tag);
		2186	const air_datas = self.air.instructions.items(.data);
		2187	const use_old = self.target.ofmt == .coff;
2205		2188
2206	self.arg_index = 0;	2189	self.arg_index = 0;
2207	for (body) \|inst\| switch (air_tags[@intFromEnum(inst)]) {	2190	for (body) \|inst\| switch (air_tags[@intFromEnum(inst)]) {
...	@@ -2209,12 +2192,13 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2209,12 +2192,13 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2209	wip_mir_log.debug("{}", .{self.fmtAir(inst)});	2192	wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2210	verbose_tracking_log.debug("{}", .{self.fmtTracking()});	2193	verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2211		2194
2212	const old_air_bookkeeping = self.air_bookkeeping;	2195	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
2213	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);	2196	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
2214		2197
2215	try self.airArg(inst);	2198	try self.airArg(inst);
2216		2199
2217	self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);	2200	self.resetTemps();
		2201	self.checkInvariantsAfterAirInst();
2218	},	2202	},
2219	else => break,	2203	else => break,
2220	};	2204	};
...	@@ -2226,7 +2210,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2226,7 +2210,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2226	wip_mir_log.debug("{}", .{self.fmtAir(inst)});	2210	wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2227	verbose_tracking_log.debug("{}", .{self.fmtTracking()});	2211	verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2228		2212
2229	const old_air_bookkeeping = self.air_bookkeeping;	2213	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
2230	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);	2214	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
2231	switch (air_tags[@intFromEnum(inst)]) {	2215	switch (air_tags[@intFromEnum(inst)]) {
2232	// zig fmt: off	2216	// zig fmt: off
...	@@ -2260,7 +2244,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2260,7 +2244,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2260	.sub_sat => try self.airSubSat(inst),	2244	.sub_sat => try self.airSubSat(inst),
2261	.mul_sat => try self.airMulSat(inst),	2245	.mul_sat => try self.airMulSat(inst),
2262	.shl_sat => try self.airShlSat(inst),	2246	.shl_sat => try self.airShlSat(inst),
2263	.slice => try self.airSlice(inst),
2264		2247
2265	.sin,	2248	.sin,
2266	.cos,	2249	.cos,
...	@@ -2298,127 +2281,58 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2298,127 +2281,58 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2298	.cmp_vector => try self.airCmpVector(inst),	2281	.cmp_vector => try self.airCmpVector(inst),
2299	.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),	2282	.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
2300		2283
2301	.alloc => try self.airAlloc(inst),	2284	.bitcast => try self.airBitCast(inst),
2302	.ret_ptr => try self.airRetPtr(inst),	2285	.fptrunc => try self.airFptrunc(inst),
2303	.arg => try self.airDbgArg(inst),	2286	.fpext => try self.airFpext(inst),
2304	.assembly => try self.airAsm(inst),	2287	.intcast => try self.airIntCast(inst),
2305	.bitcast => try self.airBitCast(inst),	2288	.trunc => try self.airTrunc(inst),
2306	.block => try self.airBlock(inst),	2289	.is_non_null => try self.airIsNonNull(inst),
2307	.br => try self.airBr(inst),	2290	.is_null => try self.airIsNull(inst),
2308	.repeat => try self.airRepeat(inst),	2291	.is_non_err => try self.airIsNonErr(inst),
2309	.switch_dispatch => try self.airSwitchDispatch(inst),	2292	.is_err => try self.airIsErr(inst),
2310	.trap => try self.airTrap(),	2293	.load => try self.airLoad(inst),
2311	.breakpoint => try self.airBreakpoint(),	2294	.store => try self.airStore(inst, false),
2312	.ret_addr => try self.airRetAddr(inst),	2295	.store_safe => try self.airStore(inst, true),
2313	.frame_addr => try self.airFrameAddress(inst),	2296	.struct_field_val => try self.airStructFieldVal(inst),
2314	.cond_br => try self.airCondBr(inst),	2297	.float_from_int => try self.airFloatFromInt(inst),
2315	.fptrunc => try self.airFptrunc(inst),	2298	.int_from_float => try self.airIntFromFloat(inst),
2316	.fpext => try self.airFpext(inst),	2299	.cmpxchg_strong => try self.airCmpxchg(inst),
2317	.intcast => try self.airIntCast(inst),	2300	.cmpxchg_weak => try self.airCmpxchg(inst),
2318	.trunc => try self.airTrunc(inst),	2301	.atomic_rmw => try self.airAtomicRmw(inst),
2319	.int_from_bool => try self.airIntFromBool(inst),	2302	.atomic_load => try self.airAtomicLoad(inst),
2320	.is_non_null => try self.airIsNonNull(inst),	2303	.memcpy => try self.airMemcpy(inst),
2321	.is_non_null_ptr => try self.airIsNonNullPtr(inst),	2304	.memset => try self.airMemset(inst, false),
2322	.is_null => try self.airIsNull(inst),	2305	.memset_safe => try self.airMemset(inst, true),
2323	.is_null_ptr => try self.airIsNullPtr(inst),	2306	.set_union_tag => try self.airSetUnionTag(inst),
2324	.is_non_err => try self.airIsNonErr(inst),	2307	.get_union_tag => try self.airGetUnionTag(inst),
2325	.is_non_err_ptr => try self.airIsNonErrPtr(inst),	2308	.clz => try self.airClz(inst),
2326	.is_err => try self.airIsErr(inst),	2309	.ctz => try self.airCtz(inst),
2327	.is_err_ptr => try self.airIsErrPtr(inst),	2310	.popcount => try self.airPopCount(inst),
2328	.load => try self.airLoad(inst),	2311	.byte_swap => try self.airByteSwap(inst),
2329	.loop => try self.airLoop(inst),	2312	.bit_reverse => try self.airBitReverse(inst),
2330	.int_from_ptr => try self.airIntFromPtr(inst),	2313	.tag_name => try self.airTagName(inst),
2331	.ret => try self.airRet(inst, false),	2314	.error_name => try self.airErrorName(inst),
2332	.ret_safe => try self.airRet(inst, true),	2315	.splat => try self.airSplat(inst),
2333	.ret_load => try self.airRetLoad(inst),	2316	.select => try self.airSelect(inst),
2334	.store => try self.airStore(inst, false),	2317	.shuffle => try self.airShuffle(inst),
2335	.store_safe => try self.airStore(inst, true),	2318	.reduce => try self.airReduce(inst),
2336	.struct_field_ptr=> try self.airStructFieldPtr(inst),	2319	.aggregate_init => try self.airAggregateInit(inst),
2337	.struct_field_val=> try self.airStructFieldVal(inst),	2320	.union_init => try self.airUnionInit(inst),
2338	.array_to_slice => try self.airArrayToSlice(inst),	2321	.prefetch => try self.airPrefetch(inst),
2339	.float_from_int => try self.airFloatFromInt(inst),	2322	.mul_add => try self.airMulAdd(inst),
2340	.int_from_float => try self.airIntFromFloat(inst),
2341	.cmpxchg_strong => try self.airCmpxchg(inst),
2342	.cmpxchg_weak => try self.airCmpxchg(inst),
2343	.atomic_rmw => try self.airAtomicRmw(inst),
2344	.atomic_load => try self.airAtomicLoad(inst),
2345	.memcpy => try self.airMemcpy(inst),
2346	.memset => try self.airMemset(inst, false),
2347	.memset_safe => try self.airMemset(inst, true),
2348	.set_union_tag => try self.airSetUnionTag(inst),
2349	.get_union_tag => try self.airGetUnionTag(inst),
2350	.clz => try self.airClz(inst),
2351	.ctz => try self.airCtz(inst),
2352	.popcount => try self.airPopCount(inst),
2353	.byte_swap => try self.airByteSwap(inst),
2354	.bit_reverse => try self.airBitReverse(inst),
2355	.tag_name => try self.airTagName(inst),
2356	.error_name => try self.airErrorName(inst),
2357	.splat => try self.airSplat(inst),
2358	.select => try self.airSelect(inst),
2359	.shuffle => try self.airShuffle(inst),
2360	.reduce => try self.airReduce(inst),
2361	.aggregate_init => try self.airAggregateInit(inst),
2362	.union_init => try self.airUnionInit(inst),
2363	.prefetch => try self.airPrefetch(inst),
2364	.mul_add => try self.airMulAdd(inst),
2365	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
2366
2367	.@"try" => try self.airTry(inst),
2368	.try_cold => try self.airTry(inst), // TODO
2369	.try_ptr => try self.airTryPtr(inst),
2370	.try_ptr_cold => try self.airTryPtr(inst), // TODO
2371
2372	.dbg_stmt => try self.airDbgStmt(inst),
2373	.dbg_empty_stmt => try self.airDbgEmptyStmt(),
2374	.dbg_inline_block => try self.airDbgInlineBlock(inst),
2375	.dbg_var_ptr,
2376	.dbg_var_val,
2377	.dbg_arg_inline,
2378	=> try self.airDbgVar(inst),
2379
2380	.call => try self.airCall(inst, .auto),
2381	.call_always_tail => try self.airCall(inst, .always_tail),
2382	.call_never_tail => try self.airCall(inst, .never_tail),
2383	.call_never_inline => try self.airCall(inst, .never_inline),
2384		2323
2385	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),	2324	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
2386	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),	2325	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
2387	.atomic_store_release => try self.airAtomicStore(inst, .release),	2326	.atomic_store_release => try self.airAtomicStore(inst, .release),
2388	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),	2327	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
2389		2328
2390	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
2391	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
2392	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
2393	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
2394
2395	.field_parent_ptr => try self.airFieldParentPtr(inst),
2396
2397	.switch_br => try self.airSwitchBr(inst),
2398	.loop_switch_br => try self.airLoopSwitchBr(inst),
2399	.slice_ptr => try self.airSlicePtr(inst),
2400	.slice_len => try self.airSliceLen(inst),
2401
2402	.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
2403	.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
2404
2405	.array_elem_val => try self.airArrayElemVal(inst),	2329	.array_elem_val => try self.airArrayElemVal(inst),
2406	.slice_elem_val => try self.airSliceElemVal(inst),	2330	.slice_elem_val => try self.airSliceElemVal(inst),
2407	.slice_elem_ptr => try self.airSliceElemPtr(inst),
2408	.ptr_elem_val => try self.airPtrElemVal(inst),	2331	.ptr_elem_val => try self.airPtrElemVal(inst),
2409	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
2410
2411	.inferred_alloc, .inferred_alloc_comptime => unreachable,
2412	.unreach => self.finishAirBookkeeping(),
2413		2332
2414	.optional_payload => try self.airOptionalPayload(inst),	2333	.optional_payload => try self.airOptionalPayload(inst),
2415	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
2416	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
2417	.unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst),	2334	.unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst),
2418	.unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst),	2335	.unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst),
2419	.unwrap_errunion_err_ptr => try self.airUnwrapErrUnionErrPtr(inst),
2420	.unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst),
2421	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
2422	.err_return_trace => try self.airErrReturnTrace(inst),	2336	.err_return_trace => try self.airErrReturnTrace(inst),
2423	.set_err_return_trace => try self.airSetErrReturnTrace(inst),	2337	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
2424	.save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst),	2338	.save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst),
...	@@ -2426,7 +2340,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2426,7 +2340,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2426	.wrap_optional => try self.airWrapOptional(inst),	2340	.wrap_optional => try self.airWrapOptional(inst),
2427	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),	2341	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
2428	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),	2342	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
		2343	// zig fmt: on
2429		2344
		2345	.add_safe,
		2346	.sub_safe,
		2347	.mul_safe,
		2348	=> return self.fail("TODO implement safety_checked_instructions", .{}),
2430	.add_optimized,	2349	.add_optimized,
2431	.sub_optimized,	2350	.sub_optimized,
2432	.mul_optimized,	2351	.mul_optimized,
...	@@ -2448,13 +2367,429 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2448,13 +2367,429 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2448	.int_from_float_optimized,	2367	.int_from_float_optimized,
2449	=> return self.fail("TODO implement optimized float mode", .{}),	2368	=> return self.fail("TODO implement optimized float mode", .{}),
2450		2369
2451	.add_safe,	2370	.arg => try self.airDbgArg(inst),
2452	.sub_safe,	2371	.alloc => if (use_old) try self.airAlloc(inst) else {
2453	.mul_safe,	2372	var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{
2454	=> return self.fail("TODO implement safety_checked_instructions", .{}),	2373	.index = try self.allocMemPtr(inst),
		2374	} });
		2375	try slot.moveTo(inst, self);
		2376	},
		2377	.inferred_alloc => unreachable,
		2378	.inferred_alloc_comptime => unreachable,
		2379	.ret_ptr => if (use_old) try self.airRetPtr(inst) else {
		2380	var slot = switch (self.ret_mcv.long) {
		2381	else => unreachable,
		2382	.none => try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{
		2383	.index = try self.allocMemPtr(inst),
		2384	} }),
		2385	.load_frame => slot: {
		2386	var slot = try self.tempFromValue(self.typeOfIndex(inst), self.ret_mcv.long);
		2387	try slot.toOffset(self.ret_mcv.short.indirect.off, self);
		2388	break :slot slot;
		2389	},
		2390	};
		2391	try slot.moveTo(inst, self);
		2392	},
		2393	.assembly => try self.airAsm(inst),
		2394	.block => if (use_old) try self.airBlock(inst) else {
		2395	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2396	const extra = self.air.extraData(Air.Block, ty_pl.payload);
		2397	try self.asmPseudo(.pseudo_dbg_enter_block_none);
		2398	try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
		2399	try self.asmPseudo(.pseudo_dbg_leave_block_none);
		2400	},
		2401	.loop => if (use_old) try self.airLoop(inst) else {
		2402	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2403	const extra = self.air.extraData(Air.Block, ty_pl.payload);
		2404	self.scope_generation += 1;
		2405	try self.loops.putNoClobber(self.gpa, inst, .{
		2406	.state = try self.saveState(),
		2407	.target = @intCast(self.mir_instructions.len),
		2408	});
		2409	defer assert(self.loops.remove(inst));
		2410	try self.genBodyBlock(@ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
		2411	},
		2412	.repeat => if (use_old) try self.airRepeat(inst) else {
		2413	const repeat = air_datas[@intFromEnum(inst)].repeat;
		2414	const loop = self.loops.get(repeat.loop_inst).?;
		2415	try self.restoreState(loop.state, &.{}, .{
		2416	.emit_instructions = true,
		2417	.update_tracking = false,
		2418	.resurrect = false,
		2419	.close_scope = true,
		2420	});
		2421	_ = try self.asmJmpReloc(loop.target);
		2422	},
		2423	.br => try self.airBr(inst),
		2424	.trap => try self.asmOpOnly(.{ ._, .ud2 }),
		2425	.breakpoint => try self.asmOpOnly(.{ ._, .int3 }),
		2426	.ret_addr => if (use_old) try self.airRetAddr(inst) else {
		2427	var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .load_frame = .{
		2428	.index = .ret_addr,
		2429	} });
		2430	while (try slot.toAnyReg(self)) {}
		2431	try slot.moveTo(inst, self);
		2432	},
		2433	.frame_addr => if (use_old) try self.airFrameAddress(inst) else {
		2434	var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{
		2435	.index = .base_ptr,
		2436	} });
		2437	try slot.moveTo(inst, self);
		2438	},
		2439	.call => try self.airCall(inst, .auto),
		2440	.call_always_tail => try self.airCall(inst, .always_tail),
		2441	.call_never_tail => try self.airCall(inst, .never_tail),
		2442	.call_never_inline => try self.airCall(inst, .never_inline),
2455		2443
2456	.is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),	2444	.cond_br => try self.airCondBr(inst),
		2445	.switch_br => try self.airSwitchBr(inst),
		2446	.loop_switch_br => try self.airLoopSwitchBr(inst),
		2447	.switch_dispatch => try self.airSwitchDispatch(inst),
		2448	.@"try", .try_cold => try self.airTry(inst),
		2449	.try_ptr, .try_ptr_cold => try self.airTryPtr(inst),
		2450	.dbg_stmt => if (use_old) try self.airDbgStmt(inst) else {
		2451	const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt;
		2452	_ = try self.addInst(.{
		2453	.tag = .pseudo,
		2454	.ops = .pseudo_dbg_line_line_column,
		2455	.data = .{ .line_column = .{
		2456	.line = dbg_stmt.line,
		2457	.column = dbg_stmt.column,
		2458	} },
		2459	});
		2460	},
		2461	.dbg_empty_stmt => if (use_old) try self.airDbgEmptyStmt() else {
		2462	if (self.mir_instructions.len > 0) {
		2463	const prev_mir_op = &self.mir_instructions.items(.ops)[self.mir_instructions.len - 1];
		2464	if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column)
		2465	prev_mir_op.* = .pseudo_dbg_line_line_column;
		2466	}
		2467	try self.asmOpOnly(.{ ._, .nop });
		2468	},
		2469	.dbg_inline_block => if (use_old) try self.airDbgInlineBlock(inst) else {
		2470	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2471	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
		2472	const old_inline_func = self.inline_func;
		2473	defer self.inline_func = old_inline_func;
		2474	self.inline_func = extra.data.func;
		2475	_ = try self.addInst(.{
		2476	.tag = .pseudo,
		2477	.ops = .pseudo_dbg_enter_inline_func,
		2478	.data = .{ .func = extra.data.func },
		2479	});
		2480	try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
		2481	_ = try self.addInst(.{
		2482	.tag = .pseudo,
		2483	.ops = .pseudo_dbg_leave_inline_func,
		2484	.data = .{ .func = old_inline_func },
		2485	});
		2486	},
		2487	.dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try self.airDbgVar(inst) else {
		2488	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
		2489	var ops = try self.tempsFromOperands(inst, .{pl_op.operand});
		2490	try self.genLocalDebugInfo(inst, ops[0].tracking(self).short);
		2491	try ops[0].die(self);
		2492	},
		2493	.is_null_ptr => if (use_old) try self.airIsNullPtr(inst) else {
		2494	const un_op = air_datas[@intFromEnum(inst)].un_op;
		2495	const opt_ty = self.typeOf(un_op).childType(zcu);
		2496	const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
		2497	const opt_child_ty = opt_ty.optionalChild(zcu);
		2498	const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
		2499	var ops = try self.tempsFromOperands(inst, .{un_op});
		2500	if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self);
		2501	while (try ops[0].toLea(self)) {}
		2502	try self.asmMemoryImmediate(
		2503	.{ ._, .cmp },
		2504	try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)
		2505	.byte
		2506	else if (opt_child_ty.isSlice(zcu))
		2507	.qword
		2508	else
		2509	Memory.Size.fromSize(opt_child_abi_size)),
		2510	.u(0),
		2511	);
		2512	var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
		2513	try ops[0].die(self);
		2514	try is_null.moveTo(inst, self);
		2515	},
		2516	.is_non_null_ptr => if (use_old) try self.airIsNonNullPtr(inst) else {
		2517	const un_op = air_datas[@intFromEnum(inst)].un_op;
		2518	const opt_ty = self.typeOf(un_op).childType(zcu);
		2519	const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
		2520	const opt_child_ty = opt_ty.optionalChild(zcu);
		2521	const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
		2522	var ops = try self.tempsFromOperands(inst, .{un_op});
		2523	if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self);
		2524	while (try ops[0].toLea(self)) {}
		2525	try self.asmMemoryImmediate(
		2526	.{ ._, .cmp },
		2527	try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)
		2528	.byte
		2529	else if (opt_child_ty.isSlice(zcu))
		2530	.qword
		2531	else
		2532	Memory.Size.fromSize(opt_child_abi_size)),
		2533	.u(0),
		2534	);
		2535	var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
		2536	try ops[0].die(self);
		2537	try is_non_null.moveTo(inst, self);
		2538	},
		2539	.is_err_ptr => if (use_old) try self.airIsErrPtr(inst) else {
		2540	const un_op = air_datas[@intFromEnum(inst)].un_op;
		2541	const eu_ty = self.typeOf(un_op).childType(zcu);
		2542	const eu_err_ty = eu_ty.errorUnionSet(zcu);
		2543	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		2544	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		2545	var ops = try self.tempsFromOperands(inst, .{un_op});
		2546	try ops[0].toOffset(eu_err_off, self);
		2547	while (try ops[0].toLea(self)) {}
		2548	try self.asmMemoryImmediate(
		2549	.{ ._, .cmp },
		2550	try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),
		2551	.u(0),
		2552	);
		2553	var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
		2554	try ops[0].die(self);
		2555	try is_err.moveTo(inst, self);
		2556	},
		2557	.is_non_err_ptr => if (use_old) try self.airIsNonErrPtr(inst) else {
		2558	const un_op = air_datas[@intFromEnum(inst)].un_op;
		2559	const eu_ty = self.typeOf(un_op).childType(zcu);
		2560	const eu_err_ty = eu_ty.errorUnionSet(zcu);
		2561	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		2562	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		2563	var ops = try self.tempsFromOperands(inst, .{un_op});
		2564	try ops[0].toOffset(eu_err_off, self);
		2565	while (try ops[0].toLea(self)) {}
		2566	try self.asmMemoryImmediate(
		2567	.{ ._, .cmp },
		2568	try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),
		2569	.u(0),
		2570	);
		2571	var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
		2572	try ops[0].die(self);
		2573	try is_non_err.moveTo(inst, self);
		2574	},
		2575	.int_from_ptr => if (use_old) try self.airIntFromPtr(inst) else {
		2576	const un_op = air_datas[@intFromEnum(inst)].un_op;
		2577	var ops = try self.tempsFromOperands(inst, .{un_op});
		2578	try ops[0].toLimb(0, self);
		2579	try ops[0].moveTo(inst, self);
		2580	},
		2581	.int_from_bool => if (use_old) try self.airIntFromBool(inst) else {
		2582	const un_op = air_datas[@intFromEnum(inst)].un_op;
		2583	var ops = try self.tempsFromOperands(inst, .{un_op});
		2584	try ops[0].moveTo(inst, self);
		2585	},
		2586	.ret => try self.airRet(inst, false),
		2587	.ret_safe => try self.airRet(inst, true),
		2588	.ret_load => try self.airRetLoad(inst),
		2589	.unreach => {},
		2590	.optional_payload_ptr => if (use_old) try self.airOptionalPayloadPtr(inst) else {
		2591	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2592	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2593	try ops[0].moveTo(inst, self);
		2594	},
		2595	.optional_payload_ptr_set => if (use_old) try self.airOptionalPayloadPtrSet(inst) else {
		2596	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2597	const opt_ty = self.typeOf(ty_op.operand).childType(zcu);
		2598	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2599	if (!opt_ty.optionalReprIsPayload(zcu)) {
		2600	const opt_child_ty = opt_ty.optionalChild(zcu);
		2601	const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu));
		2602	try ops[0].toOffset(opt_child_abi_size, self);
		2603	var has_value = try self.tempFromValue(Type.bool, .{ .immediate = 1 });
		2604	try ops[0].store(&has_value, self);
		2605	try has_value.die(self);
		2606	try ops[0].toOffset(-opt_child_abi_size, self);
		2607	}
		2608	try ops[0].moveTo(inst, self);
		2609	},
		2610	.unwrap_errunion_payload_ptr => if (use_old) try self.airUnwrapErrUnionPayloadPtr(inst) else {
		2611	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2612	const eu_ty = self.typeOf(ty_op.operand).childType(zcu);
		2613	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		2614	const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
		2615	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2616	try ops[0].toOffset(eu_pl_off, self);
		2617	try ops[0].moveTo(inst, self);
		2618	},
		2619	.unwrap_errunion_err_ptr => if (use_old) try self.airUnwrapErrUnionErrPtr(inst) else {
		2620	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2621	const eu_ty = self.typeOf(ty_op.operand).childType(zcu);
		2622	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		2623	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		2624	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2625	try ops[0].toOffset(eu_err_off, self);
		2626	var err = try ops[0].load(eu_ty.errorUnionSet(zcu), self);
		2627	try ops[0].die(self);
		2628	try err.moveTo(inst, self);
		2629	},
		2630	.errunion_payload_ptr_set => if (use_old) try self.airErrUnionPayloadPtrSet(inst) else {
		2631	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2632	const eu_ty = self.typeOf(ty_op.operand).childType(zcu);
		2633	const eu_err_ty = eu_ty.errorUnionSet(zcu);
		2634	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		2635	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		2636	const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
		2637	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2638	try ops[0].toOffset(eu_err_off, self);
		2639	var no_err = try self.tempFromValue(eu_err_ty, .{ .immediate = 0 });
		2640	try ops[0].store(&no_err, self);
		2641	try no_err.die(self);
		2642	try ops[0].toOffset(eu_pl_off - eu_err_off, self);
		2643	try ops[0].moveTo(inst, self);
		2644	},
		2645	.struct_field_ptr => if (use_old) try self.airStructFieldPtr(inst) else {
		2646	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2647	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
		2648	var ops = try self.tempsFromOperands(inst, .{extra.struct_operand});
		2649	try ops[0].toOffset(self.fieldOffset(self.typeOf(extra.struct_operand), self.typeOfIndex(inst), extra.field_index), self);
		2650	try ops[0].moveTo(inst, self);
		2651	},
		2652	.struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else {
		2653	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2654	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2655	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 0), self);
		2656	try ops[0].moveTo(inst, self);
		2657	},
		2658	.struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else {
		2659	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2660	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2661	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 1), self);
		2662	try ops[0].moveTo(inst, self);
		2663	},
		2664	.struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else {
		2665	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2666	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2667	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 2), self);
		2668	try ops[0].moveTo(inst, self);
		2669	},
		2670	.struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else {
		2671	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2672	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2673	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 3), self);
		2674	try ops[0].moveTo(inst, self);
		2675	},
		2676	.slice => if (use_old) try self.airSlice(inst) else {
		2677	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2678	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
		2679	var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		2680	try ops[0].toPair(&ops[1], self);
		2681	try ops[0].moveTo(inst, self);
		2682	},
		2683	.slice_len => if (use_old) try self.airSliceLen(inst) else {
		2684	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2685	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2686	try ops[0].toLimb(1, self);
		2687	try ops[0].moveTo(inst, self);
		2688	},
		2689	.slice_ptr => if (use_old) try self.airSlicePtr(inst) else {
		2690	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2691	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2692	try ops[0].toLimb(0, self);
		2693	try ops[0].moveTo(inst, self);
		2694	},
		2695	.ptr_slice_len_ptr => if (use_old) try self.airPtrSliceLenPtr(inst) else {
		2696	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2697	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2698	try ops[0].toOffset(8, self);
		2699	try ops[0].moveTo(inst, self);
		2700	},
		2701	.ptr_slice_ptr_ptr => if (use_old) try self.airPtrSlicePtrPtr(inst) else {
		2702	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2703	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2704	try ops[0].toOffset(0, self);
		2705	try ops[0].moveTo(inst, self);
		2706	},
		2707	.slice_elem_ptr, .ptr_elem_ptr => \|tag\| if (use_old) switch (tag) {
		2708	else => unreachable,
		2709	.slice_elem_ptr => try self.airSliceElemPtr(inst),
		2710	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
		2711	} else {
		2712	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2713	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
		2714	var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		2715	switch (tag) {
		2716	else => unreachable,
		2717	.slice_elem_ptr => try ops[0].toLimb(0, self),
		2718	.ptr_elem_ptr => {},
		2719	}
		2720	const dst_ty = self.typeOfIndex(inst);
		2721	if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {
		2722	const elem_size = dst_ty.childType(zcu).abiSize(zcu);
		2723	if (elem_size == 0) break :zero_offset;
		2724	while (true) for (&ops) \|*op\| {
		2725	if (try op.toAnyReg(self)) break;
		2726	} else break;
		2727	const lhs_reg = ops[0].unwrap(self).temp.tracking(self).short.register.to64();
		2728	const rhs_reg = ops[1].unwrap(self).temp.tracking(self).short.register.to64();
		2729	if (!std.math.isPowerOfTwo(elem_size)) {
		2730	try self.spillEflagsIfOccupied();
		2731	try self.asmRegisterRegisterImmediate(
		2732	.{ .i_, .mul },
		2733	rhs_reg,
		2734	rhs_reg,
		2735	.u(elem_size),
		2736	);
		2737	try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		2738	.base = .{ .reg = lhs_reg },
		2739	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
		2740	});
		2741	} else if (elem_size > 8) {
		2742	try self.spillEflagsIfOccupied();
		2743	try self.asmRegisterImmediate(
		2744	.{ ._l, .sh },
		2745	rhs_reg,
		2746	.u(std.math.log2_int(u64, elem_size)),
		2747	);
		2748	try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		2749	.base = .{ .reg = lhs_reg },
		2750	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
		2751	});
		2752	} else try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		2753	.base = .{ .reg = lhs_reg },
		2754	.mod = .{ .rm = .{
		2755	.size = .qword,
		2756	.index = rhs_reg,
		2757	.scale = .fromFactor(@intCast(elem_size)),
		2758	} },
		2759	});
		2760	}
		2761	try ops[1].die(self);
		2762	try ops[0].moveTo(inst, self);
		2763	},
		2764	.array_to_slice => if (use_old) try self.airArrayToSlice(inst) else {
		2765	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2766	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2767	var len = try self.tempFromValue(Type.usize, .{
		2768	.immediate = self.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu),
		2769	});
		2770	try ops[0].toPair(&len, self);
		2771	try ops[0].moveTo(inst, self);
		2772	},
2457	.error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),	2773	.error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
		2774	.field_parent_ptr => if (use_old) try self.airFieldParentPtr(inst) else {
		2775	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2776	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
		2777	var ops = try self.tempsFromOperands(inst, .{extra.field_ptr});
		2778	try ops[0].toOffset(-self.fieldOffset(self.typeOfIndex(inst), self.typeOf(extra.field_ptr), extra.field_index), self);
		2779	try ops[0].moveTo(inst, self);
		2780	},
		2781
		2782	.is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
		2783
		2784	.wasm_memory_size => unreachable,
		2785	.wasm_memory_grow => unreachable,
		2786
		2787	.addrspace_cast => {
		2788	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		2789	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
		2790	try ops[0].moveTo(inst, self);
		2791	},
		2792
2458	.vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),	2793	.vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
2459		2794
2460	.c_va_arg => try self.airVaArg(inst),	2795	.c_va_arg => try self.airVaArg(inst),
...	@@ -2462,15 +2797,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2462,15 +2797,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2462	.c_va_end => try self.airVaEnd(inst),	2797	.c_va_end => try self.airVaEnd(inst),
2463	.c_va_start => try self.airVaStart(inst),	2798	.c_va_start => try self.airVaStart(inst),
2464		2799
2465	.wasm_memory_size => unreachable,
2466	.wasm_memory_grow => unreachable,
2467
2468	.work_item_id => unreachable,	2800	.work_item_id => unreachable,
2469	.work_group_size => unreachable,	2801	.work_group_size => unreachable,
2470	.work_group_id => unreachable,	2802	.work_group_id => unreachable,
2471	// zig fmt: on
2472	}	2803	}
2473	self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);	2804	self.resetTemps();
		2805	self.checkInvariantsAfterAirInst();
2474	}	2806	}
2475	verbose_tracking_log.debug("{}", .{self.fmtTracking()});	2807	verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2476	}	2808	}
...	@@ -2530,7 +2862,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {	...	@@ -2530,7 +2862,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2530	data_off += @intCast(tag_name_len + 1);	2862	data_off += @intCast(tag_name_len + 1);
2531	}	2863	}
2532		2864
2533	try self.airTrap();	2865	try self.asmOpOnly(.{ ._, .ud2 });
2534		2866
2535	for (exitlude_jump_relocs) \|reloc\| self.performReloc(reloc);	2867	for (exitlude_jump_relocs) \|reloc\| self.performReloc(reloc);
2536	try self.asmOpOnly(.{ ._, .ret });	2868	try self.asmOpOnly(.{ ._, .ret });
...	@@ -2544,6 +2876,10 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {	...	@@ -2544,6 +2876,10 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2544		2876
2545	fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {	2877	fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {
2546	for (value.getRegs()) \|reg\| try self.register_manager.getReg(reg, inst);	2878	for (value.getRegs()) \|reg\| try self.register_manager.getReg(reg, inst);
		2879	switch (value) {
		2880	else => {},
		2881	.eflags, .register_overflow => self.eflags_inst = inst,
		2882	}
2547	}	2883	}
2548		2884
2549	fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {	2885	fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
...	@@ -2577,26 +2913,18 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2577,26 +2913,18 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) !void {
2577	try self.inst_tracking.getPtr(inst).?.die(self, inst);	2913	try self.inst_tracking.getPtr(inst).?.die(self, inst);
2578	}	2914	}
2579		2915
2580	/// Called when there are no operands, and the instruction is always unreferenced.
2581	fn finishAirBookkeeping(self: *Self) void {
2582	if (std.debug.runtime_safety) {
2583	self.air_bookkeeping += 1;
2584	}
2585	}
2586
2587	fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {	2916	fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
2588	if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {	2917	if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {
2589	.none, .dead, .unreach => {},	2918	.none, .dead, .unreach => {},
2590	else => unreachable, // Why didn't the result die?	2919	else => unreachable, // Why didn't the result die?
2591	} else {	2920	} else {
2592	tracking_log.debug("%{d} => {} (birth)", .{ inst, result });	2921	tracking_log.debug("{} => {} (birth)", .{ inst, result });
2593	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));	2922	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
2594	// In some cases, an operand may be reused as the result.	2923	// In some cases, an operand may be reused as the result.
2595	// If that operand died and was a register, it was freed by	2924	// If that operand died and was a register, it was freed by
2596	// processDeath, so we have to "re-allocate" the register.	2925	// processDeath, so we have to "re-allocate" the register.
2597	self.getValueIfFree(result, inst);	2926	self.getValueIfFree(result, inst);
2598	}	2927	}
2599	self.finishAirBookkeeping();
2600	}	2928	}
2601		2929
2602	fn finishAir(	2930	fn finishAir(
...	@@ -2605,11 +2933,10 @@ fn finishAir(	...	@@ -2605,11 +2933,10 @@ fn finishAir(
2605	result: MCValue,	2933	result: MCValue,
2606	operands: [Liveness.bpi - 1]Air.Inst.Ref,	2934	operands: [Liveness.bpi - 1]Air.Inst.Ref,
2607	) !void {	2935	) !void {
2608	var tomb_bits = self.liveness.getTombBits(inst);	2936	const tomb_bits = self.liveness.getTombBits(inst);
2609	for (operands) \|op\| {	2937	for (0.., operands) \|op_index, op\| {
2610	const dies = @as(u1, @truncate(tomb_bits)) != 0;	2938	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
2611	tomb_bits >>= 1;	2939	if (self.reused_operands.isSet(op_index)) continue;
2612	if (!dies) continue;
2613	try self.processDeath(op.toIndexAllowNone() orelse continue);	2940	try self.processDeath(op.toIndexAllowNone() orelse continue);
2614	}	2941	}
2615	self.finishAirResult(inst, result);	2942	self.finishAirResult(inst, result);
...	@@ -2657,7 +2984,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo	...	@@ -2657,7 +2984,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2657	}	2984	}
2658	};	2985	};
2659	const sort_context = SortContext{ .frame_align = frame_align };	2986	const sort_context = SortContext{ .frame_align = frame_align };
2660	mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);	2987	std.mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
2661	}	2988	}
2662		2989
2663	const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)];	2990	const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)];
...	@@ -2697,13 +3024,13 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo	...	@@ -2697,13 +3024,13 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2697	@intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]);	3024	@intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]);
2698		3025
2699	return .{	3026	return .{
2700	.stack_mask = @as(u32, math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),	3027	.stack_mask = @as(u32, std.math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),
2701	.stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]),	3028	.stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]),
2702	.save_reg_list = save_reg_list,	3029	.save_reg_list = save_reg_list,
2703	};	3030	};
2704	}	3031	}
2705		3032
2706	fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) Alignment {	3033	fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) InternPool.Alignment {
2707	const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;	3034	const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;
2708	return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));	3035	return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));
2709	}	3036	}
...	@@ -2741,7 +3068,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {	...	@@ -2741,7 +3068,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
2741	const ptr_ty = self.typeOfIndex(inst);	3068	const ptr_ty = self.typeOfIndex(inst);
2742	const val_ty = ptr_ty.childType(zcu);	3069	const val_ty = ptr_ty.childType(zcu);
2743	return self.allocFrameIndex(FrameAlloc.init(.{	3070	return self.allocFrameIndex(FrameAlloc.init(.{
2744	.size = math.cast(u32, val_ty.abiSize(zcu)) orelse {	3071	.size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse {
2745	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});	3072	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});
2746	},	3073	},
2747	.alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),	3074	.alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),
...	@@ -2759,7 +3086,7 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {	...	@@ -2759,7 +3086,7 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {
2759	fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {	3086	fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
2760	const pt = self.pt;	3087	const pt = self.pt;
2761	const zcu = pt.zcu;	3088	const zcu = pt.zcu;
2762	const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse {	3089	const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
2763	return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(pt)});	3090	return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(pt)});
2764	};	3091	};
2765		3092
...	@@ -2857,8 +3184,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt	...	@@ -2857,8 +3184,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2857	}	3184	}
2858		3185
2859	if (opts.resurrect) for (	3186	if (opts.resurrect) for (
2860	self.inst_tracking.keys()[0..state.inst_tracking_len],	3187	self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len],
2861	self.inst_tracking.values()[0..state.inst_tracking_len],	3188	self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len],
2862	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);	3189	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);
2863	for (deaths) \|death\| try self.processDeath(death);	3190	for (deaths) \|death\| try self.processDeath(death);
2864		3191
...	@@ -3067,7 +3394,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3067,7 +3394,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3067	.{ .v_, .cvtps2ph },	3394	.{ .v_, .cvtps2ph },
3068	dst_reg,	3395	dst_reg,
3069	mat_src_reg.to128(),	3396	mat_src_reg.to128(),
3070	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	3397	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
3071	);	3398	);
3072	},	3399	},
3073	else => unreachable,	3400	else => unreachable,
...	@@ -3267,7 +3594,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3267,7 +3594,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3267		3594
3268	const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu);	3595	const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu);
3269	const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu);	3596	const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu);
3270	switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {	3597	switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) {
3271	.lt => {	3598	.lt => {
3272	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {	3599	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
3273	else => break :result null,	3600	else => break :result null,
...	@@ -3431,8 +3758,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3431,8 +3758,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3431	};	3758	};
3432		3759
3433	const dst_mcv = if (dst_int_info.bits <= src_storage_bits and	3760	const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
3434	math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==	3761	std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
3435	math.divCeil(u32, src_storage_bits, 64) catch unreachable and	3762	std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and
3436	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {	3763	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
3437	const dst_mcv = try self.allocRegOrMem(inst, true);	3764	const dst_mcv = try self.allocRegOrMem(inst, true);
3438	try self.genCopy(min_ty, dst_mcv, src_mcv, .{});	3765	try self.genCopy(min_ty, dst_mcv, src_mcv, .{});
...	@@ -3449,8 +3776,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3449,8 +3776,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3449	break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };	3776	break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
3450	}	3777	}
3451		3778
3452	const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;	3779	const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
3453	const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;	3780	const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
3454		3781
3455	const high_mcv: MCValue = if (dst_mcv.isMemory())	3782	const high_mcv: MCValue = if (dst_mcv.isMemory())
3456	dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()	3783	dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
...	@@ -3570,7 +3897,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3570,7 +3897,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3570	const dst_info = dst_elem_ty.intInfo(zcu);	3897	const dst_info = dst_elem_ty.intInfo(zcu);
3571	const src_info = src_elem_ty.intInfo(zcu);	3898	const src_info = src_elem_ty.intInfo(zcu);
3572		3899
3573	const mask_val = try pt.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));	3900	const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
3574		3901
3575	const splat_ty = try pt.vectorType(.{	3902	const splat_ty = try pt.vectorType(.{
3576	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),	3903	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
...	@@ -3607,7 +3934,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3607,7 +3934,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3607	.{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },	3934	.{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
3608	registerAlias(temp_reg, dst_abi_size),	3935	registerAlias(temp_reg, dst_abi_size),
3609	dst_alias,	3936	dst_alias,
3610	Immediate.u(1),	3937	.u(1),
3611	);	3938	);
3612	try self.asmRegisterRegisterRegister(	3939	try self.asmRegisterRegisterRegister(
3613	mir_tag,	3940	mir_tag,
...	@@ -3806,7 +4133,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3806,7 +4133,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3806	try self.asmMemoryImmediate(	4133	try self.asmMemoryImmediate(
3807	.{ ._, .mov },	4134	.{ ._, .mov },
3808	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },	4135	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
3809	Immediate.u(0),	4136	.u(0),
3810	);	4137	);
3811		4138
3812	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	4139	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	@@ -3920,11 +4247,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3920,11 +4247,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3920	.mod = .{ .rm = .{ .size = .qword } },	4247	.mod = .{ .rm = .{ .size = .qword } },
3921	},	4248	},
3922	);	4249	);
3923	try self.asmRegisterImmediate(	4250	try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0));
3924	.{ ._, .sbb },
3925	dst_mcv.register_pair[1],
3926	Immediate.u(0),
3927	);
3928	try self.freeValue(	4251	try self.freeValue(
3929	.{ .load_frame = .{ .index = signed_div_floor_state.frame_index } },	4252	.{ .load_frame = .{ .index = signed_div_floor_state.frame_index } },
3930	);	4253	);
...	@@ -4068,7 +4391,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4068,7 +4391,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
4068	break :cc .o;	4391	break :cc .o;
4069	} else cc: {	4392	} else cc: {
4070	try self.genSetReg(limit_reg, ty, .{	4393	try self.genSetReg(limit_reg, ty, .{
4071	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),	4394	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),
4072	}, .{});	4395	}, .{});
4073		4396
4074	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);	4397	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
...	@@ -4266,12 +4589,12 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4266,12 +4589,12 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4266	mat_rhs_mcv.register_pair[1],	4589	mat_rhs_mcv.register_pair[1],
4267	);	4590	);
4268		4591
4269	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));	4592	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
4270	try self.asmRegister(.{ ._, .not }, tmp_reg);	4593	try self.asmRegister(.{ ._, .not }, tmp_reg);
4271	try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));	4594	try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), .s(0));
4272	try self.freeValue(overflow);	4595	try self.freeValue(overflow);
4273	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);	4596	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
4274	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));	4597	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));
4275	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);	4598	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);
4276	break :result dst_mcv;	4599	break :result dst_mcv;
4277	}	4600	}
...	@@ -4321,7 +4644,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4321,7 +4644,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4321	break :cc .o;	4644	break :cc .o;
4322	} else cc: {	4645	} else cc: {
4323	try self.genSetReg(limit_reg, ty, .{	4646	try self.genSetReg(limit_reg, ty, .{
4324	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),	4647	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits),
4325	}, .{});	4648	}, .{});
4326	break :cc .c;	4649	break :cc .c;
4327	};	4650	};
...	@@ -4366,7 +4689,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4366,7 +4689,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4366	};	4689	};
4367		4690
4368	const tuple_ty = self.typeOfIndex(inst);	4691	const tuple_ty = self.typeOfIndex(inst);
4369	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {	4692	if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
4370	switch (partial_mcv) {	4693	switch (partial_mcv) {
4371	.register => \|reg\| {	4694	.register => \|reg\| {
4372	self.eflags_inst = inst;	4695	self.eflags_inst = inst;
...	@@ -4444,7 +4767,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4444,7 +4767,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4444	const cc = Condition.ne;	4767	const cc = Condition.ne;
4445		4768
4446	const tuple_ty = self.typeOfIndex(inst);	4769	const tuple_ty = self.typeOfIndex(inst);
4447	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {	4770	if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
4448	switch (partial_mcv) {	4771	switch (partial_mcv) {
4449	.register => \|reg\| {	4772	.register => \|reg\| {
4450	self.eflags_inst = inst;	4773	self.eflags_inst = inst;
...	@@ -4576,7 +4899,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4576,7 +4899,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4576	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {	4899	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
4577	const slow_inc = self.hasFeature(.slow_incdec);	4900	const slow_inc = self.hasFeature(.slow_incdec);
4578	const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));	4901	const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
4579	const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;	4902	const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
4580		4903
4581	try self.spillRegisters(&.{ .rax, .rcx, .rdx });	4904	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
4582	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });	4905	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
...	@@ -4618,7 +4941,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4618,7 +4941,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4618	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);	4941	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
4619		4942
4620	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	4943	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4621	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));	4944	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
4622	try self.asmMemoryRegister(.{ ._, .adc }, .{	4945	try self.asmMemoryRegister(.{ ._, .adc }, .{
4623	.base = .{ .frame = dst_mcv.load_frame.index },	4946	.base = .{ .frame = dst_mcv.load_frame.index },
4624	.mod = .{ .rm = .{	4947	.mod = .{ .rm = .{
...	@@ -4642,7 +4965,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4642,7 +4965,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4642	});	4965	});
4643	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());	4966	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
4644		4967
4645	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));	4968	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
4646	try self.asmMemoryRegister(.{ ._, .adc }, .{	4969	try self.asmMemoryRegister(.{ ._, .adc }, .{
4647	.base = .{ .frame = dst_mcv.load_frame.index },	4970	.base = .{ .frame = dst_mcv.load_frame.index },
4648	.mod = .{ .rm = .{	4971	.mod = .{ .rm = .{
...	@@ -4656,30 +4979,22 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4656,30 +4979,22 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4656	try self.asmSetccRegister(.c, .ch);	4979	try self.asmSetccRegister(.c, .ch);
4657		4980
4658	if (slow_inc) {	4981	if (slow_inc) {
4659	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));	4982	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
4660	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));	4983	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
4661	} else {	4984	} else {
4662	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());	4985	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
4663	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());	4986	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
4664	}	4987	}
4665	try self.asmRegisterImmediate(	4988	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
4666	.{ ._, .cmp },
4667	temp_regs[3].to32(),
4668	Immediate.u(limb_len),
4669	);
4670	_ = try self.asmJccReloc(.b, inner_loop);	4989	_ = try self.asmJccReloc(.b, inner_loop);
4671		4990
4672	try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx);	4991	try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx);
4673	const overflow = try self.asmJccReloc(.nz, undefined);	4992	const overflow = try self.asmJccReloc(.nz, undefined);
4674	const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	4993	const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4675	try self.asmRegisterImmediate(	4994	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len));
4676	.{ ._, .cmp },
4677	temp_regs[2].to32(),
4678	Immediate.u(limb_len),
4679	);
4680	const no_overflow = try self.asmJccReloc(.nb, undefined);	4995	const no_overflow = try self.asmJccReloc(.nb, undefined);
4681	if (slow_inc) {	4996	if (slow_inc) {
4682	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));	4997	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
4683	} else {	4998	} else {
4684	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());	4999	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
4685	}	5000	}
...	@@ -4691,7 +5006,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4691,7 +5006,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4691	.scale = .@"8",	5006	.scale = .@"8",
4692	.disp = lhs_mcv.load_frame.off - 8,	5007	.disp = lhs_mcv.load_frame.off - 8,
4693	} },	5008	} },
4694	}, Immediate.u(0));	5009	}, .u(0));
4695	_ = try self.asmJccReloc(.z, overflow_loop);	5010	_ = try self.asmJccReloc(.z, overflow_loop);
4696	self.performReloc(overflow);	5011	self.performReloc(overflow);
4697	try self.asmMemoryImmediate(.{ ._, .mov }, .{	5012	try self.asmMemoryImmediate(.{ ._, .mov }, .{
...	@@ -4701,20 +5016,16 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4701,20 +5016,16 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4701	.disp = dst_mcv.load_frame.off +	5016	.disp = dst_mcv.load_frame.off +
4702	@as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),	5017	@as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
4703	} },	5018	} },
4704	}, Immediate.u(1));	5019	}, .u(1));
4705	self.performReloc(no_overflow);	5020	self.performReloc(no_overflow);
4706		5021
4707	self.performReloc(skip_inner);	5022	self.performReloc(skip_inner);
4708	if (slow_inc) {	5023	if (slow_inc) {
4709	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	5024	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
4710	} else {	5025	} else {
4711	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	5026	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
4712	}	5027	}
4713	try self.asmRegisterImmediate(	5028	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
4714	.{ ._, .cmp },
4715	temp_regs[0].to32(),
4716	Immediate.u(limb_len),
4717	);
4718	_ = try self.asmJccReloc(.b, outer_loop);	5029	_ = try self.asmJccReloc(.b, outer_loop);
4719		5030
4720	break :result dst_mcv;	5031	break :result dst_mcv;
...	@@ -4750,7 +5061,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4750,7 +5061,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4750	try self.asmMemoryImmediate(	5061	try self.asmMemoryImmediate(
4751	.{ ._, .cmp },	5062	.{ ._, .cmp },
4752	try overflow.mem(self, self.memSize(Type.c_int)),	5063	try overflow.mem(self, self.memSize(Type.c_int)),
4753	Immediate.s(0),	5064	.s(0),
4754	);	5065	);
4755	try self.genSetMem(	5066	try self.genSetMem(
4756	.{ .frame = dst_mcv.load_frame.index },	5067	.{ .frame = dst_mcv.load_frame.index },
...	@@ -5038,7 +5349,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -5038,7 +5349,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
5038	try self.asmRegisterImmediate(	5349	try self.asmRegisterImmediate(
5039	.{ ._r, .sa },	5350	.{ ._r, .sa },
5040	registerAlias(divisor, abi_size),	5351	registerAlias(divisor, abi_size),
5041	Immediate.u(int_info.bits - 1),	5352	.u(int_info.bits - 1),
5042	);	5353	);
5043	try self.asmRegisterRegister(	5354	try self.asmRegisterRegister(
5044	.{ ._, .@"test" },	5355	.{ ._, .@"test" },
...	@@ -5217,8 +5528,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5217,8 +5528,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5217	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|	5528	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|
5218	self.register_manager.unlockReg(lock);	5529	self.register_manager.unlockReg(lock);
5219		5530
5220	const shift_imm =	5531	const shift_imm: Immediate =
5221	Immediate.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));	5532	.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));
5222	if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(	5533	if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
5223	mir_tag,	5534	mir_tag,
5224	registerAlias(dst_reg, abi_size),	5535	registerAlias(dst_reg, abi_size),
...	@@ -5434,7 +5745,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5434,7 +5745,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5434	break :result operand;	5745	break :result operand;
5435	}	5746	}
5436		5747
5437	const err_off = errUnionErrorOffset(payload_ty, zcu);	5748	const err_off = codegen.errUnionErrorOffset(payload_ty, zcu);
5438	switch (operand) {	5749	switch (operand) {
5439	.register => \|reg\| {	5750	.register => \|reg\| {
5440	// TODO reuse operand	5751	// TODO reuse operand
...	@@ -5492,7 +5803,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5492,7 +5803,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
5492	const eu_ty = src_ty.childType(zcu);	5803	const eu_ty = src_ty.childType(zcu);
5493	const pl_ty = eu_ty.errorUnionPayload(zcu);	5804	const pl_ty = eu_ty.errorUnionPayload(zcu);
5494	const err_ty = eu_ty.errorUnionSet(zcu);	5805	const err_ty = eu_ty.errorUnionSet(zcu);
5495	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	5806	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5496	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));	5807	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
5497	try self.asmRegisterMemory(	5808	try self.asmRegisterMemory(
5498	.{ ._, .mov },	5809	.{ ._, .mov },
...	@@ -5535,7 +5846,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5535,7 +5846,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5535	const eu_ty = src_ty.childType(zcu);	5846	const eu_ty = src_ty.childType(zcu);
5536	const pl_ty = eu_ty.errorUnionPayload(zcu);	5847	const pl_ty = eu_ty.errorUnionPayload(zcu);
5537	const err_ty = eu_ty.errorUnionSet(zcu);	5848	const err_ty = eu_ty.errorUnionSet(zcu);
5538	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	5849	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5539	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));	5850	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
5540	try self.asmMemoryImmediate(	5851	try self.asmMemoryImmediate(
5541	.{ ._, .mov },	5852	.{ ._, .mov },
...	@@ -5546,7 +5857,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5546,7 +5857,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5546	.disp = err_off,	5857	.disp = err_off,
5547	} },	5858	} },
5548	},	5859	},
5549	Immediate.u(0),	5860	.u(0),
5550	);	5861	);
5551		5862
5552	if (self.liveness.isUnused(inst)) break :result .unreach;	5863	if (self.liveness.isUnused(inst)) break :result .unreach;
...	@@ -5559,7 +5870,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5559,7 +5870,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5559	const dst_lock = self.register_manager.lockReg(dst_reg);	5870	const dst_lock = self.register_manager.lockReg(dst_reg);
5560	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	5871	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5561		5872
5562	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));	5873	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
5563	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));	5874	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
5564	try self.asmRegisterMemory(	5875	try self.asmRegisterMemory(
5565	.{ ._, .lea },	5876	.{ ._, .lea },
...	@@ -5587,7 +5898,7 @@ fn genUnwrapErrUnionPayloadMir(	...	@@ -5587,7 +5898,7 @@ fn genUnwrapErrUnionPayloadMir(
5587	const result: MCValue = result: {	5898	const result: MCValue = result: {
5588	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;	5899	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
5589		5900
5590	const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu));	5901	const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu));
5591	switch (err_union) {	5902	switch (err_union) {
5592	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{	5903	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
5593	.index = frame_addr.index,	5904	.index = frame_addr.index,
...	@@ -5636,7 +5947,7 @@ fn genUnwrapErrUnionPayloadPtrMir(	...	@@ -5636,7 +5947,7 @@ fn genUnwrapErrUnionPayloadPtrMir(
5636	const payload_ty = err_union_ty.errorUnionPayload(zcu);	5947	const payload_ty = err_union_ty.errorUnionPayload(zcu);
5637		5948
5638	const result: MCValue = result: {	5949	const result: MCValue = result: {
5639	const payload_off = errUnionPayloadOffset(payload_ty, zcu);	5950	const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu);
5640	const result_mcv: MCValue = if (maybe_inst) \|inst\|	5951	const result_mcv: MCValue = if (maybe_inst) \|inst\|
5641	try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)	5952	try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)
5642	else	5953	else
...	@@ -5696,7 +6007,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5696,7 +6007,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
5696	try self.asmRegisterImmediate(	6007	try self.asmRegisterImmediate(
5697	.{ ._s, .bt },	6008	.{ ._s, .bt },
5698	opt_reg,	6009	opt_reg,
5699	Immediate.u(@as(u6, @intCast(pl_abi_size * 8))),	6010	.u(@as(u6, @intCast(pl_abi_size * 8))),
5700	);	6011	);
5701	},	6012	},
5702		6013
...	@@ -5709,7 +6020,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5709,7 +6020,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
5709	.disp = frame_addr.off + pl_abi_size,	6020	.disp = frame_addr.off + pl_abi_size,
5710	} },	6021	} },
5711	},	6022	},
5712	Immediate.u(1),	6023	.u(1),
5713	),	6024	),
5714	}	6025	}
5715	}	6026	}
...	@@ -5733,8 +6044,8 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5733,8 +6044,8 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
5733	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };	6044	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
5734		6045
5735	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));	6046	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
5736	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));	6047	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
5737	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	6048	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5738	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});	6049	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
5739	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});	6050	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});
5740	break :result .{ .load_frame = .{ .index = frame_index } };	6051	break :result .{ .load_frame = .{ .index = frame_index } };
...	@@ -5756,8 +6067,8 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5756,8 +6067,8 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5756	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);	6067	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
5757		6068
5758	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));	6069	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
5759	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));	6070	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
5760	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	6071	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5761	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});	6072	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
5762	const operand = try self.resolveInst(ty_op.operand);	6073	const operand = try self.resolveInst(ty_op.operand);
5763	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});	6074	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});
...	@@ -5770,11 +6081,20 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5770,11 +6081,20 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
5770	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	6081	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5771	const result = result: {	6082	const result = result: {
5772	const src_mcv = try self.resolveInst(ty_op.operand);	6083	const src_mcv = try self.resolveInst(ty_op.operand);
5773	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;	6084	const ptr_mcv: MCValue = switch (src_mcv) {
		6085	.register_pair => \|regs\| .{ .register = regs[0] },
		6086	else => src_mcv,
		6087	};
		6088	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
		6089	switch (src_mcv) {
		6090	.register_pair => \|regs\| try self.freeValue(.{ .register = regs[1] }),
		6091	else => {},
		6092	}
		6093	break :result ptr_mcv;
		6094	}
5774		6095
5775	const dst_mcv = try self.allocRegOrMem(inst, true);	6096	const dst_mcv = try self.allocRegOrMem(inst, true);
5776	const dst_ty = self.typeOfIndex(inst);	6097	try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{});
5777	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
5778	break :result dst_mcv;	6098	break :result dst_mcv;
5779	};	6099	};
5780	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	6100	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	@@ -5782,23 +6102,28 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5782,23 +6102,28 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
5782		6102
5783	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {	6103	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
5784	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	6104	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5785		6105	const result = result: {
5786	const result: MCValue = result: {
5787	const src_mcv = try self.resolveInst(ty_op.operand);	6106	const src_mcv = try self.resolveInst(ty_op.operand);
5788	switch (src_mcv) {	6107	const len_mcv: MCValue = switch (src_mcv) {
5789	.load_frame => \|frame_addr\| {	6108	.register_pair => \|regs\| .{ .register = regs[1] },
5790	const len_mcv: MCValue = .{ .load_frame = .{	6109	.load_frame => \|frame_addr\| .{ .load_frame = .{
5791	.index = frame_addr.index,	6110	.index = frame_addr.index,
5792	.off = frame_addr.off + 8,	6111	.off = frame_addr.off + 8,
5793	} };	6112	} },
5794	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
5795
5796	const dst_mcv = try self.allocRegOrMem(inst, true);
5797	try self.genCopy(Type.usize, dst_mcv, len_mcv, .{});
5798	break :result dst_mcv;
5799	},
5800	else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),	6113	else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),
		6114	};
		6115	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
		6116	switch (src_mcv) {
		6117	.register_pair => \|regs\| try self.freeValue(.{ .register = regs[0] }),
		6118	.load_frame => {},
		6119	else => unreachable,
		6120	}
		6121	break :result len_mcv;
5801	}	6122	}
		6123
		6124	const dst_mcv = try self.allocRegOrMem(inst, true);
		6125	try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{});
		6126	break :result dst_mcv;
5802	};	6127	};
5803	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	6128	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5804	}	6129	}
...	@@ -6296,27 +6621,27 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6296,27 +6621,27 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6296	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));	6621	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
6297	const has_lzcnt = self.hasFeature(.lzcnt);	6622	const has_lzcnt = self.hasFeature(.lzcnt);
6298	if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {	6623	if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {
6299	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	6624	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6300	const extra_bits = abi_size * 8 - src_bits;	6625	const extra_bits = abi_size * 8 - src_bits;
6301		6626
6302	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	6627	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6303	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);	6628	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
6304	defer self.register_manager.unlockReg(index_lock);	6629	defer self.register_manager.unlockReg(index_lock);
6305		6630
6306	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.u(limbs_len));	6631	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len));
6307	switch (extra_bits) {	6632	switch (extra_bits) {
6308	1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),	6633	1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
6309	else => try self.asmRegisterImmediate(	6634	else => try self.asmRegisterImmediate(
6310	.{ ._, .mov },	6635	.{ ._, .mov },
6311	dst_reg.to32(),	6636	dst_reg.to32(),
6312	Immediate.s(@as(i32, extra_bits) - 1),	6637	.s(@as(i32, extra_bits) - 1),
6313	),	6638	),
6314	}	6639	}
6315	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	6640	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6316	try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());	6641	try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());
6317	const zero = try self.asmJccReloc(.z, undefined);	6642	const zero = try self.asmJccReloc(.z, undefined);
6318	if (self.hasFeature(.slow_incdec)) {	6643	if (self.hasFeature(.slow_incdec)) {
6319	try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), Immediate.u(1));	6644	try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1));
6320	} else {	6645	} else {
6321	try self.asmRegister(.{ ._, .dec }, index_reg.to32());	6646	try self.asmRegister(.{ ._, .dec }, index_reg.to32());
6322	}	6647	}
...	@@ -6328,7 +6653,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6328,7 +6653,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6328	.scale = .@"8",	6653	.scale = .@"8",
6329	.disp = src_mcv.load_frame.off,	6654	.disp = src_mcv.load_frame.off,
6330	} },	6655	} },
6331	}, Immediate.u(0));	6656	}, .u(0));
6332	_ = try self.asmJccReloc(.e, loop);	6657	_ = try self.asmJccReloc(.e, loop);
6333	try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{	6658	try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{
6334	.base = .{ .frame = src_mcv.load_frame.index },	6659	.base = .{ .frame = src_mcv.load_frame.index },
...	@@ -6340,9 +6665,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6340,9 +6665,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6340	} },	6665	} },
6341	});	6666	});
6342	self.performReloc(zero);	6667	self.performReloc(zero);
6343	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));	6668	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
6344	try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());	6669	try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());
6345	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(src_bits - 1));	6670	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1));
6346	try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());	6671	try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());
6347	break :result dst_mcv;	6672	break :result dst_mcv;
6348	}	6673	}
...	@@ -6404,7 +6729,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6404,7 +6729,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6404		6729
6405	assert(src_bits <= 64);	6730	assert(src_bits <= 64);
6406	const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);	6731	const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
6407	if (math.isPowerOfTwo(src_bits)) {	6732	if (std.math.isPowerOfTwo(src_bits)) {
6408	const imm_reg = try self.copyToTmpRegister(dst_ty, .{	6733	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
6409	.immediate = src_bits ^ (src_bits - 1),	6734	.immediate = src_bits ^ (src_bits - 1),
6410	});	6735	});
...	@@ -6429,7 +6754,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6429,7 +6754,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6429	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });	6754	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
6430	} else {	6755	} else {
6431	const imm_reg = try self.copyToTmpRegister(dst_ty, .{	6756	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
6432	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),	6757	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
6433	});	6758	});
6434	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);	6759	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
6435	defer self.register_manager.unlockReg(imm_lock);	6760	defer self.register_manager.unlockReg(imm_lock);
...	@@ -6493,30 +6818,30 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6493,30 +6818,30 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6493	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));	6818	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
6494	const has_bmi = self.hasFeature(.bmi);	6819	const has_bmi = self.hasFeature(.bmi);
6495	if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {	6820	if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {
6496	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	6821	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6497	const extra_bits = abi_size * 8 - src_bits;	6822	const extra_bits = abi_size * 8 - src_bits;
6498		6823
6499	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	6824	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6500	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);	6825	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
6501	defer self.register_manager.unlockReg(index_lock);	6826	defer self.register_manager.unlockReg(index_lock);
6502		6827
6503	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.s(-1));	6828	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1));
6504	switch (extra_bits) {	6829	switch (extra_bits) {
6505	0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),	6830	0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
6506	1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),	6831	1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),
6507	else => try self.asmRegisterImmediate(	6832	else => try self.asmRegisterImmediate(
6508	.{ ._, .mov },	6833	.{ ._, .mov },
6509	dst_reg.to32(),	6834	dst_reg.to32(),
6510	Immediate.s(-@as(i32, extra_bits)),	6835	.s(-@as(i32, extra_bits)),
6511	),	6836	),
6512	}	6837	}
6513	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	6838	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6514	if (self.hasFeature(.slow_incdec)) {	6839	if (self.hasFeature(.slow_incdec)) {
6515	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));	6840	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
6516	} else {	6841	} else {
6517	try self.asmRegister(.{ ._, .inc }, index_reg.to32());	6842	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
6518	}	6843	}
6519	try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), Immediate.u(limbs_len));	6844	try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len));
6520	const zero = try self.asmJccReloc(.nb, undefined);	6845	const zero = try self.asmJccReloc(.nb, undefined);
6521	try self.asmMemoryImmediate(.{ ._, .cmp }, .{	6846	try self.asmMemoryImmediate(.{ ._, .cmp }, .{
6522	.base = .{ .frame = src_mcv.load_frame.index },	6847	.base = .{ .frame = src_mcv.load_frame.index },
...	@@ -6526,7 +6851,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6526,7 +6851,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6526	.scale = .@"8",	6851	.scale = .@"8",
6527	.disp = src_mcv.load_frame.off,	6852	.disp = src_mcv.load_frame.off,
6528	} },	6853	} },
6529	}, Immediate.u(0));	6854	}, .u(0));
6530	_ = try self.asmJccReloc(.e, loop);	6855	_ = try self.asmJccReloc(.e, loop);
6531	try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{	6856	try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{
6532	.base = .{ .frame = src_mcv.load_frame.index },	6857	.base = .{ .frame = src_mcv.load_frame.index },
...	@@ -6538,7 +6863,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6538,7 +6863,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6538	} },	6863	} },
6539	});	6864	});
6540	self.performReloc(zero);	6865	self.performReloc(zero);
6541	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));	6866	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
6542	try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());	6867	try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());
6543	break :result dst_mcv;	6868	break :result dst_mcv;
6544	}	6869	}
...	@@ -6558,7 +6883,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6558,7 +6883,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6558	.{ ._, .@"or" },	6883	.{ ._, .@"or" },
6559	wide_ty,	6884	wide_ty,
6560	tmp_mcv,	6885	tmp_mcv,
6561	.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<	6886	.{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
6562	@intCast(src_bits) },	6887	@intCast(src_bits) },
6563	);	6888	);
6564	break :masked tmp_mcv;	6889	break :masked tmp_mcv;
...	@@ -6585,7 +6910,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6585,7 +6910,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6585	.{ ._, .@"or" },	6910	.{ ._, .@"or" },
6586	Type.u64,	6911	Type.u64,
6587	dst_mcv,	6912	dst_mcv,
6588	.{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(src_bits - 64) },	6913	.{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) },
6589	);	6914	);
6590	break :masked dst_mcv;	6915	break :masked dst_mcv;
6591	} else hi_mat_src_mcv;	6916	} else hi_mat_src_mcv;
...	@@ -6602,7 +6927,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6602,7 +6927,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6602	const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);	6927	const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
6603	defer self.register_manager.unlockReg(width_lock);	6928	defer self.register_manager.unlockReg(width_lock);
6604		6929
6605	if (src_bits <= 8 or !math.isPowerOfTwo(src_bits)) {	6930	if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) {
6606	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);	6931	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
6607	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);	6932	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
6608	defer self.register_manager.unlockReg(wide_lock);	6933	defer self.register_manager.unlockReg(wide_lock);
...	@@ -6701,11 +7026,11 @@ fn genPopCount(	...	@@ -6701,11 +7026,11 @@ fn genPopCount(
6701	},	7026	},
6702	);	7027	);
6703		7028
6704	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);	7029	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
6705	const imm_0_1 = Immediate.u(mask / 0b1_1);	7030	const imm_0_1: Immediate = .u(mask / 0b1_1);
6706	const imm_00_11 = Immediate.u(mask / 0b01_01);	7031	const imm_00_11: Immediate = .u(mask / 0b01_01);
6707	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);	7032	const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
6708	const imm_0000_0001 = Immediate.u(mask / 0b1111_1111);	7033	const imm_0000_0001: Immediate = .u(mask / 0b1111_1111);
6709		7034
6710	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	7035	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6711	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	7036	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...	@@ -6722,7 +7047,7 @@ fn genPopCount(	...	@@ -6722,7 +7047,7 @@ fn genPopCount(
6722	// dst = operand	7047	// dst = operand
6723	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	7048	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
6724	// tmp = operand	7049	// tmp = operand
6725	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));	7050	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
6726	// tmp = operand >> 1	7051	// tmp = operand >> 1
6727	if (src_abi_size > 4) {	7052	if (src_abi_size > 4) {
6728	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);	7053	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
...	@@ -6733,7 +7058,7 @@ fn genPopCount(	...	@@ -6733,7 +7058,7 @@ fn genPopCount(
6733	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)	7058	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)
6734	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	7059	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
6735	// tmp = temp1	7060	// tmp = temp1
6736	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));	7061	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
6737	// dst = temp1 >> 2	7062	// dst = temp1 >> 2
6738	if (src_abi_size > 4) {	7063	if (src_abi_size > 4) {
6739	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);	7064	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
...	@@ -6749,7 +7074,7 @@ fn genPopCount(	...	@@ -6749,7 +7074,7 @@ fn genPopCount(
6749	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)	7074	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
6750	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);	7075	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
6751	// dst = temp2	7076	// dst = temp2
6752	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));	7077	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4));
6753	// tmp = temp2 >> 4	7078	// tmp = temp2 >> 4
6754	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);	7079	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
6755	// dst = temp2 + (temp2 >> 4)	7080	// dst = temp2 + (temp2 >> 4)
...	@@ -6767,7 +7092,7 @@ fn genPopCount(	...	@@ -6767,7 +7092,7 @@ fn genPopCount(
6767	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f	7092	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
6768	// dst = temp3 * 0x01...01	7093	// dst = temp3 * 0x01...01
6769	if (src_abi_size > 1) {	7094	if (src_abi_size > 1) {
6770	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));	7095	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8));
6771	}	7096	}
6772	// dst = (temp3 * 0x01...01) >> (bits - 8)	7097	// dst = (temp3 * 0x01...01) >> (bits - 8)
6773	}	7098	}
...	@@ -6847,7 +7172,7 @@ fn genByteSwap(	...	@@ -6847,7 +7172,7 @@ fn genByteSwap(
6847	return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } };	7172	return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } };
6848	},	7173	},
6849	else => {	7174	else => {
6850	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	7175	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6851		7176
6852	const temp_regs =	7177	const temp_regs =
6853	try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);	7178	try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
...	@@ -6856,11 +7181,7 @@ fn genByteSwap(	...	@@ -6856,11 +7181,7 @@ fn genByteSwap(
6856		7181
6857	const dst_mcv = try self.allocRegOrMem(inst, false);	7182	const dst_mcv = try self.allocRegOrMem(inst, false);
6858	try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());	7183	try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
6859	try self.asmRegisterImmediate(	7184	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
6860	.{ ._, .mov },
6861	temp_regs[1].to32(),
6862	Immediate.u(limbs_len - 1),
6863	);
6864		7185
6865	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	7186	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6866	try self.asmRegisterMemory(	7187	try self.asmRegisterMemory(
...	@@ -6912,8 +7233,8 @@ fn genByteSwap(	...	@@ -6912,8 +7233,8 @@ fn genByteSwap(
6912	} },	7233	} },
6913	}, temp_regs[2].to64());	7234	}, temp_regs[2].to64());
6914	if (self.hasFeature(.slow_incdec)) {	7235	if (self.hasFeature(.slow_incdec)) {
6915	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	7236	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
6916	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));	7237	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
6917	} else {	7238	} else {
6918	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	7239	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
6919	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());	7240	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
...	@@ -6994,10 +7315,10 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6994,10 +7315,10 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
6994	else	7315	else
6995	undefined;	7316	undefined;
6996		7317
6997	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);	7318	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);
6998	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);	7319	const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
6999	const imm_00_11 = Immediate.u(mask / 0b01_01);	7320	const imm_00_11: Immediate = .u(mask / 0b01_01);
7000	const imm_0_1 = Immediate.u(mask / 0b1_1);	7321	const imm_0_1: Immediate = .u(mask / 0b1_1);
7001		7322
7002	for (dst_mcv.getRegs()) \|dst_reg\| {	7323	for (dst_mcv.getRegs()) \|dst_reg\| {
7003	const dst = registerAlias(dst_reg, limb_abi_size);	7324	const dst = registerAlias(dst_reg, limb_abi_size);
...	@@ -7005,7 +7326,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7005,7 +7326,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7005	// dst = temp1 = bswap(operand)	7326	// dst = temp1 = bswap(operand)
7006	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	7327	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
7007	// tmp = temp1	7328	// tmp = temp1
7008	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));	7329	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4));
7009	// dst = temp1 >> 4	7330	// dst = temp1 >> 4
7010	if (limb_abi_size > 4) {	7331	if (limb_abi_size > 4) {
7011	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);	7332	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
...	@@ -7017,13 +7338,13 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7017,13 +7338,13 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7017	}	7338	}
7018	// tmp = temp1 & 0x0F...0F	7339	// tmp = temp1 & 0x0F...0F
7019	// dst = (temp1 >> 4) & 0x0F...0F	7340	// dst = (temp1 >> 4) & 0x0F...0F
7020	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));	7341	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4));
7021	// tmp = (temp1 & 0x0F...0F) << 4	7342	// tmp = (temp1 & 0x0F...0F) << 4
7022	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);	7343	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
7023	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)	7344	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)
7024	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	7345	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
7025	// tmp = temp2	7346	// tmp = temp2
7026	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));	7347	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
7027	// dst = temp2 >> 2	7348	// dst = temp2 >> 2
7028	if (limb_abi_size > 4) {	7349	if (limb_abi_size > 4) {
7029	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);	7350	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
...	@@ -7050,7 +7371,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7050,7 +7371,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7050	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)	7371	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
7051	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);	7372	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
7052	// dst = temp3	7373	// dst = temp3
7053	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));	7374	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
7054	// tmp = temp3 >> 1	7375	// tmp = temp3 >> 1
7055	if (limb_abi_size > 4) {	7376	if (limb_abi_size > 4) {
7056	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);	7377	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
...	@@ -7337,7 +7658,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro	...	@@ -7337,7 +7658,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7337	dst_alias,	7658	dst_alias,
7338	dst_alias,	7659	dst_alias,
7339	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),	7660	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7340	Immediate.u(@as(u5, @bitCast(mode))),	7661	.u(@as(u5, @bitCast(mode))),
7341	) else try self.asmRegisterRegisterRegisterImmediate(	7662	) else try self.asmRegisterRegisterRegisterImmediate(
7342	mir_tag,	7663	mir_tag,
7343	dst_alias,	7664	dst_alias,
...	@@ -7346,13 +7667,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro	...	@@ -7346,13 +7667,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7346	src_mcv.getReg().?	7667	src_mcv.getReg().?
7347	else	7668	else
7348	try self.copyToTmpRegister(ty, src_mcv), abi_size),	7669	try self.copyToTmpRegister(ty, src_mcv), abi_size),
7349	Immediate.u(@as(u5, @bitCast(mode))),	7670	.u(@as(u5, @bitCast(mode))),
7350	),	7671	),
7351	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	7672	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
7352	mir_tag,	7673	mir_tag,
7353	dst_alias,	7674	dst_alias,
7354	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),	7675	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7355	Immediate.u(@as(u5, @bitCast(mode))),	7676	.u(@as(u5, @bitCast(mode))),
7356	) else try self.asmRegisterRegisterImmediate(	7677	) else try self.asmRegisterRegisterImmediate(
7357	mir_tag,	7678	mir_tag,
7358	dst_alias,	7679	dst_alias,
...	@@ -7360,7 +7681,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro	...	@@ -7360,7 +7681,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7360	src_mcv.getReg().?	7681	src_mcv.getReg().?
7361	else	7682	else
7362	try self.copyToTmpRegister(ty, src_mcv), abi_size),	7683	try self.copyToTmpRegister(ty, src_mcv), abi_size),
7363	Immediate.u(@as(u5, @bitCast(mode))),	7684	.u(@as(u5, @bitCast(mode))),
7364	),	7685	),
7365	}	7686	}
7366	}	7687	}
...	@@ -7433,7 +7754,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7433,7 +7754,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7433	defer self.register_manager.unlockReg(tmp_lock);	7754	defer self.register_manager.unlockReg(tmp_lock);
7434		7755
7435	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);	7756	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
7436	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));	7757	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
7437	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg);	7758	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg);
7438	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg);	7759	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg);
7439	try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg);	7760	try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg);
...	@@ -7443,7 +7764,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7443,7 +7764,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7443	},	7764	},
7444	else => {	7765	else => {
7445	const abi_size: u31 = @intCast(ty.abiSize(zcu));	7766	const abi_size: u31 = @intCast(ty.abiSize(zcu));
7446	const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;	7767	const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
7447		7768
7448	const tmp_regs =	7769	const tmp_regs =
7449	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);	7770	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
...	@@ -7460,7 +7781,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7460,7 +7781,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7460	try self.asmMemoryImmediate(	7781	try self.asmMemoryImmediate(
7461	.{ ._, .cmp },	7782	.{ ._, .cmp },
7462	try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword),	7783	try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword),
7463	Immediate.u(0),	7784	.u(0),
7464	);	7785	);
7465	const positive = try self.asmJccReloc(.ns, undefined);	7786	const positive = try self.asmJccReloc(.ns, undefined);
7466		7787
...	@@ -7469,7 +7790,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7469,7 +7790,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7469		7790
7470	const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	7791	const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
7471	try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32());	7792	try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32());
7472	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), Immediate.u(1));	7793	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1));
7473	try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{	7794	try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{
7474	.base = .{ .frame = dst_mcv.load_frame.index },	7795	.base = .{ .frame = dst_mcv.load_frame.index },
7475	.mod = .{ .rm = .{	7796	.mod = .{ .rm = .{
...	@@ -7491,11 +7812,11 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7491,11 +7812,11 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7491	}, tmp_regs[2].to64());	7812	}, tmp_regs[2].to64());
7492		7813
7493	if (self.hasFeature(.slow_incdec)) {	7814	if (self.hasFeature(.slow_incdec)) {
7494	try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), Immediate.u(1));	7815	try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1));
7495	} else {	7816	} else {
7496	try self.asmRegister(.{ ._, .inc }, tmp_regs[0].to32());	7817	try self.asmRegister(.{ ._, .inc }, tmp_regs[0].to32());
7497	}	7818	}
7498	try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), Immediate.u(limb_len));	7819	try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len));
7499	_ = try self.asmJccReloc(.b, neg_loop);	7820	_ = try self.asmJccReloc(.b, neg_loop);
7500		7821
7501	self.performReloc(positive);	7822	self.performReloc(positive);
...	@@ -7620,7 +7941,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7620,7 +7941,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7620	.{ .v_, .cvtps2ph },	7941	.{ .v_, .cvtps2ph },
7621	dst_reg,	7942	dst_reg,
7622	dst_reg,	7943	dst_reg,
7623	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	7944	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7624	);	7945	);
7625	break :result dst_mcv;	7946	break :result dst_mcv;
7626	},	7947	},
...	@@ -7650,7 +7971,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7650,7 +7971,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7650	.{ .v_, .cvtps2ph },	7971	.{ .v_, .cvtps2ph },
7651	dst_reg,	7972	dst_reg,
7652	dst_reg,	7973	dst_reg,
7653	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	7974	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7654	);	7975	);
7655	break :result dst_mcv;	7976	break :result dst_mcv;
7656	},	7977	},
...	@@ -7675,7 +7996,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7675,7 +7996,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7675	.{ .v_, .cvtps2ph },	7996	.{ .v_, .cvtps2ph },
7676	dst_reg,	7997	dst_reg,
7677	wide_reg,	7998	wide_reg,
7678	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	7999	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7679	);	8000	);
7680	break :result dst_mcv;	8001	break :result dst_mcv;
7681	},	8002	},
...	@@ -7699,9 +8020,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7699,9 +8020,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7699	else => unreachable,	8020	else => unreachable,
7700	},	8021	},
7701	else => unreachable,	8022	else => unreachable,
7702	}) orelse return self.fail("TODO implement airSqrt for {}", .{	8023	}) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)});
7703	ty.fmt(pt),
7704	});
7705	switch (mir_tag[0]) {	8024	switch (mir_tag[0]) {
7706	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(	8025	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
7707	mir_tag,	8026	mir_tag,
...	@@ -7805,7 +8124,7 @@ fn reuseOperandAdvanced(	...	@@ -7805,7 +8124,7 @@ fn reuseOperandAdvanced(
7805	}	8124	}
7806		8125
7807	// Prevent the operand deaths processing code from deallocating it.	8126	// Prevent the operand deaths processing code from deallocating it.
7808	self.liveness.clearOperandDeath(inst, op_index);	8127	self.reused_operands.set(op_index);
7809	const op_inst = operand.toIndex().?;	8128	const op_inst = operand.toIndex().?;
7810	self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst);	8129	self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst);
7811		8130
...	@@ -7890,7 +8209,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -7890,7 +8209,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7890	} },	8209	} },
7891	});	8210	});
7892	try self.spillEflagsIfOccupied();	8211	try self.spillEflagsIfOccupied();
7893	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));	8212	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(val_bit_off));
7894	} else {	8213	} else {
7895	const tmp_reg =	8214	const tmp_reg =
7896	registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size);	8215	registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size);
...	@@ -7913,12 +8232,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -7913,12 +8232,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7913	} },	8232	} },
7914	});	8233	});
7915	try self.spillEflagsIfOccupied();	8234	try self.spillEflagsIfOccupied();
7916	try self.asmRegisterRegisterImmediate(	8235	try self.asmRegisterRegisterImmediate(.{ ._rd, .sh }, dst_alias, tmp_reg, .u(val_bit_off));
7917	.{ ._rd, .sh },
7918	dst_alias,
7919	tmp_reg,
7920	Immediate.u(val_bit_off),
7921	);
7922	}	8236	}
7923		8237
7924	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);	8238	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
...	@@ -8064,16 +8378,16 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -8064,16 +8378,16 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8064	} },	8378	} },
8065	};	8379	};
8066		8380
8067	const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<	8381	const part_mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
8068	@intCast(part_bit_off);	8382	@intCast(part_bit_off);
8069	const part_mask_not = part_mask ^ (@as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));	8383	const part_mask_not = part_mask ^ (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));
8070	if (limb_abi_size <= 4) {	8384	if (limb_abi_size <= 4) {
8071	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));	8385	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .u(part_mask_not));
8072	} else if (math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {	8386	} else if (std.math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {
8073	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));	8387	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .s(small));
8074	} else {	8388	} else {
8075	const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	8389	const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
8076	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));	8390	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, .u(part_mask_not));
8077	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);	8391	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
8078	}	8392	}
8079		8393
...	@@ -8209,25 +8523,14 @@ fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8209,25 +8523,14 @@ fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
8209	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });	8523	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
8210	}	8524	}
8211		8525
8212	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {	8526	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, field_index: u8) !void {
8213	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	8527	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8214	const result = try self.fieldPtr(inst, ty_op.operand, index);	8528	const result = try self.fieldPtr(inst, ty_op.operand, field_index);
8215	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	8529	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8216	}	8530	}
8217		8531
8218	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {	8532	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {
8219	const pt = self.pt;
8220	const zcu = pt.zcu;
8221	const ptr_field_ty = self.typeOfIndex(inst);	8533	const ptr_field_ty = self.typeOfIndex(inst);
8222	const ptr_container_ty = self.typeOf(operand);
8223	const container_ty = ptr_container_ty.childType(zcu);
8224
8225	const field_off: i32 = switch (container_ty.containerLayout(zcu)) {
8226	.auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)),
8227	.@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8228	(if (zcu.typeToStruct(container_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, index) else 0) -
8229	ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
8230	};
8231		8534
8232	const src_mcv = try self.resolveInst(operand);	8535	const src_mcv = try self.resolveInst(operand);
8233	const dst_mcv = if (switch (src_mcv) {	8536	const dst_mcv = if (switch (src_mcv) {
...	@@ -8235,7 +8538,19 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32	...	@@ -8235,7 +8538,19 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
8235	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),	8538	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),
8236	else => false,	8539	else => false,
8237	}) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv);	8540	}) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv);
8238	return dst_mcv.offset(field_off);	8541	return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index));
		8542	}
		8543
		8544	fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {
		8545	const pt = self.pt;
		8546	const zcu = pt.zcu;
		8547	const agg_ty = ptr_agg_ty.childType(zcu);
		8548	return switch (agg_ty.containerLayout(zcu)) {
		8549	.auto, .@"extern" => @intCast(agg_ty.structFieldOffset(field_index, zcu)),
		8550	.@"packed" => @divExact(@as(i32, ptr_agg_ty.ptrInfo(zcu).packed_offset.bit_offset) +
		8551	(if (zcu.typeToStruct(agg_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, field_index) else 0) -
		8552	ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
		8553	};
8239	}	8554	}
8240		8555
8241	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	8556	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -8476,7 +8791,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8476,7 +8791,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8476	} },	8791	} },
8477	});	8792	});
8478	try self.spillEflagsIfOccupied();	8793	try self.spillEflagsIfOccupied();
8479	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));	8794	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(field_bit_off));
8480	} else {	8795	} else {
8481	const tmp_reg = registerAlias(	8796	const tmp_reg = registerAlias(
8482	try self.register_manager.allocReg(null, abi.RegisterClass.gp),	8797	try self.register_manager.allocReg(null, abi.RegisterClass.gp),
...	@@ -8509,7 +8824,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8509,7 +8824,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8509	.{ ._rd, .sh },	8824	.{ ._rd, .sh },
8510	dst_alias,	8825	dst_alias,
8511	tmp_reg,	8826	tmp_reg,
8512	Immediate.u(field_bit_off),	8827	.u(field_bit_off),
8513	);	8828	);
8514	}	8829	}
8515		8830
...	@@ -8528,27 +8843,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8528,27 +8843,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8528	}	8843	}
8529		8844
8530	fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {	8845	fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
8531	const pt = self.pt;
8532	const zcu = pt.zcu;
8533	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	8846	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8534	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;	8847	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
8535		8848
8536	const inst_ty = self.typeOfIndex(inst);	8849	const ptr_agg_ty = self.typeOfIndex(inst);
8537	const parent_ty = inst_ty.childType(zcu);
8538	const field_off: i32 = switch (parent_ty.containerLayout(zcu)) {
8539	.auto, .@"extern" => @intCast(parent_ty.structFieldOffset(extra.field_index, zcu)),
8540	.@"packed" => @divExact(@as(i32, inst_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8541	(if (zcu.typeToStruct(parent_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0) -
8542	self.typeOf(extra.field_ptr).ptrInfo(zcu).packed_offset.bit_offset, 8),
8543	};
8544
8545	const src_mcv = try self.resolveInst(extra.field_ptr);	8850	const src_mcv = try self.resolveInst(extra.field_ptr);
8546	const dst_mcv = if (src_mcv.isRegisterOffset() and	8851	const dst_mcv = if (src_mcv.isRegisterOffset() and
8547	self.reuseOperand(inst, extra.field_ptr, 0, src_mcv))	8852	self.reuseOperand(inst, extra.field_ptr, 0, src_mcv))
8548	src_mcv	8853	src_mcv
8549	else	8854	else
8550	try self.copyToRegisterWithInstTracking(inst, inst_ty, src_mcv);	8855	try self.copyToRegisterWithInstTracking(inst, ptr_agg_ty, src_mcv);
8551	const result = dst_mcv.offset(-field_off);	8856	const result = dst_mcv.offset(-self.fieldOffset(ptr_agg_ty, self.typeOf(extra.field_ptr), extra.field_index));
8552	return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });	8857	return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
8553	}	8858	}
8554		8859
...	@@ -8613,7 +8918,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:	...	@@ -8613,7 +8918,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
8613	};	8918	};
8614		8919
8615	if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {	8920	if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {
8616	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_bits);	8921	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits);
8617	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });	8922	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
8618	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);	8923	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
8619	}	8924	}
...	@@ -8698,7 +9003,7 @@ fn genShiftBinOpMir(	...	@@ -8698,7 +9003,7 @@ fn genShiftBinOpMir(
8698	try self.spillEflagsIfOccupied();	9003	try self.spillEflagsIfOccupied();
8699		9004
8700	if (abi_size > 16) {	9005	if (abi_size > 16) {
8701	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	9006	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
8702	assert(shift_abi_size >= 1 and shift_abi_size <= 2);	9007	assert(shift_abi_size >= 1 and shift_abi_size <= 2);
8703		9008
8704	const rcx_lock: ?RegisterLock = switch (rhs_mcv) {	9009	const rcx_lock: ?RegisterLock = switch (rhs_mcv) {
...	@@ -8725,12 +9030,12 @@ fn genShiftBinOpMir(	...	@@ -8725,12 +9030,12 @@ fn genShiftBinOpMir(
8725		9030
8726	switch (tag[0]) {	9031	switch (tag[0]) {
8727	._l => {	9032	._l => {
8728	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), Immediate.u(limbs_len - 1));	9033	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
8729	switch (rhs_mcv) {	9034	switch (rhs_mcv) {
8730	.immediate => \|shift_imm\| try self.asmRegisterImmediate(	9035	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8731	.{ ._, .mov },	9036	.{ ._, .mov },
8732	temp_regs[0].to32(),	9037	temp_regs[0].to32(),
8733	Immediate.u(limbs_len - (shift_imm >> 6) - 1),	9038	.u(limbs_len - (shift_imm >> 6) - 1),
8734	),	9039	),
8735	else => {	9040	else => {
8736	try self.asmRegisterRegister(	9041	try self.asmRegisterRegister(
...	@@ -8738,16 +9043,8 @@ fn genShiftBinOpMir(	...	@@ -8738,16 +9043,8 @@ fn genShiftBinOpMir(
8738	temp_regs[2].to32(),	9043	temp_regs[2].to32(),
8739	registerAlias(.rcx, shift_abi_size),	9044	registerAlias(.rcx, shift_abi_size),
8740	);	9045	);
8741	try self.asmRegisterImmediate(	9046	try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
8742	.{ ._, .@"and" },	9047	try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[2].to32(), .u(6));
8743	.cl,
8744	Immediate.u(math.maxInt(u6)),
8745	);
8746	try self.asmRegisterImmediate(
8747	.{ ._r, .sh },
8748	temp_regs[2].to32(),
8749	Immediate.u(6),
8750	);
8751	try self.asmRegisterRegister(	9048	try self.asmRegisterRegister(
8752	.{ ._, .mov },	9049	.{ ._, .mov },
8753	temp_regs[0].to32(),	9050	temp_regs[0].to32(),
...	@@ -8767,7 +9064,7 @@ fn genShiftBinOpMir(	...	@@ -8767,7 +9064,7 @@ fn genShiftBinOpMir(
8767	.immediate => \|shift_imm\| try self.asmRegisterImmediate(	9064	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8768	.{ ._, .mov },	9065	.{ ._, .mov },
8769	temp_regs[0].to32(),	9066	temp_regs[0].to32(),
8770	Immediate.u(shift_imm >> 6),	9067	.u(shift_imm >> 6),
8771	),	9068	),
8772	else => {	9069	else => {
8773	try self.asmRegisterRegister(	9070	try self.asmRegisterRegister(
...	@@ -8775,16 +9072,8 @@ fn genShiftBinOpMir(	...	@@ -8775,16 +9072,8 @@ fn genShiftBinOpMir(
8775	temp_regs[0].to32(),	9072	temp_regs[0].to32(),
8776	registerAlias(.rcx, shift_abi_size),	9073	registerAlias(.rcx, shift_abi_size),
8777	);	9074	);
8778	try self.asmRegisterImmediate(	9075	try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
8779	.{ ._, .@"and" },	9076	try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[0].to32(), .u(6));
8780	.cl,
8781	Immediate.u(math.maxInt(u6)),
8782	);
8783	try self.asmRegisterImmediate(
8784	.{ ._r, .sh },
8785	temp_regs[0].to32(),
8786	Immediate.u(6),
8787	);
8788	},	9077	},
8789	}	9078	}
8790	},	9079	},
...	@@ -8813,7 +9102,7 @@ fn genShiftBinOpMir(	...	@@ -8813,7 +9102,7 @@ fn genShiftBinOpMir(
8813	try self.asmRegisterImmediate(	9102	try self.asmRegisterImmediate(
8814	.{ ._, .cmp },	9103	.{ ._, .cmp },
8815	temp_regs[0].to32(),	9104	temp_regs[0].to32(),
8816	Immediate.u(limbs_len - 1),	9105	.u(limbs_len - 1),
8817	);	9106	);
8818	break :skip try self.asmJccReloc(.nb, undefined);	9107	break :skip try self.asmJccReloc(.nb, undefined);
8819	},	9108	},
...	@@ -8843,7 +9132,7 @@ fn genShiftBinOpMir(	...	@@ -8843,7 +9132,7 @@ fn genShiftBinOpMir(
8843	}, .sh },	9132	}, .sh },
8844	temp_regs[2].to64(),	9133	temp_regs[2].to64(),
8845	temp_regs[3].to64(),	9134	temp_regs[3].to64(),
8846	Immediate.u(shift_imm & math.maxInt(u6)),	9135	.u(shift_imm & std.math.maxInt(u6)),
8847	),	9136	),
8848	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {	9137	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
8849	._l => ._ld,	9138	._l => ._ld,
...	@@ -8864,8 +9153,8 @@ fn genShiftBinOpMir(	...	@@ -8864,8 +9153,8 @@ fn genShiftBinOpMir(
8864	switch (tag[0]) {	9153	switch (tag[0]) {
8865	._l => {	9154	._l => {
8866	if (slow_inc_dec) {	9155	if (slow_inc_dec) {
8867	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));	9156	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
8868	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), Immediate.u(1));	9157	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), .u(1));
8869	} else {	9158	} else {
8870	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());	9159	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
8871	try self.asmRegister(.{ ._, .dec }, temp_regs[0].to32());	9160	try self.asmRegister(.{ ._, .dec }, temp_regs[0].to32());
...	@@ -8874,8 +9163,8 @@ fn genShiftBinOpMir(	...	@@ -8874,8 +9163,8 @@ fn genShiftBinOpMir(
8874	},	9163	},
8875	._r => {	9164	._r => {
8876	if (slow_inc_dec) {	9165	if (slow_inc_dec) {
8877	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));	9166	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
8878	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	9167	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
8879	} else {	9168	} else {
8880	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());	9169	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
8881	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	9170	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
...	@@ -8883,7 +9172,7 @@ fn genShiftBinOpMir(	...	@@ -8883,7 +9172,7 @@ fn genShiftBinOpMir(
8883	try self.asmRegisterImmediate(	9172	try self.asmRegisterImmediate(
8884	.{ ._, .cmp },	9173	.{ ._, .cmp },
8885	temp_regs[0].to32(),	9174	temp_regs[0].to32(),
8886	Immediate.u(limbs_len - 1),	9175	.u(limbs_len - 1),
8887	);	9176	);
8888	_ = try self.asmJccReloc(.b, loop);	9177	_ = try self.asmJccReloc(.b, loop);
8889	},	9178	},
...	@@ -8898,7 +9187,7 @@ fn genShiftBinOpMir(	...	@@ -8898,7 +9187,7 @@ fn genShiftBinOpMir(
8898	.immediate => \|shift_imm\| try self.asmRegisterImmediate(	9187	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8899	tag,	9188	tag,
8900	temp_regs[2].to64(),	9189	temp_regs[2].to64(),
8901	Immediate.u(shift_imm & math.maxInt(u6)),	9190	.u(shift_imm & std.math.maxInt(u6)),
8902	),	9191	),
8903	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),	9192	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
8904	}	9193	}
...	@@ -8914,7 +9203,7 @@ fn genShiftBinOpMir(	...	@@ -8914,7 +9203,7 @@ fn genShiftBinOpMir(
8914	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(	9203	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
8915	tag,	9204	tag,
8916	temp_regs[2].to64(),	9205	temp_regs[2].to64(),
8917	Immediate.u(63),	9206	.u(63),
8918	);	9207	);
8919	if (switch (rhs_mcv) {	9208	if (switch (rhs_mcv) {
8920	.immediate => \|shift_imm\| shift_imm >> 6 > 0,	9209	.immediate => \|shift_imm\| shift_imm >> 6 > 0,
...	@@ -8935,7 +9224,7 @@ fn genShiftBinOpMir(	...	@@ -8935,7 +9224,7 @@ fn genShiftBinOpMir(
8935	try self.asmRegisterImmediate(	9224	try self.asmRegisterImmediate(
8936	.{ ._, .cmp },	9225	.{ ._, .cmp },
8937	temp_regs[1].to32(),	9226	temp_regs[1].to32(),
8938	Immediate.u(limbs_len - 1),	9227	.u(limbs_len - 1),
8939	);	9228	);
8940	break :skip try self.asmJccReloc(.nb, undefined);	9229	break :skip try self.asmJccReloc(.nb, undefined);
8941	},	9230	},
...	@@ -8945,12 +9234,12 @@ fn genShiftBinOpMir(	...	@@ -8945,12 +9234,12 @@ fn genShiftBinOpMir(
8945	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	9234	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
8946	switch (tag[0]) {	9235	switch (tag[0]) {
8947	._l => if (slow_inc_dec) {	9236	._l => if (slow_inc_dec) {
8948	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));	9237	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
8949	} else {	9238	} else {
8950	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());	9239	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
8951	},	9240	},
8952	._r => if (slow_inc_dec) {	9241	._r => if (slow_inc_dec) {
8953	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));	9242	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
8954	} else {	9243	} else {
8955	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());	9244	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
8956	},	9245	},
...	@@ -8972,14 +9261,14 @@ fn genShiftBinOpMir(	...	@@ -8972,14 +9261,14 @@ fn genShiftBinOpMir(
8972	.scale = .@"8",	9261	.scale = .@"8",
8973	.disp = lhs_mcv.load_frame.off,	9262	.disp = lhs_mcv.load_frame.off,
8974	} },	9263	} },
8975	}, Immediate.u(0));	9264	}, .u(0));
8976	switch (tag[0]) {	9265	switch (tag[0]) {
8977	._l => _ = try self.asmJccReloc(.nz, loop),	9266	._l => _ = try self.asmJccReloc(.nz, loop),
8978	._r => {	9267	._r => {
8979	try self.asmRegisterImmediate(	9268	try self.asmRegisterImmediate(
8980	.{ ._, .cmp },	9269	.{ ._, .cmp },
8981	temp_regs[1].to32(),	9270	temp_regs[1].to32(),
8982	Immediate.u(limbs_len - 1),	9271	.u(limbs_len - 1),
8983	);	9272	);
8984	_ = try self.asmJccReloc(.b, loop);	9273	_ = try self.asmJccReloc(.b, loop);
8985	},	9274	},
...	@@ -9021,12 +9310,12 @@ fn genShiftBinOpMir(	...	@@ -9021,12 +9310,12 @@ fn genShiftBinOpMir(
9021	info.double_tag,	9310	info.double_tag,
9022	lhs_regs[info.indices[1]],	9311	lhs_regs[info.indices[1]],
9023	lhs_regs[info.indices[0]],	9312	lhs_regs[info.indices[0]],
9024	Immediate.u(shift_imm),	9313	.u(shift_imm),
9025	);	9314	);
9026	try self.asmRegisterImmediate(	9315	try self.asmRegisterImmediate(
9027	tag,	9316	tag,
9028	lhs_regs[info.indices[0]],	9317	lhs_regs[info.indices[0]],
9029	Immediate.u(shift_imm),	9318	.u(shift_imm),
9030	);	9319	);
9031	return;	9320	return;
9032	} else {	9321	} else {
...	@@ -9039,7 +9328,7 @@ fn genShiftBinOpMir(	...	@@ -9039,7 +9328,7 @@ fn genShiftBinOpMir(
9039	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(	9328	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
9040	tag,	9329	tag,
9041	lhs_regs[info.indices[0]],	9330	lhs_regs[info.indices[0]],
9042	Immediate.u(63),	9331	.u(63),
9043	) else try self.asmRegisterRegister(	9332	) else try self.asmRegisterRegister(
9044	.{ ._, .xor },	9333	.{ ._, .xor },
9045	lhs_regs[info.indices[0]],	9334	lhs_regs[info.indices[0]],
...	@@ -9048,7 +9337,7 @@ fn genShiftBinOpMir(	...	@@ -9048,7 +9337,7 @@ fn genShiftBinOpMir(
9048	if (shift_imm > 64) try self.asmRegisterImmediate(	9337	if (shift_imm > 64) try self.asmRegisterImmediate(
9049	tag,	9338	tag,
9050	lhs_regs[info.indices[1]],	9339	lhs_regs[info.indices[1]],
9051	Immediate.u(shift_imm - 64),	9340	.u(shift_imm - 64),
9052	);	9341	);
9053	return;	9342	return;
9054	},	9343	},
...	@@ -9059,7 +9348,7 @@ fn genShiftBinOpMir(	...	@@ -9059,7 +9348,7 @@ fn genShiftBinOpMir(
9059		9348
9060	if (tag[0] == ._r and tag[1] == .sa) {	9349	if (tag[0] == ._r and tag[1] == .sa) {
9061	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]);	9350	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]);
9062	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));	9351	try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
9063	} else try self.asmRegisterRegister(	9352	} else try self.asmRegisterRegister(
9064	.{ ._, .xor },	9353	.{ ._, .xor },
9065	tmp_reg.to32(),	9354	tmp_reg.to32(),
...	@@ -9076,11 +9365,7 @@ fn genShiftBinOpMir(	...	@@ -9076,11 +9365,7 @@ fn genShiftBinOpMir(
9076	lhs_regs[info.indices[0]],	9365	lhs_regs[info.indices[0]],
9077	registerAlias(shift_reg, 1),	9366	registerAlias(shift_reg, 1),
9078	);	9367	);
9079	try self.asmRegisterImmediate(	9368	try self.asmRegisterImmediate(.{ ._, .cmp }, registerAlias(shift_reg, 1), .u(64));
9080	.{ ._, .cmp },
9081	registerAlias(shift_reg, 1),
9082	Immediate.u(64),
9083	);
9084	try self.asmCmovccRegisterRegister(	9369	try self.asmCmovccRegisterRegister(
9085	.ae,	9370	.ae,
9086	lhs_regs[info.indices[1]],	9371	lhs_regs[info.indices[1]],
...	@@ -9119,7 +9404,7 @@ fn genShiftBinOpMir(	...	@@ -9119,7 +9404,7 @@ fn genShiftBinOpMir(
9119	} },	9404	} },
9120	},	9405	},
9121	tmp_reg,	9406	tmp_reg,
9122	Immediate.u(shift_imm),	9407	.u(shift_imm),
9123	);	9408	);
9124	try self.asmMemoryImmediate(	9409	try self.asmMemoryImmediate(
9125	tag,	9410	tag,
...	@@ -9130,7 +9415,7 @@ fn genShiftBinOpMir(	...	@@ -9130,7 +9415,7 @@ fn genShiftBinOpMir(
9130	.disp = dst_frame_addr.off + info.indices[0] * 8,	9415	.disp = dst_frame_addr.off + info.indices[0] * 8,
9131	} },	9416	} },
9132	},	9417	},
9133	Immediate.u(shift_imm),	9418	.u(shift_imm),
9134	);	9419	);
9135	return;	9420	return;
9136	} else {	9421	} else {
...	@@ -9149,7 +9434,7 @@ fn genShiftBinOpMir(	...	@@ -9149,7 +9434,7 @@ fn genShiftBinOpMir(
9149	if (shift_imm > 64) try self.asmRegisterImmediate(	9434	if (shift_imm > 64) try self.asmRegisterImmediate(
9150	tag,	9435	tag,
9151	tmp_reg,	9436	tmp_reg,
9152	Immediate.u(shift_imm - 64),	9437	.u(shift_imm - 64),
9153	);	9438	);
9154	try self.asmMemoryRegister(	9439	try self.asmMemoryRegister(
9155	.{ ._, .mov },	9440	.{ ._, .mov },
...	@@ -9171,7 +9456,7 @@ fn genShiftBinOpMir(	...	@@ -9171,7 +9456,7 @@ fn genShiftBinOpMir(
9171	.disp = dst_frame_addr.off + info.indices[0] * 8,	9456	.disp = dst_frame_addr.off + info.indices[0] * 8,
9172	} },	9457	} },
9173	},	9458	},
9174	Immediate.u(63),	9459	.u(63),
9175	) else {	9460	) else {
9176	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());	9461	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
9177	try self.asmMemoryRegister(	9462	try self.asmMemoryRegister(
...	@@ -9223,7 +9508,7 @@ fn genShiftBinOpMir(	...	@@ -9223,7 +9508,7 @@ fn genShiftBinOpMir(
9223	);	9508	);
9224	if (tag[0] == ._r and tag[1] == .sa) {	9509	if (tag[0] == ._r and tag[1] == .sa) {
9225	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);	9510	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
9226	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));	9511	try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
9227	} else try self.asmRegisterRegister(	9512	} else try self.asmRegisterRegister(
9228	.{ ._, .xor },	9513	.{ ._, .xor },
9229	tmp_reg.to32(),	9514	tmp_reg.to32(),
...	@@ -9239,7 +9524,7 @@ fn genShiftBinOpMir(	...	@@ -9239,7 +9524,7 @@ fn genShiftBinOpMir(
9239	try self.asmRegisterImmediate(	9524	try self.asmRegisterImmediate(
9240	.{ ._, .cmp },	9525	.{ ._, .cmp },
9241	registerAlias(shift_reg, 1),	9526	registerAlias(shift_reg, 1),
9242	Immediate.u(64),	9527	.u(64),
9243	);	9528	);
9244	try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg);	9529	try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg);
9245	try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);	9530	try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);
...	@@ -9277,7 +9562,7 @@ fn genShiftBinOpMir(	...	@@ -9277,7 +9562,7 @@ fn genShiftBinOpMir(
9277	.immediate => \|shift_imm\| return self.asmRegisterImmediate(	9562	.immediate => \|shift_imm\| return self.asmRegisterImmediate(
9278	tag,	9563	tag,
9279	registerAlias(lhs_reg, abi_size),	9564	registerAlias(lhs_reg, abi_size),
9280	Immediate.u(shift_imm),	9565	.u(shift_imm),
9281	),	9566	),
9282	.register => \|shift_reg\| return self.asmRegisterRegister(	9567	.register => \|shift_reg\| return self.asmRegisterRegister(
9283	tag,	9568	tag,
...	@@ -9292,7 +9577,7 @@ fn genShiftBinOpMir(	...	@@ -9292,7 +9577,7 @@ fn genShiftBinOpMir(
9292	.base = .{ .reg = .ds },	9577	.base = .{ .reg = .ds },
9293	.mod = .{ .rm = .{	9578	.mod = .{ .rm = .{
9294	.size = Memory.Size.fromSize(abi_size),	9579	.size = Memory.Size.fromSize(abi_size),
9295	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse	9580	.disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
9296	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{	9581	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
9297	@tagName(lhs_mcv),	9582	@tagName(lhs_mcv),
9298	@tagName(shift_mcv),	9583	@tagName(shift_mcv),
...	@@ -9316,11 +9601,7 @@ fn genShiftBinOpMir(	...	@@ -9316,11 +9601,7 @@ fn genShiftBinOpMir(
9316	else => unreachable,	9601	else => unreachable,
9317	};	9602	};
9318	switch (shift_mcv) {	9603	switch (shift_mcv) {
9319	.immediate => \|shift_imm\| return self.asmMemoryImmediate(	9604	.immediate => \|shift_imm\| return self.asmMemoryImmediate(tag, lhs_mem, .u(shift_imm)),
9320	tag,
9321	lhs_mem,
9322	Immediate.u(shift_imm),
9323	),
9324	.register => \|shift_reg\| return self.asmMemoryRegister(	9605	.register => \|shift_reg\| return self.asmMemoryRegister(
9325	tag,	9606	tag,
9326	lhs_mem,	9607	lhs_mem,
...	@@ -9495,7 +9776,7 @@ fn genMulDivBinOp(	...	@@ -9495,7 +9776,7 @@ fn genMulDivBinOp(
9495	switch (tag) {	9776	switch (tag) {
9496	.mul, .mul_wrap => {	9777	.mul, .mul_wrap => {
9497	const slow_inc = self.hasFeature(.slow_incdec);	9778	const slow_inc = self.hasFeature(.slow_incdec);
9498	const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;	9779	const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;
9499		9780
9500	try self.spillRegisters(&.{ .rax, .rcx, .rdx });	9781	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
9501	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });	9782	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
...	@@ -9536,7 +9817,7 @@ fn genMulDivBinOp(	...	@@ -9536,7 +9817,7 @@ fn genMulDivBinOp(
9536	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);	9817	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
9537		9818
9538	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	9819	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
9539	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));	9820	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
9540	try self.asmMemoryRegister(.{ ._, .adc }, .{	9821	try self.asmMemoryRegister(.{ ._, .adc }, .{
9541	.base = .{ .frame = dst_mcv.load_frame.index },	9822	.base = .{ .frame = dst_mcv.load_frame.index },
9542	.mod = .{ .rm = .{	9823	.mod = .{ .rm = .{
...	@@ -9559,7 +9840,7 @@ fn genMulDivBinOp(	...	@@ -9559,7 +9840,7 @@ fn genMulDivBinOp(
9559	});	9840	});
9560	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());	9841	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
9561		9842
9562	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));	9843	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
9563	try self.asmMemoryRegister(.{ ._, .adc }, .{	9844	try self.asmMemoryRegister(.{ ._, .adc }, .{
9564	.base = .{ .frame = dst_mcv.load_frame.index },	9845	.base = .{ .frame = dst_mcv.load_frame.index },
9565	.mod = .{ .rm = .{	9846	.mod = .{ .rm = .{
...	@@ -9572,30 +9853,22 @@ fn genMulDivBinOp(	...	@@ -9572,30 +9853,22 @@ fn genMulDivBinOp(
9572	try self.asmSetccRegister(.c, .ch);	9853	try self.asmSetccRegister(.c, .ch);
9573		9854
9574	if (slow_inc) {	9855	if (slow_inc) {
9575	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));	9856	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
9576	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));	9857	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
9577	} else {	9858	} else {
9578	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());	9859	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
9579	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());	9860	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
9580	}	9861	}
9581	try self.asmRegisterImmediate(	9862	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
9582	.{ ._, .cmp },
9583	temp_regs[3].to32(),
9584	Immediate.u(limb_len),
9585	);
9586	_ = try self.asmJccReloc(.b, inner_loop);	9863	_ = try self.asmJccReloc(.b, inner_loop);
9587		9864
9588	self.performReloc(skip_inner);	9865	self.performReloc(skip_inner);
9589	if (slow_inc) {	9866	if (slow_inc) {
9590	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	9867	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
9591	} else {	9868	} else {
9592	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	9869	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
9593	}	9870	}
9594	try self.asmRegisterImmediate(	9871	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
9595	.{ ._, .cmp },
9596	temp_regs[0].to32(),
9597	Immediate.u(limb_len),
9598	);
9599	_ = try self.asmJccReloc(.b, outer_loop);	9872	_ = try self.asmJccReloc(.b, outer_loop);
9600		9873
9601	return dst_mcv;	9874	return dst_mcv;
...	@@ -9911,7 +10184,7 @@ fn genBinOp(	...	@@ -9911,7 +10184,7 @@ fn genBinOp(
9911	dst_reg,	10184	dst_reg,
9912	dst_reg,	10185	dst_reg,
9913	try rhs_mcv.mem(self, .word),	10186	try rhs_mcv.mem(self, .word),
9914	Immediate.u(1),	10187	.u(1),
9915	) else try self.asmRegisterRegisterRegister(	10188	) else try self.asmRegisterRegisterRegister(
9916	.{ .vp_, .unpcklwd },	10189	.{ .vp_, .unpcklwd },
9917	dst_reg,	10190	dst_reg,
...	@@ -9969,7 +10242,7 @@ fn genBinOp(	...	@@ -9969,7 +10242,7 @@ fn genBinOp(
9969	.{ .v_, .cvtps2ph },	10242	.{ .v_, .cvtps2ph },
9970	dst_reg,	10243	dst_reg,
9971	dst_reg,	10244	dst_reg,
9972	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	10245	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
9973	);	10246	);
9974	break :adjusted .{ .register = dst_reg };	10247	break :adjusted .{ .register = dst_reg };
9975	},	10248	},
...	@@ -10278,7 +10551,7 @@ fn genBinOp(	...	@@ -10278,7 +10551,7 @@ fn genBinOp(
10278	.lea_tlv,	10551	.lea_tlv,
10279	.lea_frame,	10552	.lea_frame,
10280	=> true,	10553	=> true,
10281	.memory => \|addr\| math.cast(i32, @as(i64, @bitCast(addr))) == null,	10554	.memory => \|addr\| std.math.cast(i32, @as(i64, @bitCast(addr))) == null,
10282	else => false,	10555	else => false,
10283	.register_pair,	10556	.register_pair,
10284	.register_overflow,	10557	.register_overflow,
...	@@ -10410,7 +10683,7 @@ fn genBinOp(	...	@@ -10410,7 +10683,7 @@ fn genBinOp(
10410	dst_reg,	10683	dst_reg,
10411	dst_reg,	10684	dst_reg,
10412	try src_mcv.mem(self, .word),	10685	try src_mcv.mem(self, .word),
10413	Immediate.u(1),	10686	.u(1),
10414	) else try self.asmRegisterRegisterRegister(	10687	) else try self.asmRegisterRegisterRegister(
10415	.{ .vp_, .unpcklwd },	10688	.{ .vp_, .unpcklwd },
10416	dst_reg,	10689	dst_reg,
...	@@ -10442,7 +10715,7 @@ fn genBinOp(	...	@@ -10442,7 +10715,7 @@ fn genBinOp(
10442	dst_reg,	10715	dst_reg,
10443	dst_reg,	10716	dst_reg,
10444	dst_reg,	10717	dst_reg,
10445	Immediate.u(@as(u5, @bitCast(RoundMode{	10718	.u(@as(u5, @bitCast(RoundMode{
10446	.mode = switch (air_tag) {	10719	.mode = switch (air_tag) {
10447	.div_trunc => .zero,	10720	.div_trunc => .zero,
10448	.div_floor => .down,	10721	.div_floor => .down,
...	@@ -10457,7 +10730,7 @@ fn genBinOp(	...	@@ -10457,7 +10730,7 @@ fn genBinOp(
10457	.{ .v_, .cvtps2ph },	10730	.{ .v_, .cvtps2ph },
10458	dst_reg,	10731	dst_reg,
10459	dst_reg,	10732	dst_reg,
10460	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	10733	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10461	);	10734	);
10462	return dst_mcv;	10735	return dst_mcv;
10463	},	10736	},
...	@@ -10856,7 +11129,7 @@ fn genBinOp(	...	@@ -10856,7 +11129,7 @@ fn genBinOp(
10856	dst_reg,	11129	dst_reg,
10857	dst_reg,	11130	dst_reg,
10858	try src_mcv.mem(self, .word),	11131	try src_mcv.mem(self, .word),
10859	Immediate.u(1),	11132	.u(1),
10860	) else try self.asmRegisterRegisterRegister(	11133	) else try self.asmRegisterRegisterRegister(
10861	.{ .vp_, .unpcklwd },	11134	.{ .vp_, .unpcklwd },
10862	dst_reg,	11135	dst_reg,
...	@@ -10886,7 +11159,7 @@ fn genBinOp(	...	@@ -10886,7 +11159,7 @@ fn genBinOp(
10886	.{ .v_, .cvtps2ph },	11159	.{ .v_, .cvtps2ph },
10887	dst_reg,	11160	dst_reg,
10888	dst_reg,	11161	dst_reg,
10889	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	11162	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10890	);	11163	);
10891	return dst_mcv;	11164	return dst_mcv;
10892	},	11165	},
...	@@ -10902,7 +11175,7 @@ fn genBinOp(	...	@@ -10902,7 +11175,7 @@ fn genBinOp(
10902	.{ .vp_d, .insr },	11175	.{ .vp_d, .insr },
10903	dst_reg,	11176	dst_reg,
10904	try src_mcv.mem(self, .dword),	11177	try src_mcv.mem(self, .dword),
10905	Immediate.u(1),	11178	.u(1),
10906	) else try self.asmRegisterRegisterRegister(	11179	) else try self.asmRegisterRegisterRegister(
10907	.{ .v_ps, .unpckl },	11180	.{ .v_ps, .unpckl },
10908	dst_reg,	11181	dst_reg,
...	@@ -10937,7 +11210,7 @@ fn genBinOp(	...	@@ -10937,7 +11210,7 @@ fn genBinOp(
10937	.{ .v_, .cvtps2ph },	11210	.{ .v_, .cvtps2ph },
10938	dst_reg,	11211	dst_reg,
10939	dst_reg,	11212	dst_reg,
10940	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	11213	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10941	);	11214	);
10942	return dst_mcv;	11215	return dst_mcv;
10943	},	11216	},
...	@@ -10980,7 +11253,7 @@ fn genBinOp(	...	@@ -10980,7 +11253,7 @@ fn genBinOp(
10980	.{ .v_, .cvtps2ph },	11253	.{ .v_, .cvtps2ph },
10981	dst_reg,	11254	dst_reg,
10982	dst_reg,	11255	dst_reg,
10983	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	11256	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10984	);	11257	);
10985	return dst_mcv;	11258	return dst_mcv;
10986	},	11259	},
...	@@ -11023,7 +11296,7 @@ fn genBinOp(	...	@@ -11023,7 +11296,7 @@ fn genBinOp(
11023	.{ .v_, .cvtps2ph },	11296	.{ .v_, .cvtps2ph },
11024	dst_reg,	11297	dst_reg,
11025	dst_reg.to256(),	11298	dst_reg.to256(),
11026	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	11299	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
11027	);	11300	);
11028	return dst_mcv;	11301	return dst_mcv;
11029	},	11302	},
...	@@ -11191,7 +11464,7 @@ fn genBinOp(	...	@@ -11191,7 +11464,7 @@ fn genBinOp(
11191	);	11464	);
11192	},	11465	},
11193	.cmp => {	11466	.cmp => {
11194	const imm = Immediate.u(switch (air_tag) {	11467	const imm: Immediate = .u(switch (air_tag) {
11195	.cmp_eq => 0,	11468	.cmp_eq => 0,
11196	.cmp_lt, .cmp_gt => 1,	11469	.cmp_lt, .cmp_gt => 1,
11197	.cmp_lte, .cmp_gte => 2,	11470	.cmp_lte, .cmp_gte => 2,
...	@@ -11289,7 +11562,7 @@ fn genBinOp(	...	@@ -11289,7 +11562,7 @@ fn genBinOp(
11289	mask_reg,	11562	mask_reg,
11290	rhs_copy_reg,	11563	rhs_copy_reg,
11291	rhs_copy_reg,	11564	rhs_copy_reg,
11292	Immediate.u(3), // unord	11565	.u(3), // unord
11293	);	11566	);
11294	try self.asmRegisterRegisterRegisterRegister(	11567	try self.asmRegisterRegisterRegisterRegister(
11295	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {	11568	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
...	@@ -11356,7 +11629,7 @@ fn genBinOp(	...	@@ -11356,7 +11629,7 @@ fn genBinOp(
11356	}),	11629	}),
11357	mask_reg,	11630	mask_reg,
11358	mask_reg,	11631	mask_reg,
11359	Immediate.u(if (has_blend) 3 else 7), // unord, ord	11632	.u(if (has_blend) 3 else 7), // unord, ord
11360	);	11633	);
11361	if (has_blend) try self.asmRegisterRegisterRegister(	11634	if (has_blend) try self.asmRegisterRegisterRegister(
11362	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {	11635	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
...	@@ -11567,29 +11840,29 @@ fn genBinOpMir(	...	@@ -11567,29 +11840,29 @@ fn genBinOpMir(
11567	8 => try self.asmRegisterImmediate(	11840	8 => try self.asmRegisterImmediate(
11568	mir_limb_tag,	11841	mir_limb_tag,
11569	dst_alias,	11842	dst_alias,
11570	if (math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|	11843	if (std.math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
11571	Immediate.s(small)	11844	.s(small)
11572	else	11845	else
11573	Immediate.u(@as(u8, @intCast(imm))),	11846	.u(@as(u8, @intCast(imm))),
11574	),	11847	),
11575	16 => try self.asmRegisterImmediate(	11848	16 => try self.asmRegisterImmediate(
11576	mir_limb_tag,	11849	mir_limb_tag,
11577	dst_alias,	11850	dst_alias,
11578	if (math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|	11851	if (std.math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
11579	Immediate.s(small)	11852	.s(small)
11580	else	11853	else
11581	Immediate.u(@as(u16, @intCast(imm))),	11854	.u(@as(u16, @intCast(imm))),
11582	),	11855	),
11583	32 => try self.asmRegisterImmediate(	11856	32 => try self.asmRegisterImmediate(
11584	mir_limb_tag,	11857	mir_limb_tag,
11585	dst_alias,	11858	dst_alias,
11586	if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	11859	if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11587	Immediate.s(small)	11860	.s(small)
11588	else	11861	else
11589	Immediate.u(@as(u32, @intCast(imm))),	11862	.u(@as(u32, @intCast(imm))),
11590	),	11863	),
11591	64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	11864	64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11592	try self.asmRegisterImmediate(mir_limb_tag, dst_alias, Immediate.s(small))	11865	try self.asmRegisterImmediate(mir_limb_tag, dst_alias, .s(small))
11593	else	11866	else
11594	try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias(	11867	try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias(
11595	try self.copyToTmpRegister(ty, src_mcv),	11868	try self.copyToTmpRegister(ty, src_mcv),
...	@@ -11619,7 +11892,7 @@ fn genBinOpMir(	...	@@ -11619,7 +11892,7 @@ fn genBinOpMir(
11619	.base = .{ .reg = .ds },	11892	.base = .{ .reg = .ds },
11620	.mod = .{ .rm = .{	11893	.mod = .{ .rm = .{
11621	.size = Memory.Size.fromSize(limb_abi_size),	11894	.size = Memory.Size.fromSize(limb_abi_size),
11622	.disp = math.cast(i32, addr + off) orelse break :direct,	11895	.disp = std.math.cast(i32, addr + off) orelse break :direct,
11623	} },	11896	} },
11624	},	11897	},
11625	.indirect => \|reg_off\| .{	11898	.indirect => \|reg_off\| .{
...	@@ -11731,8 +12004,8 @@ fn genBinOpMir(	...	@@ -11731,8 +12004,8 @@ fn genBinOpMir(
11731	=> null,	12004	=> null,
11732	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {	12005	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
11733	switch (resolved_src_mcv) {	12006	switch (resolved_src_mcv) {
11734	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr))) != null and	12007	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr))) != null and
11735	math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)	12008	std.math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)
11736	break :src null,	12009	break :src null,
11737	.load_symbol, .load_got, .load_direct, .load_tlv => {},	12010	.load_symbol, .load_got, .load_direct, .load_tlv => {},
11738	else => unreachable,	12011	else => unreachable,
...	@@ -11823,33 +12096,29 @@ fn genBinOpMir(	...	@@ -11823,33 +12096,29 @@ fn genBinOpMir(
11823	8 => try self.asmMemoryImmediate(	12096	8 => try self.asmMemoryImmediate(
11824	mir_limb_tag,	12097	mir_limb_tag,
11825	dst_limb_mem,	12098	dst_limb_mem,
11826	if (math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|	12099	if (std.math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
11827	Immediate.s(small)	12100	.s(small)
11828	else	12101	else
11829	Immediate.u(@as(u8, @intCast(imm))),	12102	.u(@as(u8, @intCast(imm))),
11830	),	12103	),
11831	16 => try self.asmMemoryImmediate(	12104	16 => try self.asmMemoryImmediate(
11832	mir_limb_tag,	12105	mir_limb_tag,
11833	dst_limb_mem,	12106	dst_limb_mem,
11834	if (math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|	12107	if (std.math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
11835	Immediate.s(small)	12108	.s(small)
11836	else	12109	else
11837	Immediate.u(@as(u16, @intCast(imm))),	12110	.u(@as(u16, @intCast(imm))),
11838	),	12111	),
11839	32 => try self.asmMemoryImmediate(	12112	32 => try self.asmMemoryImmediate(
11840	mir_limb_tag,	12113	mir_limb_tag,
11841	dst_limb_mem,	12114	dst_limb_mem,
11842	if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	12115	if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11843	Immediate.s(small)	12116	.s(small)
11844	else	12117	else
11845	Immediate.u(@as(u32, @intCast(imm))),	12118	.u(@as(u32, @intCast(imm))),
11846	),	12119	),
11847	64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	12120	64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11848	try self.asmMemoryImmediate(	12121	try self.asmMemoryImmediate(mir_limb_tag, dst_limb_mem, .s(small))
11849	mir_limb_tag,
11850	dst_limb_mem,
11851	Immediate.s(small),
11852	)
11853	else	12122	else
11854	try self.asmMemoryRegister(	12123	try self.asmMemoryRegister(
11855	mir_limb_tag,	12124	mir_limb_tag,
...	@@ -11973,12 +12242,12 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -11973,12 +12242,12 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
11973	registerAlias(src_reg, abi_size),	12242	registerAlias(src_reg, abi_size),
11974	),	12243	),
11975	.immediate => \|imm\| {	12244	.immediate => \|imm\| {
11976	if (math.cast(i32, imm)) \|small\| {	12245	if (std.math.cast(i32, imm)) \|small\| {
11977	try self.asmRegisterRegisterImmediate(	12246	try self.asmRegisterRegisterImmediate(
11978	.{ .i_, .mul },	12247	.{ .i_, .mul },
11979	dst_alias,	12248	dst_alias,
11980	dst_alias,	12249	dst_alias,
11981	Immediate.s(small),	12250	.s(small),
11982	);	12251	);
11983	} else {	12252	} else {
11984	const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);	12253	const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
...	@@ -12009,7 +12278,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -12009,7 +12278,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
12009	.base = .{ .reg = .ds },	12278	.base = .{ .reg = .ds },
12010	.mod = .{ .rm = .{	12279	.mod = .{ .rm = .{
12011	.size = Memory.Size.fromSize(abi_size),	12280	.size = Memory.Size.fromSize(abi_size),
12012	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse	12281	.disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
12013	return self.asmRegisterRegister(	12282	return self.asmRegisterRegister(
12014	.{ .i_, .mul },	12283	.{ .i_, .mul },
12015	dst_alias,	12284	dst_alias,
...	@@ -12087,11 +12356,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12087,11 +12356,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12087	) \|dst_reg, elem_index\| {	12356	) \|dst_reg, elem_index\| {
12088	assert(self.register_manager.isRegFree(dst_reg));	12357	assert(self.register_manager.isRegFree(dst_reg));
12089	if (elem_index > 0) {	12358	if (elem_index > 0) {
12090	try self.asmRegisterImmediate(	12359	try self.asmRegisterImmediate(.{ ._l, .sh }, dst_reg.to8(), .u(elem_index));
12091	.{ ._l, .sh },
12092	dst_reg.to8(),
12093	Immediate.u(elem_index),
12094	);
12095	try self.asmRegisterRegister(	12360	try self.asmRegisterRegister(
12096	.{ ._, .@"or" },	12361	.{ ._, .@"or" },
12097	dst_reg.to8(),	12362	dst_reg.to8(),
...	@@ -12127,7 +12392,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12127,7 +12392,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12127	try self.asmRegisterImmediate(	12392	try self.asmRegisterImmediate(
12128	.{ ._, .mov },	12393	.{ ._, .mov },
12129	index_reg.to32(),	12394	index_reg.to32(),
12130	Immediate.u(regs_frame_addr.regs),	12395	.u(regs_frame_addr.regs),
12131	);	12396	);
12132	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	12397	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12133	try self.asmMemoryImmediate(.{ ._, .cmp }, .{	12398	try self.asmMemoryImmediate(.{ ._, .cmp }, .{
...	@@ -12147,14 +12412,14 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12147,14 +12412,14 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12147	);	12412	);
12148	self.performReloc(unset);	12413	self.performReloc(unset);
12149	if (self.hasFeature(.slow_incdec)) {	12414	if (self.hasFeature(.slow_incdec)) {
12150	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));	12415	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
12151	} else {	12416	} else {
12152	try self.asmRegister(.{ ._, .inc }, index_reg.to32());	12417	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
12153	}	12418	}
12154	try self.asmRegisterImmediate(	12419	try self.asmRegisterImmediate(
12155	.{ ._, .cmp },	12420	.{ ._, .cmp },
12156	index_reg.to32(),	12421	index_reg.to32(),
12157	Immediate.u(arg_ty.vectorLen(zcu)),	12422	.u(arg_ty.vectorLen(zcu)),
12158	);	12423	);
12159	_ = try self.asmJccReloc(.b, loop);	12424	_ = try self.asmJccReloc(.b, loop);
12160		12425
...	@@ -12180,7 +12445,6 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12180,7 +12445,6 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
12180	for (self.args[self.arg_index..]) \|arg\| {	12445	for (self.args[self.arg_index..]) \|arg\| {
12181	if (arg != .none) break;	12446	if (arg != .none) break;
12182	} else try self.airDbgVarArgs();	12447	} else try self.airDbgVarArgs();
12183	self.finishAirBookkeeping();
12184	}	12448	}
12185		12449
12186	fn airDbgVarArgs(self: *Self) !void {	12450	fn airDbgVarArgs(self: *Self) !void {
...	@@ -12200,9 +12464,9 @@ fn genLocalDebugInfo(	...	@@ -12200,9 +12464,9 @@ fn genLocalDebugInfo(
12200	switch (mcv) {	12464	switch (mcv) {
12201	.none => try self.asmAir(.dbg_local, inst),	12465	.none => try self.asmAir(.dbg_local, inst),
12202	.unreach, .dead, .elementwise_regs_then_frame, .reserved_frame, .air_ref => unreachable,	12466	.unreach, .dead, .elementwise_regs_then_frame, .reserved_frame, .air_ref => unreachable,
12203	.immediate => \|imm\| try self.asmAirImmediate(.dbg_local, inst, Immediate.u(imm)),	12467	.immediate => \|imm\| try self.asmAirImmediate(.dbg_local, inst, .u(imm)),
12204	.lea_frame => \|frame_addr\| try self.asmAirFrameAddress(.dbg_local, inst, frame_addr),	12468	.lea_frame => \|frame_addr\| try self.asmAirFrameAddress(.dbg_local, inst, frame_addr),
12205	.lea_symbol => \|sym_off\| try self.asmAirImmediate(.dbg_local, inst, Immediate.rel(sym_off)),	12469	.lea_symbol => \|sym_off\| try self.asmAirImmediate(.dbg_local, inst, .rel(sym_off)),
12206	else => {	12470	else => {
12207	const ty = switch (tag) {	12471	const ty = switch (tag) {
12208	else => unreachable,	12472	else => unreachable,
...	@@ -12245,16 +12509,6 @@ fn genLocalDebugInfo(	...	@@ -12245,16 +12509,6 @@ fn genLocalDebugInfo(
12245	}	12509	}
12246	}	12510	}
12247		12511
12248	fn airTrap(self: *Self) !void {
12249	try self.asmOpOnly(.{ ._, .ud2 });
12250	self.finishAirBookkeeping();
12251	}
12252
12253	fn airBreakpoint(self: *Self) !void {
12254	try self.asmOpOnly(.{ ._, .int3 });
12255	self.finishAirBookkeeping();
12256	}
12257
12258	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {	12512	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
12259	const dst_mcv = try self.allocRegOrMem(inst, true);	12513	const dst_mcv = try self.allocRegOrMem(inst, true);
12260	try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});	12514	try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
...	@@ -12426,7 +12680,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12426,7 +12680,7 @@ fn genCall(self: *Self, info: union(enum) {
12426	try self.asmRegisterImmediate(	12680	try self.asmRegisterImmediate(
12427	.{ ._, .mov },	12681	.{ ._, .mov },
12428	index_reg.to32(),	12682	index_reg.to32(),
12429	Immediate.u(regs_frame_addr.regs),	12683	.u(regs_frame_addr.regs),
12430	);	12684	);
12431	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	12685	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12432	try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32());	12686	try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32());
...	@@ -12440,14 +12694,14 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12440,14 +12694,14 @@ fn genCall(self: *Self, info: union(enum) {
12440	} },	12694	} },
12441	});	12695	});
12442	if (self.hasFeature(.slow_incdec)) {	12696	if (self.hasFeature(.slow_incdec)) {
12443	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));	12697	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
12444	} else {	12698	} else {
12445	try self.asmRegister(.{ ._, .inc }, index_reg.to32());	12699	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
12446	}	12700	}
12447	try self.asmRegisterImmediate(	12701	try self.asmRegisterImmediate(
12448	.{ ._, .cmp },	12702	.{ ._, .cmp },
12449	index_reg.to32(),	12703	index_reg.to32(),
12450	Immediate.u(arg_ty.vectorLen(zcu)),	12704	.u(arg_ty.vectorLen(zcu)),
12451	);	12705	);
12452	_ = try self.asmJccReloc(.b, loop);	12706	_ = try self.asmJccReloc(.b, loop);
12453		12707
...	@@ -12521,11 +12775,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12521,11 +12775,7 @@ fn genCall(self: *Self, info: union(enum) {
12521	0..,	12775	0..,
12522	) \|dst_reg, elem_index\| {	12776	) \|dst_reg, elem_index\| {
12523	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());	12777	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12524	try self.asmMemoryImmediate(	12778	try self.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index));
12525	.{ ._, .bt },
12526	src_mem,
12527	Immediate.u(elem_index),
12528	);
12529	try self.asmSetccRegister(.c, dst_reg.to8());	12779	try self.asmSetccRegister(.c, dst_reg.to8());
12530	}	12780	}
12531	},	12781	},
...	@@ -12533,7 +12783,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12533,7 +12783,7 @@ fn genCall(self: *Self, info: union(enum) {
12533	};	12783	};
12534		12784
12535	if (fn_info.is_var_args)	12785	if (fn_info.is_var_args)
12536	try self.asmRegisterImmediate(.{ ._, .mov }, .al, Immediate.u(call_info.fp_count));	12786	try self.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count));
12537		12787
12538	// Due to incremental compilation, how function calls are generated depends	12788	// Due to incremental compilation, how function calls are generated depends
12539	// on linking.	12789	// on linking.
...	@@ -12551,7 +12801,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12551,7 +12801,7 @@ fn genCall(self: *Self, info: union(enum) {
12551	if (self.bin_file.cast(.elf)) \|elf_file\| {	12801	if (self.bin_file.cast(.elf)) \|elf_file\| {
12552	const zo = elf_file.zigObjectPtr().?;	12802	const zo = elf_file.zigObjectPtr().?;
12553	const sym_index = try zo.getOrCreateMetadataForNav(zcu, func.owner_nav);	12803	const sym_index = try zo.getOrCreateMetadataForNav(zcu, func.owner_nav);
12554	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index }));	12804	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index }));
12555	} else if (self.bin_file.cast(.coff)) \|coff_file\| {	12805	} else if (self.bin_file.cast(.coff)) \|coff_file\| {
12556	const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);	12806	const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);
12557	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;	12807	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
...	@@ -12561,7 +12811,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12561,7 +12811,7 @@ fn genCall(self: *Self, info: union(enum) {
12561	const zo = macho_file.getZigObject().?;	12811	const zo = macho_file.getZigObject().?;
12562	const sym_index = try zo.getOrCreateMetadataForNav(macho_file, func.owner_nav);	12812	const sym_index = try zo.getOrCreateMetadataForNav(macho_file, func.owner_nav);
12563	const sym = zo.symbols.items[sym_index];	12813	const sym = zo.symbols.items[sym_index];
12564	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym.nlist_idx }));	12814	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym.nlist_idx }));
12565	} else if (self.bin_file.cast(.plan9)) \|p9\| {	12815	} else if (self.bin_file.cast(.plan9)) \|p9\| {
12566	const atom_index = try p9.seeNav(pt, func.owner_nav);	12816	const atom_index = try p9.seeNav(pt, func.owner_nav);
12567	const atom = p9.getAtom(atom_index);	12817	const atom = p9.getAtom(atom_index);
...	@@ -12579,13 +12829,13 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12579,13 +12829,13 @@ fn genCall(self: *Self, info: union(enum) {
12579	@"extern".name.toSlice(ip),	12829	@"extern".name.toSlice(ip),
12580	@"extern".lib_name.toSlice(ip),	12830	@"extern".lib_name.toSlice(ip),
12581	);	12831	);
12582	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	12832	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12583	} else if (self.bin_file.cast(.macho)) \|macho_file\| {	12833	} else if (self.bin_file.cast(.macho)) \|macho_file\| {
12584	const target_sym_index = try macho_file.getGlobalSymbol(	12834	const target_sym_index = try macho_file.getGlobalSymbol(
12585	@"extern".name.toSlice(ip),	12835	@"extern".name.toSlice(ip),
12586	@"extern".lib_name.toSlice(ip),	12836	@"extern".lib_name.toSlice(ip),
12587	);	12837	);
12588	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	12838	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12589	} else try self.genExternSymbolRef(	12839	} else try self.genExternSymbolRef(
12590	.call,	12840	.call,
12591	@"extern".lib_name.toSlice(ip),	12841	@"extern".lib_name.toSlice(ip),
...	@@ -12600,10 +12850,10 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12600,10 +12850,10 @@ fn genCall(self: *Self, info: union(enum) {
12600	},	12850	},
12601	.lib => \|lib\| if (self.bin_file.cast(.elf)) \|elf_file\| {	12851	.lib => \|lib\| if (self.bin_file.cast(.elf)) \|elf_file\| {
12602	const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib);	12852	const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib);
12603	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	12853	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12604	} else if (self.bin_file.cast(.macho)) \|macho_file\| {	12854	} else if (self.bin_file.cast(.macho)) \|macho_file\| {
12605	const target_sym_index = try macho_file.getGlobalSymbol(lib.callee, lib.lib);	12855	const target_sym_index = try macho_file.getGlobalSymbol(lib.callee, lib.lib);
12606	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	12856	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12607	} else try self.genExternSymbolRef(.call, lib.lib, lib.callee),	12857	} else try self.genExternSymbolRef(.call, lib.lib, lib.callee),
12608	}	12858	}
12609	return call_info.return_value.short;	12859	return call_info.return_value.short;
...	@@ -12665,7 +12915,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12665,7 +12915,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
12665	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);	12915	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
12666	}	12916	}
12667		12917
12668	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	12918	fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
12669	const pt = self.pt;	12919	const pt = self.pt;
12670	const zcu = pt.zcu;	12920	const zcu = pt.zcu;
12671	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	12921	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -12754,7 +13004,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12754,7 +13004,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12754	try self.asmRegisterImmediate(	13004	try self.asmRegisterImmediate(
12755	.{ ._r, .sh },	13005	.{ ._r, .sh },
12756	registerAlias(temp_lhs_reg, opt_abi_size),	13006	registerAlias(temp_lhs_reg, opt_abi_size),
12757	Immediate.u(payload_abi_size * 8),	13007	.u(payload_abi_size * 8),
12758	);	13008	);
12759	}	13009	}
12760		13010
...	@@ -12775,7 +13025,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12775,7 +13025,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12775	try self.asmRegisterImmediate(	13025	try self.asmRegisterImmediate(
12776	.{ ._r, .sh },	13026	.{ ._r, .sh },
12777	registerAlias(temp_rhs_reg, opt_abi_size),	13027	registerAlias(temp_rhs_reg, opt_abi_size),
12778	Immediate.u(payload_abi_size * 8),	13028	.u(payload_abi_size * 8),
12779	);	13029	);
12780	try self.asmRegisterRegister(	13030	try self.asmRegisterRegister(
12781	.{ ._, .@"test" },	13031	.{ ._, .@"test" },
...	@@ -12867,10 +13117,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12867,10 +13117,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12867	.register_pair, .load_frame => null,	13117	.register_pair, .load_frame => null,
12868	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: {	13118	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: {
12869	switch (resolved_dst_mcv) {	13119	switch (resolved_dst_mcv) {
12870	.memory => \|addr\| if (math.cast(	13120	.memory => \|addr\| if (std.math.cast(
12871	i32,	13121	i32,
12872	@as(i64, @bitCast(addr)),	13122	@as(i64, @bitCast(addr)),
12873	) != null and math.cast(	13123	) != null and std.math.cast(
12874	i32,	13124	i32,
12875	@as(i64, @bitCast(addr)) + abi_size - 8,	13125	@as(i64, @bitCast(addr)) + abi_size - 8,
12876	) != null) break :dst null,	13126	) != null) break :dst null,
...	@@ -12928,10 +13178,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12928,10 +13178,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12928	.register_pair, .load_frame => null,	13178	.register_pair, .load_frame => null,
12929	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {	13179	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
12930	switch (resolved_src_mcv) {	13180	switch (resolved_src_mcv) {
12931	.memory => \|addr\| if (math.cast(	13181	.memory => \|addr\| if (std.math.cast(
12932	i32,	13182	i32,
12933	@as(i64, @bitCast(addr)),	13183	@as(i64, @bitCast(addr)),
12934	) != null and math.cast(	13184	) != null and std.math.cast(
12935	i32,	13185	i32,
12936	@as(i64, @bitCast(addr)) + abi_size - 8,	13186	@as(i64, @bitCast(addr)) + abi_size - 8,
12937	) != null) break :src null,	13187	) != null) break :src null,
...	@@ -12971,7 +13221,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12971,7 +13221,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12971	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);	13221	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
12972	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);	13222	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);
12973		13223
12974	const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;	13224	const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
12975	var limb_i: u16 = 0;	13225	var limb_i: u16 = 0;
12976	while (limb_i < limbs_len) : (limb_i += 1) {	13226	while (limb_i < limbs_len) : (limb_i += 1) {
12977	const off = limb_i * 8;	13227	const off = limb_i * 8;
...	@@ -13067,7 +13317,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -13067,7 +13317,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
13067	tmp1_reg,	13317	tmp1_reg,
13068	dst_reg.to128(),	13318	dst_reg.to128(),
13069	try src_mcv.mem(self, .word),	13319	try src_mcv.mem(self, .word),
13070	Immediate.u(1),	13320	.u(1),
13071	) else try self.asmRegisterRegisterRegister(	13321	) else try self.asmRegisterRegisterRegister(
13072	.{ .vp_, .unpcklwd },	13322	.{ .vp_, .unpcklwd },
13073	tmp1_reg,	13323	tmp1_reg,
...	@@ -13232,7 +13482,6 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13232,7 +13482,6 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
13232	.column = dbg_stmt.column,	13482	.column = dbg_stmt.column,
13233	} },	13483	} },
13234	});	13484	});
13235	self.finishAirBookkeeping();
13236	}	13485	}
13237		13486
13238	fn airDbgEmptyStmt(self: *Self) !void {	13487	fn airDbgEmptyStmt(self: *Self) !void {
...	@@ -13240,7 +13489,6 @@ fn airDbgEmptyStmt(self: *Self) !void {	...	@@ -13240,7 +13489,6 @@ fn airDbgEmptyStmt(self: *Self) !void {
13240	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)	13489	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)
13241	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;	13490	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;
13242	try self.asmOpOnly(.{ ._, .nop });	13491	try self.asmOpOnly(.{ ._, .nop });
13243	self.finishAirBookkeeping();
13244	}	13492	}
13245		13493
13246	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {	13494	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -13278,7 +13526,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {	...	@@ -13278,7 +13526,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13278	},	13526	},
13279	.register => \|reg\| {	13527	.register => \|reg\| {
13280	try self.spillEflagsIfOccupied();	13528	try self.spillEflagsIfOccupied();
13281	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));	13529	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), .u(1));
13282	return self.asmJccReloc(.z, undefined);	13530	return self.asmJccReloc(.z, undefined);
13283	},	13531	},
13284	.immediate,	13532	.immediate,
...	@@ -13338,7 +13586,6 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13338,7 +13586,6 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
13338	});	13586	});
13339		13587
13340	// We already took care of pl_op.operand earlier, so there's nothing left to do.	13588	// We already took care of pl_op.operand earlier, so there's nothing left to do.
13341	self.finishAirBookkeeping();
13342	}	13589	}
13343		13590
13344	fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {	13591	fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
...	@@ -13353,7 +13600,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13353,7 +13600,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13353		13600
13354	const pl_ty = opt_ty.optionalChild(zcu);	13601	const pl_ty = opt_ty.optionalChild(zcu);
13355		13602
13356	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))	13603	const some_info: struct { off: u31, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
13357	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }	13604	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
13358	else	13605	else
13359	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };	13606	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
...	@@ -13366,7 +13613,6 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13366,7 +13613,6 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13366	.undef,	13613	.undef,
13367	.immediate,	13614	.immediate,
13368	.eflags,	13615	.eflags,
13369	.register_pair,
13370	.register_offset,	13616	.register_offset,
13371	.register_overflow,	13617	.register_overflow,
13372	.lea_direct,	13618	.lea_direct,
...	@@ -13396,7 +13642,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13396,7 +13642,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13396	try self.asmRegisterImmediate(	13642	try self.asmRegisterImmediate(
13397	.{ ._, .bt },	13643	.{ ._, .bt },
13398	registerAlias(opt_reg, opt_abi_size),	13644	registerAlias(opt_reg, opt_abi_size),
13399	Immediate.u(@as(u6, @intCast(some_info.off * 8))),	13645	.u(@as(u6, @intCast(some_info.off * 8))),
		13646	);
		13647	return .{ .eflags = .nc };
		13648	},
		13649
		13650	.register_pair => \|opt_regs\| {
		13651	if (some_info.off == 0) {
		13652	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));
		13653	const alias_reg = registerAlias(opt_regs[0], some_abi_size);
		13654	assert(some_abi_size * 8 == alias_reg.bitSize());
		13655	try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
		13656	return .{ .eflags = .z };
		13657	}
		13658	assert(some_info.ty.ip_index == .bool_type);
		13659	const opt_abi_size: u32 = @intCast(opt_ty.abiSize(zcu));
		13660	try self.asmRegisterImmediate(
		13661	.{ ._, .bt },
		13662	registerAlias(opt_regs[some_info.off / 8], opt_abi_size),
		13663	.u(@as(u6, @truncate(some_info.off * 8))),
13400	);	13664	);
13401	return .{ .eflags = .nc };	13665	return .{ .eflags = .nc };
13402	},	13666	},
...	@@ -13422,7 +13686,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13422,7 +13686,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13422	.disp = some_info.off,	13686	.disp = some_info.off,
13423	} },	13687	} },
13424	},	13688	},
13425	Immediate.u(0),	13689	.u(0),
13426	);	13690	);
13427	return .{ .eflags = .e };	13691	return .{ .eflags = .e };
13428	},	13692	},
...	@@ -13448,7 +13712,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13448,7 +13712,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13448	},	13712	},
13449	else => unreachable,	13713	else => unreachable,
13450	},	13714	},
13451	Immediate.u(0),	13715	.u(0),
13452	);	13716	);
13453	return .{ .eflags = .e };	13717	return .{ .eflags = .e };
13454	},	13718	},
...	@@ -13485,7 +13749,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)	...	@@ -13485,7 +13749,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
13485	.disp = some_info.off,	13749	.disp = some_info.off,
13486	} },	13750	} },
13487	},	13751	},
13488	Immediate.u(0),	13752	.u(0),
13489	);	13753	);
13490		13754
13491	self.eflags_inst = inst;	13755	self.eflags_inst = inst;
...	@@ -13500,7 +13764,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)	...	@@ -13500,7 +13764,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
13500		13764
13501	try self.spillEflagsIfOccupied();	13765	try self.spillEflagsIfOccupied();
13502		13766
13503	const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));	13767	const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
13504	switch (eu_mcv) {	13768	switch (eu_mcv) {
13505	.register => \|reg\| {	13769	.register => \|reg\| {
13506	const eu_lock = self.register_manager.lockReg(reg);	13770	const eu_lock = self.register_manager.lockReg(reg);
...	@@ -13557,7 +13821,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV	...	@@ -13557,7 +13821,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
13557	const ptr_lock = self.register_manager.lockReg(ptr_reg);	13821	const ptr_lock = self.register_manager.lockReg(ptr_reg);
13558	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	13822	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
13559		13823
13560	const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));	13824	const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
13561	try self.asmMemoryImmediate(	13825	try self.asmMemoryImmediate(
13562	.{ ._, .cmp },	13826	.{ ._, .cmp },
13563	.{	13827	.{
...	@@ -13567,7 +13831,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV	...	@@ -13567,7 +13831,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
13567	.disp = err_off,	13831	.disp = err_off,
13568	} },	13832	} },
13569	},	13833	},
13570	Immediate.u(0),	13834	.u(0),
13571	);	13835	);
13572		13836
13573	if (maybe_inst) \|inst\| self.eflags_inst = inst;	13837	if (maybe_inst) \|inst\| self.eflags_inst = inst;
...	@@ -13686,12 +13950,11 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13686,12 +13950,11 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
13686		13950
13687	try self.loops.putNoClobber(self.gpa, inst, .{	13951	try self.loops.putNoClobber(self.gpa, inst, .{
13688	.state = state,	13952	.state = state,
13689	.jmp_target = @intCast(self.mir_instructions.len),	13953	.target = @intCast(self.mir_instructions.len),
13690	});	13954	});
13691	defer assert(self.loops.remove(inst));	13955	defer assert(self.loops.remove(inst));
13692		13956
13693	try self.genBodyBlock(body);	13957	try self.genBodyBlock(body);
13694	self.finishAirBookkeeping();
13695	}	13958	}
13696		13959
13697	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {	13960	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -13729,7 +13992,6 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !	...	@@ -13729,7 +13992,6 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
13729	const tracking = &self.inst_tracking.values()[inst_tracking_i];	13992	const tracking = &self.inst_tracking.values()[inst_tracking_i];
13730	if (self.liveness.isUnused(inst)) try tracking.die(self, inst);	13993	if (self.liveness.isUnused(inst)) try tracking.die(self, inst);
13731	self.getValueIfFree(tracking.short, inst);	13994	self.getValueIfFree(tracking.short, inst);
13732	self.finishAirBookkeeping();
13733	}	13995	}
13734		13996
13735	fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {	13997	fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {
...	@@ -13864,7 +14126,6 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13864,7 +14126,6 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13864	try self.lowerSwitchBr(inst, switch_br, condition);	14126	try self.lowerSwitchBr(inst, switch_br, condition);
13865		14127
13866	// We already took care of pl_op.operand earlier, so there's nothing left to do	14128	// We already took care of pl_op.operand earlier, so there's nothing left to do
13867	self.finishAirBookkeeping();
13868	}	14129	}
13869		14130
13870	fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	14131	fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -13893,7 +14154,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13893,7 +14154,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13893		14154
13894	try self.loops.putNoClobber(self.gpa, inst, .{	14155	try self.loops.putNoClobber(self.gpa, inst, .{
13895	.state = state,	14156	.state = state,
13896	.jmp_target = @intCast(self.mir_instructions.len),	14157	.target = @intCast(self.mir_instructions.len),
13897	});	14158	});
13898	defer assert(self.loops.remove(inst));	14159	defer assert(self.loops.remove(inst));
13899		14160
...	@@ -13903,7 +14164,6 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13903,7 +14164,6 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13903	try self.lowerSwitchBr(inst, switch_br, mat_cond);	14164	try self.lowerSwitchBr(inst, switch_br, mat_cond);
13904		14165
13905	try self.processDeath(inst);	14166	try self.processDeath(inst);
13906	self.finishAirBookkeeping();
13907	}	14167	}
13908		14168
13909	fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {	14169	fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -13945,12 +14205,10 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13945,12 +14205,10 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
13945		14205
13946	// Emit a jump with a relocation. It will be patched up after the block ends.	14206	// Emit a jump with a relocation. It will be patched up after the block ends.
13947	// Leave the jump offset undefined	14207	// Leave the jump offset undefined
13948	_ = try self.asmJmpReloc(loop_data.jmp_target);	14208	_ = try self.asmJmpReloc(loop_data.target);
13949		14209
13950	// Stop tracking block result without forgetting tracking info	14210	// Stop tracking block result without forgetting tracking info
13951	try self.freeValue(block_tracking.short);	14211	try self.freeValue(block_tracking.short);
13952
13953	self.finishAirBookkeeping();
13954	}	14212	}
13955		14213
13956	fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {	14214	fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
...	@@ -14023,8 +14281,6 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14023,8 +14281,6 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
14023		14281
14024	// Stop tracking block result without forgetting tracking info	14282	// Stop tracking block result without forgetting tracking info
14025	try self.freeValue(block_tracking.short);	14283	try self.freeValue(block_tracking.short);
14026
14027	self.finishAirBookkeeping();
14028	}	14284	}
14029		14285
14030	fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {	14286	fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -14036,8 +14292,7 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14036,8 +14292,7 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
14036	.resurrect = false,	14292	.resurrect = false,
14037	.close_scope = true,	14293	.close_scope = true,
14038	});	14294	});
14039	_ = try self.asmJmpReloc(repeat_info.jmp_target);	14295	_ = try self.asmJmpReloc(repeat_info.target);
14040	self.finishAirBookkeeping();
14041	}	14296	}
14042		14297
14043	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	14298	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -14068,9 +14323,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14068,9 +14323,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14068		14323
14069	var outputs_extra_i = extra_i;	14324	var outputs_extra_i = extra_i;
14070	for (outputs) \|output\| {	14325	for (outputs) \|output\| {
14071	const extra_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);	14326	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
14072	const constraint = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	14327	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
14073	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	14328	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
14074	// This equation accounts for the fact that even if we have exactly 4 bytes	14329	// This equation accounts for the fact that even if we have exactly 4 bytes
14075	// for the string, we still use the next u32 for the null terminator.	14330	// for the string, we still use the next u32 for the null terminator.
14076	extra_i += (constraint.len + name.len + (2 + 3)) / 4;	14331	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	@@ -14097,8 +14352,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14097,8 +14352,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14097	const is_early_clobber = constraint[1] == '&';	14352	const is_early_clobber = constraint[1] == '&';
14098	const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];	14353	const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
14099	const arg_mcv: MCValue = arg_mcv: {	14354	const arg_mcv: MCValue = arg_mcv: {
14100	const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "r") or	14355	const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or
14101	mem.eql(u8, rest, "f") or mem.eql(u8, rest, "x"))	14356	std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x"))
14102	registerAlias(	14357	registerAlias(
14103	self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) {	14358	self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) {
14104	'r' => abi.RegisterClass.gp,	14359	'r' => abi.RegisterClass.gp,
...	@@ -14108,20 +14363,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14108,20 +14363,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14108	}) orelse return self.fail("ran out of registers lowering inline asm", .{}),	14363	}) orelse return self.fail("ran out of registers lowering inline asm", .{}),
14109	@intCast(ty.abiSize(zcu)),	14364	@intCast(ty.abiSize(zcu)),
14110	)	14365	)
14111	else if (mem.eql(u8, rest, "m"))	14366	else if (std.mem.eql(u8, rest, "m"))
14112	if (output != .none) null else return self.fail(	14367	if (output != .none) null else return self.fail(
14113	"memory constraint unsupported for asm result: '{s}'",	14368	"memory constraint unsupported for asm result: '{s}'",
14114	.{constraint},	14369	.{constraint},
14115	)	14370	)
14116	else if (mem.eql(u8, rest, "g") or	14371	else if (std.mem.eql(u8, rest, "g") or
14117	mem.eql(u8, rest, "rm") or mem.eql(u8, rest, "mr") or	14372	std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or
14118	mem.eql(u8, rest, "r,m") or mem.eql(u8, rest, "m,r"))	14373	std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r"))
14119	self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse	14374	self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse
14120	if (output != .none)	14375	if (output != .none)
14121	null	14376	null
14122	else	14377	else
14123	return self.fail("ran out of registers lowering inline asm", .{})	14378	return self.fail("ran out of registers lowering inline asm", .{})
14124	else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}"))	14379	else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}"))
14125	parseRegName(rest["{".len .. rest.len - "}".len]) orelse	14380	parseRegName(rest["{".len .. rest.len - "}".len]) orelse
14126	return self.fail("invalid register constraint: '{s}'", .{constraint})	14381	return self.fail("invalid register constraint: '{s}'", .{constraint})
14127	else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {	14382	else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {
...	@@ -14134,7 +14389,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14134,7 +14389,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14134	break :arg_mcv if (arg_maybe_reg) \|reg\| .{ .register = reg } else arg: {	14389	break :arg_mcv if (arg_maybe_reg) \|reg\| .{ .register = reg } else arg: {
14135	const ptr_mcv = try self.resolveInst(output);	14390	const ptr_mcv = try self.resolveInst(output);
14136	switch (ptr_mcv) {	14391	switch (ptr_mcv) {
14137	.immediate => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|	14392	.immediate => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14138	break :arg ptr_mcv.deref(),	14393	break :arg ptr_mcv.deref(),
14139	.register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),	14394	.register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),
14140	else => {},	14395	else => {},
...	@@ -14145,7 +14400,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14145,7 +14400,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14145	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {	14400	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
14146	_ = self.register_manager.lockReg(reg);	14401	_ = self.register_manager.lockReg(reg);
14147	};	14402	};
14148	if (!mem.eql(u8, name, "_"))	14403	if (!std.mem.eql(u8, name, "_"))
14149	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));	14404	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
14150	args.appendAssumeCapacity(arg_mcv);	14405	args.appendAssumeCapacity(arg_mcv);
14151	if (output == .none) result = arg_mcv;	14406	if (output == .none) result = arg_mcv;
...	@@ -14153,17 +14408,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14153,17 +14408,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14153	}	14408	}
14154		14409
14155	for (inputs) \|input\| {	14410	for (inputs) \|input\| {
14156	const input_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);	14411	const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
14157	const constraint = mem.sliceTo(input_bytes, 0);	14412	const constraint = std.mem.sliceTo(input_bytes, 0);
14158	const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);	14413	const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
14159	// This equation accounts for the fact that even if we have exactly 4 bytes	14414	// This equation accounts for the fact that even if we have exactly 4 bytes
14160	// for the string, we still use the next u32 for the null terminator.	14415	// for the string, we still use the next u32 for the null terminator.
14161	extra_i += (constraint.len + name.len + (2 + 3)) / 4;	14416	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
14162		14417
14163	const ty = self.typeOf(input);	14418	const ty = self.typeOf(input);
14164	const input_mcv = try self.resolveInst(input);	14419	const input_mcv = try self.resolveInst(input);
14165	const arg_mcv: MCValue = if (mem.eql(u8, constraint, "r") or	14420	const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or
14166	mem.eql(u8, constraint, "f") or mem.eql(u8, constraint, "x"))	14421	std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x"))
14167	arg: {	14422	arg: {
14168	const rc = switch (constraint[0]) {	14423	const rc = switch (constraint[0]) {
14169	'r' => abi.RegisterClass.gp,	14424	'r' => abi.RegisterClass.gp,
...	@@ -14177,16 +14432,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14177,16 +14432,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14177	const reg = try self.register_manager.allocReg(null, rc);	14432	const reg = try self.register_manager.allocReg(null, rc);
14178	try self.genSetReg(reg, ty, input_mcv, .{});	14433	try self.genSetReg(reg, ty, input_mcv, .{});
14179	break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) };	14434	break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) };
14180	} else if (mem.eql(u8, constraint, "i") or mem.eql(u8, constraint, "n"))	14435	} else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n"))
14181	switch (input_mcv) {	14436	switch (input_mcv) {
14182	.immediate => \|imm\| .{ .immediate = imm },	14437	.immediate => \|imm\| .{ .immediate = imm },
14183	else => return self.fail("immediate operand requires comptime value: '{s}'", .{	14438	else => return self.fail("immediate operand requires comptime value: '{s}'", .{
14184	constraint,	14439	constraint,
14185	}),	14440	}),
14186	}	14441	}
14187	else if (mem.eql(u8, constraint, "m")) arg: {	14442	else if (std.mem.eql(u8, constraint, "m")) arg: {
14188	switch (input_mcv) {	14443	switch (input_mcv) {
14189	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|	14444	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14190	break :arg input_mcv,	14445	break :arg input_mcv,
14191	.indirect, .load_frame => break :arg input_mcv,	14446	.indirect, .load_frame => break :arg input_mcv,
14192	.load_symbol, .load_direct, .load_got, .load_tlv => {},	14447	.load_symbol, .load_direct, .load_got, .load_tlv => {},
...	@@ -14203,22 +14458,22 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14203,22 +14458,22 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14203	};	14458	};
14204	try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{});	14459	try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{});
14205	break :arg .{ .indirect = .{ .reg = addr_reg } };	14460	break :arg .{ .indirect = .{ .reg = addr_reg } };
14206	} else if (mem.eql(u8, constraint, "g") or	14461	} else if (std.mem.eql(u8, constraint, "g") or
14207	mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or	14462	std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or
14208	mem.eql(u8, constraint, "r,m") or mem.eql(u8, constraint, "m,r"))	14463	std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r"))
14209	arg: {	14464	arg: {
14210	switch (input_mcv) {	14465	switch (input_mcv) {
14211	.register, .indirect, .load_frame => break :arg input_mcv,	14466	.register, .indirect, .load_frame => break :arg input_mcv,
14212	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|	14467	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14213	break :arg input_mcv,	14468	break :arg input_mcv,
14214	else => {},	14469	else => {},
14215	}	14470	}
14216	const temp_mcv = try self.allocTempRegOrMem(ty, true);	14471	const temp_mcv = try self.allocTempRegOrMem(ty, true);
14217	try self.genCopy(ty, temp_mcv, input_mcv, .{});	14472	try self.genCopy(ty, temp_mcv, input_mcv, .{});
14218	break :arg temp_mcv;	14473	break :arg temp_mcv;
14219	} else if (mem.eql(u8, constraint, "X"))	14474	} else if (std.mem.eql(u8, constraint, "X"))
14220	input_mcv	14475	input_mcv
14221	else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: {	14476	else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: {
14222	const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse	14477	const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
14223	return self.fail("invalid register constraint: '{s}'", .{constraint});	14478	return self.fail("invalid register constraint: '{s}'", .{constraint});
14224	try self.register_manager.getReg(reg, null);	14479	try self.register_manager.getReg(reg, null);
...	@@ -14233,7 +14488,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14233,7 +14488,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14233	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {	14488	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
14234	_ = self.register_manager.lockReg(reg);	14489	_ = self.register_manager.lockReg(reg);
14235	};	14490	};
14236	if (!mem.eql(u8, name, "_"))	14491	if (!std.mem.eql(u8, name, "_"))
14237	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));	14492	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
14238	args.appendAssumeCapacity(arg_mcv);	14493	args.appendAssumeCapacity(arg_mcv);
14239	}	14494	}
...	@@ -14241,7 +14496,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14241,7 +14496,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14241	{	14496	{
14242	var clobber_i: u32 = 0;	14497	var clobber_i: u32 = 0;
14243	while (clobber_i < clobbers_len) : (clobber_i += 1) {	14498	while (clobber_i < clobbers_len) : (clobber_i += 1) {
14244	const clobber = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	14499	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
14245	// This equation accounts for the fact that even if we have exactly 4 bytes	14500	// This equation accounts for the fact that even if we have exactly 4 bytes
14246	// for the string, we still use the next u32 for the null terminator.	14501	// for the string, we still use the next u32 for the null terminator.
14247	extra_i += clobber.len / 4 + 1;	14502	extra_i += clobber.len / 4 + 1;
...	@@ -14294,20 +14549,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14294,20 +14549,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14294	labels.deinit(self.gpa);	14549	labels.deinit(self.gpa);
14295	}	14550	}
14296		14551
14297	const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];	14552	const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
14298	var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");	14553	var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;");
14299	next_line: while (line_it.next()) \|line\| {	14554	next_line: while (line_it.next()) \|line\| {
14300	var mnem_it = mem.tokenizeAny(u8, line, " \t");	14555	var mnem_it = std.mem.tokenizeAny(u8, line, " \t");
14301	var prefix: Instruction.Prefix = .none;	14556	var prefix: encoder.Instruction.Prefix = .none;
14302	const mnem_str = while (mnem_it.next()) \|mnem_str\| {	14557	const mnem_str = while (mnem_it.next()) \|mnem_str\| {
14303	if (mnem_str[0] == '#') continue :next_line;	14558	if (mnem_str[0] == '#') continue :next_line;
14304	if (mem.startsWith(u8, mnem_str, "//")) continue :next_line;	14559	if (std.mem.startsWith(u8, mnem_str, "//")) continue :next_line;
14305	if (std.meta.stringToEnum(Instruction.Prefix, mnem_str)) \|pre\| {	14560	if (std.meta.stringToEnum(encoder.Instruction.Prefix, mnem_str)) \|pre\| {
14306	if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});	14561	if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});
14307	prefix = pre;	14562	prefix = pre;
14308	continue;	14563	continue;
14309	}	14564	}
14310	if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;	14565	if (!std.mem.endsWith(u8, mnem_str, ":")) break mnem_str;
14311	const label_name = mnem_str[0 .. mnem_str.len - ":".len];	14566	const label_name = mnem_str[0 .. mnem_str.len - ":".len];
14312	if (!Label.isValid(.definition, label_name))	14567	if (!Label.isValid(.definition, label_name))
14313	return self.fail("invalid label: '{s}'", .{label_name});	14568	return self.fail("invalid label: '{s}'", .{label_name});
...	@@ -14332,21 +14587,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14332,21 +14587,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14332		14587
14333	var mnem_size: ?Memory.Size = if (prefix == .directive)	14588	var mnem_size: ?Memory.Size = if (prefix == .directive)
14334	null	14589	null
14335	else if (mem.endsWith(u8, mnem_str, "b"))	14590	else if (std.mem.endsWith(u8, mnem_str, "b"))
14336	.byte	14591	.byte
14337	else if (mem.endsWith(u8, mnem_str, "w"))	14592	else if (std.mem.endsWith(u8, mnem_str, "w"))
14338	.word	14593	.word
14339	else if (mem.endsWith(u8, mnem_str, "l"))	14594	else if (std.mem.endsWith(u8, mnem_str, "l"))
14340	.dword	14595	.dword
14341	else if (mem.endsWith(u8, mnem_str, "q") and	14596	else if (std.mem.endsWith(u8, mnem_str, "q") and
14342	(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))	14597	(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !std.mem.endsWith(u8, mnem_str, "dq")))
14343	.qword	14598	.qword
14344	else if (mem.endsWith(u8, mnem_str, "t"))	14599	else if (std.mem.endsWith(u8, mnem_str, "t"))
14345	.tbyte	14600	.tbyte
14346	else	14601	else
14347	null;	14602	null;
14348	const mnem_tag = while (true) break std.meta.stringToEnum(	14603	const mnem_tag = while (true) break std.meta.stringToEnum(
14349	Instruction.Mnemonic,	14604	encoder.Instruction.Mnemonic,
14350	mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],	14605	mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
14351	) orelse if (mnem_size) \|_\| {	14606	) orelse if (mnem_size) \|_\| {
14352	mnem_size = null;	14607	mnem_size = null;
...	@@ -14367,18 +14622,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14367,18 +14622,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14367	.{ ._, .pseudo }	14622	.{ ._, .pseudo }
14368	else for (std.enums.values(Mir.Inst.Fixes)) \|fixes\| {	14623	else for (std.enums.values(Mir.Inst.Fixes)) \|fixes\| {
14369	const fixes_name = @tagName(fixes);	14624	const fixes_name = @tagName(fixes);
14370	const space_i = mem.indexOfScalar(u8, fixes_name, ' ');	14625	const space_i = std.mem.indexOfScalar(u8, fixes_name, ' ');
14371	const fixes_prefix = if (space_i) \|i\|	14626	const fixes_prefix = if (space_i) \|i\|
14372	std.meta.stringToEnum(Instruction.Prefix, fixes_name[0..i]).?	14627	std.meta.stringToEnum(encoder.Instruction.Prefix, fixes_name[0..i]).?
14373	else	14628	else
14374	.none;	14629	.none;
14375	if (fixes_prefix != prefix) continue;	14630	if (fixes_prefix != prefix) continue;
14376	const pattern = fixes_name[if (space_i) \|i\| i + " ".len else 0..];	14631	const pattern = fixes_name[if (space_i) \|i\| i + " ".len else 0..];
14377	const wildcard_i = mem.indexOfScalar(u8, pattern, '_').?;	14632	const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
14378	const mnem_prefix = pattern[0..wildcard_i];	14633	const mnem_prefix = pattern[0..wildcard_i];
14379	const mnem_suffix = pattern[wildcard_i + "_".len ..];	14634	const mnem_suffix = pattern[wildcard_i + "_".len ..];
14380	if (!mem.startsWith(u8, mnem_name, mnem_prefix)) continue;	14635	if (!std.mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
14381	if (!mem.endsWith(u8, mnem_name, mnem_suffix)) continue;	14636	if (!std.mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
14382	break .{ fixes, std.meta.stringToEnum(	14637	break .{ fixes, std.meta.stringToEnum(
14383	Mir.Inst.Tag,	14638	Mir.Inst.Tag,
14384	mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],	14639	mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],
...	@@ -14400,21 +14655,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14400,21 +14655,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14400	var ops: [4]Operand = .{.none} ** 4;	14655	var ops: [4]Operand = .{.none} ** 4;
14401		14656
14402	var last_op = false;	14657	var last_op = false;
14403	var op_it = mem.splitScalar(u8, mnem_it.rest(), ',');	14658	var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ',');
14404	next_op: for (&ops) \|*op\| {	14659	next_op: for (&ops) \|*op\| {
14405	const op_str = while (!last_op) {	14660	const op_str = while (!last_op) {
14406	const full_str = op_it.next() orelse break :next_op;	14661	const full_str = op_it.next() orelse break :next_op;
14407	const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse	14662	const code_str = if (std.mem.indexOfScalar(u8, full_str, '#') orelse
14408	mem.indexOf(u8, full_str, "//")) \|comment\|	14663	std.mem.indexOf(u8, full_str, "//")) \|comment\|
14409	code: {	14664	code: {
14410	last_op = true;	14665	last_op = true;
14411	break :code full_str[0..comment];	14666	break :code full_str[0..comment];
14412	} else full_str;	14667	} else full_str;
14413	const trim_str = mem.trim(u8, code_str, " \t*");	14668	const trim_str = std.mem.trim(u8, code_str, " \t*");
14414	if (trim_str.len > 0) break trim_str;	14669	if (trim_str.len > 0) break trim_str;
14415	} else break;	14670	} else break;
14416	if (mem.startsWith(u8, op_str, "%%")) {	14671	if (std.mem.startsWith(u8, op_str, "%%")) {
14417	const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');	14672	const colon = std.mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
14418	const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse	14673	const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
14419	return self.fail("invalid register: '{s}'", .{op_str});	14674	return self.fail("invalid register: '{s}'", .{op_str});
14420	if (colon) \|colon_pos\| {	14675	if (colon) \|colon_pos\| {
...	@@ -14432,8 +14687,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14432,8 +14687,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14432	return self.fail("invalid register size: '{s}'", .{op_str});	14687	return self.fail("invalid register size: '{s}'", .{op_str});
14433	op.* = .{ .reg = reg };	14688	op.* = .{ .reg = reg };
14434	}	14689	}
14435	} else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {	14690	} else if (std.mem.startsWith(u8, op_str, "%[") and std.mem.endsWith(u8, op_str, "]")) {
14436	const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':');	14691	const colon = std.mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
14437	const modifier = if (colon) \|colon_pos\|	14692	const modifier = if (colon) \|colon_pos\|
14438	op_str[colon_pos + ":".len .. op_str.len - "]".len]	14693	op_str[colon_pos + ":".len .. op_str.len - "]".len]
14439	else	14694	else
...	@@ -14442,15 +14697,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14442,15 +14697,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14442	arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse	14697	arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse
14443	return self.fail("no matching constraint: '{s}'", .{op_str})	14698	return self.fail("no matching constraint: '{s}'", .{op_str})
14444	]) {	14699	]) {
14445	.immediate => \|imm\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "c"))	14700	.immediate => \|imm\| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "c"))
14446	.{ .imm = Immediate.u(imm) }	14701	.{ .imm = .u(imm) }
14447	else	14702	else
14448	return self.fail("invalid modifier: '{s}'", .{modifier}),	14703	return self.fail("invalid modifier: '{s}'", .{modifier}),
14449	.register => \|reg\| if (mem.eql(u8, modifier, ""))	14704	.register => \|reg\| if (std.mem.eql(u8, modifier, ""))
14450	.{ .reg = reg }	14705	.{ .reg = reg }
14451	else	14706	else
14452	return self.fail("invalid modifier: '{s}'", .{modifier}),	14707	return self.fail("invalid modifier: '{s}'", .{modifier}),
14453	.memory => \|addr\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "P"))	14708	.memory => \|addr\| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "P"))
14454	.{ .mem = .{	14709	.{ .mem = .{
14455	.base = .{ .reg = .ds },	14710	.base = .{ .reg = .ds },
14456	.mod = .{ .rm = .{	14711	.mod = .{ .rm = .{
...	@@ -14461,7 +14716,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14461,7 +14716,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14461	} }	14716	} }
14462	else	14717	else
14463	return self.fail("invalid modifier: '{s}'", .{modifier}),	14718	return self.fail("invalid modifier: '{s}'", .{modifier}),
14464	.indirect => \|reg_off\| if (mem.eql(u8, modifier, ""))	14719	.indirect => \|reg_off\| if (std.mem.eql(u8, modifier, ""))
14465	.{ .mem = .{	14720	.{ .mem = .{
14466	.base = .{ .reg = reg_off.reg },	14721	.base = .{ .reg = reg_off.reg },
14467	.mod = .{ .rm = .{	14722	.mod = .{ .rm = .{
...	@@ -14472,7 +14727,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14472,7 +14727,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14472	} }	14727	} }
14473	else	14728	else
14474	return self.fail("invalid modifier: '{s}'", .{modifier}),	14729	return self.fail("invalid modifier: '{s}'", .{modifier}),
14475	.load_frame => \|frame_addr\| if (mem.eql(u8, modifier, ""))	14730	.load_frame => \|frame_addr\| if (std.mem.eql(u8, modifier, ""))
14476	.{ .mem = .{	14731	.{ .mem = .{
14477	.base = .{ .frame = frame_addr.index },	14732	.base = .{ .frame = frame_addr.index },
14478	.mod = .{ .rm = .{	14733	.mod = .{ .rm = .{
...	@@ -14483,42 +14738,42 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14483,42 +14738,42 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14483	} }	14738	} }
14484	else	14739	else
14485	return self.fail("invalid modifier: '{s}'", .{modifier}),	14740	return self.fail("invalid modifier: '{s}'", .{modifier}),
14486	.lea_got => \|sym_index\| if (mem.eql(u8, modifier, "P"))	14741	.lea_got => \|sym_index\| if (std.mem.eql(u8, modifier, "P"))
14487	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }	14742	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }
14488	else	14743	else
14489	return self.fail("invalid modifier: '{s}'", .{modifier}),	14744	return self.fail("invalid modifier: '{s}'", .{modifier}),
14490	.lea_symbol => \|sym_off\| if (mem.eql(u8, modifier, "P"))	14745	.lea_symbol => \|sym_off\| if (std.mem.eql(u8, modifier, "P"))
14491	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) }	14746	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) }
14492	else	14747	else
14493	return self.fail("invalid modifier: '{s}'", .{modifier}),	14748	return self.fail("invalid modifier: '{s}'", .{modifier}),
14494	else => return self.fail("invalid constraint: '{s}'", .{op_str}),	14749	else => return self.fail("invalid constraint: '{s}'", .{op_str}),
14495	};	14750	};
14496	} else if (mem.startsWith(u8, op_str, "$")) {	14751	} else if (std.mem.startsWith(u8, op_str, "$")) {
14497	if (std.fmt.parseInt(i32, op_str["$".len..], 0)) \|s\| {	14752	if (std.fmt.parseInt(i32, op_str["$".len..], 0)) \|s\| {
14498	if (mnem_size) \|size\| {	14753	if (mnem_size) \|size\| {
14499	const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));	14754	const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));
14500	if ((if (s < 0) ~s else s) > max)	14755	if ((if (s < 0) ~s else s) > max)
14501	return self.fail("invalid immediate size: '{s}'", .{op_str});	14756	return self.fail("invalid immediate size: '{s}'", .{op_str});
14502	}	14757	}
14503	op.* = .{ .imm = Immediate.s(s) };	14758	op.* = .{ .imm = .s(s) };
14504	} else \|_\| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) \|u\| {	14759	} else \|_\| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) \|u\| {
14505	if (mnem_size) \|size\| {	14760	if (mnem_size) \|size\| {
14506	const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - size.bitSize());	14761	const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize());
14507	if (u > max)	14762	if (u > max)
14508	return self.fail("invalid immediate size: '{s}'", .{op_str});	14763	return self.fail("invalid immediate size: '{s}'", .{op_str});
14509	}	14764	}
14510	op.* = .{ .imm = Immediate.u(u) };	14765	op.* = .{ .imm = .u(u) };
14511	} else \|_\| return self.fail("invalid immediate: '{s}'", .{op_str});	14766	} else \|_\| return self.fail("invalid immediate: '{s}'", .{op_str});
14512	} else if (mem.endsWith(u8, op_str, ")")) {	14767	} else if (std.mem.endsWith(u8, op_str, ")")) {
14513	const open = mem.indexOfScalar(u8, op_str, '(') orelse	14768	const open = std.mem.indexOfScalar(u8, op_str, '(') orelse
14514	return self.fail("invalid operand: '{s}'", .{op_str});	14769	return self.fail("invalid operand: '{s}'", .{op_str});
14515	var sib_it = mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');	14770	var sib_it = std.mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
14516	const base_str = sib_it.next() orelse	14771	const base_str = sib_it.next() orelse
14517	return self.fail("invalid memory operand: '{s}'", .{op_str});	14772	return self.fail("invalid memory operand: '{s}'", .{op_str});
14518	if (base_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))	14773	if (base_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
14519	return self.fail("invalid memory operand: '{s}'", .{op_str});	14774	return self.fail("invalid memory operand: '{s}'", .{op_str});
14520	const index_str = sib_it.next() orelse "";	14775	const index_str = sib_it.next() orelse "";
14521	if (index_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))	14776	if (index_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
14522	return self.fail("invalid memory operand: '{s}'", .{op_str});	14777	return self.fail("invalid memory operand: '{s}'", .{op_str});
14523	const scale_str = sib_it.next() orelse "";	14778	const scale_str = sib_it.next() orelse "";
14524	if (index_str.len == 0 and scale_str.len > 0)	14779	if (index_str.len == 0 and scale_str.len > 0)
...	@@ -14550,10 +14805,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14550,10 +14805,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14550	else	14805	else
14551	.none,	14806	.none,
14552	.scale = scale,	14807	.scale = scale,
14553	.disp = if (mem.startsWith(u8, op_str[0..open], "%[") and	14808	.disp = if (std.mem.startsWith(u8, op_str[0..open], "%[") and
14554	mem.endsWith(u8, op_str[0..open], "]"))	14809	std.mem.endsWith(u8, op_str[0..open], "]"))
14555	disp: {	14810	disp: {
14556	const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');	14811	const colon = std.mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
14557	const modifier = if (colon) \|colon_pos\|	14812	const modifier = if (colon) \|colon_pos\|
14558	op_str[colon_pos + ":".len .. open - "]".len]	14813	op_str[colon_pos + ":".len .. open - "]".len]
14559	else	14814	else
...	@@ -14562,9 +14817,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14562,9 +14817,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14562	arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse	14817	arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
14563	return self.fail("no matching constraint: '{s}'", .{op_str})	14818	return self.fail("no matching constraint: '{s}'", .{op_str})
14564	]) {	14819	]) {
14565	.immediate => \|imm\| if (mem.eql(u8, modifier, "") or	14820	.immediate => \|imm\| if (std.mem.eql(u8, modifier, "") or
14566	mem.eql(u8, modifier, "c"))	14821	std.mem.eql(u8, modifier, "c"))
14567	math.cast(i32, @as(i64, @bitCast(imm))) orelse	14822	std.math.cast(i32, @as(i64, @bitCast(imm))) orelse
14568	return self.fail("invalid displacement: '{s}'", .{op_str})	14823	return self.fail("invalid displacement: '{s}'", .{op_str})
14569	else	14824	else
14570	return self.fail("invalid modifier: '{s}'", .{modifier}),	14825	return self.fail("invalid modifier: '{s}'", .{modifier}),
...	@@ -14730,10 +14985,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14730,10 +14985,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14730	return self.fail("undefined label: '{s}'", .{label.key_ptr.*});	14985	return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
14731		14986
14732	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {	14987	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {
14733	const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);	14988	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
14734	const constraint =	14989	const constraint =
14735	mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);	14990	std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
14736	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	14991	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
14737	// This equation accounts for the fact that even if we have exactly 4 bytes	14992	// This equation accounts for the fact that even if we have exactly 4 bytes
14738	// for the string, we still use the next u32 for the null terminator.	14993	// for the string, we still use the next u32 for the null terminator.
14739	outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;	14994	outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	@@ -14777,7 +15032,10 @@ const MoveStrategy = union(enum) {	...	@@ -14777,7 +15032,10 @@ const MoveStrategy = union(enum) {
14777		15032
14778	pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {	15033	pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {
14779	switch (strat) {	15034	switch (strat) {
14780	.move => \|tag\| try self.asmRegisterMemory(tag, dst_reg, src_mem),	15035	.move => \|tag\| try self.asmRegisterMemory(tag, switch (tag[1]) {
		15036	else => dst_reg,
		15037	.lea => if (dst_reg.bitSize() >= 32) dst_reg else dst_reg.to32(),
		15038	}, src_mem),
14781	.x87_load_store => {	15039	.x87_load_store => {
14782	try self.asmMemory(.{ .f_, .ld }, src_mem);	15040	try self.asmMemory(.{ .f_, .ld }, src_mem);
14783	assert(dst_reg != .st7);	15041	assert(dst_reg != .st7);
...	@@ -14787,14 +15045,14 @@ const MoveStrategy = union(enum) {	...	@@ -14787,14 +15045,14 @@ const MoveStrategy = union(enum) {
14787	ie.insert,	15045	ie.insert,
14788	dst_reg,	15046	dst_reg,
14789	src_mem,	15047	src_mem,
14790	Immediate.u(0),	15048	.u(0),
14791	),	15049	),
14792	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(	15050	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
14793	ie.insert,	15051	ie.insert,
14794	dst_reg,	15052	dst_reg,
14795	dst_reg,	15053	dst_reg,
14796	src_mem,	15054	src_mem,
14797	Immediate.u(0),	15055	.u(0),
14798	),	15056	),
14799	}	15057	}
14800	}	15058	}
...	@@ -14809,7 +15067,7 @@ const MoveStrategy = union(enum) {	...	@@ -14809,7 +15067,7 @@ const MoveStrategy = union(enum) {
14809	ie.extract,	15067	ie.extract,
14810	dst_mem,	15068	dst_mem,
14811	src_reg,	15069	src_reg,
14812	Immediate.u(0),	15070	.u(0),
14813	),	15071	),
14814	}	15072	}
14815	}	15073	}
...	@@ -14823,7 +15081,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14823,7 +15081,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14823	.mmx => {},	15081	.mmx => {},
14824	.sse => switch (ty.zigTypeTag(zcu)) {	15082	.sse => switch (ty.zigTypeTag(zcu)) {
14825	else => {	15083	else => {
14826	const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);	15084	const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
14827	assert(std.mem.indexOfNone(abi.Class, classes, &.{	15085	assert(std.mem.indexOfNone(abi.Class, classes, &.{
14828	.integer, .sse, .sseup, .memory, .float, .float_combine,	15086	.integer, .sse, .sseup, .memory, .float, .float_combine,
14829	}) == null);	15087	}) == null);
...	@@ -15135,7 +15393,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy	...	@@ -15135,7 +15393,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
15135	),	15393	),
15136	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {	15394	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
15137	switch (dst_mcv) {	15395	switch (dst_mcv) {
15138	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|	15396	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
15139	return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts),	15397	return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts),
15140	.load_symbol, .load_direct, .load_got, .load_tlv => {},	15398	.load_symbol, .load_direct, .load_got, .load_tlv => {},
15141	else => unreachable,	15399	else => unreachable,
...	@@ -15179,17 +15437,17 @@ fn genSetReg(	...	@@ -15179,17 +15437,17 @@ fn genSetReg(
15179	=> unreachable,	15437	=> unreachable,
15180	.undef => if (opts.safety) switch (dst_reg.class()) {	15438	.undef => if (opts.safety) switch (dst_reg.class()) {
15181	.general_purpose => switch (abi_size) {	15439	.general_purpose => switch (abi_size) {
15182	1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), Immediate.u(0xAA)),	15440	1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xAA)),
15183	2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), Immediate.u(0xAAAA)),	15441	2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xAAAA)),
15184	3...4 => try self.asmRegisterImmediate(	15442	3...4 => try self.asmRegisterImmediate(
15185	.{ ._, .mov },	15443	.{ ._, .mov },
15186	dst_reg.to32(),	15444	dst_reg.to32(),
15187	Immediate.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),	15445	.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
15188	),	15446	),
15189	5...8 => try self.asmRegisterImmediate(	15447	5...8 => try self.asmRegisterImmediate(
15190	.{ ._, .mov },	15448	.{ ._, .mov },
15191	dst_reg.to64(),	15449	dst_reg.to64(),
15192	Immediate.u(0xAAAAAAAAAAAAAAAA),	15450	.u(0xAAAAAAAAAAAAAAAA),
15193	),	15451	),
15194	else => unreachable,	15452	else => unreachable,
15195	},	15453	},
...	@@ -15203,20 +15461,20 @@ fn genSetReg(	...	@@ -15203,20 +15461,20 @@ fn genSetReg(
15203	// register is the fastest way to zero a register.	15461	// register is the fastest way to zero a register.
15204	try self.spillEflagsIfOccupied();	15462	try self.spillEflagsIfOccupied();
15205	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());	15463	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
15206	} else if (abi_size > 4 and math.cast(u32, imm) != null) {	15464	} else if (abi_size > 4 and std.math.cast(u32, imm) != null) {
15207	// 32-bit moves zero-extend to 64-bit.	15465	// 32-bit moves zero-extend to 64-bit.
15208	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));	15466	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(imm));
15209	} else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) {	15467	} else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) {
15210	try self.asmRegisterImmediate(	15468	try self.asmRegisterImmediate(
15211	.{ ._, .mov },	15469	.{ ._, .mov },
15212	registerAlias(dst_reg, abi_size),	15470	registerAlias(dst_reg, abi_size),
15213	Immediate.s(@intCast(@as(i64, @bitCast(imm)))),	15471	.s(@intCast(@as(i64, @bitCast(imm)))),
15214	);	15472	);
15215	} else {	15473	} else {
15216	try self.asmRegisterImmediate(	15474	try self.asmRegisterImmediate(
15217	.{ ._, .mov },	15475	.{ ._, .mov },
15218	registerAlias(dst_reg, abi_size),	15476	registerAlias(dst_reg, abi_size),
15219	Immediate.u(imm),	15477	.u(imm),
15220	);	15478	);
15221	}	15479	}
15222	},	15480	},
...	@@ -15325,15 +15583,15 @@ fn genSetReg(	...	@@ -15325,15 +15583,15 @@ fn genSetReg(
15325	.load_frame => \|frame_addr\| try self.moveStrategy(	15583	.load_frame => \|frame_addr\| try self.moveStrategy(
15326	ty,	15584	ty,
15327	dst_reg.class(),	15585	dst_reg.class(),
15328	self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(	15586	self.getFrameAddrAlignment(frame_addr).compare(.gte, InternPool.Alignment.fromLog2Units(
15329	math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),	15587	std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
15330	)),	15588	)),
15331	),	15589	),
15332	.lea_frame => .{ .move = .{ ._, .lea } },	15590	.lea_frame => .{ .move = .{ ._, .lea } },
15333	else => unreachable,	15591	else => unreachable,
15334	}).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {	15592	}).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {
15335	.register_offset, .indirect => \|reg_off\| .{	15593	.register_offset, .indirect => \|reg_off\| .{
15336	.base = .{ .reg = reg_off.reg },	15594	.base = .{ .reg = reg_off.reg.to64() },
15337	.mod = .{ .rm = .{	15595	.mod = .{ .rm = .{
15338	.size = self.memSize(ty),	15596	.size = self.memSize(ty),
15339	.disp = reg_off.off,	15597	.disp = reg_off.off,
...	@@ -15350,7 +15608,7 @@ fn genSetReg(	...	@@ -15350,7 +15608,7 @@ fn genSetReg(
15350	}),	15608	}),
15351	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {	15609	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
15352	switch (src_mcv) {	15610	switch (src_mcv) {
15353	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|	15611	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
15354	return (try self.moveStrategy(	15612	return (try self.moveStrategy(
15355	ty,	15613	ty,
15356	dst_reg.class(),	15614	dst_reg.class(),
...	@@ -15400,14 +15658,10 @@ fn genSetReg(	...	@@ -15400,14 +15658,10 @@ fn genSetReg(
15400	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	15658	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
15401	defer self.register_manager.unlockReg(addr_lock);	15659	defer self.register_manager.unlockReg(addr_lock);
15402		15660
15403	try (try self.moveStrategy(ty, dst_reg.class(), false)).read(	15661	try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, registerAlias(dst_reg, abi_size), .{
15404	self,	15662	.base = .{ .reg = addr_reg.to64() },
15405	registerAlias(dst_reg, abi_size),	15663	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
15406	.{	15664	});
15407	.base = .{ .reg = addr_reg },
15408	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
15409	},
15410	);
15411	},	15665	},
15412	.lea_symbol => \|sym_off\| switch (self.bin_file.tag) {	15666	.lea_symbol => \|sym_off\| switch (self.bin_file.tag) {
15413	.elf, .macho => try self.asmRegisterMemory(	15667	.elf, .macho => try self.asmRegisterMemory(
...	@@ -15478,12 +15732,12 @@ fn genSetMem(	...	@@ -15478,12 +15732,12 @@ fn genSetMem(
15478	),	15732	),
15479	.immediate => \|imm\| switch (abi_size) {	15733	.immediate => \|imm\| switch (abi_size) {
15480	1, 2, 4 => {	15734	1, 2, 4 => {
15481	const immediate = switch (if (ty.isAbiInt(zcu))	15735	const immediate: Immediate = switch (if (ty.isAbiInt(zcu))
15482	ty.intInfo(zcu).signedness	15736	ty.intInfo(zcu).signedness
15483	else	15737	else
15484	.unsigned) {	15738	.unsigned) {
15485	.signed => Immediate.s(@truncate(@as(i64, @bitCast(imm)))),	15739	.signed => .s(@truncate(@as(i64, @bitCast(imm)))),
15486	.unsigned => Immediate.u(@as(u32, @intCast(imm))),	15740	.unsigned => .u(@as(u32, @intCast(imm))),
15487	};	15741	};
15488	try self.asmMemoryImmediate(	15742	try self.asmMemoryImmediate(
15489	.{ ._, .mov },	15743	.{ ._, .mov },
...	@@ -15495,14 +15749,14 @@ fn genSetMem(	...	@@ -15495,14 +15749,14 @@ fn genSetMem(
15495	);	15749	);
15496	},	15750	},
15497	3, 5...7 => unreachable,	15751	3, 5...7 => unreachable,
15498	else => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {	15752	else => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {
15499	try self.asmMemoryImmediate(	15753	try self.asmMemoryImmediate(
15500	.{ ._, .mov },	15754	.{ ._, .mov },
15501	.{ .base = base, .mod = .{ .rm = .{	15755	.{ .base = base, .mod = .{ .rm = .{
15502	.size = Memory.Size.fromSize(abi_size),	15756	.size = Memory.Size.fromSize(abi_size),
15503	.disp = disp,	15757	.disp = disp,
15504	} } },	15758	} } },
15505	Immediate.s(small),	15759	.s(small),
15506	);	15760	);
15507	} else {	15761	} else {
15508	var offset: i32 = 0;	15762	var offset: i32 = 0;
...	@@ -15512,10 +15766,10 @@ fn genSetMem(	...	@@ -15512,10 +15766,10 @@ fn genSetMem(
15512	.size = .dword,	15766	.size = .dword,
15513	.disp = disp + offset,	15767	.disp = disp + offset,
15514	} } },	15768	} } },
15515	if (ty.isSignedInt(zcu)) Immediate.s(	15769	if (ty.isSignedInt(zcu)) .s(
15516	@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),	15770	@truncate(@as(i64, @bitCast(imm)) >> (std.math.cast(u6, offset * 8) orelse 63)),
15517	) else Immediate.u(	15771	) else .u(
15518	@as(u32, @truncate(if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),	15772	@as(u32, @truncate(if (std.math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),
15519	),	15773	),
15520	);	15774	);
15521	},	15775	},
...	@@ -15542,7 +15796,9 @@ fn genSetMem(	...	@@ -15542,7 +15796,9 @@ fn genSetMem(
15542	.general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),	15796	.general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),
15543	.mmx, .sse => abi_size,	15797	.mmx, .sse => abi_size,
15544	});	15798	});
15545	const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size));	15799	const src_align = InternPool.Alignment.fromNonzeroByteUnits(
		15800	std.math.ceilPowerOfTwoAssert(u32, src_size),
		15801	);
15546	if (src_size > mem_size) {	15802	if (src_size > mem_size) {
15547	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{	15803	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
15548	.size = src_size,	15804	.size = src_size,
...	@@ -15755,7 +16011,7 @@ fn genLazySymbolRef(	...	@@ -15755,7 +16011,7 @@ fn genLazySymbolRef(
15755	.base = .{ .reloc = sym_index },	16011	.base = .{ .reloc = sym_index },
15756	.mod = .{ .rm = .{ .size = .qword } },	16012	.mod = .{ .rm = .{ .size = .qword } },
15757	}),	16013	}),
15758	.call => try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index })),	16014	.call => try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index })),
15759	else => unreachable,	16015	else => unreachable,
15760	}	16016	}
15761	} else if (self.bin_file.cast(.plan9)) \|p9_file\| {	16017	} else if (self.bin_file.cast(.plan9)) \|p9_file\| {
...	@@ -15861,7 +16117,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15861,7 +16117,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
15861	const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and	16117	const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and
15862	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {	16118	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
15863	const dst_mcv = try self.allocRegOrMem(inst, true);	16119	const dst_mcv = try self.allocRegOrMem(inst, true);
15864	try self.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {	16120	try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
15865	.lt => dst_ty,	16121	.lt => dst_ty,
15866	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,	16122	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
15867	.gt => src_ty,	16123	.gt => src_ty,
...	@@ -15878,7 +16134,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15878,7 +16134,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
15878	const bit_size = dst_ty.bitSize(zcu);	16134	const bit_size = dst_ty.bitSize(zcu);
15879	if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;	16135	if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
15880		16136
15881	const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;	16137	const dst_limbs_len = std.math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
15882	const high_mcv: MCValue = switch (dst_mcv) {	16138	const high_mcv: MCValue = switch (dst_mcv) {
15883	.register => \|dst_reg\| .{ .register = dst_reg },	16139	.register => \|dst_reg\| .{ .register = dst_reg },
15884	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },	16140	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
...	@@ -15940,7 +16196,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15940,7 +16196,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
15940	const src_bits: u32 = @intCast(src_ty.bitSize(zcu));	16196	const src_bits: u32 = @intCast(src_ty.bitSize(zcu));
15941	const src_signedness =	16197	const src_signedness =
15942	if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;	16198	if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;
15943	const src_size = math.divCeil(u32, @max(switch (src_signedness) {	16199	const src_size = std.math.divCeil(u32, @max(switch (src_signedness) {
15944	.signed => src_bits,	16200	.signed => src_bits,
15945	.unsigned => src_bits + 1,	16201	.unsigned => src_bits + 1,
15946	}, 32), 8) catch unreachable;	16202	}, 32), 8) catch unreachable;
...	@@ -16017,7 +16273,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16017,7 +16273,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16017	const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu));	16273	const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu));
16018	const dst_signedness =	16274	const dst_signedness =
16019	if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned;	16275	if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned;
16020	const dst_size = math.divCeil(u32, @max(switch (dst_signedness) {	16276	const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) {
16021	.signed => dst_bits,	16277	.signed => dst_bits,
16022	.unsigned => dst_bits + 1,	16278	.unsigned => dst_bits + 1,
16023	}, 32), 8) catch unreachable;	16279	}, 32), 8) catch unreachable;
...	@@ -16593,13 +16849,17 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16593,13 +16849,17 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16593	const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx });	16849	const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx });
16594	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);	16850	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
16595		16851
16596	const dst_ptr = try self.resolveInst(bin_op.lhs);	16852	const dst = try self.resolveInst(bin_op.lhs);
16597	const dst_ptr_ty = self.typeOf(bin_op.lhs);	16853	const dst_ty = self.typeOf(bin_op.lhs);
16598	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {	16854	const dst_locks: [2]?RegisterLock = switch (dst) {
16599	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	16855	.register => \|dst_reg\| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
16600	else => null,	16856	.register_pair => \|dst_regs\| .{
		16857	self.register_manager.lockRegAssumeUnused(dst_regs[0]),
		16858	self.register_manager.lockRegAssumeUnused(dst_regs[1]),
		16859	},
		16860	else => .{ null, null },
16601	};	16861	};
16602	defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	16862	for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
16603		16863
16604	const src_val = try self.resolveInst(bin_op.rhs);	16864	const src_val = try self.resolveInst(bin_op.rhs);
16605	const elem_ty = self.typeOf(bin_op.rhs);	16865	const elem_ty = self.typeOf(bin_op.rhs);
...	@@ -16612,16 +16872,20 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16612,16 +16872,20 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16612	const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));	16872	const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));
16613		16873
16614	if (elem_abi_size == 1) {	16874	if (elem_abi_size == 1) {
16615	const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {	16875	const dst_ptr: MCValue = switch (dst_ty.ptrSize(zcu)) {
16616	// TODO: this only handles slices stored in the stack	16876	.slice => switch (dst) {
16617	.slice => dst_ptr,	16877	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
16618	.one => dst_ptr,	16878	else => dst,
		16879	},
		16880	.one => dst,
16619	.c, .many => unreachable,	16881	.c, .many => unreachable,
16620	};	16882	};
16621	const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {	16883	const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
16622	// TODO: this only handles slices stored in the stack	16884	.slice => switch (dst) {
16623	.slice => dst_ptr.address().offset(8).deref(),	16885	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
16624	.one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) },	16886	else => dst.address().offset(8).deref(),
		16887	},
		16888	.one => .{ .immediate = dst_ty.childType(zcu).arrayLen(zcu) },
16625	.c, .many => unreachable,	16889	.c, .many => unreachable,
16626	};	16890	};
16627	const len_lock: ?RegisterLock = switch (len) {	16891	const len_lock: ?RegisterLock = switch (len) {
...	@@ -16630,20 +16894,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16630,20 +16894,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16630	};	16894	};
16631	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);	16895	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
16632		16896
16633	try self.genInlineMemset(ptr, src_val, len, .{ .safety = safety });	16897	try self.genInlineMemset(dst_ptr, src_val, len, .{ .safety = safety });
16634	break :result;	16898	break :result;
16635	}	16899	}
16636		16900
16637	// Store the first element, and then rely on memcpy copying forwards.	16901	// Store the first element, and then rely on memcpy copying forwards.
16638	// Length zero requires a runtime check - so we handle arrays specially	16902	// Length zero requires a runtime check - so we handle arrays specially
16639	// here to elide it.	16903	// here to elide it.
16640	switch (dst_ptr_ty.ptrSize(zcu)) {	16904	switch (dst_ty.ptrSize(zcu)) {
16641	.slice => {	16905	.slice => {
16642	const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(zcu);	16906	const slice_ptr_ty = dst_ty.slicePtrFieldType(zcu);
16643		16907
16644	// TODO: this only handles slices stored in the stack	16908	const dst_ptr: MCValue = switch (dst) {
16645	const ptr = dst_ptr;	16909	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
16646	const len = dst_ptr.address().offset(8).deref();	16910	else => dst,
		16911	};
		16912	const len: MCValue = switch (dst) {
		16913	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
		16914	else => dst.address().offset(8).deref(),
		16915	};
16647		16916
16648	// Used to store the number of elements for comparison.	16917	// Used to store the number of elements for comparison.
16649	// After comparison, updated to store number of bytes needed to copy.	16918	// After comparison, updated to store number of bytes needed to copy.
...	@@ -16656,7 +16925,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16656,7 +16925,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16656	try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);	16925	try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);
16657		16926
16658	const skip_reloc = try self.asmJccReloc(.z, undefined);	16927	const skip_reloc = try self.asmJccReloc(.z, undefined);
16659	try self.store(slice_ptr_ty, ptr, src_val, .{ .safety = safety });	16928	try self.store(slice_ptr_ty, dst_ptr, src_val, .{ .safety = safety });
16660		16929
16661	const second_elem_ptr_reg =	16930	const second_elem_ptr_reg =
16662	try self.register_manager.allocReg(null, abi.RegisterClass.gp);	16931	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	@@ -16666,7 +16935,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16666,7 +16935,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16666	defer self.register_manager.unlockReg(second_elem_ptr_lock);	16935	defer self.register_manager.unlockReg(second_elem_ptr_lock);
16667		16936
16668	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{	16937	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
16669	.reg = try self.copyToTmpRegister(Type.usize, ptr),	16938	.reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
16670	.off = elem_abi_size,	16939	.off = elem_abi_size,
16671	} }, .{});	16940	} }, .{});
16672		16941
...	@@ -16675,19 +16944,19 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16675,19 +16944,19 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16675	.{ .i_, .mul },	16944	.{ .i_, .mul },
16676	len_reg,	16945	len_reg,
16677	len_reg,	16946	len_reg,
16678	Immediate.s(elem_abi_size),	16947	.s(elem_abi_size),
16679	);	16948	);
16680	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);	16949	try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv);
16681		16950
16682	self.performReloc(skip_reloc);	16951	self.performReloc(skip_reloc);
16683	},	16952	},
16684	.one => {	16953	.one => {
16685	const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);	16954	const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);
16686		16955
16687	const len = dst_ptr_ty.childType(zcu).arrayLen(zcu);	16956	const len = dst_ty.childType(zcu).arrayLen(zcu);
16688		16957
16689	assert(len != 0); // prevented by Sema	16958	assert(len != 0); // prevented by Sema
16690	try self.store(elem_ptr_ty, dst_ptr, src_val, .{ .safety = safety });	16959	try self.store(elem_ptr_ty, dst, src_val, .{ .safety = safety });
16691		16960
16692	const second_elem_ptr_reg =	16961	const second_elem_ptr_reg =
16693	try self.register_manager.allocReg(null, abi.RegisterClass.gp);	16962	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	@@ -16697,12 +16966,12 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16697,12 +16966,12 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16697	defer self.register_manager.unlockReg(second_elem_ptr_lock);	16966	defer self.register_manager.unlockReg(second_elem_ptr_lock);
16698		16967
16699	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{	16968	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
16700	.reg = try self.copyToTmpRegister(Type.usize, dst_ptr),	16969	.reg = try self.copyToTmpRegister(Type.usize, dst),
16701	.off = elem_abi_size,	16970	.off = elem_abi_size,
16702	} }, .{});	16971	} }, .{});
16703		16972
16704	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };	16973	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
16705	try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);	16974	try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy);
16706	},	16975	},
16707	.c, .many => unreachable,	16976	.c, .many => unreachable,
16708	}	16977	}
...	@@ -16719,48 +16988,72 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16719,48 +16988,72 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
16719	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });	16988	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
16720	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);	16989	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
16721		16990
16722	const dst_ptr = try self.resolveInst(bin_op.lhs);	16991	const dst = try self.resolveInst(bin_op.lhs);
16723	const dst_ptr_ty = self.typeOf(bin_op.lhs);	16992	const dst_ty = self.typeOf(bin_op.lhs);
16724	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {	16993	const dst_locks: [2]?RegisterLock = switch (dst) {
16725	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	16994	.register => \|dst_reg\| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
16726	else => null,	16995	.register_pair => \|dst_regs\| .{
		16996	self.register_manager.lockRegAssumeUnused(dst_regs[0]),
		16997	self.register_manager.lockReg(dst_regs[1]),
		16998	},
		16999	else => .{ null, null },
16727	};	17000	};
16728	defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	17001	for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
16729		17002
16730	const src_ptr = try self.resolveInst(bin_op.rhs);	17003	const src = try self.resolveInst(bin_op.rhs);
16731	const src_ptr_lock: ?RegisterLock = switch (src_ptr) {	17004	const src_locks: [2]?RegisterLock = switch (src) {
16732	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	17005	.register => \|src_reg\| .{ self.register_manager.lockReg(src_reg), null },
16733	else => null,	17006	.register_pair => \|src_regs\| .{
		17007	self.register_manager.lockRegAssumeUnused(src_regs[0]),
		17008	self.register_manager.lockRegAssumeUnused(src_regs[1]),
		17009	},
		17010	else => .{ null, null },
16734	};	17011	};
16735	defer if (src_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	17012	for (src_locks) \|src_lock\| if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
16736		17013
16737	const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {	17014	const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
16738	.slice => len: {	17015	.slice => len: {
16739	const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	17016	const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
16740	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);	17017	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
16741	defer self.register_manager.unlockReg(len_lock);	17018	defer self.register_manager.unlockReg(len_lock);
16742		17019
16743	try self.asmRegisterMemoryImmediate(	17020	switch (dst) {
16744	.{ .i_, .mul },	17021	.register_pair => \|dst_regs\| try self.asmRegisterRegisterImmediate(
16745	len_reg,	17022	.{ .i_, .mul },
16746	try dst_ptr.address().offset(8).deref().mem(self, .qword),	17023	len_reg,
16747	Immediate.s(@intCast(dst_ptr_ty.childType(zcu).abiSize(zcu))),	17024	dst_regs[1],
16748	);	17025	.s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
		17026	),
		17027	else => try self.asmRegisterMemoryImmediate(
		17028	.{ .i_, .mul },
		17029	len_reg,
		17030	try dst.address().offset(8).deref().mem(self, .qword),
		17031	.s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
		17032	),
		17033	}
16749	break :len .{ .register = len_reg };	17034	break :len .{ .register = len_reg };
16750	},	17035	},
16751	.one => len: {	17036	.one => len: {
16752	const array_ty = dst_ptr_ty.childType(zcu);	17037	const array_ty = dst_ty.childType(zcu);
16753	break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };	17038	break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };
16754	},	17039	},
16755	.c, .many => unreachable,	17040	.c, .many => unreachable,
16756	};	17041	};
16757	const len_lock: ?RegisterLock = switch (len) {	17042	const len_lock: ?RegisterLock = switch (len) {
16758	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	17043	.register => \|reg\| self.register_manager.lockReg(reg),
16759	else => null,	17044	else => null,
16760	};	17045	};
16761	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);	17046	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
16762		17047
16763	// TODO: dst_ptr and src_ptr could be slices rather than raw pointers	17048	const dst_ptr: MCValue = switch (dst) {
		17049	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
		17050	else => dst,
		17051	};
		17052	const src_ptr: MCValue = switch (src) {
		17053	.register_pair => \|src_regs\| .{ .register = src_regs[0] },
		17054	else => src,
		17055	};
		17056
16764	try self.genInlineMemcpy(dst_ptr, src_ptr, len);	17057	try self.genInlineMemcpy(dst_ptr, src_ptr, len);
16765		17058
16766	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });	17059	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
...	@@ -16930,27 +17223,23 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16930,27 +17223,23 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16930	try self.genSetReg(	17223	try self.genSetReg(
16931	regs[1],	17224	regs[1],
16932	vector_ty,	17225	vector_ty,
16933	.{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },	17226	.{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) },
16934	.{},	17227	.{},
16935	);	17228	);
16936	const src_mcv = try self.resolveInst(ty_op.operand);	17229	const src_mcv = try self.resolveInst(ty_op.operand);
16937	const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);	17230	const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4);
16938	try self.asmCmovccRegisterRegister(	17231	try self.asmCmovccRegisterRegister(
16939	switch (src_mcv) {	17232	switch (src_mcv) {
16940	.eflags => \|cc\| cc,	17233	.eflags => \|cc\| cc,
16941	.register => \|src_reg\| cc: {	17234	.register => \|src_reg\| cc: {
16942	try self.asmRegisterImmediate(	17235	try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1));
16943	.{ ._, .@"test" },
16944	src_reg.to8(),
16945	Immediate.u(1),
16946	);
16947	break :cc .nz;	17236	break :cc .nz;
16948	},	17237	},
16949	else => cc: {	17238	else => cc: {
16950	try self.asmMemoryImmediate(	17239	try self.asmMemoryImmediate(
16951	.{ ._, .@"test" },	17240	.{ ._, .@"test" },
16952	try src_mcv.mem(self, .byte),	17241	try src_mcv.mem(self, .byte),
16953	Immediate.u(1),	17242	.u(1),
16954	);	17243	);
16955	break :cc .nz;	17244	break :cc .nz;
16956	},	17245	},
...	@@ -17037,7 +17326,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17037,7 +17326,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17037	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },	17326	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
17038	dst_alias,	17327	dst_alias,
17039	dst_alias,	17328	dst_alias,
17040	Immediate.u(0b00_00_00_00),	17329	.u(0b00_00_00_00),
17041	);	17330	);
17042	if (switch (scalar_bits) {	17331	if (switch (scalar_bits) {
17043	1...8 => vector_len > 4,	17332	1...8 => vector_len > 4,
...	@@ -17049,7 +17338,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17049,7 +17338,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17049	.{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },	17338	.{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
17050	dst_alias,	17339	dst_alias,
17051	dst_alias,	17340	dst_alias,
17052	Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),	17341	.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
17053	);	17342	);
17054	break :result .{ .register = dst_reg };	17343	break :result .{ .register = dst_reg };
17055	},	17344	},
...	@@ -17080,7 +17369,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17080,7 +17369,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17080	dst_reg.to128(),	17369	dst_reg.to128(),
17081	src_reg.to128(),	17370	src_reg.to128(),
17082	src_reg.to128(),	17371	src_reg.to128(),
17083	Immediate.u(0),	17372	.u(0),
17084	);	17373	);
17085	}	17374	}
17086	break :result .{ .register = dst_reg };	17375	break :result .{ .register = dst_reg };
...	@@ -17095,7 +17384,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17095,7 +17384,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17095	.{ ._ps, .shuf },	17384	.{ ._ps, .shuf },
17096	dst_reg.to128(),	17385	dst_reg.to128(),
17097	dst_reg.to128(),	17386	dst_reg.to128(),
17098	Immediate.u(0),	17387	.u(0),
17099	);	17388	);
17100	break :result dst_mcv;	17389	break :result dst_mcv;
17101	}	17390	}
...	@@ -17122,14 +17411,14 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17122,14 +17411,14 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17122	dst_reg.to128(),	17411	dst_reg.to128(),
17123	src_reg.to128(),	17412	src_reg.to128(),
17124	src_reg.to128(),	17413	src_reg.to128(),
17125	Immediate.u(0),	17414	.u(0),
17126	);	17415	);
17127	try self.asmRegisterRegisterRegisterImmediate(	17416	try self.asmRegisterRegisterRegisterImmediate(
17128	.{ .v_f128, .insert },	17417	.{ .v_f128, .insert },
17129	dst_reg.to256(),	17418	dst_reg.to256(),
17130	dst_reg.to256(),	17419	dst_reg.to256(),
17131	dst_reg.to128(),	17420	dst_reg.to128(),
17132	Immediate.u(1),	17421	.u(1),
17133	);	17422	);
17134	}	17423	}
17135	}	17424	}
...	@@ -17198,7 +17487,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17198,7 +17487,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17198	dst_reg.to256(),	17487	dst_reg.to256(),
17199	dst_reg.to256(),	17488	dst_reg.to256(),
17200	dst_reg.to128(),	17489	dst_reg.to128(),
17201	Immediate.u(1),	17490	.u(1),
17202	);	17491	);
17203	}	17492	}
17204	}	17493	}
...	@@ -17231,7 +17520,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17231,7 +17520,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17231	dst_reg.to256(),	17520	dst_reg.to256(),
17232	src_reg.to256(),	17521	src_reg.to256(),
17233	src_reg.to128(),	17522	src_reg.to128(),
17234	Immediate.u(1),	17523	.u(1),
17235	);	17524	);
17236	}	17525	}
17237	break :result .{ .register = dst_reg };	17526	break :result .{ .register = dst_reg };
...	@@ -17308,7 +17597,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17308,7 +17597,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17308	mask_alias,	17597	mask_alias,
17309	mask_alias,	17598	mask_alias,
17310	mask_reg.to128(),	17599	mask_reg.to128(),
17311	Immediate.u(1),	17600	.u(1),
17312	);	17601	);
17313	break :broadcast;	17602	break :broadcast;
17314	},	17603	},
...	@@ -17362,7 +17651,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17362,7 +17651,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17362	.{ if (has_avx) .vp_w else .p_w, .shufl },	17651	.{ if (has_avx) .vp_w else .p_w, .shufl },
17363	mask_alias,	17652	mask_alias,
17364	mask_alias,	17653	mask_alias,
17365	Immediate.u(0b00_00_00_00),	17654	.u(0b00_00_00_00),
17366	);	17655	);
17367	if (abi_size <= 8) break :broadcast;	17656	if (abi_size <= 8) break :broadcast;
17368	}	17657	}
...	@@ -17370,7 +17659,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17370,7 +17659,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17370	.{ if (has_avx) .vp_d else .p_d, .shuf },	17659	.{ if (has_avx) .vp_d else .p_d, .shuf },
17371	mask_alias,	17660	mask_alias,
17372	mask_alias,	17661	mask_alias,
17373	Immediate.u(switch (elem_abi_size) {	17662	.u(switch (elem_abi_size) {
17374	1...2, 5...8 => 0b01_00_01_00,	17663	1...2, 5...8 => 0b01_00_01_00,
17375	3...4 => 0b00_00_00_00,	17664	3...4 => 0b00_00_00_00,
17376	else => unreachable,	17665	else => unreachable,
...	@@ -17649,7 +17938,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17649,7 +17938,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17649	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {	17938	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17650	const mask_elem = maybe_mask_elem orelse continue;	17939	const mask_elem = maybe_mask_elem orelse continue;
17651	const mask_elem_index =	17940	const mask_elem_index =
17652	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;	17941	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
17653	const elem_byte = (elem_index >> 1) * elem_abi_size;	17942	const elem_byte = (elem_index >> 1) * elem_abi_size;
17654	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {	17943	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {
17655	.unpckl => 0b0000,	17944	.unpckl => 0b0000,
...	@@ -17746,10 +18035,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17746,10 +18035,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17746	} else sources[(elem_index & 0b010) >> 1] = source;	18035	} else sources[(elem_index & 0b010) >> 1] = source;
17747		18036
17748	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);	18037	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17749	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;	18038	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17750	if (elem_index & 0b100 == 0)	18039	if (elem_index & 0b100 == 0)
17751	control \|= select	18040	control \|= select_mask
17752	else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;	18041	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd;
17753	}	18042	}
17754		18043
17755	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };	18044	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
...	@@ -17767,7 +18056,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17767,7 +18056,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17767	.{ if (has_avx) .vp_d else .p_d, .shuf },	18056	.{ if (has_avx) .vp_d else .p_d, .shuf },
17768	dst_alias,	18057	dst_alias,
17769	try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	18058	try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17770	Immediate.u(control),	18059	.u(control),
17771	) else try self.asmRegisterRegisterImmediate(	18060	) else try self.asmRegisterRegisterImmediate(
17772	.{ if (has_avx) .vp_d else .p_d, .shuf },	18061	.{ if (has_avx) .vp_d else .p_d, .shuf },
17773	dst_alias,	18062	dst_alias,
...	@@ -17775,7 +18064,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17775,7 +18064,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17775	src_mcv.getReg().?	18064	src_mcv.getReg().?
17776	else	18065	else
17777	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),	18066	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
17778	Immediate.u(control),	18067	.u(control),
17779	);	18068	);
17780	break :result .{ .register = dst_reg };	18069	break :result .{ .register = dst_reg };
17781	}	18070	}
...	@@ -17797,10 +18086,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17797,10 +18086,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17797	} else sources[(elem_index & 0b010) >> 1] = source;	18086	} else sources[(elem_index & 0b010) >> 1] = source;
17798		18087
17799	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);	18088	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17800	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;	18089	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17801	if (elem_index & 0b100 == 0)	18090	if (elem_index & 0b100 == 0)
17802	control \|= select	18091	control \|= select_mask
17803	else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;	18092	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps;
17804	}	18093	}
17805	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;	18094	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
17806		18095
...	@@ -17824,7 +18113,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17824,7 +18113,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17824	dst_alias,	18113	dst_alias,
17825	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),	18114	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17826	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	18115	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17827	Immediate.u(control),	18116	.u(control),
17828	) else try self.asmRegisterRegisterRegisterImmediate(	18117	) else try self.asmRegisterRegisterRegisterImmediate(
17829	.{ .v_ps, .shuf },	18118	.{ .v_ps, .shuf },
17830	dst_alias,	18119	dst_alias,
...	@@ -17833,12 +18122,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17833,12 +18122,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17833	rhs_mcv.getReg().?	18122	rhs_mcv.getReg().?
17834	else	18123	else
17835	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	18124	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17836	Immediate.u(control),	18125	.u(control),
17837	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	18126	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17838	.{ ._ps, .shuf },	18127	.{ ._ps, .shuf },
17839	dst_alias,	18128	dst_alias,
17840	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	18129	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17841	Immediate.u(control),	18130	.u(control),
17842	) else try self.asmRegisterRegisterImmediate(	18131	) else try self.asmRegisterRegisterImmediate(
17843	.{ ._ps, .shuf },	18132	.{ ._ps, .shuf },
17844	dst_alias,	18133	dst_alias,
...	@@ -17846,7 +18135,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17846,7 +18135,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17846	rhs_mcv.getReg().?	18135	rhs_mcv.getReg().?
17847	else	18136	else
17848	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	18137	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17849	Immediate.u(control),	18138	.u(control),
17850	);	18139	);
17851	break :result dst_mcv;	18140	break :result dst_mcv;
17852	}	18141	}
...	@@ -17891,7 +18180,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17891,7 +18180,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17891	dst_alias,	18180	dst_alias,
17892	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),	18181	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17893	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	18182	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17894	Immediate.u(control),	18183	.u(control),
17895	) else try self.asmRegisterRegisterRegisterImmediate(	18184	) else try self.asmRegisterRegisterRegisterImmediate(
17896	.{ .v_pd, .shuf },	18185	.{ .v_pd, .shuf },
17897	dst_alias,	18186	dst_alias,
...	@@ -17900,12 +18189,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17900,12 +18189,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17900	rhs_mcv.getReg().?	18189	rhs_mcv.getReg().?
17901	else	18190	else
17902	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	18191	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17903	Immediate.u(control),	18192	.u(control),
17904	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	18193	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17905	.{ ._pd, .shuf },	18194	.{ ._pd, .shuf },
17906	dst_alias,	18195	dst_alias,
17907	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	18196	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17908	Immediate.u(control),	18197	.u(control),
17909	) else try self.asmRegisterRegisterImmediate(	18198	) else try self.asmRegisterRegisterImmediate(
17910	.{ ._pd, .shuf },	18199	.{ ._pd, .shuf },
17911	dst_alias,	18200	dst_alias,
...	@@ -17913,7 +18202,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17913,7 +18202,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17913	rhs_mcv.getReg().?	18202	rhs_mcv.getReg().?
17914	else	18203	else
17915	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	18204	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17916	Immediate.u(control),	18205	.u(control),
17917	);	18206	);
17918	break :result dst_mcv;	18207	break :result dst_mcv;
17919	}	18208	}
...	@@ -17927,13 +18216,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17927,13 +18216,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17927	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {	18216	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17928	const mask_elem = maybe_mask_elem orelse continue;	18217	const mask_elem = maybe_mask_elem orelse continue;
17929	const mask_elem_index =	18218	const mask_elem_index =
17930	math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;	18219	std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
17931	if (mask_elem_index != elem_index) break :blend;	18220	if (mask_elem_index != elem_index) break :blend;
17932		18221
17933	const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);	18222	const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
17934	if (elem_index & 0b1000 == 0)	18223	if (elem_index & 0b1000 == 0)
17935	control \|= select	18224	control \|= select_mask
17936	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;	18225	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend;
17937	}	18226	}
17938		18227
17939	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {	18228	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
...	@@ -17961,7 +18250,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17961,7 +18250,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17961	registerAlias(dst_reg, dst_abi_size),	18250	registerAlias(dst_reg, dst_abi_size),
17962	registerAlias(lhs_reg, dst_abi_size),	18251	registerAlias(lhs_reg, dst_abi_size),
17963	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	18252	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17964	Immediate.u(expanded_control),	18253	.u(expanded_control),
17965	) else try self.asmRegisterRegisterRegisterImmediate(	18254	) else try self.asmRegisterRegisterRegisterImmediate(
17966	.{ .vp_d, .blend },	18255	.{ .vp_d, .blend },
17967	registerAlias(dst_reg, dst_abi_size),	18256	registerAlias(dst_reg, dst_abi_size),
...	@@ -17970,7 +18259,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17970,7 +18259,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17970	rhs_mcv.getReg().?	18259	rhs_mcv.getReg().?
17971	else	18260	else
17972	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	18261	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17973	Immediate.u(expanded_control),	18262	.u(expanded_control),
17974	);	18263	);
17975	break :result .{ .register = dst_reg };	18264	break :result .{ .register = dst_reg };
17976	}	18265	}
...	@@ -18016,7 +18305,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18016,7 +18305,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18016	else	18305	else
18017	dst_reg, dst_abi_size),	18306	dst_reg, dst_abi_size),
18018	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	18307	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18019	Immediate.u(expanded_control),	18308	.u(expanded_control),
18020	) else try self.asmRegisterRegisterRegisterImmediate(	18309	) else try self.asmRegisterRegisterRegisterImmediate(
18021	.{ .vp_w, .blend },	18310	.{ .vp_w, .blend },
18022	registerAlias(dst_reg, dst_abi_size),	18311	registerAlias(dst_reg, dst_abi_size),
...	@@ -18028,12 +18317,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18028,12 +18317,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18028	rhs_mcv.getReg().?	18317	rhs_mcv.getReg().?
18029	else	18318	else
18030	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	18319	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18031	Immediate.u(expanded_control),	18320	.u(expanded_control),
18032	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	18321	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18033	.{ .p_w, .blend },	18322	.{ .p_w, .blend },
18034	registerAlias(dst_reg, dst_abi_size),	18323	registerAlias(dst_reg, dst_abi_size),
18035	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	18324	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18036	Immediate.u(expanded_control),	18325	.u(expanded_control),
18037	) else try self.asmRegisterRegisterImmediate(	18326	) else try self.asmRegisterRegisterImmediate(
18038	.{ .p_w, .blend },	18327	.{ .p_w, .blend },
18039	registerAlias(dst_reg, dst_abi_size),	18328	registerAlias(dst_reg, dst_abi_size),
...	@@ -18041,7 +18330,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18041,7 +18330,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18041	rhs_mcv.getReg().?	18330	rhs_mcv.getReg().?
18042	else	18331	else
18043	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	18332	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18044	Immediate.u(expanded_control),	18333	.u(expanded_control),
18045	);	18334	);
18046	break :result .{ .register = dst_reg };	18335	break :result .{ .register = dst_reg };
18047	}	18336	}
...	@@ -18077,7 +18366,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18077,7 +18366,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18077	else	18366	else
18078	dst_reg, dst_abi_size),	18367	dst_reg, dst_abi_size),
18079	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	18368	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18080	Immediate.u(expanded_control),	18369	.u(expanded_control),
18081	) else try self.asmRegisterRegisterRegisterImmediate(	18370	) else try self.asmRegisterRegisterRegisterImmediate(
18082	switch (elem_abi_size) {	18371	switch (elem_abi_size) {
18083	4 => .{ .v_ps, .blend },	18372	4 => .{ .v_ps, .blend },
...	@@ -18093,7 +18382,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18093,7 +18382,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18093	rhs_mcv.getReg().?	18382	rhs_mcv.getReg().?
18094	else	18383	else
18095	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	18384	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18096	Immediate.u(expanded_control),	18385	.u(expanded_control),
18097	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	18386	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18098	switch (elem_abi_size) {	18387	switch (elem_abi_size) {
18099	4 => .{ ._ps, .blend },	18388	4 => .{ ._ps, .blend },
...	@@ -18102,7 +18391,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18102,7 +18391,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18102	},	18391	},
18103	registerAlias(dst_reg, dst_abi_size),	18392	registerAlias(dst_reg, dst_abi_size),
18104	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	18393	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18105	Immediate.u(expanded_control),	18394	.u(expanded_control),
18106	) else try self.asmRegisterRegisterImmediate(	18395	) else try self.asmRegisterRegisterImmediate(
18107	switch (elem_abi_size) {	18396	switch (elem_abi_size) {
18108	4 => .{ ._ps, .blend },	18397	4 => .{ ._ps, .blend },
...	@@ -18114,7 +18403,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18114,7 +18403,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18114	rhs_mcv.getReg().?	18403	rhs_mcv.getReg().?
18115	else	18404	else
18116	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	18405	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18117	Immediate.u(expanded_control),	18406	.u(expanded_control),
18118	);	18407	);
18119	break :result .{ .register = dst_reg };	18408	break :result .{ .register = dst_reg };
18120	}	18409	}
...	@@ -18138,7 +18427,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18138,7 +18427,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18138	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {	18427	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {
18139	const mask_elem = maybe_mask_elem orelse continue;	18428	const mask_elem = maybe_mask_elem orelse continue;
18140	const mask_elem_index =	18429	const mask_elem_index =
18141	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;	18430	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
18142	if (mask_elem_index != elem_index) break :blendv;	18431	if (mask_elem_index != elem_index) break :blendv;
18143		18432
18144	select_mask_elem.* = (if (mask_elem < 0)	18433	select_mask_elem.* = (if (mask_elem < 0)
...	@@ -18380,7 +18669,9 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18380,7 +18669,9 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18380		18669
18381	break :result null;	18670	break :result null;
18382	}) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{	18671	}) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
18383	lhs_ty.fmt(pt), rhs_ty.fmt(pt), dst_ty.fmt(pt),	18672	lhs_ty.fmt(pt),
		18673	rhs_ty.fmt(pt),
		18674	dst_ty.fmt(pt),
18384	Value.fromInterned(extra.mask).fmtValue(pt),	18675	Value.fromInterned(extra.mask).fmtValue(pt),
18385	});	18676	});
18386	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });	18677	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
...	@@ -18397,7 +18688,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18397,7 +18688,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18397	try self.spillEflagsIfOccupied();	18688	try self.spillEflagsIfOccupied();
18398		18689
18399	const operand_mcv = try self.resolveInst(reduce.operand);	18690	const operand_mcv = try self.resolveInst(reduce.operand);
18400	const mask_len = (math.cast(u6, operand_ty.vectorLen(zcu)) orelse	18691	const mask_len = (std.math.cast(u6, operand_ty.vectorLen(zcu)) orelse
18401	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}));	18692	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}));
18402	const mask = (@as(u64, 1) << mask_len) - 1;	18693	const mask = (@as(u64, 1) << mask_len) - 1;
18403	const abi_size: u32 = @intCast(operand_ty.abiSize(zcu));	18694	const abi_size: u32 = @intCast(operand_ty.abiSize(zcu));
...	@@ -18406,7 +18697,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18406,7 +18697,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18406	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(	18697	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
18407	.{ ._, .@"test" },	18698	.{ ._, .@"test" },
18408	try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),	18699	try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
18409	Immediate.u(mask),	18700	.u(mask),
18410	) else {	18701	) else {
18411	const operand_reg = registerAlias(if (operand_mcv.isRegister())	18702	const operand_reg = registerAlias(if (operand_mcv.isRegister())
18412	operand_mcv.getReg().?	18703	operand_mcv.getReg().?
...	@@ -18415,7 +18706,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18415,7 +18706,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18415	if (mask_len < abi_size * 8) try self.asmRegisterImmediate(	18706	if (mask_len < abi_size * 8) try self.asmRegisterImmediate(
18416	.{ ._, .@"test" },	18707	.{ ._, .@"test" },
18417	operand_reg,	18708	operand_reg,
18418	Immediate.u(mask),	18709	.u(mask),
18419	) else try self.asmRegisterRegister(	18710	) else try self.asmRegisterRegister(
18420	.{ ._, .@"test" },	18711	.{ ._, .@"test" },
18421	operand_reg,	18712	operand_reg,
...	@@ -18431,7 +18722,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18431,7 +18722,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18431		18722
18432	try self.asmRegister(.{ ._, .not }, tmp_reg);	18723	try self.asmRegister(.{ ._, .not }, tmp_reg);
18433	if (mask_len < abi_size * 8)	18724	if (mask_len < abi_size * 8)
18434	try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, Immediate.u(mask))	18725	try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, .u(mask))
18435	else	18726	else
18436	try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg);	18727	try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg);
18437	break :result .{ .eflags = .z };	18728	break :result .{ .eflags = .z };
...	@@ -18579,12 +18870,12 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18579,12 +18870,12 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
18579	try self.asmRegisterImmediate(	18870	try self.asmRegisterImmediate(
18580	.{ ._, .@"and" },	18871	.{ ._, .@"and" },
18581	registerAlias(elem_reg, @min(result_size, 4)),	18872	registerAlias(elem_reg, @min(result_size, 4)),
18582	Immediate.u(1),	18873	.u(1),
18583	);	18874	);
18584	if (elem_i > 0) try self.asmRegisterImmediate(	18875	if (elem_i > 0) try self.asmRegisterImmediate(
18585	.{ ._l, .sh },	18876	.{ ._l, .sh },
18586	registerAlias(elem_reg, result_size),	18877	registerAlias(elem_reg, result_size),
18587	Immediate.u(@intCast(elem_i)),	18878	.u(@intCast(elem_i)),
18588	);	18879	);
18589	try self.asmRegisterRegister(	18880	try self.asmRegisterRegister(
18590	.{ ._, .@"or" },	18881	.{ ._, .@"or" },
...	@@ -18748,8 +19039,8 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18748,8 +19039,8 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
18748	lock.* = self.register_manager.lockRegAssumeUnused(reg);	19039	lock.* = self.register_manager.lockRegAssumeUnused(reg);
18749	}	19040	}
18750		19041
18751	const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or	19042	const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or
18752	mem.eql(u2, &order, &.{ 3, 1, 2 }))	19043	std.mem.eql(u2, &order, &.{ 3, 1, 2 }))
18753	switch (ty.zigTypeTag(zcu)) {	19044	switch (ty.zigTypeTag(zcu)) {
18754	.float => switch (ty.floatBits(self.target.*)) {	19045	.float => switch (ty.floatBits(self.target.*)) {
18755	32 => .{ .v_ss, .fmadd132 },	19046	32 => .{ .v_ss, .fmadd132 },
...	@@ -18776,7 +19067,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18776,7 +19067,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
18776	},	19067	},
18777	else => unreachable,	19068	else => unreachable,
18778	}	19069	}
18779	else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))	19070	else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 }))
18780	switch (ty.zigTypeTag(zcu)) {	19071	switch (ty.zigTypeTag(zcu)) {
18781	.float => switch (ty.floatBits(self.target.*)) {	19072	.float => switch (ty.floatBits(self.target.*)) {
18782	32 => .{ .v_ss, .fmadd213 },	19073	32 => .{ .v_ss, .fmadd213 },
...	@@ -18803,7 +19094,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18803,7 +19094,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
18803	},	19094	},
18804	else => unreachable,	19095	else => unreachable,
18805	}	19096	}
18806	else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))	19097	else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 }))
18807	switch (ty.zigTypeTag(zcu)) {	19098	switch (ty.zigTypeTag(zcu)) {
18808	.float => switch (ty.floatBits(self.target.*)) {	19099	.float => switch (ty.floatBits(self.target.*)) {
18809	32 => .{ .v_ss, .fmadd231 },	19100	32 => .{ .v_ss, .fmadd231 },
...	@@ -18953,13 +19244,13 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18953,13 +19244,13 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
18953	const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };	19244	const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
18954	const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };	19245	const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
18955		19246
18956	const classes = mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);	19247	const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
18957	switch (classes[0]) {	19248	switch (classes[0]) {
18958	.integer => {	19249	.integer => {
18959	assert(classes.len == 1);	19250	assert(classes.len == 1);
18960		19251
18961	try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{});	19252	try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{});
18962	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(	19253	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
18963	abi.SysV.c_abi_int_param_regs.len * 8,	19254	abi.SysV.c_abi_int_param_regs.len * 8,
18964	));	19255	));
18965	const mem_reloc = try self.asmJccReloc(.ae, undefined);	19256	const mem_reloc = try self.asmJccReloc(.ae, undefined);
...	@@ -19007,7 +19298,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -19007,7 +19298,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19007	assert(classes.len == 1);	19298	assert(classes.len == 1);
19008		19299
19009	try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{});	19300	try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{});
19010	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(	19301	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
19011	abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,	19302	abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
19012	));	19303	));
19013	const mem_reloc = try self.asmJccReloc(.ae, undefined);	19304	const mem_reloc = try self.asmJccReloc(.ae, undefined);
...	@@ -19055,9 +19346,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -19055,9 +19346,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19055	assert(classes.len == 1);	19346	assert(classes.len == 1);
19056	unreachable;	19347	unreachable;
19057	},	19348	},
19058	else => return self.fail("TODO implement c_va_arg for {} on SysV", .{	19349	else => return self.fail("TODO implement c_va_arg for {} on SysV", .{promote_ty.fmt(pt)}),
19059	promote_ty.fmt(pt),
19060	}),
19061	}	19350	}
19062		19351
19063	if (unused) break :result .unreach;	19352	if (unused) break :result .unreach;
...	@@ -19194,7 +19483,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV	...	@@ -19194,7 +19483,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV
19194	.immediate => \|imm\| {	19483	.immediate => \|imm\| {
19195	// This immediate is unsigned.	19484	// This immediate is unsigned.
19196	const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));	19485	const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));
19197	if (imm >= math.maxInt(U)) {	19486	if (imm >= std.math.maxInt(U)) {
19198	return MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) };	19487	return MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) };
19199	}	19488	}
19200	},	19489	},
...	@@ -19226,7 +19515,7 @@ const CallMCValues = struct {	...	@@ -19226,7 +19515,7 @@ const CallMCValues = struct {
19226	args: []MCValue,	19515	args: []MCValue,
19227	return_value: InstTracking,	19516	return_value: InstTracking,
19228	stack_byte_count: u31,	19517	stack_byte_count: u31,
19229	stack_align: Alignment,	19518	stack_align: InternPool.Alignment,
19230	gp_count: u32,	19519	gp_count: u32,
19231	fp_count: u32,	19520	fp_count: u32,
19232		19521
...	@@ -19303,7 +19592,7 @@ fn resolveCallingConventionValues(	...	@@ -19303,7 +19592,7 @@ fn resolveCallingConventionValues(
19303	var ret_tracking_i: usize = 0;	19592	var ret_tracking_i: usize = 0;
19304		19593
19305	const classes = switch (resolved_cc) {	19594	const classes = switch (resolved_cc) {
19306	.x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),	19595	.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),
19307	.x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},	19596	.x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},
19308	else => unreachable,	19597	else => unreachable,
19309	};	19598	};
...	@@ -19378,7 +19667,7 @@ fn resolveCallingConventionValues(	...	@@ -19378,7 +19667,7 @@ fn resolveCallingConventionValues(
19378	var arg_mcv_i: usize = 0;	19667	var arg_mcv_i: usize = 0;
19379		19668
19380	const classes = switch (resolved_cc) {	19669	const classes = switch (resolved_cc) {
19381	.x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),	19670	.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),
19382	.x86_64_win => &.{abi.classifyWindows(ty, zcu)},	19671	.x86_64_win => &.{abi.classifyWindows(ty, zcu)},
19383	else => unreachable,	19672	else => unreachable,
19384	};	19673	};
...	@@ -19435,7 +19724,7 @@ fn resolveCallingConventionValues(	...	@@ -19435,7 +19724,7 @@ fn resolveCallingConventionValues(
19435		19724
19436	const frame_elem_align = 8;	19725	const frame_elem_align = 8;
19437	const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs;	19726	const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs;
19438	const frame_elem_size = mem.alignForward(	19727	const frame_elem_size = std.mem.alignForward(
19439	u64,	19728	u64,
19440	ty.childType(zcu).abiSize(zcu),	19729	ty.childType(zcu).abiSize(zcu),
19441	frame_elem_align,	19730	frame_elem_align,
...	@@ -19443,7 +19732,7 @@ fn resolveCallingConventionValues(	...	@@ -19443,7 +19732,7 @@ fn resolveCallingConventionValues(
19443	const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size);	19732	const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size);
19444		19733
19445	result.stack_byte_count =	19734	result.stack_byte_count =
19446	mem.alignForward(u31, result.stack_byte_count, frame_elem_align);	19735	std.mem.alignForward(u31, result.stack_byte_count, frame_elem_align);
19447	arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{	19736	arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{
19448	.regs = remaining_param_int_regs,	19737	.regs = remaining_param_int_regs,
19449	.frame_off = @intCast(result.stack_byte_count),	19738	.frame_off = @intCast(result.stack_byte_count),
...	@@ -19461,19 +19750,14 @@ fn resolveCallingConventionValues(	...	@@ -19461,19 +19750,14 @@ fn resolveCallingConventionValues(
19461	continue;	19750	continue;
19462	}	19751	}
19463		19752
19464	const param_size: u31 = @intCast(ty.abiSize(zcu));
19465	const param_align = ty.abiAlignment(zcu).max(.@"8");	19753	const param_align = ty.abiAlignment(zcu).max(.@"8");
19466	result.stack_byte_count = mem.alignForward(	19754	result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
19467	u31,
19468	result.stack_byte_count,
19469	@intCast(param_align.toByteUnits().?),
19470	);
19471	result.stack_align = result.stack_align.max(param_align);	19755	result.stack_align = result.stack_align.max(param_align);
19472	arg.* = .{ .load_frame = .{	19756	arg.* = .{ .load_frame = .{
19473	.index = stack_frame_base,	19757	.index = stack_frame_base,
19474	.off = result.stack_byte_count,	19758	.off = result.stack_byte_count,
19475	} };	19759	} };
19476	result.stack_byte_count += param_size;	19760	result.stack_byte_count += @intCast(ty.abiSize(zcu));
19477	}	19761	}
19478	assert(param_int_reg_i <= 6);	19762	assert(param_int_reg_i <= 6);
19479	result.gp_count = param_int_reg_i;	19763	result.gp_count = param_int_reg_i;
...	@@ -19509,19 +19793,14 @@ fn resolveCallingConventionValues(	...	@@ -19509,19 +19793,14 @@ fn resolveCallingConventionValues(
19509	arg.* = .none;	19793	arg.* = .none;
19510	continue;	19794	continue;
19511	}	19795	}
19512	const param_size: u31 = @intCast(ty.abiSize(zcu));
19513	const param_align = ty.abiAlignment(zcu);	19796	const param_align = ty.abiAlignment(zcu);
19514	result.stack_byte_count = mem.alignForward(	19797	result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
19515	u31,
19516	result.stack_byte_count,
19517	@intCast(param_align.toByteUnits().?),
19518	);
19519	result.stack_align = result.stack_align.max(param_align);	19798	result.stack_align = result.stack_align.max(param_align);
19520	arg.* = .{ .load_frame = .{	19799	arg.* = .{ .load_frame = .{
19521	.index = stack_frame_base,	19800	.index = stack_frame_base,
19522	.off = result.stack_byte_count,	19801	.off = result.stack_byte_count,
19523	} };	19802	} };
19524	result.stack_byte_count += param_size;	19803	result.stack_byte_count += @intCast(ty.abiSize(zcu));
19525	}	19804	}
19526	},	19805	},
19527	else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),	19806	else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
...	@@ -19541,7 +19820,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMem	...	@@ -19541,7 +19820,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMem
19541	return error.CodegenFail;	19820	return error.CodegenFail;
19542	}	19821	}
19543		19822
19544	fn failMsg(self: Self, msg: ErrorMsg) error{ OutOfMemory, CodegenFail } {	19823	fn failMsg(self: Self, msg: Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
19545	@branchHint(.cold);	19824	@branchHint(.cold);
19546	const zcu = self.pt.zcu;	19825	const zcu = self.pt.zcu;
19547	switch (self.owner) {	19826	switch (self.owner) {
...	@@ -19603,8 +19882,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {	...	@@ -19603,8 +19882,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
19603	}	19882	}
19604		19883
19605	fn memSize(self: *Self, ty: Type) Memory.Size {	19884	fn memSize(self: *Self, ty: Type) Memory.Size {
19606	const pt = self.pt;	19885	const zcu = self.pt.zcu;
19607	const zcu = pt.zcu;
19608	return switch (ty.zigTypeTag(zcu)) {	19886	return switch (ty.zigTypeTag(zcu)) {
19609	.float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),	19887	.float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),
19610	else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))),	19888	else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))),
...	@@ -19614,7 +19892,7 @@ fn memSize(self: *Self, ty: Type) Memory.Size {	...	@@ -19614,7 +19892,7 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
19614	fn splitType(self: *Self, ty: Type) ![2]Type {	19892	fn splitType(self: *Self, ty: Type) ![2]Type {
19615	const pt = self.pt;	19893	const pt = self.pt;
19616	const zcu = pt.zcu;	19894	const zcu = pt.zcu;
19617	const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);	19895	const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
19618	var parts: [2]Type = undefined;	19896	var parts: [2]Type = undefined;
19619	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {	19897	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {
19620	part.* = switch (class) {	19898	part.* = switch (class) {
...	@@ -19648,7 +19926,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	...	@@ -19648,7 +19926,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
19648	.signedness = .unsigned,	19926	.signedness = .unsigned,
19649	.bits = @intCast(ty.bitSize(zcu)),	19927	.bits = @intCast(ty.bitSize(zcu)),
19650	};	19928	};
19651	const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return;	19929	const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
19652	try self.spillEflagsIfOccupied();	19930	try self.spillEflagsIfOccupied();
19653	switch (int_info.signedness) {	19931	switch (int_info.signedness) {
19654	.signed => {	19932	.signed => {
...	@@ -19690,8 +19968,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	...	@@ -19690,8 +19968,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
19690	}	19968	}
19691		19969
19692	fn regBitSize(self: *Self, ty: Type) u64 {	19970	fn regBitSize(self: *Self, ty: Type) u64 {
19693	const pt = self.pt;	19971	const zcu = self.pt.zcu;
19694	const zcu = pt.zcu;
19695	const abi_size = ty.abiSize(zcu);	19972	const abi_size = ty.abiSize(zcu);
19696	return switch (ty.zigTypeTag(zcu)) {	19973	return switch (ty.zigTypeTag(zcu)) {
19697	else => switch (abi_size) {	19974	else => switch (abi_size) {
...	@@ -19713,14 +19990,14 @@ fn regExtraBits(self: *Self, ty: Type) u64 {	...	@@ -19713,14 +19990,14 @@ fn regExtraBits(self: *Self, ty: Type) u64 {
19713	return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);	19990	return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
19714	}	19991	}
19715		19992
19716	fn hasFeature(self: *Self, feature: Target.x86.Feature) bool {	19993	fn hasFeature(self: *Self, feature: std.Target.x86.Feature) bool {
19717	return Target.x86.featureSetHas(self.target.cpu.features, feature);	19994	return std.Target.x86.featureSetHas(self.target.cpu.features, feature);
19718	}	19995	}
19719	fn hasAnyFeatures(self: *Self, features: anytype) bool {	19996	fn hasAnyFeatures(self: *Self, features: anytype) bool {
19720	return Target.x86.featureSetHasAny(self.target.cpu.features, features);	19997	return std.Target.x86.featureSetHasAny(self.target.cpu.features, features);
19721	}	19998	}
19722	fn hasAllFeatures(self: *Self, features: anytype) bool {	19999	fn hasAllFeatures(self: *Self, features: anytype) bool {
19723	return Target.x86.featureSetHasAll(self.target.cpu.features, features);	20000	return std.Target.x86.featureSetHasAll(self.target.cpu.features, features);
19724	}	20001	}
19725		20002
19726	fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {	20003	fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
...	@@ -19732,9 +20009,13 @@ fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {	...	@@ -19732,9 +20009,13 @@ fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
19732	fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {	20009	fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
19733	const pt = self.pt;	20010	const pt = self.pt;
19734	const zcu = pt.zcu;	20011	const zcu = pt.zcu;
19735	return switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {	20012	const temp: Temp = .{ .index = inst };
19736	.loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),	20013	return switch (temp.unwrap(self)) {
19737	else => self.air.typeOfIndex(inst, &zcu.intern_pool),	20014	.ref => switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {
		20015	.loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),
		20016	else => self.air.typeOfIndex(inst, &zcu.intern_pool),
		20017	},
		20018	.temp => temp.typeOf(self),
19738	};	20019	};
19739	}	20020	}
19740		20021
...	@@ -19815,3 +20096,613 @@ fn promoteVarArg(self: *Self, ty: Type) Type {	...	@@ -19815,3 +20096,613 @@ fn promoteVarArg(self: *Self, ty: Type) Type {
19815	},	20096	},
19816	}	20097	}
19817	}	20098	}
		20099
		20100	// ====================================== rewrite starts here ======================================
		20101
		20102	const Temp = struct {
		20103	index: Air.Inst.Index,
		20104
		20105	fn unwrap(temp: Temp, self: *Self) union(enum) {
		20106	ref: Air.Inst.Ref,
		20107	temp: Index,
		20108	} {
		20109	switch (temp.index.unwrap()) {
		20110	.ref => \|ref\| return .{ .ref = ref },
		20111	.target => \|target_index\| {
		20112	const temp_index: Index = @enumFromInt(target_index);
		20113	assert(temp_index.isValid(self));
		20114	return .{ .temp = temp_index };
		20115	},
		20116	}
		20117	}
		20118
		20119	fn typeOf(temp: Temp, self: *Self) Type {
		20120	return switch (temp.unwrap(self)) {
		20121	.ref => \|ref\| self.typeOf(ref),
		20122	.temp => \|temp_index\| temp_index.typeOf(self),
		20123	};
		20124	}
		20125
		20126	fn isMut(temp: Temp, self: *Self) bool {
		20127	return temp.unwrap(self) == .temp;
		20128	}
		20129
		20130	fn tracking(temp: Temp, self: *Self) InstTracking {
		20131	return self.inst_tracking.get(temp.index).?;
		20132	}
		20133
		20134	fn getOffset(temp: Temp, off: i32, self: *Self) !Temp {
		20135	const new_temp_index = self.next_temp_index;
		20136	self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
		20137	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		20138	switch (temp.tracking(self).short) {
		20139	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
		20140	.register => \|reg\| {
		20141	const new_reg =
		20142	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20143	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20144	try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
		20145	.base = .{ .reg = reg.to64() },
		20146	.mod = .{ .rm = .{
		20147	.size = .qword,
		20148	.disp = off,
		20149	} },
		20150	});
		20151	},
		20152	.register_offset => \|reg_off\| {
		20153	const new_reg =
		20154	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20155	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20156	try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
		20157	.base = .{ .reg = reg_off.reg.to64() },
		20158	.mod = .{ .rm = .{
		20159	.size = .qword,
		20160	.disp = reg_off.off + off,
		20161	} },
		20162	});
		20163	},
		20164	.lea_symbol => \|sym_off\| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = .{
		20165	.sym_index = sym_off.sym_index,
		20166	.off = sym_off.off + off,
		20167	} }),
		20168	.load_frame => \|frame_addr\| {
		20169	const new_reg =
		20170	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20171	new_temp_index.tracking(self).* = InstTracking.init(.{ .register_offset = .{
		20172	.reg = new_reg,
		20173	.off = off,
		20174	} });
		20175	try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
		20176	.base = .{ .frame = frame_addr.index },
		20177	.mod = .{ .rm = .{
		20178	.size = .qword,
		20179	.disp = frame_addr.off,
		20180	} },
		20181	});
		20182	},
		20183	.lea_frame => \|frame_addr\| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = .{
		20184	.index = frame_addr.index,
		20185	.off = frame_addr.off + off,
		20186	} }),
		20187	}
		20188	return .{ .index = new_temp_index.toIndex() };
		20189	}
		20190
		20191	fn toOffset(temp: Temp, off: i32, self: Self) !void {
		20192	if (off == 0) return;
		20193	switch (temp.unwrap(self)) {
		20194	.ref => {},
		20195	.temp => \|temp_index\| {
		20196	const temp_tracking = temp_index.tracking(self);
		20197	switch (temp_tracking.short) {
		20198	else => {},
		20199	.register => \|reg\| {
		20200	try self.freeValue(temp_tracking.long);
		20201	temp_tracking.* = InstTracking.init(.{ .register_offset = .{
		20202	.reg = reg,
		20203	.off = off,
		20204	} });
		20205	return;
		20206	},
		20207	.register_offset => \|reg_off\| {
		20208	try self.freeValue(temp_tracking.long);
		20209	temp_tracking.* = InstTracking.init(.{ .register_offset = .{
		20210	.reg = reg_off.reg,
		20211	.off = reg_off.off + off,
		20212	} });
		20213	return;
		20214	},
		20215	.lea_symbol => \|sym_off\| {
		20216	assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
		20217	temp_tracking.* = InstTracking.init(.{ .lea_symbol = .{
		20218	.sym_index = sym_off.sym_index,
		20219	.off = sym_off.off + off,
		20220	} });
		20221	return;
		20222	},
		20223	.lea_frame => \|frame_addr\| {
		20224	assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
		20225	temp_tracking.* = InstTracking.init(.{ .lea_frame = .{
		20226	.index = frame_addr.index,
		20227	.off = frame_addr.off + off,
		20228	} });
		20229	return;
		20230	},
		20231	}
		20232	},
		20233	}
		20234	const new_temp = try temp.getOffset(off, self);
		20235	try temp.die(self);
		20236	temp.* = new_temp;
		20237	}
		20238
		20239	fn getLimb(temp: Temp, limb_index: u28, self: *Self) !Temp {
		20240	const new_temp_index = self.next_temp_index;
		20241	self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
		20242	switch (temp.tracking(self).short) {
		20243	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
		20244	.immediate => \|imm\| {
		20245	assert(limb_index == 0);
		20246	new_temp_index.tracking(self).* = InstTracking.init(.{ .immediate = imm });
		20247	},
		20248	.register => \|reg\| {
		20249	assert(limb_index == 0);
		20250	const new_reg =
		20251	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20252	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20253	try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
		20254	},
		20255	.register_pair => \|regs\| {
		20256	const new_reg =
		20257	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20258	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20259	try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
		20260	},
		20261	.register_offset => \|reg_off\| {
		20262	assert(limb_index == 0);
		20263	const new_reg =
		20264	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20265	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20266	try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
		20267	.base = .{ .reg = reg_off.reg.to64() },
		20268	.mod = .{ .rm = .{
		20269	.size = .qword,
		20270	.disp = reg_off.off + @as(u31, limb_index) * 8,
		20271	} },
		20272	});
		20273	},
		20274	.load_symbol => \|sym_off\| {
		20275	const new_reg =
		20276	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20277	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20278	try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
		20279	.base = .{ .reloc = sym_off.sym_index },
		20280	.mod = .{ .rm = .{
		20281	.size = .qword,
		20282	.disp = sym_off.off + @as(u31, limb_index) * 8,
		20283	} },
		20284	});
		20285	},
		20286	.lea_symbol => \|sym_off\| {
		20287	assert(limb_index == 0);
		20288	new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = sym_off });
		20289	},
		20290	.load_frame => \|frame_addr\| {
		20291	const new_reg =
		20292	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		20293	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20294	try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
		20295	.base = .{ .frame = frame_addr.index },
		20296	.mod = .{ .rm = .{
		20297	.size = .qword,
		20298	.disp = frame_addr.off + @as(u31, limb_index) * 8,
		20299	} },
		20300	});
		20301	},
		20302	.lea_frame => \|frame_addr\| {
		20303	assert(limb_index == 0);
		20304	new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = frame_addr });
		20305	},
		20306	}
		20307	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		20308	return .{ .index = new_temp_index.toIndex() };
		20309	}
		20310
		20311	fn toLimb(temp: Temp, limb_index: u28, self: Self) !void {
		20312	switch (temp.unwrap(self)) {
		20313	.ref => {},
		20314	.temp => \|temp_index\| {
		20315	const temp_tracking = temp_index.tracking(self);
		20316	switch (temp_tracking.short) {
		20317	else => {},
		20318	.register, .lea_symbol, .lea_frame => {
		20319	assert(limb_index == 0);
		20320	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
		20321	return;
		20322	},
		20323	.register_pair => \|regs\| {
		20324	switch (temp_tracking.long) {
		20325	.none, .reserved_frame => {},
		20326	else => temp_tracking.long =
		20327	temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
		20328	}
		20329	for (regs, 0..) \|reg, reg_index\| if (reg_index != limb_index)
		20330	self.register_manager.freeReg(reg);
		20331	temp_tracking.* = InstTracking.init(.{ .register = regs[limb_index] });
		20332	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
		20333	return;
		20334	},
		20335	.load_symbol => \|sym_off\| {
		20336	assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
		20337	temp_tracking.* = InstTracking.init(.{ .load_symbol = .{
		20338	.sym_index = sym_off.sym_index,
		20339	.off = sym_off.off + @as(u31, limb_index) * 8,
		20340	} });
		20341	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
		20342	return;
		20343	},
		20344	.load_frame => \|frame_addr\| if (!frame_addr.index.isNamed()) {
		20345	assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
		20346	temp_tracking.* = InstTracking.init(.{ .load_frame = .{
		20347	.index = frame_addr.index,
		20348	.off = frame_addr.off + @as(u31, limb_index) * 8,
		20349	} });
		20350	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
		20351	return;
		20352	},
		20353	}
		20354	},
		20355	}
		20356	const new_temp = try temp.getLimb(limb_index, self);
		20357	try temp.die(self);
		20358	temp.* = new_temp;
		20359	}
		20360
		20361	fn toReg(temp: Temp, new_reg: Register, self: Self) !bool {
		20362	const val, const ty = switch (temp.unwrap(self)) {
		20363	.ref => \|ref\| .{ temp.tracking(self).short, self.typeOf(ref) },
		20364	.temp => \|temp_index\| val: {
		20365	const temp_tracking = temp_index.tracking(self);
		20366	if (temp_tracking.short == .register and
		20367	temp_tracking.short.register == new_reg) return false;
		20368	break :val .{ temp_tracking.short, temp_index.typeOf(self) };
		20369	},
		20370	};
		20371	const new_temp_index = self.next_temp_index;
		20372	self.temp_type[@intFromEnum(new_temp_index)] = ty;
		20373	try self.genSetReg(new_reg, ty, val, .{});
		20374	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20375	try temp.die(self);
		20376	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		20377	temp.* = .{ .index = new_temp_index.toIndex() };
		20378	return true;
		20379	}
		20380
		20381	fn toAnyReg(temp: Temp, self: Self) !bool {
		20382	const val, const ty = switch (temp.unwrap(self)) {
		20383	.ref => \|ref\| .{ temp.tracking(self).short, self.typeOf(ref) },
		20384	.temp => \|temp_index\| val: {
		20385	const temp_tracking = temp_index.tracking(self);
		20386	if (temp_tracking.short == .register) return false;
		20387	break :val .{ temp_tracking.short, temp_index.typeOf(self) };
		20388	},
		20389	};
		20390	const new_temp_index = self.next_temp_index;
		20391	self.temp_type[@intFromEnum(new_temp_index)] = ty;
		20392	const new_reg =
		20393	try self.register_manager.allocReg(new_temp_index.toIndex(), self.regClassForType(ty));
		20394	try self.genSetReg(new_reg, ty, val, .{});
		20395	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
		20396	try temp.die(self);
		20397	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		20398	temp.* = .{ .index = new_temp_index.toIndex() };
		20399	return true;
		20400	}
		20401
		20402	fn toPair(first_temp: Temp, second_temp: Temp, self: *Self) !void {
		20403	while (true) for ([_]*Temp{ first_temp, second_temp }) \|part_temp\| {
		20404	if (try part_temp.toAnyReg(self)) break;
		20405	} else break;
		20406	const first_temp_tracking = first_temp.unwrap(self).temp.tracking(self);
		20407	const second_temp_tracking = second_temp.unwrap(self).temp.tracking(self);
		20408	const result: MCValue = .{ .register_pair = .{
		20409	first_temp_tracking.short.register,
		20410	second_temp_tracking.short.register,
		20411	} };
		20412	const result_temp_index = self.next_temp_index;
		20413	const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
		20414	assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
		20415	assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
		20416	self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
		20417	result_temp_index.tracking(self).* = InstTracking.init(result);
		20418	first_temp.* = result_temp;
		20419	}
		20420
		20421	fn toLea(temp: Temp, self: Self) !bool {
		20422	switch (temp.tracking(self).short) {
		20423	.none,
		20424	.unreach,
		20425	.dead,
		20426	.undef,
		20427	.eflags,
		20428	.register_pair,
		20429	.register_overflow,
		20430	.elementwise_regs_then_frame,
		20431	.reserved_frame,
		20432	.air_ref,
		20433	=> unreachable, // not a valid pointer
		20434	.immediate,
		20435	.register,
		20436	.register_offset,
		20437	.lea_direct,
		20438	.lea_got,
		20439	.lea_tlv,
		20440	.lea_frame,
		20441	=> return false,
		20442	.memory,
		20443	.indirect,
		20444	.load_symbol,
		20445	.load_direct,
		20446	.load_got,
		20447	.load_tlv,
		20448	.load_frame,
		20449	=> return temp.toAnyReg(self),
		20450	.lea_symbol => \|sym_off\| {
		20451	const off = sym_off.off;
		20452	if (off == 0) return false;
		20453	try temp.toOffset(-off, self);
		20454	while (try temp.toAnyReg(self)) {}
		20455	try temp.toOffset(off, self);
		20456	return true;
		20457	},
		20458	}
		20459	}
		20460
		20461	fn load(ptr: Temp, val_ty: Type, self: Self) !Temp {
		20462	const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));
		20463	const val = try self.tempAlloc(val_ty);
		20464	switch (val.tracking(self).short) {
		20465	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
		20466	.register => \|val_reg\| {
		20467	while (try ptr.toLea(self)) {}
		20468	switch (val_reg.class()) {
		20469	.general_purpose => try self.asmRegisterMemory(
		20470	.{ ._, .mov },
		20471	registerAlias(val_reg, val_abi_size),
		20472	try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
		20473	),
		20474	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
		20475	}
		20476	},
		20477	.load_frame => \|val_frame_addr\| {
		20478	var val_ptr = try self.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr });
		20479	var len = try self.tempFromValue(Type.usize, .{ .immediate = val_abi_size });
		20480	try val_ptr.memcpy(ptr, &len, self);
		20481	try val_ptr.die(self);
		20482	try len.die(self);
		20483	},
		20484	}
		20485	return val;
		20486	}
		20487
		20488	fn store(ptr: Temp, val: Temp, self: *Self) !void {
		20489	const val_ty = val.typeOf(self);
		20490	const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));
		20491	val: switch (val.tracking(self).short) {
		20492	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
		20493	.immediate => \|imm\| if (std.math.cast(i32, imm)) \|s\| {
		20494	while (try ptr.toLea(self)) {}
		20495	try self.asmMemoryImmediate(
		20496	.{ ._, .mov },
		20497	try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
		20498	.s(s),
		20499	);
		20500	} else continue :val .{ .register = undefined },
		20501	.register => {
		20502	while (try ptr.toLea(self) or try val.toAnyReg(self)) {}
		20503	const val_reg = val.tracking(self).short.register;
		20504	switch (val_reg.class()) {
		20505	.general_purpose => try self.asmMemoryRegister(
		20506	.{ ._, .mov },
		20507	try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
		20508	registerAlias(val_reg, val_abi_size),
		20509	),
		20510	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
		20511	}
		20512	},
		20513	}
		20514	}
		20515
		20516	fn memcpy(dst: Temp, src: Temp, len: Temp, self: Self) !void {
		20517	while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) \|temp, reg\| {
		20518	if (try temp.toReg(reg, self)) break;
		20519	} else break;
		20520	try self.asmOpOnly(.{ .@"rep _sb", .mov });
		20521	}
		20522
		20523	fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void {
		20524	if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) {
		20525	.ref => {
		20526	const result = try self.allocRegOrMem(inst, true);
		20527	try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{});
		20528	tracking_log.debug("{} => {} (birth)", .{ inst, result });
		20529	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
		20530	},
		20531	.temp => \|temp_index\| {
		20532	const temp_tracking = temp_index.tracking(self);
		20533	tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });
		20534	self.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
		20535	assert(self.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
		20536	},
		20537	}
		20538	}
		20539
		20540	fn die(temp: Temp, self: *Self) !void {
		20541	switch (temp.unwrap(self)) {
		20542	.ref => {},
		20543	.temp => \|temp_index\| try temp_index.tracking(self).die(self, temp_index.toIndex()),
		20544	}
		20545	}
		20546
		20547	const Index = enum(u4) {
		20548	_,
		20549
		20550	fn toIndex(index: Index) Air.Inst.Index {
		20551	return Air.Inst.Index.fromTargetIndex(@intFromEnum(index));
		20552	}
		20553
		20554	fn fromIndex(index: Air.Inst.Index) Index {
		20555	return @enumFromInt(index.toTargetIndex());
		20556	}
		20557
		20558	fn tracking(index: Index, self: Self) InstTracking {
		20559	return &self.inst_tracking.values()[@intFromEnum(index)];
		20560	}
		20561
		20562	fn isValid(index: Index, self: *Self) bool {
		20563	return index.tracking(self).short != .dead;
		20564	}
		20565
		20566	fn typeOf(index: Index, self: *Self) Type {
		20567	assert(index.isValid(self));
		20568	return self.temp_type[@intFromEnum(index)];
		20569	}
		20570
		20571	const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type);
		20572	const Set = std.StaticBitSet(max);
		20573	const SafetySet = if (std.debug.runtime_safety) Set else struct {
		20574	inline fn initEmpty() @This() {
		20575	return .{};
		20576	}
		20577
		20578	inline fn isSet(_: @This(), index: usize) bool {
		20579	assert(index < max);
		20580	return true;
		20581	}
		20582
		20583	inline fn set(_: @This(), index: usize) void {
		20584	assert(index < max);
		20585	}
		20586
		20587	inline fn eql(_: @This(), _: @This()) bool {
		20588	return true;
		20589	}
		20590	};
		20591	};
		20592	};
		20593
		20594	fn resetTemps(self: *Self) void {
		20595	for (0..@intFromEnum(self.next_temp_index)) \|temp_index\| {
		20596	const temp: Temp.Index = @enumFromInt(temp_index);
		20597	assert(!temp.isValid(self));
		20598	self.temp_type[temp_index] = undefined;
		20599	}
		20600	self.next_temp_index = @enumFromInt(0);
		20601	}
		20602
		20603	fn reuseTemp(
		20604	self: *Self,
		20605	new_inst: Air.Inst.Index,
		20606	old_inst: Air.Inst.Index,
		20607	tracking: *InstTracking,
		20608	) bool {
		20609	switch (tracking.short) {
		20610	.register,
		20611	.register_pair,
		20612	.register_offset,
		20613	.register_overflow,
		20614	=> for (tracking.short.getRegs()) \|tracked_reg\| {
		20615	if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) \|tracked_index\| {
		20616	self.register_manager.registers[tracked_index] = new_inst;
		20617	}
		20618	},
		20619	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
		20620	else => {},
		20621	}
		20622	switch (tracking.short) {
		20623	.eflags, .register_overflow => self.eflags_inst = new_inst,
		20624	else => {},
		20625	}
		20626	tracking.reuse(self, new_inst, old_inst);
		20627	return true;
		20628	}
		20629
		20630	fn tempAlloc(self: *Self, ty: Type) !Temp {
		20631	const temp_index = self.next_temp_index;
		20632	temp_index.tracking(self).* = InstTracking.init(try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true));
		20633	self.temp_type[@intFromEnum(temp_index)] = ty;
		20634	self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
		20635	return .{ .index = temp_index.toIndex() };
		20636	}
		20637
		20638	fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp {
		20639	const temp_index = self.next_temp_index;
		20640	temp_index.tracking(self).* = InstTracking.init(value);
		20641	self.temp_type[@intFromEnum(temp_index)] = ty;
		20642	try self.getValue(value, temp_index.toIndex());
		20643	self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
		20644	return .{ .index = temp_index.toIndex() };
		20645	}
		20646
		20647	fn tempFromOperand(
		20648	self: *Self,
		20649	inst: Air.Inst.Index,
		20650	op_index: Liveness.OperandInt,
		20651	op_ref: Air.Inst.Ref,
		20652	) !Temp {
		20653	const zcu = self.pt.zcu;
		20654	const ip = &zcu.intern_pool;
		20655
		20656	if (!self.liveness.operandDies(inst, op_index)) {
		20657	if (op_ref.toIndex()) \|op_inst\| return .{ .index = op_inst };
		20658	const val = op_ref.toInterned().?;
		20659	const gop = try self.const_tracking.getOrPut(self.gpa, val);
		20660	if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {
		20661	const const_mcv = try self.genTypedValue(Value.fromInterned(val));
		20662	switch (const_mcv) {
		20663	.lea_tlv => \|tlv_sym\| switch (self.bin_file.tag) {
		20664	.elf, .macho => {
		20665	if (self.mod.pic) {
		20666	try self.spillRegisters(&.{ .rdi, .rax });
		20667	} else {
		20668	try self.spillRegisters(&.{.rax});
		20669	}
		20670	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
		20671	.size = 8,
		20672	.alignment = .@"8",
		20673	}));
		20674	try self.genSetMem(
		20675	.{ .frame = frame_index },
		20676	0,
		20677	Type.usize,
		20678	.{ .lea_symbol = .{ .sym_index = tlv_sym } },
		20679	.{},
		20680	);
		20681	break :init .{ .load_frame = .{ .index = frame_index } };
		20682	},
		20683	else => break :init const_mcv,
		20684	},
		20685	else => break :init const_mcv,
		20686	}
		20687	});
		20688	return self.tempFromValue(Type.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
		20689	}
		20690
		20691	const temp_index = self.next_temp_index;
		20692	const temp: Temp = .{ .index = temp_index.toIndex() };
		20693	const op_inst = op_ref.toIndex().?;
		20694	const tracking = self.getResolvedInstValue(op_inst);
		20695	temp_index.tracking(self).* = tracking.*;
		20696	if (!self.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
		20697	self.temp_type[@intFromEnum(temp_index)] = self.typeOf(op_ref);
		20698	self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
		20699	return temp;
		20700	}
		20701
		20702	inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
		20703	var temps: [op_refs.len]Temp = undefined;
		20704	inline for (&temps, 0.., op_refs) \|*temp, op_index, op_ref\| {
		20705	temp.* = try self.tempFromOperand(inst, op_index, op_ref);
		20706	}
		20707	return temps;
		20708	}

src/arch/x86_64/abi.zig+2-3

 @@ -250,9 +250,8 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8
             return memory_class;
         },
         .optional => {
-            if (ty.isPtrLikeOptional(zcu)) {
+            if (ty.optionalReprIsPayload(zcu)) {
-                result[0] = .integer;
+                return classifySystemV(ty.optionalChild(zcu), zcu, target, ctx);
-                return result;
             }
             return memory_class;
         },

src/arch/x86_64/bits.zig+33-1

...	@@ -547,7 +547,39 @@ pub const Memory = struct {	...	@@ -547,7 +547,39 @@ pub const Memory = struct {
547	}	547	}
548	};	548	};
549		549
550	pub const Scale = enum(u2) { @"1", @"2", @"4", @"8" };	550	pub const Scale = enum(u2) {
		551	@"1",
		552	@"2",
		553	@"4",
		554	@"8",
		555
		556	pub fn fromFactor(factor: u4) Scale {
		557	return switch (factor) {
		558	else => unreachable,
		559	1 => .@"1",
		560	2 => .@"2",
		561	4 => .@"4",
		562	8 => .@"8",
		563	};
		564	}
		565
		566	pub fn toFactor(scale: Scale) u4 {
		567	return switch (scale) {
		568	.@"1" => 1,
		569	.@"2" => 2,
		570	.@"4" => 4,
		571	.@"8" => 8,
		572	};
		573	}
		574
		575	pub fn fromLog2(log2: u2) Scale {
		576	return @enumFromInt(log2);
		577	}
		578
		579	pub fn toLog2(scale: Scale) u2 {
		580	return @intFromEnum(scale);
		581	}
		582	};
551	};	583	};
552		584
553	pub const Immediate = union(enum) {	585	pub const Immediate = union(enum) {

src/print_air.zig+12-12

...	@@ -96,8 +96,8 @@ const Writer = struct {	...	@@ -96,8 +96,8 @@ const Writer = struct {
96	fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {	96	fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
97	const tag = w.air.instructions.items(.tag)[@intFromEnum(inst)];	97	const tag = w.air.instructions.items(.tag)[@intFromEnum(inst)];
98	try s.writeByteNTimes(' ', w.indent);	98	try s.writeByteNTimes(' ', w.indent);
99	try s.print("%{d}{c}= {s}(", .{	99	try s.print("{}{c}= {s}(", .{
100	@intFromEnum(inst),	100	inst,
101	@as(u8, if (if (w.liveness) \|liveness\| liveness.isUnused(inst) else false) '!' else ' '),	101	@as(u8, if (if (w.liveness) \|liveness\| liveness.isUnused(inst) else false) '!' else ' '),
102	@tagName(tag),	102	@tagName(tag),
103	});	103	});
...	@@ -409,7 +409,7 @@ const Writer = struct {	...	@@ -409,7 +409,7 @@ const Writer = struct {
409	try s.writeAll("}");	409	try s.writeAll("}");
410		410
411	for (liveness_block.deaths) \|operand\| {	411	for (liveness_block.deaths) \|operand\| {
412	try s.print(" %{d}!", .{@intFromEnum(operand)});	412	try s.print(" {}!", .{operand});
413	}	413	}
414	}	414	}
415		415
...	@@ -728,7 +728,7 @@ const Writer = struct {	...	@@ -728,7 +728,7 @@ const Writer = struct {
728	try s.writeByteNTimes(' ', w.indent);	728	try s.writeByteNTimes(' ', w.indent);
729	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {	729	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
730	if (i != 0) try s.writeAll(" ");	730	if (i != 0) try s.writeAll(" ");
731	try s.print("%{d}!", .{@intFromEnum(operand)});	731	try s.print("{}!", .{operand});
732	}	732	}
733	try s.writeAll("\n");	733	try s.writeAll("\n");
734	}	734	}
...	@@ -739,7 +739,7 @@ const Writer = struct {	...	@@ -739,7 +739,7 @@ const Writer = struct {
739	try s.writeAll("}");	739	try s.writeAll("}");
740		740
741	for (liveness_condbr.then_deaths) \|operand\| {	741	for (liveness_condbr.then_deaths) \|operand\| {
742	try s.print(" %{d}!", .{@intFromEnum(operand)});	742	try s.print(" {}!", .{operand});
743	}	743	}
744	}	744	}
745		745
...	@@ -765,7 +765,7 @@ const Writer = struct {	...	@@ -765,7 +765,7 @@ const Writer = struct {
765	try s.writeByteNTimes(' ', w.indent);	765	try s.writeByteNTimes(' ', w.indent);
766	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {	766	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
767	if (i != 0) try s.writeAll(" ");	767	if (i != 0) try s.writeAll(" ");
768	try s.print("%{d}!", .{@intFromEnum(operand)});	768	try s.print("{}!", .{operand});
769	}	769	}
770	try s.writeAll("\n");	770	try s.writeAll("\n");
771	}	771	}
...	@@ -776,7 +776,7 @@ const Writer = struct {	...	@@ -776,7 +776,7 @@ const Writer = struct {
776	try s.writeAll("}");	776	try s.writeAll("}");
777		777
778	for (liveness_condbr.then_deaths) \|operand\| {	778	for (liveness_condbr.then_deaths) \|operand\| {
779	try s.print(" %{d}!", .{@intFromEnum(operand)});	779	try s.print(" {}!", .{operand});
780	}	780	}
781	}	781	}
782		782
...	@@ -807,7 +807,7 @@ const Writer = struct {	...	@@ -807,7 +807,7 @@ const Writer = struct {
807	try s.writeByteNTimes(' ', w.indent);	807	try s.writeByteNTimes(' ', w.indent);
808	for (liveness_condbr.then_deaths, 0..) \|operand, i\| {	808	for (liveness_condbr.then_deaths, 0..) \|operand, i\| {
809	if (i != 0) try s.writeAll(" ");	809	if (i != 0) try s.writeAll(" ");
810	try s.print("%{d}!", .{@intFromEnum(operand)});	810	try s.print("{}!", .{operand});
811	}	811	}
812	try s.writeAll("\n");	812	try s.writeAll("\n");
813	}	813	}
...	@@ -827,7 +827,7 @@ const Writer = struct {	...	@@ -827,7 +827,7 @@ const Writer = struct {
827	try s.writeByteNTimes(' ', w.indent);	827	try s.writeByteNTimes(' ', w.indent);
828	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {	828	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
829	if (i != 0) try s.writeAll(" ");	829	if (i != 0) try s.writeAll(" ");
830	try s.print("%{d}!", .{@intFromEnum(operand)});	830	try s.print("{}!", .{operand});
831	}	831	}
832	try s.writeAll("\n");	832	try s.writeAll("\n");
833	}	833	}
...	@@ -884,7 +884,7 @@ const Writer = struct {	...	@@ -884,7 +884,7 @@ const Writer = struct {
884	try s.writeByteNTimes(' ', w.indent);	884	try s.writeByteNTimes(' ', w.indent);
885	for (deaths, 0..) \|operand, i\| {	885	for (deaths, 0..) \|operand, i\| {
886	if (i != 0) try s.writeAll(" ");	886	if (i != 0) try s.writeAll(" ");
887	try s.print("%{d}!", .{@intFromEnum(operand)});	887	try s.print("{}!", .{operand});
888	}	888	}
889	try s.writeAll("\n");	889	try s.writeAll("\n");
890	}	890	}
...	@@ -910,7 +910,7 @@ const Writer = struct {	...	@@ -910,7 +910,7 @@ const Writer = struct {
910	try s.writeByteNTimes(' ', w.indent);	910	try s.writeByteNTimes(' ', w.indent);
911	for (deaths, 0..) \|operand, i\| {	911	for (deaths, 0..) \|operand, i\| {
912	if (i != 0) try s.writeAll(" ");	912	if (i != 0) try s.writeAll(" ");
913	try s.print("%{d}!", .{@intFromEnum(operand)});	913	try s.print("{}!", .{operand});
914	}	914	}
915	try s.writeAll("\n");	915	try s.writeAll("\n");
916	}	916	}
...	@@ -994,7 +994,7 @@ const Writer = struct {	...	@@ -994,7 +994,7 @@ const Writer = struct {
994	dies: bool,	994	dies: bool,
995	) @TypeOf(s).Error!void {	995	) @TypeOf(s).Error!void {
996	_ = w;	996	_ = w;
997	try s.print("%{d}", .{@intFromEnum(inst)});	997	try s.print("{}", .{inst});
998	if (dies) try s.writeByte('!');	998	if (dies) try s.writeByte('!');
999	}	999	}
1000		1000

tools/lldb_pretty_printers.py+1-1

...	@@ -383,7 +383,7 @@ def InstRef_SummaryProvider(value, _=None):	...	@@ -383,7 +383,7 @@ def InstRef_SummaryProvider(value, _=None):
383	'InternPool.Index(%d)' % value.unsigned if value.unsigned < 0x80000000 else 'instructions[%d]' % (value.unsigned - 0x80000000))	383	'InternPool.Index(%d)' % value.unsigned if value.unsigned < 0x80000000 else 'instructions[%d]' % (value.unsigned - 0x80000000))
384		384
385	def InstIndex_SummaryProvider(value, _=None):	385	def InstIndex_SummaryProvider(value, _=None):
386	return 'instructions[%d]' % value.unsigned	386	return 'instructions[%d]' % value.unsigned if value.unsigned < 0x80000000 else 'temps[%d]' % (value.unsigned - 0x80000000)
387		387
388	class zig_DeclIndex_SynthProvider:	388	class zig_DeclIndex_SynthProvider:
389	def __init__(self, value, _=None): self.value = value	389	def __init__(self, value, _=None): self.value = value