Commit af1191ea8ba55a6eab53e0b561355bb116bdbf2d | zig

author	Jacob Young <jacobly@ziglang.org> 2024-11-25 02:58:30-05:00
committer	Jacob Young <jacobly@ziglang.org> 2025-01-16 20:42:07-05:00
log	af1191ea8ba55a6eab53e0b561355bb116bdbf2d
tree	0bd2977ad2f8e379dcac7fdec5ba58e294bb06f5
parent	257054a1467b2612725bd66852d84496024cf66c

x86_64: rewrite

12 files changed, 1885 insertions(+), 948 deletions(-)

src
- Air.zig
- Compilation.zig
- Liveness.zig
- arch
  - aarch64
    CodeGen.zig
  - arm
    CodeGen.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - x86_64
    CodeGen.zig
    abi.zig
    bits.zig
- print_air.zig
tools
- lldb_pretty_printers.py

src/Air.zig+30-6

...	...	@@ -893,14 +893,38 @@ pub const Inst = struct {
893	893	pub const Index = enum(u32) {
894	894	_,
895	895
896		pub fn toRef(i: Index) Inst.Ref {
897		assert(@intFromEnum(i) >> 31 == 0);
898		return @enumFromInt((1 << 31) \| @intFromEnum(i));
	896	pub fn unwrap(index: Index) union(enum) { ref: Inst.Ref, target: u31 } {
	897	const low_index: u31 = @truncate(@intFromEnum(index));
	898	return switch (@as(u1, @intCast(@intFromEnum(index) >> 31))) {
	899	0 => .{ .ref = @enumFromInt(@as(u32, 1 << 31) \| low_index) },
	900	1 => .{ .target = low_index },
	901	};
	902	}
	903
	904	pub fn toRef(index: Index) Inst.Ref {
	905	return index.unwrap().ref;
	906	}
	907
	908	pub fn fromTargetIndex(index: u31) Index {
	909	return @enumFromInt((1 << 31) \| @as(u32, index));
	910	}
	911
	912	pub fn toTargetIndex(index: Index) u31 {
	913	return index.unwrap().target;
899	914	}
900	915
901		pub fn toTargetIndex(i: Index) u31 {
902		assert(@intFromEnum(i) >> 31 == 1);
903		return @truncate(@intFromEnum(i));
	916	pub fn format(
	917	index: Index,
	918	comptime _: []const u8,
	919	_: std.fmt.FormatOptions,
	920	writer: anytype,
	921	) @TypeOf(writer).Error!void {
	922	try writer.writeByte('%');
	923	switch (index.unwrap()) {
	924	.ref => {},
	925	.target => try writer.writeByte('t'),
	926	}
	927	try writer.print("{d}", .{@as(u31, @truncate(@intFromEnum(index)))});
904	928	}
905	929	};
906	930

src/Compilation.zig+1

...	...	@@ -3067,6 +3067,7 @@ pub fn saveState(comp: *Compilation) !void {
3067	3067	// linker state
3068	3068	switch (lf.tag) {
3069	3069	.wasm => {
	3070	dev.check(link.File.Tag.wasm.devFeature());
3070	3071	const wasm = lf.cast(.wasm).?;
3071	3072	const is_obj = comp.config.output_mode == .Obj;
3072	3073	try bufs.ensureUnusedCapacity(85);

src/Liveness.zig-14

...	...	@@ -202,14 +202,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool
202	202	return (l.tomb_bits[usize_index] & mask) != 0;
203	203	}
204	204
205		pub fn clearOperandDeath(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) void {
206		assert(operand < bpi - 1);
207		const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
208		const mask = @as(usize, 1) <<
209		@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
210		l.tomb_bits[usize_index] &= ~mask;
211		}
212
213	205	const OperandCategory = enum {
214	206	/// The operand lives on, but this instruction cannot possibly mutate memory.
215	207	none,
...	...	@@ -844,12 +836,6 @@ const Analysis = struct {
844	836	special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
845	837	extra: std.ArrayListUnmanaged(u32),
846	838
847		fn storeTombBits(a: *Analysis, inst: Air.Inst.Index, tomb_bits: Bpi) void {
848		const usize_index = (inst * bpi) / @bitSizeOf(usize);
849		a.tomb_bits[usize_index] \|= @as(usize, tomb_bits) <<
850		@as(Log2Int(usize), @intCast((inst % (@bitSizeOf(usize) / bpi)) * bpi));
851		}
852
853	839	fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
854	840	const fields = std.meta.fields(@TypeOf(extra));
855	841	try a.extra.ensureUnusedCapacity(a.gpa, fields.len);

src/arch/aarch64/CodeGen.zig+10-10

...	...	@@ -71,6 +71,8 @@ end_di_column: u32,
71	71	/// which is a relative jump, based on the address following the reloc.
72	72	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
73	73
	74	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
	75
74	76	/// We postpone the creation of debug info for function args and locals
75	77	/// until after all Mir instructions have been generated. Only then we
76	78	/// will know saved_regs_stack_space which is necessary in order to
...	...	@@ -646,6 +648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
646	648	const old_air_bookkeeping = self.air_bookkeeping;
647	649	try self.ensureProcessDeathCapacity(Liveness.bpi);
648	650
	651	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
649	652	switch (air_tags[@intFromEnum(inst)]) {
650	653	// zig fmt: off
651	654	.add => try self.airBinOp(inst, .add),
...	...	@@ -927,16 +930,13 @@ fn finishAirBookkeeping(self: *Self) void {
927	930	}
928	931
929	932	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
930		var tomb_bits = self.liveness.getTombBits(inst);
931		for (operands) \|op\| {
932		const dies = @as(u1, @truncate(tomb_bits)) != 0;
933		tomb_bits >>= 1;
934		if (!dies) continue;
935		const op_index = op.toIndex() orelse continue;
936		self.processDeath(op_index);
	933	const tomb_bits = self.liveness.getTombBits(inst);
	934	for (0.., operands) \|op_index, op\| {
	935	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
	936	if (self.reused_operands.isSet(op_index)) continue;
	937	self.processDeath(op.toIndexAllowNone() orelse continue);
937	938	}
938		const is_used = @as(u1, @truncate(tomb_bits)) == 0;
939		if (is_used) {
	939	if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
940	940	log.debug("%{d} => {}", .{ inst, result });
941	941	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
942	942	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
...	...	@@ -3614,7 +3614,7 @@ fn reuseOperand(
3614	3614	}
3615	3615
3616	3616	// Prevent the operand deaths processing code from deallocating it.
3617		self.liveness.clearOperandDeath(inst, op_index);
	3617	self.reused_operands.set(op_index);
3618	3618
3619	3619	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
3620	3620	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/arm/CodeGen.zig+10-10

...	...	@@ -72,6 +72,8 @@ end_di_column: u32,
72	72	/// which is a relative jump, based on the address following the reloc.
73	73	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
74	74
	75	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
	76
75	77	/// We postpone the creation of debug info for function args and locals
76	78	/// until after all Mir instructions have been generated. Only then we
77	79	/// will know saved_regs_stack_space which is necessary in order to
...	...	@@ -635,6 +637,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
635	637	const old_air_bookkeeping = self.air_bookkeeping;
636	638	try self.ensureProcessDeathCapacity(Liveness.bpi);
637	639
	640	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
638	641	switch (air_tags[@intFromEnum(inst)]) {
639	642	// zig fmt: off
640	643	.add, => try self.airBinOp(inst, .add),
...	...	@@ -918,16 +921,13 @@ fn finishAirBookkeeping(self: *Self) void {
918	921	}
919	922
920	923	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
921		var tomb_bits = self.liveness.getTombBits(inst);
922		for (operands) \|op\| {
923		const dies = @as(u1, @truncate(tomb_bits)) != 0;
924		tomb_bits >>= 1;
925		if (!dies) continue;
926		const op_index = op.toIndex() orelse continue;
927		self.processDeath(op_index);
	924	const tomb_bits = self.liveness.getTombBits(inst);
	925	for (0.., operands) \|op_index, op\| {
	926	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
	927	if (self.reused_operands.isSet(op_index)) continue;
	928	self.processDeath(op.toIndexAllowNone() orelse continue);
928	929	}
929		const is_used = @as(u1, @truncate(tomb_bits)) == 0;
930		if (is_used) {
	930	if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
931	931	log.debug("%{d} => {}", .{ inst, result });
932	932	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
933	933	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
...	...	@@ -2650,7 +2650,7 @@ fn reuseOperand(
2650	2650	}
2651	2651
2652	2652	// Prevent the operand deaths processing code from deallocating it.
2653		self.liveness.clearOperandDeath(inst, op_index);
	2653	self.reused_operands.set(op_index);
2654	2654
2655	2655	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
2656	2656	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/riscv64/CodeGen.zig+11-7

...	...	@@ -82,6 +82,8 @@ scope_generation: u32,
82	82	/// which is a relative jump, based on the address following the reloc.
83	83	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
84	84
	85	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
	86
85	87	/// Whenever there is a runtime branch, we push a Branch onto this stack,
86	88	/// and pop it off when the runtime branch joins. This provides an "overlay"
87	89	/// of the table of mappings from instructions to `MCValue` from within the branch.
...	...	@@ -1443,8 +1445,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1443	1445	verbose_tracking_log.debug("{}", .{func.fmtTracking()});
1444	1446
1445	1447	const old_air_bookkeeping = func.air_bookkeeping;
	1448	try func.ensureProcessDeathCapacity(Liveness.bpi);
	1449
	1450	func.reused_operands = @TypeOf(func.reused_operands).initEmpty();
1446	1451	try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
1447		const tag: Air.Inst.Tag = air_tags[@intFromEnum(inst)];
	1452	const tag = air_tags[@intFromEnum(inst)];
1448	1453	switch (tag) {
1449	1454	// zig fmt: off
1450	1455	.add,
...	...	@@ -1783,11 +1788,10 @@ fn finishAir(
1783	1788	result: MCValue,
1784	1789	operands: [Liveness.bpi - 1]Air.Inst.Ref,
1785	1790	) !void {
1786		var tomb_bits = func.liveness.getTombBits(inst);
1787		for (operands) \|op\| {
1788		const dies = @as(u1, @truncate(tomb_bits)) != 0;
1789		tomb_bits >>= 1;
1790		if (!dies) continue;
	1791	const tomb_bits = func.liveness.getTombBits(inst);
	1792	for (0.., operands) \|op_index, op\| {
	1793	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
	1794	if (func.reused_operands.isSet(op_index)) continue;
1791	1795	try func.processDeath(op.toIndexAllowNone() orelse continue);
1792	1796	}
1793	1797	func.finishAirResult(inst, result);
...	...	@@ -4424,7 +4428,7 @@ fn reuseOperandAdvanced(
4424	4428	}
4425	4429
4426	4430	// Prevent the operand deaths processing code from deallocating it.
4427		func.liveness.clearOperandDeath(inst, op_index);
	4431	func.reused_operands.set(op_index);
4428	4432	const op_inst = operand.toIndex().?;
4429	4433	func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst);
4430	4434

src/arch/sparc64/CodeGen.zig+10-10

...	...	@@ -78,6 +78,8 @@ end_di_column: u32,
78	78	/// which is a relative jump, based on the address following the reloc.
79	79	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
80	80
	81	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
	82
81	83	/// Whenever there is a runtime branch, we push a Branch onto this stack,
82	84	/// and pop it off when the runtime branch joins. This provides an "overlay"
83	85	/// of the table of mappings from instructions to `MCValue` from within the branch.
...	...	@@ -493,6 +495,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
493	495	const old_air_bookkeeping = self.air_bookkeeping;
494	496	try self.ensureProcessDeathCapacity(Liveness.bpi);
495	497
	498	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
496	499	switch (air_tags[@intFromEnum(inst)]) {
497	500	// zig fmt: off
498	501	.ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
...	...	@@ -3523,16 +3526,13 @@ fn finishAirBookkeeping(self: *Self) void {
3523	3526	}
3524	3527
3525	3528	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
3526		var tomb_bits = self.liveness.getTombBits(inst);
3527		for (operands) \|op\| {
3528		const dies = @as(u1, @truncate(tomb_bits)) != 0;
3529		tomb_bits >>= 1;
3530		if (!dies) continue;
3531		const op_index = op.toIndex() orelse continue;
3532		self.processDeath(op_index);
	3529	const tomb_bits = self.liveness.getTombBits(inst);
	3530	for (0.., operands) \|op_index, op\| {
	3531	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
	3532	if (self.reused_operands.isSet(op_index)) continue;
	3533	self.processDeath(op.toIndexAllowNone() orelse continue);
3533	3534	}
3534		const is_used = @as(u1, @truncate(tomb_bits)) == 0;
3535		if (is_used) {
	3535	if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
3536	3536	log.debug("%{d} => {}", .{ inst, result });
3537	3537	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
3538	3538	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
...	...	@@ -4568,7 +4568,7 @@ fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_ind
4568	4568	}
4569	4569
4570	4570	// Prevent the operand deaths processing code from deallocating it.
4571		self.liveness.clearOperandDeath(inst, op_index);
	4571	self.reused_operands.set(op_index);
4572	4572
4573	4573	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
4574	4574	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/x86_64/CodeGen.zig+1765-874

...	...	@@ -1,41 +1,26 @@
1	1	const std = @import("std");
2		const build_options = @import("build_options");
3		const builtin = @import("builtin");
4	2	const assert = std.debug.assert;
5	3	const codegen = @import("../../codegen.zig");
6		const leb128 = std.leb;
7	4	const link = @import("../../link.zig");
8	5	const log = std.log.scoped(.codegen);
9	6	const tracking_log = std.log.scoped(.tracking);
10	7	const verbose_tracking_log = std.log.scoped(.verbose_tracking);
11	8	const wip_mir_log = std.log.scoped(.wip_mir);
12		const math = std.math;
13		const mem = std.mem;
14		const target_util = @import("../../target.zig");
15		const trace = @import("../../tracy.zig").trace;
16	9
17	10	const Air = @import("../../Air.zig");
18		const Allocator = mem.Allocator;
19		const CodeGenError = codegen.CodeGenError;
20		const Compilation = @import("../../Compilation.zig");
21		const ErrorMsg = Zcu.ErrorMsg;
	11	const Allocator = std.mem.Allocator;
22	12	const Emit = @import("Emit.zig");
23	13	const Liveness = @import("../../Liveness.zig");
24	14	const Lower = @import("Lower.zig");
25	15	const Mir = @import("Mir.zig");
26		const Package = @import("../../Package.zig");
27	16	const Zcu = @import("../../Zcu.zig");
	17	const Module = @import("../../Package/Module.zig");
28	18	const InternPool = @import("../../InternPool.zig");
29		const Alignment = InternPool.Alignment;
30		const Target = std.Target;
31	19	const Type = @import("../../Type.zig");
32	20	const Value = @import("../../Value.zig");
33		const Instruction = @import("encoder.zig").Instruction;
34	21
35	22	const abi = @import("abi.zig");
36	23	const bits = @import("bits.zig");
37		const errUnionErrorOffset = codegen.errUnionErrorOffset;
38		const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
39	24	const encoder = @import("encoder.zig");
40	25
41	26	const Condition = bits.Condition;
...	...	@@ -46,7 +31,7 @@ const RegisterManager = abi.RegisterManager;
46	31	const RegisterLock = RegisterManager.RegisterLock;
47	32	const FrameIndex = bits.FrameIndex;
48	33
49		const InnerError = CodeGenError \|\| error{OutOfRegisters};
	34	const InnerError = codegen.CodeGenError \|\| error{OutOfRegisters};
50	35
51	36	gpa: Allocator,
52	37	pt: Zcu.PerThread,
...	...	@@ -57,7 +42,7 @@ debug_output: link.File.DebugInfoOutput,
57	42	target: *const std.Target,
58	43	owner: Owner,
59	44	inline_func: InternPool.Index,
60		mod: *Package.Module,
	45	mod: *Module,
61	46	arg_index: u32,
62	47	args: []MCValue,
63	48	va_info: union {
...	...	@@ -89,6 +74,7 @@ end_di_column: u32,
89	74	/// which is a relative jump, based on the address following the reloc.
90	75	exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
91	76
	77	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
92	78	const_tracking: ConstTrackingMap = .{},
93	79	inst_tracking: InstTrackingMap = .{},
94	80
...	...	@@ -108,13 +94,11 @@ loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
108	94	/// The state to restore before branching.
109	95	state: State,
110	96	/// The branch target.
111		jmp_target: Mir.Inst.Index,
	97	target: Mir.Inst.Index,
112	98	}) = .{},
113	99
114		/// Debug field, used to find bugs in the compiler.
115		air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
116
117		const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
	100	next_temp_index: Temp.Index = @enumFromInt(0),
	101	temp_type: [Temp.Index.max]Type = undefined,
118	102
119	103	const Owner = union(enum) {
120	104	nav_index: InternPool.Nav.Index,
...	...	@@ -433,7 +417,7 @@ pub const MCValue = union(enum) {
433	417	.reserved_frame,
434	418	.lea_symbol,
435	419	=> unreachable,
436		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
	420	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
437	421	.base = .{ .reg = .ds },
438	422	.mod = .{ .rm = .{
439	423	.size = size,
...	...	@@ -484,8 +468,8 @@ pub const MCValue = union(enum) {
484	468	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{
485	469	@tagName(pl.eflags), @tagName(pl.reg),
486	470	}),
487		.load_symbol => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.sym_index, pl.off }),
488		.lea_symbol => \|pl\| try writer.print("{} + 0x{x}", .{ pl.sym_index, pl.off }),
	471	.load_symbol => \|pl\| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }),
	472	.lea_symbol => \|pl\| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }),
489	473	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
490	474	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),
491	475	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),
...	...	@@ -562,7 +546,7 @@ const InstTracking = struct {
562	546	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
563	547	else => unreachable,
564	548	}
565		tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
	549	tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long });
566	550	try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{});
567	551	}
568	552
...	...	@@ -605,7 +589,7 @@ const InstTracking = struct {
605	589	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
606	590	try function.freeValue(self.short);
607	591	self.reuseFrame();
608		tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
	592	tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });
609	593	}
610	594
611	595	fn verifyMaterialize(self: InstTracking, target: InstTracking) void {
...	...	@@ -678,14 +662,14 @@ const InstTracking = struct {
678	662	else => target.long,
679	663	} else target.long;
680	664	self.short = target.short;
681		tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });
	665	tracking_log.debug("{} => {} (materialize)", .{ inst, self.* });
682	666	}
683	667
684	668	fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
685	669	switch (self.short) {
686	670	.dead => \|die_generation\| if (die_generation >= scope_generation) {
687	671	self.reuseFrame();
688		tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });
	672	tracking_log.debug("{} => {} (resurrect)", .{ inst, self.* });
689	673	},
690	674	else => {},
691	675	}
...	...	@@ -695,7 +679,7 @@ const InstTracking = struct {
695	679	if (self.short == .dead) return;
696	680	try function.freeValue(self.short);
697	681	self.short = .{ .dead = function.scope_generation };
698		tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
	682	tracking_log.debug("{} => {} (death)", .{ inst, self.* });
699	683	}
700	684
701	685	fn reuse(
...	...	@@ -705,16 +689,13 @@ const InstTracking = struct {
705	689	old_inst: Air.Inst.Index,
706	690	) void {
707	691	self.short = .{ .dead = function.scope_generation };
708		if (new_inst) \|inst\|
709		tracking_log.debug("%{d} => {} (reuse %{d})", .{ inst, self.*, old_inst })
710		else
711		tracking_log.debug("tmp => {} (reuse %{d})", .{ self.*, old_inst });
	692	tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst });
712	693	}
713	694
714	695	fn liveOut(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
715	696	for (self.getRegs()) \|reg\| {
716	697	if (function.register_manager.isRegFree(reg)) {
717		tracking_log.debug("%{d} => {} (live-out)", .{ inst, self.* });
	698	tracking_log.debug("{} => {} (live-out)", .{ inst, self.* });
718	699	continue;
719	700	}
720	701
...	...	@@ -741,7 +722,7 @@ const InstTracking = struct {
741	722	// Perform side-effects of freeValue manually.
742	723	function.register_manager.freeReg(reg);
743	724
744		tracking_log.debug("%{d} => {} (live-out %{d})", .{ inst, self.*, tracked_inst });
	725	tracking_log.debug("{} => {} (live-out {})", .{ inst, self.*, tracked_inst });
745	726	}
746	727	}
747	728
...	...	@@ -759,10 +740,10 @@ const InstTracking = struct {
759	740	const FrameAlloc = struct {
760	741	abi_size: u31,
761	742	spill_pad: u3,
762		abi_align: Alignment,
	743	abi_align: InternPool.Alignment,
763	744	ref_count: u16,
764	745
765		fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
	746	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc {
766	747	return .{
767	748	.abi_size = @intCast(alloc_abi.size),
768	749	.spill_pad = alloc_abi.pad,
...	...	@@ -779,14 +760,14 @@ const FrameAlloc = struct {
779	760	fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {
780	761	const abi_size = ty.abiSize(zcu);
781	762	const spill_size = if (abi_size < 8)
782		math.ceilPowerOfTwoAssert(u64, abi_size)
	763	std.math.ceilPowerOfTwoAssert(u64, abi_size)
783	764	else
784	765	std.mem.alignForward(u64, abi_size, 8);
785	766	return init(.{
786	767	.size = spill_size,
787	768	.pad = @intCast(spill_size - abi_size),
788	769	.alignment = ty.abiAlignment(zcu).maxStrict(
789		Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
	770	InternPool.Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
790	771	),
791	772	});
792	773	}
...	...	@@ -819,7 +800,7 @@ pub fn generate(
819	800	liveness: Liveness,
820	801	code: *std.ArrayListUnmanaged(u8),
821	802	debug_output: link.File.DebugInfoOutput,
822		) CodeGenError!void {
	803	) codegen.CodeGenError!void {
823	804	const zcu = pt.zcu;
824	805	const comp = zcu.comp;
825	806	const gpa = zcu.gpa;
...	...	@@ -862,6 +843,11 @@ pub fn generate(
862	843	function.mir_instructions.deinit(gpa);
863	844	function.mir_extra.deinit(gpa);
864	845	}
	846	try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);
	847	for (0..Temp.Index.max) \|temp_index\| {
	848	const temp: Temp.Index = @enumFromInt(temp_index);
	849	function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), InstTracking.init(.none));
	850	}
865	851
866	852	wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});
867	853
...	...	@@ -891,9 +877,9 @@ pub fn generate(
891	877	}));
892	878	function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{
893	879	.size = Type.usize.abiSize(zcu),
894		.alignment = Alignment.min(
	880	.alignment = InternPool.Alignment.min(
895	881	call_info.stack_align,
896		Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
	882	InternPool.Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
897	883	),
898	884	}));
899	885	function.frame_allocs.set(
...	...	@@ -972,7 +958,7 @@ pub fn generateLazy(
972	958	lazy_sym: link.File.LazySymbol,
973	959	code: *std.ArrayListUnmanaged(u8),
974	960	debug_output: link.File.DebugInfoOutput,
975		) CodeGenError!void {
	961	) codegen.CodeGenError!void {
976	962	const comp = bin_file.comp;
977	963	const gpa = comp.gpa;
978	964	// This function is for generating global code, so we use the root module.
...	...	@@ -1169,14 +1155,14 @@ fn formatWipMir(
1169	1155	lower.mir.extraData(Mir.Imm64, mir_inst.data.ai.i).data.decode(),
1170	1156	}),
1171	1157	.pseudo_dbg_local_as => {
1172		const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
	1158	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1173	1159	.base = .{ .reloc = mir_inst.data.as.sym_index },
1174	1160	}) };
1175	1161	try writer.print(" {}, {}", .{ mir_inst.data.as.air_inst, mem_op.fmt(.m) });
1176	1162	},
1177	1163	.pseudo_dbg_local_aso => {
1178	1164	const sym_off = lower.mir.extraData(bits.SymbolOffset, mir_inst.data.ax.payload).data;
1179		const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
	1165	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1180	1166	.base = .{ .reloc = sym_off.sym_index },
1181	1167	.disp = sym_off.off,
1182	1168	}) };
...	...	@@ -1184,7 +1170,7 @@ fn formatWipMir(
1184	1170	},
1185	1171	.pseudo_dbg_local_aro => {
1186	1172	const air_off = lower.mir.extraData(Mir.AirOffset, mir_inst.data.rx.payload).data;
1187		const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
	1173	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1188	1174	.base = .{ .reg = mir_inst.data.rx.r1 },
1189	1175	.disp = air_off.off,
1190	1176	}) };
...	...	@@ -1192,14 +1178,14 @@ fn formatWipMir(
1192	1178	},
1193	1179	.pseudo_dbg_local_af => {
1194	1180	const frame_addr = lower.mir.extraData(bits.FrameAddr, mir_inst.data.ax.payload).data;
1195		const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{
	1181	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1196	1182	.base = .{ .frame = frame_addr.index },
1197	1183	.disp = frame_addr.off,
1198	1184	}) };
1199		try writer.print(" {}, {d}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
	1185	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
1200	1186	},
1201	1187	.pseudo_dbg_local_am => {
1202		const mem_op: Instruction.Operand = .{
	1188	const mem_op: encoder.Instruction.Operand = .{
1203	1189	.mem = lower.mir.extraData(Mir.Memory, mir_inst.data.ax.payload).data.decode(),
1204	1190	};
1205	1191	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
...	...	@@ -1221,7 +1207,7 @@ fn formatTracking(
1221	1207	writer: anytype,
1222	1208	) @TypeOf(writer).Error!void {
1223	1209	var it = data.self.inst_tracking.iterator();
1224		while (it.next()) \|entry\| try writer.print("\n%{d} = {}", .{ entry.key_ptr., entry.value_ptr. });
	1210	while (it.next()) \|entry\| try writer.print("\n{} = {}", .{ entry.key_ptr., entry.value_ptr. });
1225	1211	}
1226	1212	fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {
1227	1213	return .{ .data = .{ .self = self } };
...	...	@@ -1427,7 +1413,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
1427	1413	.i = @bitCast(s),
1428	1414	} },
1429	1415	}),
1430		.unsigned => \|u\| _ = if (math.cast(u32, u)) \|small\| try self.addInst(.{
	1416	.unsigned => \|u\| _ = if (std.math.cast(u32, u)) \|small\| try self.addInst(.{
1431	1417	.tag = .pseudo,
1432	1418	.ops = switch (tag) {
1433	1419	.dbg_local => .pseudo_dbg_local_ai_u,
...	...	@@ -1632,7 +1618,7 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2
1632	1618	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1633	1619	const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {
1634	1620	.signed => \|s\| .{ .ri_s, @bitCast(s) },
1635		.unsigned => \|u\| if (math.cast(u32, u)) \|small\|
	1621	.unsigned => \|u\| if (std.math.cast(u32, u)) \|small\|
1636	1622	.{ .ri_u, small }
1637	1623	else
1638	1624	.{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) },
...	...	@@ -1831,8 +1817,8 @@ fn asmRegisterMemoryImmediate(
1831	1817	imm: Immediate,
1832	1818	) !void {
1833	1819	if (switch (imm) {
1834		.signed => \|s\| if (math.cast(i16, s)) \|x\| @as(u16, @bitCast(x)) else null,
1835		.unsigned => \|u\| math.cast(u16, u),
	1820	.signed => \|s\| if (std.math.cast(i16, s)) \|x\| @as(u16, @bitCast(x)) else null,
	1821	.unsigned => \|u\| std.math.cast(u16, u),
1836	1822	.reloc => unreachable,
1837	1823	}) \|small_imm\| {
1838	1824	_ = try self.addInst(.{
...	...	@@ -1967,8 +1953,8 @@ fn gen(self: *Self) InnerError!void {
1967	1953	const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
1968	1954	if (cc != .naked) {
1969	1955	try self.asmRegister(.{ ._, .push }, .rbp);
1970		try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, Immediate.s(8));
1971		try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, Immediate.s(0));
	1956	try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, .s(8));
	1957	try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, .s(0));
1972	1958	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1973	1959	try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp);
1974	1960	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
...	...	@@ -2016,7 +2002,7 @@ fn gen(self: *Self) InnerError!void {
2016	2002	.{},
2017	2003	);
2018	2004
2019		try self.asmRegisterImmediate(.{ ._, .cmp }, .al, Immediate.u(info.fp_count));
	2005	try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count));
2020	2006	const skip_sse_reloc = try self.asmJccReloc(.na, undefined);
2021	2007
2022	2008	const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type });
...	...	@@ -2055,15 +2041,15 @@ fn gen(self: *Self) InnerError!void {
2055	2041	const backpatch_stack_dealloc = try self.asmPlaceholder();
2056	2042	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
2057	2043	try self.asmRegister(.{ ._, .pop }, .rbp);
2058		try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, Immediate.s(8));
	2044	try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, .s(8));
2059	2045	try self.asmOpOnly(.{ ._, .ret });
2060	2046
2061	2047	const frame_layout = try self.computeFrameLayout(cc);
2062		const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
	2048	const need_frame_align = frame_layout.stack_mask != std.math.maxInt(u32);
2063	2049	const need_stack_adjust = frame_layout.stack_adjust > 0;
2064	2050	const need_save_reg = frame_layout.save_reg_list.count() > 0;
2065	2051	if (need_frame_align) {
2066		const page_align = @as(u32, math.maxInt(u32)) << 12;
	2052	const page_align = @as(u32, std.math.maxInt(u32)) << 12;
2067	2053	self.mir_instructions.set(backpatch_frame_align, .{
2068	2054	.tag = .@"and",
2069	2055	.ops = .ri_s,
...	...	@@ -2170,23 +2156,18 @@ fn gen(self: *Self) InnerError!void {
2170	2156	});
2171	2157	}
2172	2158
2173		fn checkInvariantsAfterAirInst(self: *Self, inst: Air.Inst.Index, old_air_bookkeeping: @TypeOf(air_bookkeeping_init)) void {
	2159	fn checkInvariantsAfterAirInst(self: *Self) void {
2174	2160	assert(!self.register_manager.lockedRegsExist());
2175	2161
2176	2162	if (std.debug.runtime_safety) {
2177		if (self.air_bookkeeping < old_air_bookkeeping + 1) {
2178		std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, self.air.instructions.items(.tag)[@intFromEnum(inst)] });
2179		}
2180
2181		{ // check consistency of tracked registers
2182		var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
2183		while (it.next()) \|index\| {
2184		const tracked_inst = self.register_manager.registers[index];
2185		const tracking = self.getResolvedInstValue(tracked_inst);
2186		for (tracking.getRegs()) \|reg\| {
2187		if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
2188		} else unreachable; // tracked register not in use
2189		}
	2163	// check consistency of tracked registers
	2164	var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
	2165	while (it.next()) \|index\| {
	2166	const tracked_inst = self.register_manager.registers[index];
	2167	const tracking = self.getResolvedInstValue(tracked_inst);
	2168	for (tracking.getRegs()) \|reg\| {
	2169	if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
	2170	} else unreachable; // tracked register not in use
2190	2171	}
2191	2172	}
2192	2173	}
...	...	@@ -2202,6 +2183,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2202	2183	const zcu = pt.zcu;
2203	2184	const ip = &zcu.intern_pool;
2204	2185	const air_tags = self.air.instructions.items(.tag);
	2186	const air_datas = self.air.instructions.items(.data);
	2187	const use_old = self.target.ofmt == .coff;
2205	2188
2206	2189	self.arg_index = 0;
2207	2190	for (body) \|inst\| switch (air_tags[@intFromEnum(inst)]) {
...	...	@@ -2209,12 +2192,13 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2209	2192	wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2210	2193	verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2211	2194
2212		const old_air_bookkeeping = self.air_bookkeeping;
	2195	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
2213	2196	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
2214	2197
2215	2198	try self.airArg(inst);
2216	2199
2217		self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);
	2200	self.resetTemps();
	2201	self.checkInvariantsAfterAirInst();
2218	2202	},
2219	2203	else => break,
2220	2204	};
...	...	@@ -2226,7 +2210,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2226	2210	wip_mir_log.debug("{}", .{self.fmtAir(inst)});
2227	2211	verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2228	2212
2229		const old_air_bookkeeping = self.air_bookkeeping;
	2213	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
2230	2214	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
2231	2215	switch (air_tags[@intFromEnum(inst)]) {
2232	2216	// zig fmt: off
...	...	@@ -2260,7 +2244,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2260	2244	.sub_sat => try self.airSubSat(inst),
2261	2245	.mul_sat => try self.airMulSat(inst),
2262	2246	.shl_sat => try self.airShlSat(inst),
2263		.slice => try self.airSlice(inst),
2264	2247
2265	2248	.sin,
2266	2249	.cos,
...	...	@@ -2298,127 +2281,58 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2298	2281	.cmp_vector => try self.airCmpVector(inst),
2299	2282	.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
2300	2283
2301		.alloc => try self.airAlloc(inst),
2302		.ret_ptr => try self.airRetPtr(inst),
2303		.arg => try self.airDbgArg(inst),
2304		.assembly => try self.airAsm(inst),
2305		.bitcast => try self.airBitCast(inst),
2306		.block => try self.airBlock(inst),
2307		.br => try self.airBr(inst),
2308		.repeat => try self.airRepeat(inst),
2309		.switch_dispatch => try self.airSwitchDispatch(inst),
2310		.trap => try self.airTrap(),
2311		.breakpoint => try self.airBreakpoint(),
2312		.ret_addr => try self.airRetAddr(inst),
2313		.frame_addr => try self.airFrameAddress(inst),
2314		.cond_br => try self.airCondBr(inst),
2315		.fptrunc => try self.airFptrunc(inst),
2316		.fpext => try self.airFpext(inst),
2317		.intcast => try self.airIntCast(inst),
2318		.trunc => try self.airTrunc(inst),
2319		.int_from_bool => try self.airIntFromBool(inst),
2320		.is_non_null => try self.airIsNonNull(inst),
2321		.is_non_null_ptr => try self.airIsNonNullPtr(inst),
2322		.is_null => try self.airIsNull(inst),
2323		.is_null_ptr => try self.airIsNullPtr(inst),
2324		.is_non_err => try self.airIsNonErr(inst),
2325		.is_non_err_ptr => try self.airIsNonErrPtr(inst),
2326		.is_err => try self.airIsErr(inst),
2327		.is_err_ptr => try self.airIsErrPtr(inst),
2328		.load => try self.airLoad(inst),
2329		.loop => try self.airLoop(inst),
2330		.int_from_ptr => try self.airIntFromPtr(inst),
2331		.ret => try self.airRet(inst, false),
2332		.ret_safe => try self.airRet(inst, true),
2333		.ret_load => try self.airRetLoad(inst),
2334		.store => try self.airStore(inst, false),
2335		.store_safe => try self.airStore(inst, true),
2336		.struct_field_ptr=> try self.airStructFieldPtr(inst),
2337		.struct_field_val=> try self.airStructFieldVal(inst),
2338		.array_to_slice => try self.airArrayToSlice(inst),
2339		.float_from_int => try self.airFloatFromInt(inst),
2340		.int_from_float => try self.airIntFromFloat(inst),
2341		.cmpxchg_strong => try self.airCmpxchg(inst),
2342		.cmpxchg_weak => try self.airCmpxchg(inst),
2343		.atomic_rmw => try self.airAtomicRmw(inst),
2344		.atomic_load => try self.airAtomicLoad(inst),
2345		.memcpy => try self.airMemcpy(inst),
2346		.memset => try self.airMemset(inst, false),
2347		.memset_safe => try self.airMemset(inst, true),
2348		.set_union_tag => try self.airSetUnionTag(inst),
2349		.get_union_tag => try self.airGetUnionTag(inst),
2350		.clz => try self.airClz(inst),
2351		.ctz => try self.airCtz(inst),
2352		.popcount => try self.airPopCount(inst),
2353		.byte_swap => try self.airByteSwap(inst),
2354		.bit_reverse => try self.airBitReverse(inst),
2355		.tag_name => try self.airTagName(inst),
2356		.error_name => try self.airErrorName(inst),
2357		.splat => try self.airSplat(inst),
2358		.select => try self.airSelect(inst),
2359		.shuffle => try self.airShuffle(inst),
2360		.reduce => try self.airReduce(inst),
2361		.aggregate_init => try self.airAggregateInit(inst),
2362		.union_init => try self.airUnionInit(inst),
2363		.prefetch => try self.airPrefetch(inst),
2364		.mul_add => try self.airMulAdd(inst),
2365		.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
2366
2367		.@"try" => try self.airTry(inst),
2368		.try_cold => try self.airTry(inst), // TODO
2369		.try_ptr => try self.airTryPtr(inst),
2370		.try_ptr_cold => try self.airTryPtr(inst), // TODO
2371
2372		.dbg_stmt => try self.airDbgStmt(inst),
2373		.dbg_empty_stmt => try self.airDbgEmptyStmt(),
2374		.dbg_inline_block => try self.airDbgInlineBlock(inst),
2375		.dbg_var_ptr,
2376		.dbg_var_val,
2377		.dbg_arg_inline,
2378		=> try self.airDbgVar(inst),
2379
2380		.call => try self.airCall(inst, .auto),
2381		.call_always_tail => try self.airCall(inst, .always_tail),
2382		.call_never_tail => try self.airCall(inst, .never_tail),
2383		.call_never_inline => try self.airCall(inst, .never_inline),
	2284	.bitcast => try self.airBitCast(inst),
	2285	.fptrunc => try self.airFptrunc(inst),
	2286	.fpext => try self.airFpext(inst),
	2287	.intcast => try self.airIntCast(inst),
	2288	.trunc => try self.airTrunc(inst),
	2289	.is_non_null => try self.airIsNonNull(inst),
	2290	.is_null => try self.airIsNull(inst),
	2291	.is_non_err => try self.airIsNonErr(inst),
	2292	.is_err => try self.airIsErr(inst),
	2293	.load => try self.airLoad(inst),
	2294	.store => try self.airStore(inst, false),
	2295	.store_safe => try self.airStore(inst, true),
	2296	.struct_field_val => try self.airStructFieldVal(inst),
	2297	.float_from_int => try self.airFloatFromInt(inst),
	2298	.int_from_float => try self.airIntFromFloat(inst),
	2299	.cmpxchg_strong => try self.airCmpxchg(inst),
	2300	.cmpxchg_weak => try self.airCmpxchg(inst),
	2301	.atomic_rmw => try self.airAtomicRmw(inst),
	2302	.atomic_load => try self.airAtomicLoad(inst),
	2303	.memcpy => try self.airMemcpy(inst),
	2304	.memset => try self.airMemset(inst, false),
	2305	.memset_safe => try self.airMemset(inst, true),
	2306	.set_union_tag => try self.airSetUnionTag(inst),
	2307	.get_union_tag => try self.airGetUnionTag(inst),
	2308	.clz => try self.airClz(inst),
	2309	.ctz => try self.airCtz(inst),
	2310	.popcount => try self.airPopCount(inst),
	2311	.byte_swap => try self.airByteSwap(inst),
	2312	.bit_reverse => try self.airBitReverse(inst),
	2313	.tag_name => try self.airTagName(inst),
	2314	.error_name => try self.airErrorName(inst),
	2315	.splat => try self.airSplat(inst),
	2316	.select => try self.airSelect(inst),
	2317	.shuffle => try self.airShuffle(inst),
	2318	.reduce => try self.airReduce(inst),
	2319	.aggregate_init => try self.airAggregateInit(inst),
	2320	.union_init => try self.airUnionInit(inst),
	2321	.prefetch => try self.airPrefetch(inst),
	2322	.mul_add => try self.airMulAdd(inst),
2384	2323
2385	2324	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
2386	2325	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
2387	2326	.atomic_store_release => try self.airAtomicStore(inst, .release),
2388	2327	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
2389	2328
2390		.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
2391		.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
2392		.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
2393		.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
2394
2395		.field_parent_ptr => try self.airFieldParentPtr(inst),
2396
2397		.switch_br => try self.airSwitchBr(inst),
2398		.loop_switch_br => try self.airLoopSwitchBr(inst),
2399		.slice_ptr => try self.airSlicePtr(inst),
2400		.slice_len => try self.airSliceLen(inst),
2401
2402		.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
2403		.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
2404
2405	2329	.array_elem_val => try self.airArrayElemVal(inst),
2406	2330	.slice_elem_val => try self.airSliceElemVal(inst),
2407		.slice_elem_ptr => try self.airSliceElemPtr(inst),
2408	2331	.ptr_elem_val => try self.airPtrElemVal(inst),
2409		.ptr_elem_ptr => try self.airPtrElemPtr(inst),
2410
2411		.inferred_alloc, .inferred_alloc_comptime => unreachable,
2412		.unreach => self.finishAirBookkeeping(),
2413	2332
2414	2333	.optional_payload => try self.airOptionalPayload(inst),
2415		.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
2416		.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
2417	2334	.unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst),
2418	2335	.unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst),
2419		.unwrap_errunion_err_ptr => try self.airUnwrapErrUnionErrPtr(inst),
2420		.unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst),
2421		.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
2422	2336	.err_return_trace => try self.airErrReturnTrace(inst),
2423	2337	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
2424	2338	.save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst),
...	...	@@ -2426,7 +2340,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2426	2340	.wrap_optional => try self.airWrapOptional(inst),
2427	2341	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
2428	2342	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
	2343	// zig fmt: on
2429	2344
	2345	.add_safe,
	2346	.sub_safe,
	2347	.mul_safe,
	2348	=> return self.fail("TODO implement safety_checked_instructions", .{}),
2430	2349	.add_optimized,
2431	2350	.sub_optimized,
2432	2351	.mul_optimized,
...	...	@@ -2448,13 +2367,429 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2448	2367	.int_from_float_optimized,
2449	2368	=> return self.fail("TODO implement optimized float mode", .{}),
2450	2369
2451		.add_safe,
2452		.sub_safe,
2453		.mul_safe,
2454		=> return self.fail("TODO implement safety_checked_instructions", .{}),
	2370	.arg => try self.airDbgArg(inst),
	2371	.alloc => if (use_old) try self.airAlloc(inst) else {
	2372	var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{
	2373	.index = try self.allocMemPtr(inst),
	2374	} });
	2375	try slot.moveTo(inst, self);
	2376	},
	2377	.inferred_alloc => unreachable,
	2378	.inferred_alloc_comptime => unreachable,
	2379	.ret_ptr => if (use_old) try self.airRetPtr(inst) else {
	2380	var slot = switch (self.ret_mcv.long) {
	2381	else => unreachable,
	2382	.none => try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{
	2383	.index = try self.allocMemPtr(inst),
	2384	} }),
	2385	.load_frame => slot: {
	2386	var slot = try self.tempFromValue(self.typeOfIndex(inst), self.ret_mcv.long);
	2387	try slot.toOffset(self.ret_mcv.short.indirect.off, self);
	2388	break :slot slot;
	2389	},
	2390	};
	2391	try slot.moveTo(inst, self);
	2392	},
	2393	.assembly => try self.airAsm(inst),
	2394	.block => if (use_old) try self.airBlock(inst) else {
	2395	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
	2396	const extra = self.air.extraData(Air.Block, ty_pl.payload);
	2397	try self.asmPseudo(.pseudo_dbg_enter_block_none);
	2398	try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
	2399	try self.asmPseudo(.pseudo_dbg_leave_block_none);
	2400	},
	2401	.loop => if (use_old) try self.airLoop(inst) else {
	2402	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
	2403	const extra = self.air.extraData(Air.Block, ty_pl.payload);
	2404	self.scope_generation += 1;
	2405	try self.loops.putNoClobber(self.gpa, inst, .{
	2406	.state = try self.saveState(),
	2407	.target = @intCast(self.mir_instructions.len),
	2408	});
	2409	defer assert(self.loops.remove(inst));
	2410	try self.genBodyBlock(@ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
	2411	},
	2412	.repeat => if (use_old) try self.airRepeat(inst) else {
	2413	const repeat = air_datas[@intFromEnum(inst)].repeat;
	2414	const loop = self.loops.get(repeat.loop_inst).?;
	2415	try self.restoreState(loop.state, &.{}, .{
	2416	.emit_instructions = true,
	2417	.update_tracking = false,
	2418	.resurrect = false,
	2419	.close_scope = true,
	2420	});
	2421	_ = try self.asmJmpReloc(loop.target);
	2422	},
	2423	.br => try self.airBr(inst),
	2424	.trap => try self.asmOpOnly(.{ ._, .ud2 }),
	2425	.breakpoint => try self.asmOpOnly(.{ ._, .int3 }),
	2426	.ret_addr => if (use_old) try self.airRetAddr(inst) else {
	2427	var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .load_frame = .{
	2428	.index = .ret_addr,
	2429	} });
	2430	while (try slot.toAnyReg(self)) {}
	2431	try slot.moveTo(inst, self);
	2432	},
	2433	.frame_addr => if (use_old) try self.airFrameAddress(inst) else {
	2434	var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{
	2435	.index = .base_ptr,
	2436	} });
	2437	try slot.moveTo(inst, self);
	2438	},
	2439	.call => try self.airCall(inst, .auto),
	2440	.call_always_tail => try self.airCall(inst, .always_tail),
	2441	.call_never_tail => try self.airCall(inst, .never_tail),
	2442	.call_never_inline => try self.airCall(inst, .never_inline),
2455	2443
2456		.is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
	2444	.cond_br => try self.airCondBr(inst),
	2445	.switch_br => try self.airSwitchBr(inst),
	2446	.loop_switch_br => try self.airLoopSwitchBr(inst),
	2447	.switch_dispatch => try self.airSwitchDispatch(inst),
	2448	.@"try", .try_cold => try self.airTry(inst),
	2449	.try_ptr, .try_ptr_cold => try self.airTryPtr(inst),
	2450	.dbg_stmt => if (use_old) try self.airDbgStmt(inst) else {
	2451	const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt;
	2452	_ = try self.addInst(.{
	2453	.tag = .pseudo,
	2454	.ops = .pseudo_dbg_line_line_column,
	2455	.data = .{ .line_column = .{
	2456	.line = dbg_stmt.line,
	2457	.column = dbg_stmt.column,
	2458	} },
	2459	});
	2460	},
	2461	.dbg_empty_stmt => if (use_old) try self.airDbgEmptyStmt() else {
	2462	if (self.mir_instructions.len > 0) {
	2463	const prev_mir_op = &self.mir_instructions.items(.ops)[self.mir_instructions.len - 1];
	2464	if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column)
	2465	prev_mir_op.* = .pseudo_dbg_line_line_column;
	2466	}
	2467	try self.asmOpOnly(.{ ._, .nop });
	2468	},
	2469	.dbg_inline_block => if (use_old) try self.airDbgInlineBlock(inst) else {
	2470	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
	2471	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
	2472	const old_inline_func = self.inline_func;
	2473	defer self.inline_func = old_inline_func;
	2474	self.inline_func = extra.data.func;
	2475	_ = try self.addInst(.{
	2476	.tag = .pseudo,
	2477	.ops = .pseudo_dbg_enter_inline_func,
	2478	.data = .{ .func = extra.data.func },
	2479	});
	2480	try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
	2481	_ = try self.addInst(.{
	2482	.tag = .pseudo,
	2483	.ops = .pseudo_dbg_leave_inline_func,
	2484	.data = .{ .func = old_inline_func },
	2485	});
	2486	},
	2487	.dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try self.airDbgVar(inst) else {
	2488	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
	2489	var ops = try self.tempsFromOperands(inst, .{pl_op.operand});
	2490	try self.genLocalDebugInfo(inst, ops[0].tracking(self).short);
	2491	try ops[0].die(self);
	2492	},
	2493	.is_null_ptr => if (use_old) try self.airIsNullPtr(inst) else {
	2494	const un_op = air_datas[@intFromEnum(inst)].un_op;
	2495	const opt_ty = self.typeOf(un_op).childType(zcu);
	2496	const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
	2497	const opt_child_ty = opt_ty.optionalChild(zcu);
	2498	const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
	2499	var ops = try self.tempsFromOperands(inst, .{un_op});
	2500	if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self);
	2501	while (try ops[0].toLea(self)) {}
	2502	try self.asmMemoryImmediate(
	2503	.{ ._, .cmp },
	2504	try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)
	2505	.byte
	2506	else if (opt_child_ty.isSlice(zcu))
	2507	.qword
	2508	else
	2509	Memory.Size.fromSize(opt_child_abi_size)),
	2510	.u(0),
	2511	);
	2512	var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
	2513	try ops[0].die(self);
	2514	try is_null.moveTo(inst, self);
	2515	},
	2516	.is_non_null_ptr => if (use_old) try self.airIsNonNullPtr(inst) else {
	2517	const un_op = air_datas[@intFromEnum(inst)].un_op;
	2518	const opt_ty = self.typeOf(un_op).childType(zcu);
	2519	const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
	2520	const opt_child_ty = opt_ty.optionalChild(zcu);
	2521	const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
	2522	var ops = try self.tempsFromOperands(inst, .{un_op});
	2523	if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self);
	2524	while (try ops[0].toLea(self)) {}
	2525	try self.asmMemoryImmediate(
	2526	.{ ._, .cmp },
	2527	try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)
	2528	.byte
	2529	else if (opt_child_ty.isSlice(zcu))
	2530	.qword
	2531	else
	2532	Memory.Size.fromSize(opt_child_abi_size)),
	2533	.u(0),
	2534	);
	2535	var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
	2536	try ops[0].die(self);
	2537	try is_non_null.moveTo(inst, self);
	2538	},
	2539	.is_err_ptr => if (use_old) try self.airIsErrPtr(inst) else {
	2540	const un_op = air_datas[@intFromEnum(inst)].un_op;
	2541	const eu_ty = self.typeOf(un_op).childType(zcu);
	2542	const eu_err_ty = eu_ty.errorUnionSet(zcu);
	2543	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
	2544	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
	2545	var ops = try self.tempsFromOperands(inst, .{un_op});
	2546	try ops[0].toOffset(eu_err_off, self);
	2547	while (try ops[0].toLea(self)) {}
	2548	try self.asmMemoryImmediate(
	2549	.{ ._, .cmp },
	2550	try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),
	2551	.u(0),
	2552	);
	2553	var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
	2554	try ops[0].die(self);
	2555	try is_err.moveTo(inst, self);
	2556	},
	2557	.is_non_err_ptr => if (use_old) try self.airIsNonErrPtr(inst) else {
	2558	const un_op = air_datas[@intFromEnum(inst)].un_op;
	2559	const eu_ty = self.typeOf(un_op).childType(zcu);
	2560	const eu_err_ty = eu_ty.errorUnionSet(zcu);
	2561	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
	2562	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
	2563	var ops = try self.tempsFromOperands(inst, .{un_op});
	2564	try ops[0].toOffset(eu_err_off, self);
	2565	while (try ops[0].toLea(self)) {}
	2566	try self.asmMemoryImmediate(
	2567	.{ ._, .cmp },
	2568	try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),
	2569	.u(0),
	2570	);
	2571	var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
	2572	try ops[0].die(self);
	2573	try is_non_err.moveTo(inst, self);
	2574	},
	2575	.int_from_ptr => if (use_old) try self.airIntFromPtr(inst) else {
	2576	const un_op = air_datas[@intFromEnum(inst)].un_op;
	2577	var ops = try self.tempsFromOperands(inst, .{un_op});
	2578	try ops[0].toLimb(0, self);
	2579	try ops[0].moveTo(inst, self);
	2580	},
	2581	.int_from_bool => if (use_old) try self.airIntFromBool(inst) else {
	2582	const un_op = air_datas[@intFromEnum(inst)].un_op;
	2583	var ops = try self.tempsFromOperands(inst, .{un_op});
	2584	try ops[0].moveTo(inst, self);
	2585	},
	2586	.ret => try self.airRet(inst, false),
	2587	.ret_safe => try self.airRet(inst, true),
	2588	.ret_load => try self.airRetLoad(inst),
	2589	.unreach => {},
	2590	.optional_payload_ptr => if (use_old) try self.airOptionalPayloadPtr(inst) else {
	2591	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2592	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2593	try ops[0].moveTo(inst, self);
	2594	},
	2595	.optional_payload_ptr_set => if (use_old) try self.airOptionalPayloadPtrSet(inst) else {
	2596	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2597	const opt_ty = self.typeOf(ty_op.operand).childType(zcu);
	2598	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2599	if (!opt_ty.optionalReprIsPayload(zcu)) {
	2600	const opt_child_ty = opt_ty.optionalChild(zcu);
	2601	const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu));
	2602	try ops[0].toOffset(opt_child_abi_size, self);
	2603	var has_value = try self.tempFromValue(Type.bool, .{ .immediate = 1 });
	2604	try ops[0].store(&has_value, self);
	2605	try has_value.die(self);
	2606	try ops[0].toOffset(-opt_child_abi_size, self);
	2607	}
	2608	try ops[0].moveTo(inst, self);
	2609	},
	2610	.unwrap_errunion_payload_ptr => if (use_old) try self.airUnwrapErrUnionPayloadPtr(inst) else {
	2611	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2612	const eu_ty = self.typeOf(ty_op.operand).childType(zcu);
	2613	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
	2614	const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
	2615	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2616	try ops[0].toOffset(eu_pl_off, self);
	2617	try ops[0].moveTo(inst, self);
	2618	},
	2619	.unwrap_errunion_err_ptr => if (use_old) try self.airUnwrapErrUnionErrPtr(inst) else {
	2620	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2621	const eu_ty = self.typeOf(ty_op.operand).childType(zcu);
	2622	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
	2623	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
	2624	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2625	try ops[0].toOffset(eu_err_off, self);
	2626	var err = try ops[0].load(eu_ty.errorUnionSet(zcu), self);
	2627	try ops[0].die(self);
	2628	try err.moveTo(inst, self);
	2629	},
	2630	.errunion_payload_ptr_set => if (use_old) try self.airErrUnionPayloadPtrSet(inst) else {
	2631	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2632	const eu_ty = self.typeOf(ty_op.operand).childType(zcu);
	2633	const eu_err_ty = eu_ty.errorUnionSet(zcu);
	2634	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
	2635	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
	2636	const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
	2637	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2638	try ops[0].toOffset(eu_err_off, self);
	2639	var no_err = try self.tempFromValue(eu_err_ty, .{ .immediate = 0 });
	2640	try ops[0].store(&no_err, self);
	2641	try no_err.die(self);
	2642	try ops[0].toOffset(eu_pl_off - eu_err_off, self);
	2643	try ops[0].moveTo(inst, self);
	2644	},
	2645	.struct_field_ptr => if (use_old) try self.airStructFieldPtr(inst) else {
	2646	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
	2647	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
	2648	var ops = try self.tempsFromOperands(inst, .{extra.struct_operand});
	2649	try ops[0].toOffset(self.fieldOffset(self.typeOf(extra.struct_operand), self.typeOfIndex(inst), extra.field_index), self);
	2650	try ops[0].moveTo(inst, self);
	2651	},
	2652	.struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else {
	2653	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2654	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2655	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 0), self);
	2656	try ops[0].moveTo(inst, self);
	2657	},
	2658	.struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else {
	2659	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2660	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2661	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 1), self);
	2662	try ops[0].moveTo(inst, self);
	2663	},
	2664	.struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else {
	2665	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2666	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2667	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 2), self);
	2668	try ops[0].moveTo(inst, self);
	2669	},
	2670	.struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else {
	2671	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2672	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2673	try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 3), self);
	2674	try ops[0].moveTo(inst, self);
	2675	},
	2676	.slice => if (use_old) try self.airSlice(inst) else {
	2677	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
	2678	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	2679	var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
	2680	try ops[0].toPair(&ops[1], self);
	2681	try ops[0].moveTo(inst, self);
	2682	},
	2683	.slice_len => if (use_old) try self.airSliceLen(inst) else {
	2684	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2685	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2686	try ops[0].toLimb(1, self);
	2687	try ops[0].moveTo(inst, self);
	2688	},
	2689	.slice_ptr => if (use_old) try self.airSlicePtr(inst) else {
	2690	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2691	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2692	try ops[0].toLimb(0, self);
	2693	try ops[0].moveTo(inst, self);
	2694	},
	2695	.ptr_slice_len_ptr => if (use_old) try self.airPtrSliceLenPtr(inst) else {
	2696	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2697	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2698	try ops[0].toOffset(8, self);
	2699	try ops[0].moveTo(inst, self);
	2700	},
	2701	.ptr_slice_ptr_ptr => if (use_old) try self.airPtrSlicePtrPtr(inst) else {
	2702	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2703	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2704	try ops[0].toOffset(0, self);
	2705	try ops[0].moveTo(inst, self);
	2706	},
	2707	.slice_elem_ptr, .ptr_elem_ptr => \|tag\| if (use_old) switch (tag) {
	2708	else => unreachable,
	2709	.slice_elem_ptr => try self.airSliceElemPtr(inst),
	2710	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
	2711	} else {
	2712	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
	2713	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	2714	var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
	2715	switch (tag) {
	2716	else => unreachable,
	2717	.slice_elem_ptr => try ops[0].toLimb(0, self),
	2718	.ptr_elem_ptr => {},
	2719	}
	2720	const dst_ty = self.typeOfIndex(inst);
	2721	if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {
	2722	const elem_size = dst_ty.childType(zcu).abiSize(zcu);
	2723	if (elem_size == 0) break :zero_offset;
	2724	while (true) for (&ops) \|*op\| {
	2725	if (try op.toAnyReg(self)) break;
	2726	} else break;
	2727	const lhs_reg = ops[0].unwrap(self).temp.tracking(self).short.register.to64();
	2728	const rhs_reg = ops[1].unwrap(self).temp.tracking(self).short.register.to64();
	2729	if (!std.math.isPowerOfTwo(elem_size)) {
	2730	try self.spillEflagsIfOccupied();
	2731	try self.asmRegisterRegisterImmediate(
	2732	.{ .i_, .mul },
	2733	rhs_reg,
	2734	rhs_reg,
	2735	.u(elem_size),
	2736	);
	2737	try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
	2738	.base = .{ .reg = lhs_reg },
	2739	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
	2740	});
	2741	} else if (elem_size > 8) {
	2742	try self.spillEflagsIfOccupied();
	2743	try self.asmRegisterImmediate(
	2744	.{ ._l, .sh },
	2745	rhs_reg,
	2746	.u(std.math.log2_int(u64, elem_size)),
	2747	);
	2748	try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
	2749	.base = .{ .reg = lhs_reg },
	2750	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
	2751	});
	2752	} else try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
	2753	.base = .{ .reg = lhs_reg },
	2754	.mod = .{ .rm = .{
	2755	.size = .qword,
	2756	.index = rhs_reg,
	2757	.scale = .fromFactor(@intCast(elem_size)),
	2758	} },
	2759	});
	2760	}
	2761	try ops[1].die(self);
	2762	try ops[0].moveTo(inst, self);
	2763	},
	2764	.array_to_slice => if (use_old) try self.airArrayToSlice(inst) else {
	2765	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2766	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2767	var len = try self.tempFromValue(Type.usize, .{
	2768	.immediate = self.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu),
	2769	});
	2770	try ops[0].toPair(&len, self);
	2771	try ops[0].moveTo(inst, self);
	2772	},
2457	2773	.error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
	2774	.field_parent_ptr => if (use_old) try self.airFieldParentPtr(inst) else {
	2775	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
	2776	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
	2777	var ops = try self.tempsFromOperands(inst, .{extra.field_ptr});
	2778	try ops[0].toOffset(-self.fieldOffset(self.typeOfIndex(inst), self.typeOf(extra.field_ptr), extra.field_index), self);
	2779	try ops[0].moveTo(inst, self);
	2780	},
	2781
	2782	.is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
	2783
	2784	.wasm_memory_size => unreachable,
	2785	.wasm_memory_grow => unreachable,
	2786
	2787	.addrspace_cast => {
	2788	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
	2789	var ops = try self.tempsFromOperands(inst, .{ty_op.operand});
	2790	try ops[0].moveTo(inst, self);
	2791	},
	2792
2458	2793	.vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
2459	2794
2460	2795	.c_va_arg => try self.airVaArg(inst),
...	...	@@ -2462,15 +2797,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2462	2797	.c_va_end => try self.airVaEnd(inst),
2463	2798	.c_va_start => try self.airVaStart(inst),
2464	2799
2465		.wasm_memory_size => unreachable,
2466		.wasm_memory_grow => unreachable,
2467
2468	2800	.work_item_id => unreachable,
2469	2801	.work_group_size => unreachable,
2470	2802	.work_group_id => unreachable,
2471		// zig fmt: on
2472	2803	}
2473		self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);
	2804	self.resetTemps();
	2805	self.checkInvariantsAfterAirInst();
2474	2806	}
2475	2807	verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2476	2808	}
...	...	@@ -2530,7 +2862,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2530	2862	data_off += @intCast(tag_name_len + 1);
2531	2863	}
2532	2864
2533		try self.airTrap();
	2865	try self.asmOpOnly(.{ ._, .ud2 });
2534	2866
2535	2867	for (exitlude_jump_relocs) \|reloc\| self.performReloc(reloc);
2536	2868	try self.asmOpOnly(.{ ._, .ret });
...	...	@@ -2544,6 +2876,10 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2544	2876
2545	2877	fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {
2546	2878	for (value.getRegs()) \|reg\| try self.register_manager.getReg(reg, inst);
	2879	switch (value) {
	2880	else => {},
	2881	.eflags, .register_overflow => self.eflags_inst = inst,
	2882	}
2547	2883	}
2548	2884
2549	2885	fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
...	...	@@ -2577,26 +2913,18 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) !void {
2577	2913	try self.inst_tracking.getPtr(inst).?.die(self, inst);
2578	2914	}
2579	2915
2580		/// Called when there are no operands, and the instruction is always unreferenced.
2581		fn finishAirBookkeeping(self: *Self) void {
2582		if (std.debug.runtime_safety) {
2583		self.air_bookkeeping += 1;
2584		}
2585		}
2586
2587	2916	fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
2588	2917	if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {
2589	2918	.none, .dead, .unreach => {},
2590	2919	else => unreachable, // Why didn't the result die?
2591	2920	} else {
2592		tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
	2921	tracking_log.debug("{} => {} (birth)", .{ inst, result });
2593	2922	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
2594	2923	// In some cases, an operand may be reused as the result.
2595	2924	// If that operand died and was a register, it was freed by
2596	2925	// processDeath, so we have to "re-allocate" the register.
2597	2926	self.getValueIfFree(result, inst);
2598	2927	}
2599		self.finishAirBookkeeping();
2600	2928	}
2601	2929
2602	2930	fn finishAir(
...	...	@@ -2605,11 +2933,10 @@ fn finishAir(
2605	2933	result: MCValue,
2606	2934	operands: [Liveness.bpi - 1]Air.Inst.Ref,
2607	2935	) !void {
2608		var tomb_bits = self.liveness.getTombBits(inst);
2609		for (operands) \|op\| {
2610		const dies = @as(u1, @truncate(tomb_bits)) != 0;
2611		tomb_bits >>= 1;
2612		if (!dies) continue;
	2936	const tomb_bits = self.liveness.getTombBits(inst);
	2937	for (0.., operands) \|op_index, op\| {
	2938	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
	2939	if (self.reused_operands.isSet(op_index)) continue;
2613	2940	try self.processDeath(op.toIndexAllowNone() orelse continue);
2614	2941	}
2615	2942	self.finishAirResult(inst, result);
...	...	@@ -2657,7 +2984,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2657	2984	}
2658	2985	};
2659	2986	const sort_context = SortContext{ .frame_align = frame_align };
2660		mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
	2987	std.mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
2661	2988	}
2662	2989
2663	2990	const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)];
...	...	@@ -2697,13 +3024,13 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2697	3024	@intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]);
2698	3025
2699	3026	return .{
2700		.stack_mask = @as(u32, math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),
	3027	.stack_mask = @as(u32, std.math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),
2701	3028	.stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]),
2702	3029	.save_reg_list = save_reg_list,
2703	3030	};
2704	3031	}
2705	3032
2706		fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) Alignment {
	3033	fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) InternPool.Alignment {
2707	3034	const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;
2708	3035	return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));
2709	3036	}
...	...	@@ -2741,7 +3068,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
2741	3068	const ptr_ty = self.typeOfIndex(inst);
2742	3069	const val_ty = ptr_ty.childType(zcu);
2743	3070	return self.allocFrameIndex(FrameAlloc.init(.{
2744		.size = math.cast(u32, val_ty.abiSize(zcu)) orelse {
	3071	.size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse {
2745	3072	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});
2746	3073	},
2747	3074	.alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),
...	...	@@ -2759,7 +3086,7 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {
2759	3086	fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
2760	3087	const pt = self.pt;
2761	3088	const zcu = pt.zcu;
2762		const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse {
	3089	const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
2763	3090	return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(pt)});
2764	3091	};
2765	3092
...	...	@@ -2857,8 +3184,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2857	3184	}
2858	3185
2859	3186	if (opts.resurrect) for (
2860		self.inst_tracking.keys()[0..state.inst_tracking_len],
2861		self.inst_tracking.values()[0..state.inst_tracking_len],
	3187	self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len],
	3188	self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len],
2862	3189	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);
2863	3190	for (deaths) \|death\| try self.processDeath(death);
2864	3191
...	...	@@ -3067,7 +3394,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3067	3394	.{ .v_, .cvtps2ph },
3068	3395	dst_reg,
3069	3396	mat_src_reg.to128(),
3070		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	3397	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
3071	3398	);
3072	3399	},
3073	3400	else => unreachable,
...	...	@@ -3267,7 +3594,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3267	3594
3268	3595	const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu);
3269	3596	const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu);
3270		switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {
	3597	switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) {
3271	3598	.lt => {
3272	3599	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
3273	3600	else => break :result null,
...	...	@@ -3431,8 +3758,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3431	3758	};
3432	3759
3433	3760	const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
3434		math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
3435		math.divCeil(u32, src_storage_bits, 64) catch unreachable and
	3761	std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
	3762	std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and
3436	3763	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
3437	3764	const dst_mcv = try self.allocRegOrMem(inst, true);
3438	3765	try self.genCopy(min_ty, dst_mcv, src_mcv, .{});
...	...	@@ -3449,8 +3776,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3449	3776	break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
3450	3777	}
3451	3778
3452		const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
3453		const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
	3779	const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
	3780	const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
3454	3781
3455	3782	const high_mcv: MCValue = if (dst_mcv.isMemory())
3456	3783	dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
...	...	@@ -3570,7 +3897,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3570	3897	const dst_info = dst_elem_ty.intInfo(zcu);
3571	3898	const src_info = src_elem_ty.intInfo(zcu);
3572	3899
3573		const mask_val = try pt.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
	3900	const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
3574	3901
3575	3902	const splat_ty = try pt.vectorType(.{
3576	3903	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
...	...	@@ -3607,7 +3934,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3607	3934	.{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
3608	3935	registerAlias(temp_reg, dst_abi_size),
3609	3936	dst_alias,
3610		Immediate.u(1),
	3937	.u(1),
3611	3938	);
3612	3939	try self.asmRegisterRegisterRegister(
3613	3940	mir_tag,
...	...	@@ -3806,7 +4133,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3806	4133	try self.asmMemoryImmediate(
3807	4134	.{ ._, .mov },
3808	4135	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
3809		Immediate.u(0),
	4136	.u(0),
3810	4137	);
3811	4138
3812	4139	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	...	@@ -3920,11 +4247,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3920	4247	.mod = .{ .rm = .{ .size = .qword } },
3921	4248	},
3922	4249	);
3923		try self.asmRegisterImmediate(
3924		.{ ._, .sbb },
3925		dst_mcv.register_pair[1],
3926		Immediate.u(0),
3927		);
	4250	try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0));
3928	4251	try self.freeValue(
3929	4252	.{ .load_frame = .{ .index = signed_div_floor_state.frame_index } },
3930	4253	);
...	...	@@ -4068,7 +4391,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
4068	4391	break :cc .o;
4069	4392	} else cc: {
4070	4393	try self.genSetReg(limit_reg, ty, .{
4071		.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),
	4394	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),
4072	4395	}, .{});
4073	4396
4074	4397	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
...	...	@@ -4266,12 +4589,12 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4266	4589	mat_rhs_mcv.register_pair[1],
4267	4590	);
4268	4591
4269		try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
	4592	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
4270	4593	try self.asmRegister(.{ ._, .not }, tmp_reg);
4271		try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));
	4594	try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), .s(0));
4272	4595	try self.freeValue(overflow);
4273	4596	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
4274		try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));
	4597	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));
4275	4598	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);
4276	4599	break :result dst_mcv;
4277	4600	}
...	...	@@ -4321,7 +4644,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4321	4644	break :cc .o;
4322	4645	} else cc: {
4323	4646	try self.genSetReg(limit_reg, ty, .{
4324		.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),
	4647	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits),
4325	4648	}, .{});
4326	4649	break :cc .c;
4327	4650	};
...	...	@@ -4366,7 +4689,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4366	4689	};
4367	4690
4368	4691	const tuple_ty = self.typeOfIndex(inst);
4369		if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
	4692	if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
4370	4693	switch (partial_mcv) {
4371	4694	.register => \|reg\| {
4372	4695	self.eflags_inst = inst;
...	...	@@ -4444,7 +4767,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4444	4767	const cc = Condition.ne;
4445	4768
4446	4769	const tuple_ty = self.typeOfIndex(inst);
4447		if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
	4770	if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
4448	4771	switch (partial_mcv) {
4449	4772	.register => \|reg\| {
4450	4773	self.eflags_inst = inst;
...	...	@@ -4576,7 +4899,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4576	4899	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
4577	4900	const slow_inc = self.hasFeature(.slow_incdec);
4578	4901	const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
4579		const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;
	4902	const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
4580	4903
4581	4904	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
4582	4905	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
...	...	@@ -4618,7 +4941,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4618	4941	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
4619	4942
4620	4943	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4621		try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));
	4944	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
4622	4945	try self.asmMemoryRegister(.{ ._, .adc }, .{
4623	4946	.base = .{ .frame = dst_mcv.load_frame.index },
4624	4947	.mod = .{ .rm = .{
...	...	@@ -4642,7 +4965,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4642	4965	});
4643	4966	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
4644	4967
4645		try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));
	4968	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
4646	4969	try self.asmMemoryRegister(.{ ._, .adc }, .{
4647	4970	.base = .{ .frame = dst_mcv.load_frame.index },
4648	4971	.mod = .{ .rm = .{
...	...	@@ -4656,30 +4979,22 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4656	4979	try self.asmSetccRegister(.c, .ch);
4657	4980
4658	4981	if (slow_inc) {
4659		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));
4660		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));
	4982	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
	4983	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
4661	4984	} else {
4662	4985	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
4663	4986	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
4664	4987	}
4665		try self.asmRegisterImmediate(
4666		.{ ._, .cmp },
4667		temp_regs[3].to32(),
4668		Immediate.u(limb_len),
4669		);
	4988	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
4670	4989	_ = try self.asmJccReloc(.b, inner_loop);
4671	4990
4672	4991	try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx);
4673	4992	const overflow = try self.asmJccReloc(.nz, undefined);
4674	4993	const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4675		try self.asmRegisterImmediate(
4676		.{ ._, .cmp },
4677		temp_regs[2].to32(),
4678		Immediate.u(limb_len),
4679		);
	4994	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len));
4680	4995	const no_overflow = try self.asmJccReloc(.nb, undefined);
4681	4996	if (slow_inc) {
4682		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));
	4997	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
4683	4998	} else {
4684	4999	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
4685	5000	}
...	...	@@ -4691,7 +5006,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4691	5006	.scale = .@"8",
4692	5007	.disp = lhs_mcv.load_frame.off - 8,
4693	5008	} },
4694		}, Immediate.u(0));
	5009	}, .u(0));
4695	5010	_ = try self.asmJccReloc(.z, overflow_loop);
4696	5011	self.performReloc(overflow);
4697	5012	try self.asmMemoryImmediate(.{ ._, .mov }, .{
...	...	@@ -4701,20 +5016,16 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4701	5016	.disp = dst_mcv.load_frame.off +
4702	5017	@as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
4703	5018	} },
4704		}, Immediate.u(1));
	5019	}, .u(1));
4705	5020	self.performReloc(no_overflow);
4706	5021
4707	5022	self.performReloc(skip_inner);
4708	5023	if (slow_inc) {
4709		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
	5024	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
4710	5025	} else {
4711	5026	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
4712	5027	}
4713		try self.asmRegisterImmediate(
4714		.{ ._, .cmp },
4715		temp_regs[0].to32(),
4716		Immediate.u(limb_len),
4717		);
	5028	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
4718	5029	_ = try self.asmJccReloc(.b, outer_loop);
4719	5030
4720	5031	break :result dst_mcv;
...	...	@@ -4750,7 +5061,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4750	5061	try self.asmMemoryImmediate(
4751	5062	.{ ._, .cmp },
4752	5063	try overflow.mem(self, self.memSize(Type.c_int)),
4753		Immediate.s(0),
	5064	.s(0),
4754	5065	);
4755	5066	try self.genSetMem(
4756	5067	.{ .frame = dst_mcv.load_frame.index },
...	...	@@ -5038,7 +5349,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
5038	5349	try self.asmRegisterImmediate(
5039	5350	.{ ._r, .sa },
5040	5351	registerAlias(divisor, abi_size),
5041		Immediate.u(int_info.bits - 1),
	5352	.u(int_info.bits - 1),
5042	5353	);
5043	5354	try self.asmRegisterRegister(
5044	5355	.{ ._, .@"test" },
...	...	@@ -5217,8 +5528,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5217	5528	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|
5218	5529	self.register_manager.unlockReg(lock);
5219	5530
5220		const shift_imm =
5221		Immediate.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));
	5531	const shift_imm: Immediate =
	5532	.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));
5222	5533	if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
5223	5534	mir_tag,
5224	5535	registerAlias(dst_reg, abi_size),
...	...	@@ -5434,7 +5745,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5434	5745	break :result operand;
5435	5746	}
5436	5747
5437		const err_off = errUnionErrorOffset(payload_ty, zcu);
	5748	const err_off = codegen.errUnionErrorOffset(payload_ty, zcu);
5438	5749	switch (operand) {
5439	5750	.register => \|reg\| {
5440	5751	// TODO reuse operand
...	...	@@ -5492,7 +5803,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
5492	5803	const eu_ty = src_ty.childType(zcu);
5493	5804	const pl_ty = eu_ty.errorUnionPayload(zcu);
5494	5805	const err_ty = eu_ty.errorUnionSet(zcu);
5495		const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
	5806	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5496	5807	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
5497	5808	try self.asmRegisterMemory(
5498	5809	.{ ._, .mov },
...	...	@@ -5535,7 +5846,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5535	5846	const eu_ty = src_ty.childType(zcu);
5536	5847	const pl_ty = eu_ty.errorUnionPayload(zcu);
5537	5848	const err_ty = eu_ty.errorUnionSet(zcu);
5538		const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
	5849	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5539	5850	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
5540	5851	try self.asmMemoryImmediate(
5541	5852	.{ ._, .mov },
...	...	@@ -5546,7 +5857,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5546	5857	.disp = err_off,
5547	5858	} },
5548	5859	},
5549		Immediate.u(0),
	5860	.u(0),
5550	5861	);
5551	5862
5552	5863	if (self.liveness.isUnused(inst)) break :result .unreach;
...	...	@@ -5559,7 +5870,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5559	5870	const dst_lock = self.register_manager.lockReg(dst_reg);
5560	5871	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5561	5872
5562		const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
	5873	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
5563	5874	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
5564	5875	try self.asmRegisterMemory(
5565	5876	.{ ._, .lea },
...	...	@@ -5587,7 +5898,7 @@ fn genUnwrapErrUnionPayloadMir(
5587	5898	const result: MCValue = result: {
5588	5899	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
5589	5900
5590		const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
	5901	const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu));
5591	5902	switch (err_union) {
5592	5903	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
5593	5904	.index = frame_addr.index,
...	...	@@ -5636,7 +5947,7 @@ fn genUnwrapErrUnionPayloadPtrMir(
5636	5947	const payload_ty = err_union_ty.errorUnionPayload(zcu);
5637	5948
5638	5949	const result: MCValue = result: {
5639		const payload_off = errUnionPayloadOffset(payload_ty, zcu);
	5950	const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu);
5640	5951	const result_mcv: MCValue = if (maybe_inst) \|inst\|
5641	5952	try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)
5642	5953	else
...	...	@@ -5696,7 +6007,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
5696	6007	try self.asmRegisterImmediate(
5697	6008	.{ ._s, .bt },
5698	6009	opt_reg,
5699		Immediate.u(@as(u6, @intCast(pl_abi_size * 8))),
	6010	.u(@as(u6, @intCast(pl_abi_size * 8))),
5700	6011	);
5701	6012	},
5702	6013
...	...	@@ -5709,7 +6020,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
5709	6020	.disp = frame_addr.off + pl_abi_size,
5710	6021	} },
5711	6022	},
5712		Immediate.u(1),
	6023	.u(1),
5713	6024	),
5714	6025	}
5715	6026	}
...	...	@@ -5733,8 +6044,8 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
5733	6044	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
5734	6045
5735	6046	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
5736		const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
5737		const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
	6047	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
	6048	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5738	6049	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
5739	6050	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});
5740	6051	break :result .{ .load_frame = .{ .index = frame_index } };
...	...	@@ -5756,8 +6067,8 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5756	6067	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
5757	6068
5758	6069	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
5759		const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
5760		const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
	6070	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
	6071	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5761	6072	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
5762	6073	const operand = try self.resolveInst(ty_op.operand);
5763	6074	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});
...	...	@@ -5770,11 +6081,20 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
5770	6081	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5771	6082	const result = result: {
5772	6083	const src_mcv = try self.resolveInst(ty_op.operand);
5773		if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
	6084	const ptr_mcv: MCValue = switch (src_mcv) {
	6085	.register_pair => \|regs\| .{ .register = regs[0] },
	6086	else => src_mcv,
	6087	};
	6088	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
	6089	switch (src_mcv) {
	6090	.register_pair => \|regs\| try self.freeValue(.{ .register = regs[1] }),
	6091	else => {},
	6092	}
	6093	break :result ptr_mcv;
	6094	}
5774	6095
5775	6096	const dst_mcv = try self.allocRegOrMem(inst, true);
5776		const dst_ty = self.typeOfIndex(inst);
5777		try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
	6097	try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{});
5778	6098	break :result dst_mcv;
5779	6099	};
5780	6100	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -5782,23 +6102,28 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
5782	6102
5783	6103	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
5784	6104	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5785
5786		const result: MCValue = result: {
	6105	const result = result: {
5787	6106	const src_mcv = try self.resolveInst(ty_op.operand);
5788		switch (src_mcv) {
5789		.load_frame => \|frame_addr\| {
5790		const len_mcv: MCValue = .{ .load_frame = .{
5791		.index = frame_addr.index,
5792		.off = frame_addr.off + 8,
5793		} };
5794		if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
5795
5796		const dst_mcv = try self.allocRegOrMem(inst, true);
5797		try self.genCopy(Type.usize, dst_mcv, len_mcv, .{});
5798		break :result dst_mcv;
5799		},
	6107	const len_mcv: MCValue = switch (src_mcv) {
	6108	.register_pair => \|regs\| .{ .register = regs[1] },
	6109	.load_frame => \|frame_addr\| .{ .load_frame = .{
	6110	.index = frame_addr.index,
	6111	.off = frame_addr.off + 8,
	6112	} },
5800	6113	else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),
	6114	};
	6115	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
	6116	switch (src_mcv) {
	6117	.register_pair => \|regs\| try self.freeValue(.{ .register = regs[0] }),
	6118	.load_frame => {},
	6119	else => unreachable,
	6120	}
	6121	break :result len_mcv;
5801	6122	}
	6123
	6124	const dst_mcv = try self.allocRegOrMem(inst, true);
	6125	try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{});
	6126	break :result dst_mcv;
5802	6127	};
5803	6128	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5804	6129	}
...	...	@@ -6296,27 +6621,27 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6296	6621	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
6297	6622	const has_lzcnt = self.hasFeature(.lzcnt);
6298	6623	if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {
6299		const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
	6624	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6300	6625	const extra_bits = abi_size * 8 - src_bits;
6301	6626
6302	6627	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6303	6628	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
6304	6629	defer self.register_manager.unlockReg(index_lock);
6305	6630
6306		try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.u(limbs_len));
	6631	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len));
6307	6632	switch (extra_bits) {
6308	6633	1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
6309	6634	else => try self.asmRegisterImmediate(
6310	6635	.{ ._, .mov },
6311	6636	dst_reg.to32(),
6312		Immediate.s(@as(i32, extra_bits) - 1),
	6637	.s(@as(i32, extra_bits) - 1),
6313	6638	),
6314	6639	}
6315	6640	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6316	6641	try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());
6317	6642	const zero = try self.asmJccReloc(.z, undefined);
6318	6643	if (self.hasFeature(.slow_incdec)) {
6319		try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), Immediate.u(1));
	6644	try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1));
6320	6645	} else {
6321	6646	try self.asmRegister(.{ ._, .dec }, index_reg.to32());
6322	6647	}
...	...	@@ -6328,7 +6653,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6328	6653	.scale = .@"8",
6329	6654	.disp = src_mcv.load_frame.off,
6330	6655	} },
6331		}, Immediate.u(0));
	6656	}, .u(0));
6332	6657	_ = try self.asmJccReloc(.e, loop);
6333	6658	try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{
6334	6659	.base = .{ .frame = src_mcv.load_frame.index },
...	...	@@ -6340,9 +6665,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6340	6665	} },
6341	6666	});
6342	6667	self.performReloc(zero);
6343		try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
	6668	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
6344	6669	try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());
6345		try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(src_bits - 1));
	6670	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1));
6346	6671	try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());
6347	6672	break :result dst_mcv;
6348	6673	}
...	...	@@ -6404,7 +6729,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6404	6729
6405	6730	assert(src_bits <= 64);
6406	6731	const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
6407		if (math.isPowerOfTwo(src_bits)) {
	6732	if (std.math.isPowerOfTwo(src_bits)) {
6408	6733	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
6409	6734	.immediate = src_bits ^ (src_bits - 1),
6410	6735	});
...	...	@@ -6429,7 +6754,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6429	6754	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
6430	6755	} else {
6431	6756	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
6432		.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
	6757	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
6433	6758	});
6434	6759	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
6435	6760	defer self.register_manager.unlockReg(imm_lock);
...	...	@@ -6493,30 +6818,30 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6493	6818	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
6494	6819	const has_bmi = self.hasFeature(.bmi);
6495	6820	if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {
6496		const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
	6821	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6497	6822	const extra_bits = abi_size * 8 - src_bits;
6498	6823
6499	6824	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6500	6825	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
6501	6826	defer self.register_manager.unlockReg(index_lock);
6502	6827
6503		try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.s(-1));
	6828	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1));
6504	6829	switch (extra_bits) {
6505	6830	0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
6506	6831	1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),
6507	6832	else => try self.asmRegisterImmediate(
6508	6833	.{ ._, .mov },
6509	6834	dst_reg.to32(),
6510		Immediate.s(-@as(i32, extra_bits)),
	6835	.s(-@as(i32, extra_bits)),
6511	6836	),
6512	6837	}
6513	6838	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6514	6839	if (self.hasFeature(.slow_incdec)) {
6515		try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
	6840	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
6516	6841	} else {
6517	6842	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
6518	6843	}
6519		try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), Immediate.u(limbs_len));
	6844	try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len));
6520	6845	const zero = try self.asmJccReloc(.nb, undefined);
6521	6846	try self.asmMemoryImmediate(.{ ._, .cmp }, .{
6522	6847	.base = .{ .frame = src_mcv.load_frame.index },
...	...	@@ -6526,7 +6851,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6526	6851	.scale = .@"8",
6527	6852	.disp = src_mcv.load_frame.off,
6528	6853	} },
6529		}, Immediate.u(0));
	6854	}, .u(0));
6530	6855	_ = try self.asmJccReloc(.e, loop);
6531	6856	try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{
6532	6857	.base = .{ .frame = src_mcv.load_frame.index },
...	...	@@ -6538,7 +6863,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6538	6863	} },
6539	6864	});
6540	6865	self.performReloc(zero);
6541		try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
	6866	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
6542	6867	try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());
6543	6868	break :result dst_mcv;
6544	6869	}
...	...	@@ -6558,7 +6883,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6558	6883	.{ ._, .@"or" },
6559	6884	wide_ty,
6560	6885	tmp_mcv,
6561		.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
	6886	.{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
6562	6887	@intCast(src_bits) },
6563	6888	);
6564	6889	break :masked tmp_mcv;
...	...	@@ -6585,7 +6910,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6585	6910	.{ ._, .@"or" },
6586	6911	Type.u64,
6587	6912	dst_mcv,
6588		.{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(src_bits - 64) },
	6913	.{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) },
6589	6914	);
6590	6915	break :masked dst_mcv;
6591	6916	} else hi_mat_src_mcv;
...	...	@@ -6602,7 +6927,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6602	6927	const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
6603	6928	defer self.register_manager.unlockReg(width_lock);
6604	6929
6605		if (src_bits <= 8 or !math.isPowerOfTwo(src_bits)) {
	6930	if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) {
6606	6931	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
6607	6932	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
6608	6933	defer self.register_manager.unlockReg(wide_lock);
...	...	@@ -6701,11 +7026,11 @@ fn genPopCount(
6701	7026	},
6702	7027	);
6703	7028
6704		const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
6705		const imm_0_1 = Immediate.u(mask / 0b1_1);
6706		const imm_00_11 = Immediate.u(mask / 0b01_01);
6707		const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
6708		const imm_0000_0001 = Immediate.u(mask / 0b1111_1111);
	7029	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
	7030	const imm_0_1: Immediate = .u(mask / 0b1_1);
	7031	const imm_00_11: Immediate = .u(mask / 0b01_01);
	7032	const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
	7033	const imm_0000_0001: Immediate = .u(mask / 0b1111_1111);
6709	7034
6710	7035	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6711	7036	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...	...	@@ -6722,7 +7047,7 @@ fn genPopCount(
6722	7047	// dst = operand
6723	7048	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
6724	7049	// tmp = operand
6725		try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
	7050	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
6726	7051	// tmp = operand >> 1
6727	7052	if (src_abi_size > 4) {
6728	7053	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
...	...	@@ -6733,7 +7058,7 @@ fn genPopCount(
6733	7058	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)
6734	7059	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
6735	7060	// tmp = temp1
6736		try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
	7061	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
6737	7062	// dst = temp1 >> 2
6738	7063	if (src_abi_size > 4) {
6739	7064	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
...	...	@@ -6749,7 +7074,7 @@ fn genPopCount(
6749	7074	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
6750	7075	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
6751	7076	// dst = temp2
6752		try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));
	7077	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4));
6753	7078	// tmp = temp2 >> 4
6754	7079	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
6755	7080	// dst = temp2 + (temp2 >> 4)
...	...	@@ -6767,7 +7092,7 @@ fn genPopCount(
6767	7092	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
6768	7093	// dst = temp3 * 0x01...01
6769	7094	if (src_abi_size > 1) {
6770		try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));
	7095	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8));
6771	7096	}
6772	7097	// dst = (temp3 * 0x01...01) >> (bits - 8)
6773	7098	}
...	...	@@ -6847,7 +7172,7 @@ fn genByteSwap(
6847	7172	return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } };
6848	7173	},
6849	7174	else => {
6850		const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
	7175	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6851	7176
6852	7177	const temp_regs =
6853	7178	try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
...	...	@@ -6856,11 +7181,7 @@ fn genByteSwap(
6856	7181
6857	7182	const dst_mcv = try self.allocRegOrMem(inst, false);
6858	7183	try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
6859		try self.asmRegisterImmediate(
6860		.{ ._, .mov },
6861		temp_regs[1].to32(),
6862		Immediate.u(limbs_len - 1),
6863		);
	7184	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
6864	7185
6865	7186	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6866	7187	try self.asmRegisterMemory(
...	...	@@ -6912,8 +7233,8 @@ fn genByteSwap(
6912	7233	} },
6913	7234	}, temp_regs[2].to64());
6914	7235	if (self.hasFeature(.slow_incdec)) {
6915		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
6916		try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));
	7236	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
	7237	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
6917	7238	} else {
6918	7239	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
6919	7240	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
...	...	@@ -6994,10 +7315,10 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
6994	7315	else
6995	7316	undefined;
6996	7317
6997		const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);
6998		const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
6999		const imm_00_11 = Immediate.u(mask / 0b01_01);
7000		const imm_0_1 = Immediate.u(mask / 0b1_1);
	7318	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);
	7319	const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
	7320	const imm_00_11: Immediate = .u(mask / 0b01_01);
	7321	const imm_0_1: Immediate = .u(mask / 0b1_1);
7001	7322
7002	7323	for (dst_mcv.getRegs()) \|dst_reg\| {
7003	7324	const dst = registerAlias(dst_reg, limb_abi_size);
...	...	@@ -7005,7 +7326,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7005	7326	// dst = temp1 = bswap(operand)
7006	7327	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
7007	7328	// tmp = temp1
7008		try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));
	7329	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4));
7009	7330	// dst = temp1 >> 4
7010	7331	if (limb_abi_size > 4) {
7011	7332	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
...	...	@@ -7017,13 +7338,13 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7017	7338	}
7018	7339	// tmp = temp1 & 0x0F...0F
7019	7340	// dst = (temp1 >> 4) & 0x0F...0F
7020		try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));
	7341	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4));
7021	7342	// tmp = (temp1 & 0x0F...0F) << 4
7022	7343	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
7023	7344	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)
7024	7345	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
7025	7346	// tmp = temp2
7026		try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
	7347	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
7027	7348	// dst = temp2 >> 2
7028	7349	if (limb_abi_size > 4) {
7029	7350	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
...	...	@@ -7050,7 +7371,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7050	7371	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
7051	7372	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
7052	7373	// dst = temp3
7053		try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
	7374	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
7054	7375	// tmp = temp3 >> 1
7055	7376	if (limb_abi_size > 4) {
7056	7377	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
...	...	@@ -7337,7 +7658,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7337	7658	dst_alias,
7338	7659	dst_alias,
7339	7660	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7340		Immediate.u(@as(u5, @bitCast(mode))),
	7661	.u(@as(u5, @bitCast(mode))),
7341	7662	) else try self.asmRegisterRegisterRegisterImmediate(
7342	7663	mir_tag,
7343	7664	dst_alias,
...	...	@@ -7346,13 +7667,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7346	7667	src_mcv.getReg().?
7347	7668	else
7348	7669	try self.copyToTmpRegister(ty, src_mcv), abi_size),
7349		Immediate.u(@as(u5, @bitCast(mode))),
	7670	.u(@as(u5, @bitCast(mode))),
7350	7671	),
7351	7672	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
7352	7673	mir_tag,
7353	7674	dst_alias,
7354	7675	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7355		Immediate.u(@as(u5, @bitCast(mode))),
	7676	.u(@as(u5, @bitCast(mode))),
7356	7677	) else try self.asmRegisterRegisterImmediate(
7357	7678	mir_tag,
7358	7679	dst_alias,
...	...	@@ -7360,7 +7681,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7360	7681	src_mcv.getReg().?
7361	7682	else
7362	7683	try self.copyToTmpRegister(ty, src_mcv), abi_size),
7363		Immediate.u(@as(u5, @bitCast(mode))),
	7684	.u(@as(u5, @bitCast(mode))),
7364	7685	),
7365	7686	}
7366	7687	}
...	...	@@ -7433,7 +7754,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7433	7754	defer self.register_manager.unlockReg(tmp_lock);
7434	7755
7435	7756	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
7436		try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
	7757	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
7437	7758	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg);
7438	7759	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg);
7439	7760	try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg);
...	...	@@ -7443,7 +7764,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7443	7764	},
7444	7765	else => {
7445	7766	const abi_size: u31 = @intCast(ty.abiSize(zcu));
7446		const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;
	7767	const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
7447	7768
7448	7769	const tmp_regs =
7449	7770	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
...	...	@@ -7460,7 +7781,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7460	7781	try self.asmMemoryImmediate(
7461	7782	.{ ._, .cmp },
7462	7783	try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword),
7463		Immediate.u(0),
	7784	.u(0),
7464	7785	);
7465	7786	const positive = try self.asmJccReloc(.ns, undefined);
7466	7787
...	...	@@ -7469,7 +7790,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7469	7790
7470	7791	const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
7471	7792	try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32());
7472		try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), Immediate.u(1));
	7793	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1));
7473	7794	try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{
7474	7795	.base = .{ .frame = dst_mcv.load_frame.index },
7475	7796	.mod = .{ .rm = .{
...	...	@@ -7491,11 +7812,11 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7491	7812	}, tmp_regs[2].to64());
7492	7813
7493	7814	if (self.hasFeature(.slow_incdec)) {
7494		try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), Immediate.u(1));
	7815	try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1));
7495	7816	} else {
7496	7817	try self.asmRegister(.{ ._, .inc }, tmp_regs[0].to32());
7497	7818	}
7498		try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), Immediate.u(limb_len));
	7819	try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len));
7499	7820	_ = try self.asmJccReloc(.b, neg_loop);
7500	7821
7501	7822	self.performReloc(positive);
...	...	@@ -7620,7 +7941,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7620	7941	.{ .v_, .cvtps2ph },
7621	7942	dst_reg,
7622	7943	dst_reg,
7623		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	7944	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7624	7945	);
7625	7946	break :result dst_mcv;
7626	7947	},
...	...	@@ -7650,7 +7971,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7650	7971	.{ .v_, .cvtps2ph },
7651	7972	dst_reg,
7652	7973	dst_reg,
7653		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	7974	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7654	7975	);
7655	7976	break :result dst_mcv;
7656	7977	},
...	...	@@ -7675,7 +7996,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7675	7996	.{ .v_, .cvtps2ph },
7676	7997	dst_reg,
7677	7998	wide_reg,
7678		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	7999	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7679	8000	);
7680	8001	break :result dst_mcv;
7681	8002	},
...	...	@@ -7699,9 +8020,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7699	8020	else => unreachable,
7700	8021	},
7701	8022	else => unreachable,
7702		}) orelse return self.fail("TODO implement airSqrt for {}", .{
7703		ty.fmt(pt),
7704		});
	8023	}) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)});
7705	8024	switch (mir_tag[0]) {
7706	8025	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
7707	8026	mir_tag,
...	...	@@ -7805,7 +8124,7 @@ fn reuseOperandAdvanced(
7805	8124	}
7806	8125
7807	8126	// Prevent the operand deaths processing code from deallocating it.
7808		self.liveness.clearOperandDeath(inst, op_index);
	8127	self.reused_operands.set(op_index);
7809	8128	const op_inst = operand.toIndex().?;
7810	8129	self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst);
7811	8130
...	...	@@ -7890,7 +8209,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7890	8209	} },
7891	8210	});
7892	8211	try self.spillEflagsIfOccupied();
7893		try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
	8212	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(val_bit_off));
7894	8213	} else {
7895	8214	const tmp_reg =
7896	8215	registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size);
...	...	@@ -7913,12 +8232,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7913	8232	} },
7914	8233	});
7915	8234	try self.spillEflagsIfOccupied();
7916		try self.asmRegisterRegisterImmediate(
7917		.{ ._rd, .sh },
7918		dst_alias,
7919		tmp_reg,
7920		Immediate.u(val_bit_off),
7921		);
	8235	try self.asmRegisterRegisterImmediate(.{ ._rd, .sh }, dst_alias, tmp_reg, .u(val_bit_off));
7922	8236	}
7923	8237
7924	8238	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
...	...	@@ -8064,16 +8378,16 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8064	8378	} },
8065	8379	};
8066	8380
8067		const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
	8381	const part_mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
8068	8382	@intCast(part_bit_off);
8069		const part_mask_not = part_mask ^ (@as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));
	8383	const part_mask_not = part_mask ^ (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));
8070	8384	if (limb_abi_size <= 4) {
8071		try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
8072		} else if (math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {
8073		try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
	8385	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .u(part_mask_not));
	8386	} else if (std.math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {
	8387	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .s(small));
8074	8388	} else {
8075	8389	const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
8076		try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
	8390	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, .u(part_mask_not));
8077	8391	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
8078	8392	}
8079	8393
...	...	@@ -8209,25 +8523,14 @@ fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
8209	8523	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
8210	8524	}
8211	8525
8212		fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
	8526	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, field_index: u8) !void {
8213	8527	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8214		const result = try self.fieldPtr(inst, ty_op.operand, index);
	8528	const result = try self.fieldPtr(inst, ty_op.operand, field_index);
8215	8529	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8216	8530	}
8217	8531
8218		fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
8219		const pt = self.pt;
8220		const zcu = pt.zcu;
	8532	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {
8221	8533	const ptr_field_ty = self.typeOfIndex(inst);
8222		const ptr_container_ty = self.typeOf(operand);
8223		const container_ty = ptr_container_ty.childType(zcu);
8224
8225		const field_off: i32 = switch (container_ty.containerLayout(zcu)) {
8226		.auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)),
8227		.@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8228		(if (zcu.typeToStruct(container_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, index) else 0) -
8229		ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
8230		};
8231	8534
8232	8535	const src_mcv = try self.resolveInst(operand);
8233	8536	const dst_mcv = if (switch (src_mcv) {
...	...	@@ -8235,7 +8538,19 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
8235	8538	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),
8236	8539	else => false,
8237	8540	}) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv);
8238		return dst_mcv.offset(field_off);
	8541	return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index));
	8542	}
	8543
	8544	fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {
	8545	const pt = self.pt;
	8546	const zcu = pt.zcu;
	8547	const agg_ty = ptr_agg_ty.childType(zcu);
	8548	return switch (agg_ty.containerLayout(zcu)) {
	8549	.auto, .@"extern" => @intCast(agg_ty.structFieldOffset(field_index, zcu)),
	8550	.@"packed" => @divExact(@as(i32, ptr_agg_ty.ptrInfo(zcu).packed_offset.bit_offset) +
	8551	(if (zcu.typeToStruct(agg_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, field_index) else 0) -
	8552	ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
	8553	};
8239	8554	}
8240	8555
8241	8556	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -8476,7 +8791,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8476	8791	} },
8477	8792	});
8478	8793	try self.spillEflagsIfOccupied();
8479		try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
	8794	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(field_bit_off));
8480	8795	} else {
8481	8796	const tmp_reg = registerAlias(
8482	8797	try self.register_manager.allocReg(null, abi.RegisterClass.gp),
...	...	@@ -8509,7 +8824,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8509	8824	.{ ._rd, .sh },
8510	8825	dst_alias,
8511	8826	tmp_reg,
8512		Immediate.u(field_bit_off),
	8827	.u(field_bit_off),
8513	8828	);
8514	8829	}
8515	8830
...	...	@@ -8528,27 +8843,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8528	8843	}
8529	8844
8530	8845	fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
8531		const pt = self.pt;
8532		const zcu = pt.zcu;
8533	8846	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8534	8847	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
8535	8848
8536		const inst_ty = self.typeOfIndex(inst);
8537		const parent_ty = inst_ty.childType(zcu);
8538		const field_off: i32 = switch (parent_ty.containerLayout(zcu)) {
8539		.auto, .@"extern" => @intCast(parent_ty.structFieldOffset(extra.field_index, zcu)),
8540		.@"packed" => @divExact(@as(i32, inst_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8541		(if (zcu.typeToStruct(parent_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0) -
8542		self.typeOf(extra.field_ptr).ptrInfo(zcu).packed_offset.bit_offset, 8),
8543		};
8544
	8849	const ptr_agg_ty = self.typeOfIndex(inst);
8545	8850	const src_mcv = try self.resolveInst(extra.field_ptr);
8546	8851	const dst_mcv = if (src_mcv.isRegisterOffset() and
8547	8852	self.reuseOperand(inst, extra.field_ptr, 0, src_mcv))
8548	8853	src_mcv
8549	8854	else
8550		try self.copyToRegisterWithInstTracking(inst, inst_ty, src_mcv);
8551		const result = dst_mcv.offset(-field_off);
	8855	try self.copyToRegisterWithInstTracking(inst, ptr_agg_ty, src_mcv);
	8856	const result = dst_mcv.offset(-self.fieldOffset(ptr_agg_ty, self.typeOf(extra.field_ptr), extra.field_index));
8552	8857	return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
8553	8858	}
8554	8859
...	...	@@ -8613,7 +8918,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
8613	8918	};
8614	8919
8615	8920	if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {
8616		const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_bits);
	8921	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits);
8617	8922	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
8618	8923	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
8619	8924	}
...	...	@@ -8698,7 +9003,7 @@ fn genShiftBinOpMir(
8698	9003	try self.spillEflagsIfOccupied();
8699	9004
8700	9005	if (abi_size > 16) {
8701		const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
	9006	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
8702	9007	assert(shift_abi_size >= 1 and shift_abi_size <= 2);
8703	9008
8704	9009	const rcx_lock: ?RegisterLock = switch (rhs_mcv) {
...	...	@@ -8725,12 +9030,12 @@ fn genShiftBinOpMir(
8725	9030
8726	9031	switch (tag[0]) {
8727	9032	._l => {
8728		try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), Immediate.u(limbs_len - 1));
	9033	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
8729	9034	switch (rhs_mcv) {
8730	9035	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8731	9036	.{ ._, .mov },
8732	9037	temp_regs[0].to32(),
8733		Immediate.u(limbs_len - (shift_imm >> 6) - 1),
	9038	.u(limbs_len - (shift_imm >> 6) - 1),
8734	9039	),
8735	9040	else => {
8736	9041	try self.asmRegisterRegister(
...	...	@@ -8738,16 +9043,8 @@ fn genShiftBinOpMir(
8738	9043	temp_regs[2].to32(),
8739	9044	registerAlias(.rcx, shift_abi_size),
8740	9045	);
8741		try self.asmRegisterImmediate(
8742		.{ ._, .@"and" },
8743		.cl,
8744		Immediate.u(math.maxInt(u6)),
8745		);
8746		try self.asmRegisterImmediate(
8747		.{ ._r, .sh },
8748		temp_regs[2].to32(),
8749		Immediate.u(6),
8750		);
	9046	try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
	9047	try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[2].to32(), .u(6));
8751	9048	try self.asmRegisterRegister(
8752	9049	.{ ._, .mov },
8753	9050	temp_regs[0].to32(),
...	...	@@ -8767,7 +9064,7 @@ fn genShiftBinOpMir(
8767	9064	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8768	9065	.{ ._, .mov },
8769	9066	temp_regs[0].to32(),
8770		Immediate.u(shift_imm >> 6),
	9067	.u(shift_imm >> 6),
8771	9068	),
8772	9069	else => {
8773	9070	try self.asmRegisterRegister(
...	...	@@ -8775,16 +9072,8 @@ fn genShiftBinOpMir(
8775	9072	temp_regs[0].to32(),
8776	9073	registerAlias(.rcx, shift_abi_size),
8777	9074	);
8778		try self.asmRegisterImmediate(
8779		.{ ._, .@"and" },
8780		.cl,
8781		Immediate.u(math.maxInt(u6)),
8782		);
8783		try self.asmRegisterImmediate(
8784		.{ ._r, .sh },
8785		temp_regs[0].to32(),
8786		Immediate.u(6),
8787		);
	9075	try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
	9076	try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[0].to32(), .u(6));
8788	9077	},
8789	9078	}
8790	9079	},
...	...	@@ -8813,7 +9102,7 @@ fn genShiftBinOpMir(
8813	9102	try self.asmRegisterImmediate(
8814	9103	.{ ._, .cmp },
8815	9104	temp_regs[0].to32(),
8816		Immediate.u(limbs_len - 1),
	9105	.u(limbs_len - 1),
8817	9106	);
8818	9107	break :skip try self.asmJccReloc(.nb, undefined);
8819	9108	},
...	...	@@ -8843,7 +9132,7 @@ fn genShiftBinOpMir(
8843	9132	}, .sh },
8844	9133	temp_regs[2].to64(),
8845	9134	temp_regs[3].to64(),
8846		Immediate.u(shift_imm & math.maxInt(u6)),
	9135	.u(shift_imm & std.math.maxInt(u6)),
8847	9136	),
8848	9137	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
8849	9138	._l => ._ld,
...	...	@@ -8864,8 +9153,8 @@ fn genShiftBinOpMir(
8864	9153	switch (tag[0]) {
8865	9154	._l => {
8866	9155	if (slow_inc_dec) {
8867		try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));
8868		try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), Immediate.u(1));
	9156	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
	9157	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), .u(1));
8869	9158	} else {
8870	9159	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
8871	9160	try self.asmRegister(.{ ._, .dec }, temp_regs[0].to32());
...	...	@@ -8874,8 +9163,8 @@ fn genShiftBinOpMir(
8874	9163	},
8875	9164	._r => {
8876	9165	if (slow_inc_dec) {
8877		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));
8878		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
	9166	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
	9167	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
8879	9168	} else {
8880	9169	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
8881	9170	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
...	...	@@ -8883,7 +9172,7 @@ fn genShiftBinOpMir(
8883	9172	try self.asmRegisterImmediate(
8884	9173	.{ ._, .cmp },
8885	9174	temp_regs[0].to32(),
8886		Immediate.u(limbs_len - 1),
	9175	.u(limbs_len - 1),
8887	9176	);
8888	9177	_ = try self.asmJccReloc(.b, loop);
8889	9178	},
...	...	@@ -8898,7 +9187,7 @@ fn genShiftBinOpMir(
8898	9187	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8899	9188	tag,
8900	9189	temp_regs[2].to64(),
8901		Immediate.u(shift_imm & math.maxInt(u6)),
	9190	.u(shift_imm & std.math.maxInt(u6)),
8902	9191	),
8903	9192	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
8904	9193	}
...	...	@@ -8914,7 +9203,7 @@ fn genShiftBinOpMir(
8914	9203	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
8915	9204	tag,
8916	9205	temp_regs[2].to64(),
8917		Immediate.u(63),
	9206	.u(63),
8918	9207	);
8919	9208	if (switch (rhs_mcv) {
8920	9209	.immediate => \|shift_imm\| shift_imm >> 6 > 0,
...	...	@@ -8935,7 +9224,7 @@ fn genShiftBinOpMir(
8935	9224	try self.asmRegisterImmediate(
8936	9225	.{ ._, .cmp },
8937	9226	temp_regs[1].to32(),
8938		Immediate.u(limbs_len - 1),
	9227	.u(limbs_len - 1),
8939	9228	);
8940	9229	break :skip try self.asmJccReloc(.nb, undefined);
8941	9230	},
...	...	@@ -8945,12 +9234,12 @@ fn genShiftBinOpMir(
8945	9234	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
8946	9235	switch (tag[0]) {
8947	9236	._l => if (slow_inc_dec) {
8948		try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));
	9237	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
8949	9238	} else {
8950	9239	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
8951	9240	},
8952	9241	._r => if (slow_inc_dec) {
8953		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));
	9242	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
8954	9243	} else {
8955	9244	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
8956	9245	},
...	...	@@ -8972,14 +9261,14 @@ fn genShiftBinOpMir(
8972	9261	.scale = .@"8",
8973	9262	.disp = lhs_mcv.load_frame.off,
8974	9263	} },
8975		}, Immediate.u(0));
	9264	}, .u(0));
8976	9265	switch (tag[0]) {
8977	9266	._l => _ = try self.asmJccReloc(.nz, loop),
8978	9267	._r => {
8979	9268	try self.asmRegisterImmediate(
8980	9269	.{ ._, .cmp },
8981	9270	temp_regs[1].to32(),
8982		Immediate.u(limbs_len - 1),
	9271	.u(limbs_len - 1),
8983	9272	);
8984	9273	_ = try self.asmJccReloc(.b, loop);
8985	9274	},
...	...	@@ -9021,12 +9310,12 @@ fn genShiftBinOpMir(
9021	9310	info.double_tag,
9022	9311	lhs_regs[info.indices[1]],
9023	9312	lhs_regs[info.indices[0]],
9024		Immediate.u(shift_imm),
	9313	.u(shift_imm),
9025	9314	);
9026	9315	try self.asmRegisterImmediate(
9027	9316	tag,
9028	9317	lhs_regs[info.indices[0]],
9029		Immediate.u(shift_imm),
	9318	.u(shift_imm),
9030	9319	);
9031	9320	return;
9032	9321	} else {
...	...	@@ -9039,7 +9328,7 @@ fn genShiftBinOpMir(
9039	9328	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
9040	9329	tag,
9041	9330	lhs_regs[info.indices[0]],
9042		Immediate.u(63),
	9331	.u(63),
9043	9332	) else try self.asmRegisterRegister(
9044	9333	.{ ._, .xor },
9045	9334	lhs_regs[info.indices[0]],
...	...	@@ -9048,7 +9337,7 @@ fn genShiftBinOpMir(
9048	9337	if (shift_imm > 64) try self.asmRegisterImmediate(
9049	9338	tag,
9050	9339	lhs_regs[info.indices[1]],
9051		Immediate.u(shift_imm - 64),
	9340	.u(shift_imm - 64),
9052	9341	);
9053	9342	return;
9054	9343	},
...	...	@@ -9059,7 +9348,7 @@ fn genShiftBinOpMir(
9059	9348
9060	9349	if (tag[0] == ._r and tag[1] == .sa) {
9061	9350	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]);
9062		try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
	9351	try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
9063	9352	} else try self.asmRegisterRegister(
9064	9353	.{ ._, .xor },
9065	9354	tmp_reg.to32(),
...	...	@@ -9076,11 +9365,7 @@ fn genShiftBinOpMir(
9076	9365	lhs_regs[info.indices[0]],
9077	9366	registerAlias(shift_reg, 1),
9078	9367	);
9079		try self.asmRegisterImmediate(
9080		.{ ._, .cmp },
9081		registerAlias(shift_reg, 1),
9082		Immediate.u(64),
9083		);
	9368	try self.asmRegisterImmediate(.{ ._, .cmp }, registerAlias(shift_reg, 1), .u(64));
9084	9369	try self.asmCmovccRegisterRegister(
9085	9370	.ae,
9086	9371	lhs_regs[info.indices[1]],
...	...	@@ -9119,7 +9404,7 @@ fn genShiftBinOpMir(
9119	9404	} },
9120	9405	},
9121	9406	tmp_reg,
9122		Immediate.u(shift_imm),
	9407	.u(shift_imm),
9123	9408	);
9124	9409	try self.asmMemoryImmediate(
9125	9410	tag,
...	...	@@ -9130,7 +9415,7 @@ fn genShiftBinOpMir(
9130	9415	.disp = dst_frame_addr.off + info.indices[0] * 8,
9131	9416	} },
9132	9417	},
9133		Immediate.u(shift_imm),
	9418	.u(shift_imm),
9134	9419	);
9135	9420	return;
9136	9421	} else {
...	...	@@ -9149,7 +9434,7 @@ fn genShiftBinOpMir(
9149	9434	if (shift_imm > 64) try self.asmRegisterImmediate(
9150	9435	tag,
9151	9436	tmp_reg,
9152		Immediate.u(shift_imm - 64),
	9437	.u(shift_imm - 64),
9153	9438	);
9154	9439	try self.asmMemoryRegister(
9155	9440	.{ ._, .mov },
...	...	@@ -9171,7 +9456,7 @@ fn genShiftBinOpMir(
9171	9456	.disp = dst_frame_addr.off + info.indices[0] * 8,
9172	9457	} },
9173	9458	},
9174		Immediate.u(63),
	9459	.u(63),
9175	9460	) else {
9176	9461	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
9177	9462	try self.asmMemoryRegister(
...	...	@@ -9223,7 +9508,7 @@ fn genShiftBinOpMir(
9223	9508	);
9224	9509	if (tag[0] == ._r and tag[1] == .sa) {
9225	9510	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
9226		try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
	9511	try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
9227	9512	} else try self.asmRegisterRegister(
9228	9513	.{ ._, .xor },
9229	9514	tmp_reg.to32(),
...	...	@@ -9239,7 +9524,7 @@ fn genShiftBinOpMir(
9239	9524	try self.asmRegisterImmediate(
9240	9525	.{ ._, .cmp },
9241	9526	registerAlias(shift_reg, 1),
9242		Immediate.u(64),
	9527	.u(64),
9243	9528	);
9244	9529	try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg);
9245	9530	try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);
...	...	@@ -9277,7 +9562,7 @@ fn genShiftBinOpMir(
9277	9562	.immediate => \|shift_imm\| return self.asmRegisterImmediate(
9278	9563	tag,
9279	9564	registerAlias(lhs_reg, abi_size),
9280		Immediate.u(shift_imm),
	9565	.u(shift_imm),
9281	9566	),
9282	9567	.register => \|shift_reg\| return self.asmRegisterRegister(
9283	9568	tag,
...	...	@@ -9292,7 +9577,7 @@ fn genShiftBinOpMir(
9292	9577	.base = .{ .reg = .ds },
9293	9578	.mod = .{ .rm = .{
9294	9579	.size = Memory.Size.fromSize(abi_size),
9295		.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
	9580	.disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
9296	9581	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
9297	9582	@tagName(lhs_mcv),
9298	9583	@tagName(shift_mcv),
...	...	@@ -9316,11 +9601,7 @@ fn genShiftBinOpMir(
9316	9601	else => unreachable,
9317	9602	};
9318	9603	switch (shift_mcv) {
9319		.immediate => \|shift_imm\| return self.asmMemoryImmediate(
9320		tag,
9321		lhs_mem,
9322		Immediate.u(shift_imm),
9323		),
	9604	.immediate => \|shift_imm\| return self.asmMemoryImmediate(tag, lhs_mem, .u(shift_imm)),
9324	9605	.register => \|shift_reg\| return self.asmMemoryRegister(
9325	9606	tag,
9326	9607	lhs_mem,
...	...	@@ -9495,7 +9776,7 @@ fn genMulDivBinOp(
9495	9776	switch (tag) {
9496	9777	.mul, .mul_wrap => {
9497	9778	const slow_inc = self.hasFeature(.slow_incdec);
9498		const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;
	9779	const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;
9499	9780
9500	9781	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
9501	9782	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
...	...	@@ -9536,7 +9817,7 @@ fn genMulDivBinOp(
9536	9817	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
9537	9818
9538	9819	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
9539		try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));
	9820	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
9540	9821	try self.asmMemoryRegister(.{ ._, .adc }, .{
9541	9822	.base = .{ .frame = dst_mcv.load_frame.index },
9542	9823	.mod = .{ .rm = .{
...	...	@@ -9559,7 +9840,7 @@ fn genMulDivBinOp(
9559	9840	});
9560	9841	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
9561	9842
9562		try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));
	9843	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
9563	9844	try self.asmMemoryRegister(.{ ._, .adc }, .{
9564	9845	.base = .{ .frame = dst_mcv.load_frame.index },
9565	9846	.mod = .{ .rm = .{
...	...	@@ -9572,30 +9853,22 @@ fn genMulDivBinOp(
9572	9853	try self.asmSetccRegister(.c, .ch);
9573	9854
9574	9855	if (slow_inc) {
9575		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));
9576		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));
	9856	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
	9857	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
9577	9858	} else {
9578	9859	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
9579	9860	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
9580	9861	}
9581		try self.asmRegisterImmediate(
9582		.{ ._, .cmp },
9583		temp_regs[3].to32(),
9584		Immediate.u(limb_len),
9585		);
	9862	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
9586	9863	_ = try self.asmJccReloc(.b, inner_loop);
9587	9864
9588	9865	self.performReloc(skip_inner);
9589	9866	if (slow_inc) {
9590		try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));
	9867	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
9591	9868	} else {
9592	9869	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
9593	9870	}
9594		try self.asmRegisterImmediate(
9595		.{ ._, .cmp },
9596		temp_regs[0].to32(),
9597		Immediate.u(limb_len),
9598		);
	9871	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
9599	9872	_ = try self.asmJccReloc(.b, outer_loop);
9600	9873
9601	9874	return dst_mcv;
...	...	@@ -9911,7 +10184,7 @@ fn genBinOp(
9911	10184	dst_reg,
9912	10185	dst_reg,
9913	10186	try rhs_mcv.mem(self, .word),
9914		Immediate.u(1),
	10187	.u(1),
9915	10188	) else try self.asmRegisterRegisterRegister(
9916	10189	.{ .vp_, .unpcklwd },
9917	10190	dst_reg,
...	...	@@ -9969,7 +10242,7 @@ fn genBinOp(
9969	10242	.{ .v_, .cvtps2ph },
9970	10243	dst_reg,
9971	10244	dst_reg,
9972		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	10245	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
9973	10246	);
9974	10247	break :adjusted .{ .register = dst_reg };
9975	10248	},
...	...	@@ -10278,7 +10551,7 @@ fn genBinOp(
10278	10551	.lea_tlv,
10279	10552	.lea_frame,
10280	10553	=> true,
10281		.memory => \|addr\| math.cast(i32, @as(i64, @bitCast(addr))) == null,
	10554	.memory => \|addr\| std.math.cast(i32, @as(i64, @bitCast(addr))) == null,
10282	10555	else => false,
10283	10556	.register_pair,
10284	10557	.register_overflow,
...	...	@@ -10410,7 +10683,7 @@ fn genBinOp(
10410	10683	dst_reg,
10411	10684	dst_reg,
10412	10685	try src_mcv.mem(self, .word),
10413		Immediate.u(1),
	10686	.u(1),
10414	10687	) else try self.asmRegisterRegisterRegister(
10415	10688	.{ .vp_, .unpcklwd },
10416	10689	dst_reg,
...	...	@@ -10442,7 +10715,7 @@ fn genBinOp(
10442	10715	dst_reg,
10443	10716	dst_reg,
10444	10717	dst_reg,
10445		Immediate.u(@as(u5, @bitCast(RoundMode{
	10718	.u(@as(u5, @bitCast(RoundMode{
10446	10719	.mode = switch (air_tag) {
10447	10720	.div_trunc => .zero,
10448	10721	.div_floor => .down,
...	...	@@ -10457,7 +10730,7 @@ fn genBinOp(
10457	10730	.{ .v_, .cvtps2ph },
10458	10731	dst_reg,
10459	10732	dst_reg,
10460		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	10733	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10461	10734	);
10462	10735	return dst_mcv;
10463	10736	},
...	...	@@ -10856,7 +11129,7 @@ fn genBinOp(
10856	11129	dst_reg,
10857	11130	dst_reg,
10858	11131	try src_mcv.mem(self, .word),
10859		Immediate.u(1),
	11132	.u(1),
10860	11133	) else try self.asmRegisterRegisterRegister(
10861	11134	.{ .vp_, .unpcklwd },
10862	11135	dst_reg,
...	...	@@ -10886,7 +11159,7 @@ fn genBinOp(
10886	11159	.{ .v_, .cvtps2ph },
10887	11160	dst_reg,
10888	11161	dst_reg,
10889		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	11162	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10890	11163	);
10891	11164	return dst_mcv;
10892	11165	},
...	...	@@ -10902,7 +11175,7 @@ fn genBinOp(
10902	11175	.{ .vp_d, .insr },
10903	11176	dst_reg,
10904	11177	try src_mcv.mem(self, .dword),
10905		Immediate.u(1),
	11178	.u(1),
10906	11179	) else try self.asmRegisterRegisterRegister(
10907	11180	.{ .v_ps, .unpckl },
10908	11181	dst_reg,
...	...	@@ -10937,7 +11210,7 @@ fn genBinOp(
10937	11210	.{ .v_, .cvtps2ph },
10938	11211	dst_reg,
10939	11212	dst_reg,
10940		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	11213	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10941	11214	);
10942	11215	return dst_mcv;
10943	11216	},
...	...	@@ -10980,7 +11253,7 @@ fn genBinOp(
10980	11253	.{ .v_, .cvtps2ph },
10981	11254	dst_reg,
10982	11255	dst_reg,
10983		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	11256	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10984	11257	);
10985	11258	return dst_mcv;
10986	11259	},
...	...	@@ -11023,7 +11296,7 @@ fn genBinOp(
11023	11296	.{ .v_, .cvtps2ph },
11024	11297	dst_reg,
11025	11298	dst_reg.to256(),
11026		Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
	11299	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
11027	11300	);
11028	11301	return dst_mcv;
11029	11302	},
...	...	@@ -11191,7 +11464,7 @@ fn genBinOp(
11191	11464	);
11192	11465	},
11193	11466	.cmp => {
11194		const imm = Immediate.u(switch (air_tag) {
	11467	const imm: Immediate = .u(switch (air_tag) {
11195	11468	.cmp_eq => 0,
11196	11469	.cmp_lt, .cmp_gt => 1,
11197	11470	.cmp_lte, .cmp_gte => 2,
...	...	@@ -11289,7 +11562,7 @@ fn genBinOp(
11289	11562	mask_reg,
11290	11563	rhs_copy_reg,
11291	11564	rhs_copy_reg,
11292		Immediate.u(3), // unord
	11565	.u(3), // unord
11293	11566	);
11294	11567	try self.asmRegisterRegisterRegisterRegister(
11295	11568	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
...	...	@@ -11356,7 +11629,7 @@ fn genBinOp(
11356	11629	}),
11357	11630	mask_reg,
11358	11631	mask_reg,
11359		Immediate.u(if (has_blend) 3 else 7), // unord, ord
	11632	.u(if (has_blend) 3 else 7), // unord, ord
11360	11633	);
11361	11634	if (has_blend) try self.asmRegisterRegisterRegister(
11362	11635	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
...	...	@@ -11567,29 +11840,29 @@ fn genBinOpMir(
11567	11840	8 => try self.asmRegisterImmediate(
11568	11841	mir_limb_tag,
11569	11842	dst_alias,
11570		if (math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
11571		Immediate.s(small)
	11843	if (std.math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
	11844	.s(small)
11572	11845	else
11573		Immediate.u(@as(u8, @intCast(imm))),
	11846	.u(@as(u8, @intCast(imm))),
11574	11847	),
11575	11848	16 => try self.asmRegisterImmediate(
11576	11849	mir_limb_tag,
11577	11850	dst_alias,
11578		if (math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
11579		Immediate.s(small)
	11851	if (std.math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
	11852	.s(small)
11580	11853	else
11581		Immediate.u(@as(u16, @intCast(imm))),
	11854	.u(@as(u16, @intCast(imm))),
11582	11855	),
11583	11856	32 => try self.asmRegisterImmediate(
11584	11857	mir_limb_tag,
11585	11858	dst_alias,
11586		if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11587		Immediate.s(small)
	11859	if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
	11860	.s(small)
11588	11861	else
11589		Immediate.u(@as(u32, @intCast(imm))),
	11862	.u(@as(u32, @intCast(imm))),
11590	11863	),
11591		64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11592		try self.asmRegisterImmediate(mir_limb_tag, dst_alias, Immediate.s(small))
	11864	64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
	11865	try self.asmRegisterImmediate(mir_limb_tag, dst_alias, .s(small))
11593	11866	else
11594	11867	try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias(
11595	11868	try self.copyToTmpRegister(ty, src_mcv),
...	...	@@ -11619,7 +11892,7 @@ fn genBinOpMir(
11619	11892	.base = .{ .reg = .ds },
11620	11893	.mod = .{ .rm = .{
11621	11894	.size = Memory.Size.fromSize(limb_abi_size),
11622		.disp = math.cast(i32, addr + off) orelse break :direct,
	11895	.disp = std.math.cast(i32, addr + off) orelse break :direct,
11623	11896	} },
11624	11897	},
11625	11898	.indirect => \|reg_off\| .{
...	...	@@ -11731,8 +12004,8 @@ fn genBinOpMir(
11731	12004	=> null,
11732	12005	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
11733	12006	switch (resolved_src_mcv) {
11734		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr))) != null and
11735		math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)
	12007	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr))) != null and
	12008	std.math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)
11736	12009	break :src null,
11737	12010	.load_symbol, .load_got, .load_direct, .load_tlv => {},
11738	12011	else => unreachable,
...	...	@@ -11823,33 +12096,29 @@ fn genBinOpMir(
11823	12096	8 => try self.asmMemoryImmediate(
11824	12097	mir_limb_tag,
11825	12098	dst_limb_mem,
11826		if (math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
11827		Immediate.s(small)
	12099	if (std.math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
	12100	.s(small)
11828	12101	else
11829		Immediate.u(@as(u8, @intCast(imm))),
	12102	.u(@as(u8, @intCast(imm))),
11830	12103	),
11831	12104	16 => try self.asmMemoryImmediate(
11832	12105	mir_limb_tag,
11833	12106	dst_limb_mem,
11834		if (math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
11835		Immediate.s(small)
	12107	if (std.math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
	12108	.s(small)
11836	12109	else
11837		Immediate.u(@as(u16, @intCast(imm))),
	12110	.u(@as(u16, @intCast(imm))),
11838	12111	),
11839	12112	32 => try self.asmMemoryImmediate(
11840	12113	mir_limb_tag,
11841	12114	dst_limb_mem,
11842		if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11843		Immediate.s(small)
	12115	if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
	12116	.s(small)
11844	12117	else
11845		Immediate.u(@as(u32, @intCast(imm))),
	12118	.u(@as(u32, @intCast(imm))),
11846	12119	),
11847		64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11848		try self.asmMemoryImmediate(
11849		mir_limb_tag,
11850		dst_limb_mem,
11851		Immediate.s(small),
11852		)
	12120	64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
	12121	try self.asmMemoryImmediate(mir_limb_tag, dst_limb_mem, .s(small))
11853	12122	else
11854	12123	try self.asmMemoryRegister(
11855	12124	mir_limb_tag,
...	...	@@ -11973,12 +12242,12 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
11973	12242	registerAlias(src_reg, abi_size),
11974	12243	),
11975	12244	.immediate => \|imm\| {
11976		if (math.cast(i32, imm)) \|small\| {
	12245	if (std.math.cast(i32, imm)) \|small\| {
11977	12246	try self.asmRegisterRegisterImmediate(
11978	12247	.{ .i_, .mul },
11979	12248	dst_alias,
11980	12249	dst_alias,
11981		Immediate.s(small),
	12250	.s(small),
11982	12251	);
11983	12252	} else {
11984	12253	const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
...	...	@@ -12009,7 +12278,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
12009	12278	.base = .{ .reg = .ds },
12010	12279	.mod = .{ .rm = .{
12011	12280	.size = Memory.Size.fromSize(abi_size),
12012		.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
	12281	.disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
12013	12282	return self.asmRegisterRegister(
12014	12283	.{ .i_, .mul },
12015	12284	dst_alias,
...	...	@@ -12087,11 +12356,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12087	12356	) \|dst_reg, elem_index\| {
12088	12357	assert(self.register_manager.isRegFree(dst_reg));
12089	12358	if (elem_index > 0) {
12090		try self.asmRegisterImmediate(
12091		.{ ._l, .sh },
12092		dst_reg.to8(),
12093		Immediate.u(elem_index),
12094		);
	12359	try self.asmRegisterImmediate(.{ ._l, .sh }, dst_reg.to8(), .u(elem_index));
12095	12360	try self.asmRegisterRegister(
12096	12361	.{ ._, .@"or" },
12097	12362	dst_reg.to8(),
...	...	@@ -12127,7 +12392,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12127	12392	try self.asmRegisterImmediate(
12128	12393	.{ ._, .mov },
12129	12394	index_reg.to32(),
12130		Immediate.u(regs_frame_addr.regs),
	12395	.u(regs_frame_addr.regs),
12131	12396	);
12132	12397	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12133	12398	try self.asmMemoryImmediate(.{ ._, .cmp }, .{
...	...	@@ -12147,14 +12412,14 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12147	12412	);
12148	12413	self.performReloc(unset);
12149	12414	if (self.hasFeature(.slow_incdec)) {
12150		try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
	12415	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
12151	12416	} else {
12152	12417	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
12153	12418	}
12154	12419	try self.asmRegisterImmediate(
12155	12420	.{ ._, .cmp },
12156	12421	index_reg.to32(),
12157		Immediate.u(arg_ty.vectorLen(zcu)),
	12422	.u(arg_ty.vectorLen(zcu)),
12158	12423	);
12159	12424	_ = try self.asmJccReloc(.b, loop);
12160	12425
...	...	@@ -12180,7 +12445,6 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
12180	12445	for (self.args[self.arg_index..]) \|arg\| {
12181	12446	if (arg != .none) break;
12182	12447	} else try self.airDbgVarArgs();
12183		self.finishAirBookkeeping();
12184	12448	}
12185	12449
12186	12450	fn airDbgVarArgs(self: *Self) !void {
...	...	@@ -12200,9 +12464,9 @@ fn genLocalDebugInfo(
12200	12464	switch (mcv) {
12201	12465	.none => try self.asmAir(.dbg_local, inst),
12202	12466	.unreach, .dead, .elementwise_regs_then_frame, .reserved_frame, .air_ref => unreachable,
12203		.immediate => \|imm\| try self.asmAirImmediate(.dbg_local, inst, Immediate.u(imm)),
	12467	.immediate => \|imm\| try self.asmAirImmediate(.dbg_local, inst, .u(imm)),
12204	12468	.lea_frame => \|frame_addr\| try self.asmAirFrameAddress(.dbg_local, inst, frame_addr),
12205		.lea_symbol => \|sym_off\| try self.asmAirImmediate(.dbg_local, inst, Immediate.rel(sym_off)),
	12469	.lea_symbol => \|sym_off\| try self.asmAirImmediate(.dbg_local, inst, .rel(sym_off)),
12206	12470	else => {
12207	12471	const ty = switch (tag) {
12208	12472	else => unreachable,
...	...	@@ -12245,16 +12509,6 @@ fn genLocalDebugInfo(
12245	12509	}
12246	12510	}
12247	12511
12248		fn airTrap(self: *Self) !void {
12249		try self.asmOpOnly(.{ ._, .ud2 });
12250		self.finishAirBookkeeping();
12251		}
12252
12253		fn airBreakpoint(self: *Self) !void {
12254		try self.asmOpOnly(.{ ._, .int3 });
12255		self.finishAirBookkeeping();
12256		}
12257
12258	12512	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
12259	12513	const dst_mcv = try self.allocRegOrMem(inst, true);
12260	12514	try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
...	...	@@ -12426,7 +12680,7 @@ fn genCall(self: *Self, info: union(enum) {
12426	12680	try self.asmRegisterImmediate(
12427	12681	.{ ._, .mov },
12428	12682	index_reg.to32(),
12429		Immediate.u(regs_frame_addr.regs),
	12683	.u(regs_frame_addr.regs),
12430	12684	);
12431	12685	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12432	12686	try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32());
...	...	@@ -12440,14 +12694,14 @@ fn genCall(self: *Self, info: union(enum) {
12440	12694	} },
12441	12695	});
12442	12696	if (self.hasFeature(.slow_incdec)) {
12443		try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
	12697	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
12444	12698	} else {
12445	12699	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
12446	12700	}
12447	12701	try self.asmRegisterImmediate(
12448	12702	.{ ._, .cmp },
12449	12703	index_reg.to32(),
12450		Immediate.u(arg_ty.vectorLen(zcu)),
	12704	.u(arg_ty.vectorLen(zcu)),
12451	12705	);
12452	12706	_ = try self.asmJccReloc(.b, loop);
12453	12707
...	...	@@ -12521,11 +12775,7 @@ fn genCall(self: *Self, info: union(enum) {
12521	12775	0..,
12522	12776	) \|dst_reg, elem_index\| {
12523	12777	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12524		try self.asmMemoryImmediate(
12525		.{ ._, .bt },
12526		src_mem,
12527		Immediate.u(elem_index),
12528		);
	12778	try self.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index));
12529	12779	try self.asmSetccRegister(.c, dst_reg.to8());
12530	12780	}
12531	12781	},
...	...	@@ -12533,7 +12783,7 @@ fn genCall(self: *Self, info: union(enum) {
12533	12783	};
12534	12784
12535	12785	if (fn_info.is_var_args)
12536		try self.asmRegisterImmediate(.{ ._, .mov }, .al, Immediate.u(call_info.fp_count));
	12786	try self.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count));
12537	12787
12538	12788	// Due to incremental compilation, how function calls are generated depends
12539	12789	// on linking.
...	...	@@ -12551,7 +12801,7 @@ fn genCall(self: *Self, info: union(enum) {
12551	12801	if (self.bin_file.cast(.elf)) \|elf_file\| {
12552	12802	const zo = elf_file.zigObjectPtr().?;
12553	12803	const sym_index = try zo.getOrCreateMetadataForNav(zcu, func.owner_nav);
12554		try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index }));
	12804	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index }));
12555	12805	} else if (self.bin_file.cast(.coff)) \|coff_file\| {
12556	12806	const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);
12557	12807	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
...	...	@@ -12561,7 +12811,7 @@ fn genCall(self: *Self, info: union(enum) {
12561	12811	const zo = macho_file.getZigObject().?;
12562	12812	const sym_index = try zo.getOrCreateMetadataForNav(macho_file, func.owner_nav);
12563	12813	const sym = zo.symbols.items[sym_index];
12564		try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym.nlist_idx }));
	12814	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym.nlist_idx }));
12565	12815	} else if (self.bin_file.cast(.plan9)) \|p9\| {
12566	12816	const atom_index = try p9.seeNav(pt, func.owner_nav);
12567	12817	const atom = p9.getAtom(atom_index);
...	...	@@ -12579,13 +12829,13 @@ fn genCall(self: *Self, info: union(enum) {
12579	12829	@"extern".name.toSlice(ip),
12580	12830	@"extern".lib_name.toSlice(ip),
12581	12831	);
12582		try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
	12832	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12583	12833	} else if (self.bin_file.cast(.macho)) \|macho_file\| {
12584	12834	const target_sym_index = try macho_file.getGlobalSymbol(
12585	12835	@"extern".name.toSlice(ip),
12586	12836	@"extern".lib_name.toSlice(ip),
12587	12837	);
12588		try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
	12838	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12589	12839	} else try self.genExternSymbolRef(
12590	12840	.call,
12591	12841	@"extern".lib_name.toSlice(ip),
...	...	@@ -12600,10 +12850,10 @@ fn genCall(self: *Self, info: union(enum) {
12600	12850	},
12601	12851	.lib => \|lib\| if (self.bin_file.cast(.elf)) \|elf_file\| {
12602	12852	const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib);
12603		try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
	12853	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12604	12854	} else if (self.bin_file.cast(.macho)) \|macho_file\| {
12605	12855	const target_sym_index = try macho_file.getGlobalSymbol(lib.callee, lib.lib);
12606		try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));
	12856	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12607	12857	} else try self.genExternSymbolRef(.call, lib.lib, lib.callee),
12608	12858	}
12609	12859	return call_info.return_value.short;
...	...	@@ -12665,7 +12915,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
12665	12915	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
12666	12916	}
12667	12917
12668		fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
	12918	fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
12669	12919	const pt = self.pt;
12670	12920	const zcu = pt.zcu;
12671	12921	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	...	@@ -12754,7 +13004,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12754	13004	try self.asmRegisterImmediate(
12755	13005	.{ ._r, .sh },
12756	13006	registerAlias(temp_lhs_reg, opt_abi_size),
12757		Immediate.u(payload_abi_size * 8),
	13007	.u(payload_abi_size * 8),
12758	13008	);
12759	13009	}
12760	13010
...	...	@@ -12775,7 +13025,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12775	13025	try self.asmRegisterImmediate(
12776	13026	.{ ._r, .sh },
12777	13027	registerAlias(temp_rhs_reg, opt_abi_size),
12778		Immediate.u(payload_abi_size * 8),
	13028	.u(payload_abi_size * 8),
12779	13029	);
12780	13030	try self.asmRegisterRegister(
12781	13031	.{ ._, .@"test" },
...	...	@@ -12867,10 +13117,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12867	13117	.register_pair, .load_frame => null,
12868	13118	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: {
12869	13119	switch (resolved_dst_mcv) {
12870		.memory => \|addr\| if (math.cast(
	13120	.memory => \|addr\| if (std.math.cast(
12871	13121	i32,
12872	13122	@as(i64, @bitCast(addr)),
12873		) != null and math.cast(
	13123	) != null and std.math.cast(
12874	13124	i32,
12875	13125	@as(i64, @bitCast(addr)) + abi_size - 8,
12876	13126	) != null) break :dst null,
...	...	@@ -12928,10 +13178,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12928	13178	.register_pair, .load_frame => null,
12929	13179	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
12930	13180	switch (resolved_src_mcv) {
12931		.memory => \|addr\| if (math.cast(
	13181	.memory => \|addr\| if (std.math.cast(
12932	13182	i32,
12933	13183	@as(i64, @bitCast(addr)),
12934		) != null and math.cast(
	13184	) != null and std.math.cast(
12935	13185	i32,
12936	13186	@as(i64, @bitCast(addr)) + abi_size - 8,
12937	13187	) != null) break :src null,
...	...	@@ -12971,7 +13221,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12971	13221	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
12972	13222	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);
12973	13223
12974		const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;
	13224	const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
12975	13225	var limb_i: u16 = 0;
12976	13226	while (limb_i < limbs_len) : (limb_i += 1) {
12977	13227	const off = limb_i * 8;
...	...	@@ -13067,7 +13317,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
13067	13317	tmp1_reg,
13068	13318	dst_reg.to128(),
13069	13319	try src_mcv.mem(self, .word),
13070		Immediate.u(1),
	13320	.u(1),
13071	13321	) else try self.asmRegisterRegisterRegister(
13072	13322	.{ .vp_, .unpcklwd },
13073	13323	tmp1_reg,
...	...	@@ -13232,7 +13482,6 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
13232	13482	.column = dbg_stmt.column,
13233	13483	} },
13234	13484	});
13235		self.finishAirBookkeeping();
13236	13485	}
13237	13486
13238	13487	fn airDbgEmptyStmt(self: *Self) !void {
...	...	@@ -13240,7 +13489,6 @@ fn airDbgEmptyStmt(self: *Self) !void {
13240	13489	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)
13241	13490	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;
13242	13491	try self.asmOpOnly(.{ ._, .nop });
13243		self.finishAirBookkeeping();
13244	13492	}
13245	13493
13246	13494	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -13278,7 +13526,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13278	13526	},
13279	13527	.register => \|reg\| {
13280	13528	try self.spillEflagsIfOccupied();
13281		try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));
	13529	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), .u(1));
13282	13530	return self.asmJccReloc(.z, undefined);
13283	13531	},
13284	13532	.immediate,
...	...	@@ -13338,7 +13586,6 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
13338	13586	});
13339	13587
13340	13588	// We already took care of pl_op.operand earlier, so there's nothing left to do.
13341		self.finishAirBookkeeping();
13342	13589	}
13343	13590
13344	13591	fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
...	...	@@ -13353,7 +13600,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13353	13600
13354	13601	const pl_ty = opt_ty.optionalChild(zcu);
13355	13602
13356		const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
	13603	const some_info: struct { off: u31, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
13357	13604	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
13358	13605	else
13359	13606	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
...	...	@@ -13366,7 +13613,6 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13366	13613	.undef,
13367	13614	.immediate,
13368	13615	.eflags,
13369		.register_pair,
13370	13616	.register_offset,
13371	13617	.register_overflow,
13372	13618	.lea_direct,
...	...	@@ -13396,7 +13642,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13396	13642	try self.asmRegisterImmediate(
13397	13643	.{ ._, .bt },
13398	13644	registerAlias(opt_reg, opt_abi_size),
13399		Immediate.u(@as(u6, @intCast(some_info.off * 8))),
	13645	.u(@as(u6, @intCast(some_info.off * 8))),
	13646	);
	13647	return .{ .eflags = .nc };
	13648	},
	13649
	13650	.register_pair => \|opt_regs\| {
	13651	if (some_info.off == 0) {
	13652	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));
	13653	const alias_reg = registerAlias(opt_regs[0], some_abi_size);
	13654	assert(some_abi_size * 8 == alias_reg.bitSize());
	13655	try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
	13656	return .{ .eflags = .z };
	13657	}
	13658	assert(some_info.ty.ip_index == .bool_type);
	13659	const opt_abi_size: u32 = @intCast(opt_ty.abiSize(zcu));
	13660	try self.asmRegisterImmediate(
	13661	.{ ._, .bt },
	13662	registerAlias(opt_regs[some_info.off / 8], opt_abi_size),
	13663	.u(@as(u6, @truncate(some_info.off * 8))),
13400	13664	);
13401	13665	return .{ .eflags = .nc };
13402	13666	},
...	...	@@ -13422,7 +13686,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13422	13686	.disp = some_info.off,
13423	13687	} },
13424	13688	},
13425		Immediate.u(0),
	13689	.u(0),
13426	13690	);
13427	13691	return .{ .eflags = .e };
13428	13692	},
...	...	@@ -13448,7 +13712,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13448	13712	},
13449	13713	else => unreachable,
13450	13714	},
13451		Immediate.u(0),
	13715	.u(0),
13452	13716	);
13453	13717	return .{ .eflags = .e };
13454	13718	},
...	...	@@ -13485,7 +13749,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
13485	13749	.disp = some_info.off,
13486	13750	} },
13487	13751	},
13488		Immediate.u(0),
	13752	.u(0),
13489	13753	);
13490	13754
13491	13755	self.eflags_inst = inst;
...	...	@@ -13500,7 +13764,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
13500	13764
13501	13765	try self.spillEflagsIfOccupied();
13502	13766
13503		const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
	13767	const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
13504	13768	switch (eu_mcv) {
13505	13769	.register => \|reg\| {
13506	13770	const eu_lock = self.register_manager.lockReg(reg);
...	...	@@ -13557,7 +13821,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
13557	13821	const ptr_lock = self.register_manager.lockReg(ptr_reg);
13558	13822	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
13559	13823
13560		const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
	13824	const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
13561	13825	try self.asmMemoryImmediate(
13562	13826	.{ ._, .cmp },
13563	13827	.{
...	...	@@ -13567,7 +13831,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
13567	13831	.disp = err_off,
13568	13832	} },
13569	13833	},
13570		Immediate.u(0),
	13834	.u(0),
13571	13835	);
13572	13836
13573	13837	if (maybe_inst) \|inst\| self.eflags_inst = inst;
...	...	@@ -13686,12 +13950,11 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
13686	13950
13687	13951	try self.loops.putNoClobber(self.gpa, inst, .{
13688	13952	.state = state,
13689		.jmp_target = @intCast(self.mir_instructions.len),
	13953	.target = @intCast(self.mir_instructions.len),
13690	13954	});
13691	13955	defer assert(self.loops.remove(inst));
13692	13956
13693	13957	try self.genBodyBlock(body);
13694		self.finishAirBookkeeping();
13695	13958	}
13696	13959
13697	13960	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -13729,7 +13992,6 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
13729	13992	const tracking = &self.inst_tracking.values()[inst_tracking_i];
13730	13993	if (self.liveness.isUnused(inst)) try tracking.die(self, inst);
13731	13994	self.getValueIfFree(tracking.short, inst);
13732		self.finishAirBookkeeping();
13733	13995	}
13734	13996
13735	13997	fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {
...	...	@@ -13864,7 +14126,6 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13864	14126	try self.lowerSwitchBr(inst, switch_br, condition);
13865	14127
13866	14128	// We already took care of pl_op.operand earlier, so there's nothing left to do
13867		self.finishAirBookkeeping();
13868	14129	}
13869	14130
13870	14131	fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -13893,7 +14154,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13893	14154
13894	14155	try self.loops.putNoClobber(self.gpa, inst, .{
13895	14156	.state = state,
13896		.jmp_target = @intCast(self.mir_instructions.len),
	14157	.target = @intCast(self.mir_instructions.len),
13897	14158	});
13898	14159	defer assert(self.loops.remove(inst));
13899	14160
...	...	@@ -13903,7 +14164,6 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13903	14164	try self.lowerSwitchBr(inst, switch_br, mat_cond);
13904	14165
13905	14166	try self.processDeath(inst);
13906		self.finishAirBookkeeping();
13907	14167	}
13908	14168
13909	14169	fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -13945,12 +14205,10 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
13945	14205
13946	14206	// Emit a jump with a relocation. It will be patched up after the block ends.
13947	14207	// Leave the jump offset undefined
13948		_ = try self.asmJmpReloc(loop_data.jmp_target);
	14208	_ = try self.asmJmpReloc(loop_data.target);
13949	14209
13950	14210	// Stop tracking block result without forgetting tracking info
13951	14211	try self.freeValue(block_tracking.short);
13952
13953		self.finishAirBookkeeping();
13954	14212	}
13955	14213
13956	14214	fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
...	...	@@ -14023,8 +14281,6 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
14023	14281
14024	14282	// Stop tracking block result without forgetting tracking info
14025	14283	try self.freeValue(block_tracking.short);
14026
14027		self.finishAirBookkeeping();
14028	14284	}
14029	14285
14030	14286	fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -14036,8 +14292,7 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
14036	14292	.resurrect = false,
14037	14293	.close_scope = true,
14038	14294	});
14039		_ = try self.asmJmpReloc(repeat_info.jmp_target);
14040		self.finishAirBookkeeping();
	14295	_ = try self.asmJmpReloc(repeat_info.target);
14041	14296	}
14042	14297
14043	14298	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -14068,9 +14323,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14068	14323
14069	14324	var outputs_extra_i = extra_i;
14070	14325	for (outputs) \|output\| {
14071		const extra_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);
14072		const constraint = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
14073		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
	14326	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
	14327	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
	14328	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
14074	14329	// This equation accounts for the fact that even if we have exactly 4 bytes
14075	14330	// for the string, we still use the next u32 for the null terminator.
14076	14331	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	...	@@ -14097,8 +14352,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14097	14352	const is_early_clobber = constraint[1] == '&';
14098	14353	const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
14099	14354	const arg_mcv: MCValue = arg_mcv: {
14100		const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "r") or
14101		mem.eql(u8, rest, "f") or mem.eql(u8, rest, "x"))
	14355	const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or
	14356	std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x"))
14102	14357	registerAlias(
14103	14358	self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) {
14104	14359	'r' => abi.RegisterClass.gp,
...	...	@@ -14108,20 +14363,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14108	14363	}) orelse return self.fail("ran out of registers lowering inline asm", .{}),
14109	14364	@intCast(ty.abiSize(zcu)),
14110	14365	)
14111		else if (mem.eql(u8, rest, "m"))
	14366	else if (std.mem.eql(u8, rest, "m"))
14112	14367	if (output != .none) null else return self.fail(
14113	14368	"memory constraint unsupported for asm result: '{s}'",
14114	14369	.{constraint},
14115	14370	)
14116		else if (mem.eql(u8, rest, "g") or
14117		mem.eql(u8, rest, "rm") or mem.eql(u8, rest, "mr") or
14118		mem.eql(u8, rest, "r,m") or mem.eql(u8, rest, "m,r"))
	14371	else if (std.mem.eql(u8, rest, "g") or
	14372	std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or
	14373	std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r"))
14119	14374	self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse
14120	14375	if (output != .none)
14121	14376	null
14122	14377	else
14123	14378	return self.fail("ran out of registers lowering inline asm", .{})
14124		else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}"))
	14379	else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}"))
14125	14380	parseRegName(rest["{".len .. rest.len - "}".len]) orelse
14126	14381	return self.fail("invalid register constraint: '{s}'", .{constraint})
14127	14382	else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {
...	...	@@ -14134,7 +14389,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14134	14389	break :arg_mcv if (arg_maybe_reg) \|reg\| .{ .register = reg } else arg: {
14135	14390	const ptr_mcv = try self.resolveInst(output);
14136	14391	switch (ptr_mcv) {
14137		.immediate => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
	14392	.immediate => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14138	14393	break :arg ptr_mcv.deref(),
14139	14394	.register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),
14140	14395	else => {},
...	...	@@ -14145,7 +14400,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14145	14400	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
14146	14401	_ = self.register_manager.lockReg(reg);
14147	14402	};
14148		if (!mem.eql(u8, name, "_"))
	14403	if (!std.mem.eql(u8, name, "_"))
14149	14404	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
14150	14405	args.appendAssumeCapacity(arg_mcv);
14151	14406	if (output == .none) result = arg_mcv;
...	...	@@ -14153,17 +14408,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14153	14408	}
14154	14409
14155	14410	for (inputs) \|input\| {
14156		const input_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);
14157		const constraint = mem.sliceTo(input_bytes, 0);
14158		const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
	14411	const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
	14412	const constraint = std.mem.sliceTo(input_bytes, 0);
	14413	const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
14159	14414	// This equation accounts for the fact that even if we have exactly 4 bytes
14160	14415	// for the string, we still use the next u32 for the null terminator.
14161	14416	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
14162	14417
14163	14418	const ty = self.typeOf(input);
14164	14419	const input_mcv = try self.resolveInst(input);
14165		const arg_mcv: MCValue = if (mem.eql(u8, constraint, "r") or
14166		mem.eql(u8, constraint, "f") or mem.eql(u8, constraint, "x"))
	14420	const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or
	14421	std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x"))
14167	14422	arg: {
14168	14423	const rc = switch (constraint[0]) {
14169	14424	'r' => abi.RegisterClass.gp,
...	...	@@ -14177,16 +14432,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14177	14432	const reg = try self.register_manager.allocReg(null, rc);
14178	14433	try self.genSetReg(reg, ty, input_mcv, .{});
14179	14434	break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) };
14180		} else if (mem.eql(u8, constraint, "i") or mem.eql(u8, constraint, "n"))
	14435	} else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n"))
14181	14436	switch (input_mcv) {
14182	14437	.immediate => \|imm\| .{ .immediate = imm },
14183	14438	else => return self.fail("immediate operand requires comptime value: '{s}'", .{
14184	14439	constraint,
14185	14440	}),
14186	14441	}
14187		else if (mem.eql(u8, constraint, "m")) arg: {
	14442	else if (std.mem.eql(u8, constraint, "m")) arg: {
14188	14443	switch (input_mcv) {
14189		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
	14444	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14190	14445	break :arg input_mcv,
14191	14446	.indirect, .load_frame => break :arg input_mcv,
14192	14447	.load_symbol, .load_direct, .load_got, .load_tlv => {},
...	...	@@ -14203,22 +14458,22 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14203	14458	};
14204	14459	try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{});
14205	14460	break :arg .{ .indirect = .{ .reg = addr_reg } };
14206		} else if (mem.eql(u8, constraint, "g") or
14207		mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or
14208		mem.eql(u8, constraint, "r,m") or mem.eql(u8, constraint, "m,r"))
	14461	} else if (std.mem.eql(u8, constraint, "g") or
	14462	std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or
	14463	std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r"))
14209	14464	arg: {
14210	14465	switch (input_mcv) {
14211	14466	.register, .indirect, .load_frame => break :arg input_mcv,
14212		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
	14467	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14213	14468	break :arg input_mcv,
14214	14469	else => {},
14215	14470	}
14216	14471	const temp_mcv = try self.allocTempRegOrMem(ty, true);
14217	14472	try self.genCopy(ty, temp_mcv, input_mcv, .{});
14218	14473	break :arg temp_mcv;
14219		} else if (mem.eql(u8, constraint, "X"))
	14474	} else if (std.mem.eql(u8, constraint, "X"))
14220	14475	input_mcv
14221		else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: {
	14476	else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: {
14222	14477	const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
14223	14478	return self.fail("invalid register constraint: '{s}'", .{constraint});
14224	14479	try self.register_manager.getReg(reg, null);
...	...	@@ -14233,7 +14488,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14233	14488	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
14234	14489	_ = self.register_manager.lockReg(reg);
14235	14490	};
14236		if (!mem.eql(u8, name, "_"))
	14491	if (!std.mem.eql(u8, name, "_"))
14237	14492	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
14238	14493	args.appendAssumeCapacity(arg_mcv);
14239	14494	}
...	...	@@ -14241,7 +14496,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14241	14496	{
14242	14497	var clobber_i: u32 = 0;
14243	14498	while (clobber_i < clobbers_len) : (clobber_i += 1) {
14244		const clobber = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
	14499	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
14245	14500	// This equation accounts for the fact that even if we have exactly 4 bytes
14246	14501	// for the string, we still use the next u32 for the null terminator.
14247	14502	extra_i += clobber.len / 4 + 1;
...	...	@@ -14294,20 +14549,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14294	14549	labels.deinit(self.gpa);
14295	14550	}
14296	14551
14297		const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
14298		var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
	14552	const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
	14553	var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;");
14299	14554	next_line: while (line_it.next()) \|line\| {
14300		var mnem_it = mem.tokenizeAny(u8, line, " \t");
14301		var prefix: Instruction.Prefix = .none;
	14555	var mnem_it = std.mem.tokenizeAny(u8, line, " \t");
	14556	var prefix: encoder.Instruction.Prefix = .none;
14302	14557	const mnem_str = while (mnem_it.next()) \|mnem_str\| {
14303	14558	if (mnem_str[0] == '#') continue :next_line;
14304		if (mem.startsWith(u8, mnem_str, "//")) continue :next_line;
14305		if (std.meta.stringToEnum(Instruction.Prefix, mnem_str)) \|pre\| {
	14559	if (std.mem.startsWith(u8, mnem_str, "//")) continue :next_line;
	14560	if (std.meta.stringToEnum(encoder.Instruction.Prefix, mnem_str)) \|pre\| {
14306	14561	if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});
14307	14562	prefix = pre;
14308	14563	continue;
14309	14564	}
14310		if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;
	14565	if (!std.mem.endsWith(u8, mnem_str, ":")) break mnem_str;
14311	14566	const label_name = mnem_str[0 .. mnem_str.len - ":".len];
14312	14567	if (!Label.isValid(.definition, label_name))
14313	14568	return self.fail("invalid label: '{s}'", .{label_name});
...	...	@@ -14332,21 +14587,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14332	14587
14333	14588	var mnem_size: ?Memory.Size = if (prefix == .directive)
14334	14589	null
14335		else if (mem.endsWith(u8, mnem_str, "b"))
	14590	else if (std.mem.endsWith(u8, mnem_str, "b"))
14336	14591	.byte
14337		else if (mem.endsWith(u8, mnem_str, "w"))
	14592	else if (std.mem.endsWith(u8, mnem_str, "w"))
14338	14593	.word
14339		else if (mem.endsWith(u8, mnem_str, "l"))
	14594	else if (std.mem.endsWith(u8, mnem_str, "l"))
14340	14595	.dword
14341		else if (mem.endsWith(u8, mnem_str, "q") and
14342		(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))
	14596	else if (std.mem.endsWith(u8, mnem_str, "q") and
	14597	(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !std.mem.endsWith(u8, mnem_str, "dq")))
14343	14598	.qword
14344		else if (mem.endsWith(u8, mnem_str, "t"))
	14599	else if (std.mem.endsWith(u8, mnem_str, "t"))
14345	14600	.tbyte
14346	14601	else
14347	14602	null;
14348	14603	const mnem_tag = while (true) break std.meta.stringToEnum(
14349		Instruction.Mnemonic,
	14604	encoder.Instruction.Mnemonic,
14350	14605	mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
14351	14606	) orelse if (mnem_size) \|_\| {
14352	14607	mnem_size = null;
...	...	@@ -14367,18 +14622,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14367	14622	.{ ._, .pseudo }
14368	14623	else for (std.enums.values(Mir.Inst.Fixes)) \|fixes\| {
14369	14624	const fixes_name = @tagName(fixes);
14370		const space_i = mem.indexOfScalar(u8, fixes_name, ' ');
	14625	const space_i = std.mem.indexOfScalar(u8, fixes_name, ' ');
14371	14626	const fixes_prefix = if (space_i) \|i\|
14372		std.meta.stringToEnum(Instruction.Prefix, fixes_name[0..i]).?
	14627	std.meta.stringToEnum(encoder.Instruction.Prefix, fixes_name[0..i]).?
14373	14628	else
14374	14629	.none;
14375	14630	if (fixes_prefix != prefix) continue;
14376	14631	const pattern = fixes_name[if (space_i) \|i\| i + " ".len else 0..];
14377		const wildcard_i = mem.indexOfScalar(u8, pattern, '_').?;
	14632	const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
14378	14633	const mnem_prefix = pattern[0..wildcard_i];
14379	14634	const mnem_suffix = pattern[wildcard_i + "_".len ..];
14380		if (!mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
14381		if (!mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
	14635	if (!std.mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
	14636	if (!std.mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
14382	14637	break .{ fixes, std.meta.stringToEnum(
14383	14638	Mir.Inst.Tag,
14384	14639	mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],
...	...	@@ -14400,21 +14655,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14400	14655	var ops: [4]Operand = .{.none} ** 4;
14401	14656
14402	14657	var last_op = false;
14403		var op_it = mem.splitScalar(u8, mnem_it.rest(), ',');
	14658	var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ',');
14404	14659	next_op: for (&ops) \|*op\| {
14405	14660	const op_str = while (!last_op) {
14406	14661	const full_str = op_it.next() orelse break :next_op;
14407		const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse
14408		mem.indexOf(u8, full_str, "//")) \|comment\|
	14662	const code_str = if (std.mem.indexOfScalar(u8, full_str, '#') orelse
	14663	std.mem.indexOf(u8, full_str, "//")) \|comment\|
14409	14664	code: {
14410	14665	last_op = true;
14411	14666	break :code full_str[0..comment];
14412	14667	} else full_str;
14413		const trim_str = mem.trim(u8, code_str, " \t*");
	14668	const trim_str = std.mem.trim(u8, code_str, " \t*");
14414	14669	if (trim_str.len > 0) break trim_str;
14415	14670	} else break;
14416		if (mem.startsWith(u8, op_str, "%%")) {
14417		const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
	14671	if (std.mem.startsWith(u8, op_str, "%%")) {
	14672	const colon = std.mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
14418	14673	const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
14419	14674	return self.fail("invalid register: '{s}'", .{op_str});
14420	14675	if (colon) \|colon_pos\| {
...	...	@@ -14432,8 +14687,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14432	14687	return self.fail("invalid register size: '{s}'", .{op_str});
14433	14688	op.* = .{ .reg = reg };
14434	14689	}
14435		} else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {
14436		const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
	14690	} else if (std.mem.startsWith(u8, op_str, "%[") and std.mem.endsWith(u8, op_str, "]")) {
	14691	const colon = std.mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
14437	14692	const modifier = if (colon) \|colon_pos\|
14438	14693	op_str[colon_pos + ":".len .. op_str.len - "]".len]
14439	14694	else
...	...	@@ -14442,15 +14697,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14442	14697	arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse
14443	14698	return self.fail("no matching constraint: '{s}'", .{op_str})
14444	14699	]) {
14445		.immediate => \|imm\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "c"))
14446		.{ .imm = Immediate.u(imm) }
	14700	.immediate => \|imm\| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "c"))
	14701	.{ .imm = .u(imm) }
14447	14702	else
14448	14703	return self.fail("invalid modifier: '{s}'", .{modifier}),
14449		.register => \|reg\| if (mem.eql(u8, modifier, ""))
	14704	.register => \|reg\| if (std.mem.eql(u8, modifier, ""))
14450	14705	.{ .reg = reg }
14451	14706	else
14452	14707	return self.fail("invalid modifier: '{s}'", .{modifier}),
14453		.memory => \|addr\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "P"))
	14708	.memory => \|addr\| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "P"))
14454	14709	.{ .mem = .{
14455	14710	.base = .{ .reg = .ds },
14456	14711	.mod = .{ .rm = .{
...	...	@@ -14461,7 +14716,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14461	14716	} }
14462	14717	else
14463	14718	return self.fail("invalid modifier: '{s}'", .{modifier}),
14464		.indirect => \|reg_off\| if (mem.eql(u8, modifier, ""))
	14719	.indirect => \|reg_off\| if (std.mem.eql(u8, modifier, ""))
14465	14720	.{ .mem = .{
14466	14721	.base = .{ .reg = reg_off.reg },
14467	14722	.mod = .{ .rm = .{
...	...	@@ -14472,7 +14727,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14472	14727	} }
14473	14728	else
14474	14729	return self.fail("invalid modifier: '{s}'", .{modifier}),
14475		.load_frame => \|frame_addr\| if (mem.eql(u8, modifier, ""))
	14730	.load_frame => \|frame_addr\| if (std.mem.eql(u8, modifier, ""))
14476	14731	.{ .mem = .{
14477	14732	.base = .{ .frame = frame_addr.index },
14478	14733	.mod = .{ .rm = .{
...	...	@@ -14483,42 +14738,42 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14483	14738	} }
14484	14739	else
14485	14740	return self.fail("invalid modifier: '{s}'", .{modifier}),
14486		.lea_got => \|sym_index\| if (mem.eql(u8, modifier, "P"))
	14741	.lea_got => \|sym_index\| if (std.mem.eql(u8, modifier, "P"))
14487	14742	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }
14488	14743	else
14489	14744	return self.fail("invalid modifier: '{s}'", .{modifier}),
14490		.lea_symbol => \|sym_off\| if (mem.eql(u8, modifier, "P"))
	14745	.lea_symbol => \|sym_off\| if (std.mem.eql(u8, modifier, "P"))
14491	14746	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) }
14492	14747	else
14493	14748	return self.fail("invalid modifier: '{s}'", .{modifier}),
14494	14749	else => return self.fail("invalid constraint: '{s}'", .{op_str}),
14495	14750	};
14496		} else if (mem.startsWith(u8, op_str, "$")) {
	14751	} else if (std.mem.startsWith(u8, op_str, "$")) {
14497	14752	if (std.fmt.parseInt(i32, op_str["$".len..], 0)) \|s\| {
14498	14753	if (mnem_size) \|size\| {
14499		const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));
	14754	const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));
14500	14755	if ((if (s < 0) ~s else s) > max)
14501	14756	return self.fail("invalid immediate size: '{s}'", .{op_str});
14502	14757	}
14503		op.* = .{ .imm = Immediate.s(s) };
	14758	op.* = .{ .imm = .s(s) };
14504	14759	} else \|_\| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) \|u\| {
14505	14760	if (mnem_size) \|size\| {
14506		const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - size.bitSize());
	14761	const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize());
14507	14762	if (u > max)
14508	14763	return self.fail("invalid immediate size: '{s}'", .{op_str});
14509	14764	}
14510		op.* = .{ .imm = Immediate.u(u) };
	14765	op.* = .{ .imm = .u(u) };
14511	14766	} else \|_\| return self.fail("invalid immediate: '{s}'", .{op_str});
14512		} else if (mem.endsWith(u8, op_str, ")")) {
14513		const open = mem.indexOfScalar(u8, op_str, '(') orelse
	14767	} else if (std.mem.endsWith(u8, op_str, ")")) {
	14768	const open = std.mem.indexOfScalar(u8, op_str, '(') orelse
14514	14769	return self.fail("invalid operand: '{s}'", .{op_str});
14515		var sib_it = mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
	14770	var sib_it = std.mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
14516	14771	const base_str = sib_it.next() orelse
14517	14772	return self.fail("invalid memory operand: '{s}'", .{op_str});
14518		if (base_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
	14773	if (base_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
14519	14774	return self.fail("invalid memory operand: '{s}'", .{op_str});
14520	14775	const index_str = sib_it.next() orelse "";
14521		if (index_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
	14776	if (index_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
14522	14777	return self.fail("invalid memory operand: '{s}'", .{op_str});
14523	14778	const scale_str = sib_it.next() orelse "";
14524	14779	if (index_str.len == 0 and scale_str.len > 0)
...	...	@@ -14550,10 +14805,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14550	14805	else
14551	14806	.none,
14552	14807	.scale = scale,
14553		.disp = if (mem.startsWith(u8, op_str[0..open], "%[") and
14554		mem.endsWith(u8, op_str[0..open], "]"))
	14808	.disp = if (std.mem.startsWith(u8, op_str[0..open], "%[") and
	14809	std.mem.endsWith(u8, op_str[0..open], "]"))
14555	14810	disp: {
14556		const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
	14811	const colon = std.mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
14557	14812	const modifier = if (colon) \|colon_pos\|
14558	14813	op_str[colon_pos + ":".len .. open - "]".len]
14559	14814	else
...	...	@@ -14562,9 +14817,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14562	14817	arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
14563	14818	return self.fail("no matching constraint: '{s}'", .{op_str})
14564	14819	]) {
14565		.immediate => \|imm\| if (mem.eql(u8, modifier, "") or
14566		mem.eql(u8, modifier, "c"))
14567		math.cast(i32, @as(i64, @bitCast(imm))) orelse
	14820	.immediate => \|imm\| if (std.mem.eql(u8, modifier, "") or
	14821	std.mem.eql(u8, modifier, "c"))
	14822	std.math.cast(i32, @as(i64, @bitCast(imm))) orelse
14568	14823	return self.fail("invalid displacement: '{s}'", .{op_str})
14569	14824	else
14570	14825	return self.fail("invalid modifier: '{s}'", .{modifier}),
...	...	@@ -14730,10 +14985,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14730	14985	return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
14731	14986
14732	14987	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {
14733		const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
	14988	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
14734	14989	const constraint =
14735		mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
14736		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
	14990	std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
	14991	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
14737	14992	// This equation accounts for the fact that even if we have exactly 4 bytes
14738	14993	// for the string, we still use the next u32 for the null terminator.
14739	14994	outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	...	@@ -14777,7 +15032,10 @@ const MoveStrategy = union(enum) {
14777	15032
14778	15033	pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {
14779	15034	switch (strat) {
14780		.move => \|tag\| try self.asmRegisterMemory(tag, dst_reg, src_mem),
	15035	.move => \|tag\| try self.asmRegisterMemory(tag, switch (tag[1]) {
	15036	else => dst_reg,
	15037	.lea => if (dst_reg.bitSize() >= 32) dst_reg else dst_reg.to32(),
	15038	}, src_mem),
14781	15039	.x87_load_store => {
14782	15040	try self.asmMemory(.{ .f_, .ld }, src_mem);
14783	15041	assert(dst_reg != .st7);
...	...	@@ -14787,14 +15045,14 @@ const MoveStrategy = union(enum) {
14787	15045	ie.insert,
14788	15046	dst_reg,
14789	15047	src_mem,
14790		Immediate.u(0),
	15048	.u(0),
14791	15049	),
14792	15050	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
14793	15051	ie.insert,
14794	15052	dst_reg,
14795	15053	dst_reg,
14796	15054	src_mem,
14797		Immediate.u(0),
	15055	.u(0),
14798	15056	),
14799	15057	}
14800	15058	}
...	...	@@ -14809,7 +15067,7 @@ const MoveStrategy = union(enum) {
14809	15067	ie.extract,
14810	15068	dst_mem,
14811	15069	src_reg,
14812		Immediate.u(0),
	15070	.u(0),
14813	15071	),
14814	15072	}
14815	15073	}
...	...	@@ -14823,7 +15081,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14823	15081	.mmx => {},
14824	15082	.sse => switch (ty.zigTypeTag(zcu)) {
14825	15083	else => {
14826		const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
	15084	const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
14827	15085	assert(std.mem.indexOfNone(abi.Class, classes, &.{
14828	15086	.integer, .sse, .sseup, .memory, .float, .float_combine,
14829	15087	}) == null);
...	...	@@ -15135,7 +15393,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
15135	15393	),
15136	15394	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
15137	15395	switch (dst_mcv) {
15138		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
	15396	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
15139	15397	return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts),
15140	15398	.load_symbol, .load_direct, .load_got, .load_tlv => {},
15141	15399	else => unreachable,
...	...	@@ -15179,17 +15437,17 @@ fn genSetReg(
15179	15437	=> unreachable,
15180	15438	.undef => if (opts.safety) switch (dst_reg.class()) {
15181	15439	.general_purpose => switch (abi_size) {
15182		1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), Immediate.u(0xAA)),
15183		2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), Immediate.u(0xAAAA)),
	15440	1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xAA)),
	15441	2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xAAAA)),
15184	15442	3...4 => try self.asmRegisterImmediate(
15185	15443	.{ ._, .mov },
15186	15444	dst_reg.to32(),
15187		Immediate.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
	15445	.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
15188	15446	),
15189	15447	5...8 => try self.asmRegisterImmediate(
15190	15448	.{ ._, .mov },
15191	15449	dst_reg.to64(),
15192		Immediate.u(0xAAAAAAAAAAAAAAAA),
	15450	.u(0xAAAAAAAAAAAAAAAA),
15193	15451	),
15194	15452	else => unreachable,
15195	15453	},
...	...	@@ -15203,20 +15461,20 @@ fn genSetReg(
15203	15461	// register is the fastest way to zero a register.
15204	15462	try self.spillEflagsIfOccupied();
15205	15463	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
15206		} else if (abi_size > 4 and math.cast(u32, imm) != null) {
	15464	} else if (abi_size > 4 and std.math.cast(u32, imm) != null) {
15207	15465	// 32-bit moves zero-extend to 64-bit.
15208		try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));
	15466	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(imm));
15209	15467	} else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) {
15210	15468	try self.asmRegisterImmediate(
15211	15469	.{ ._, .mov },
15212	15470	registerAlias(dst_reg, abi_size),
15213		Immediate.s(@intCast(@as(i64, @bitCast(imm)))),
	15471	.s(@intCast(@as(i64, @bitCast(imm)))),
15214	15472	);
15215	15473	} else {
15216	15474	try self.asmRegisterImmediate(
15217	15475	.{ ._, .mov },
15218	15476	registerAlias(dst_reg, abi_size),
15219		Immediate.u(imm),
	15477	.u(imm),
15220	15478	);
15221	15479	}
15222	15480	},
...	...	@@ -15325,15 +15583,15 @@ fn genSetReg(
15325	15583	.load_frame => \|frame_addr\| try self.moveStrategy(
15326	15584	ty,
15327	15585	dst_reg.class(),
15328		self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(
15329		math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
	15586	self.getFrameAddrAlignment(frame_addr).compare(.gte, InternPool.Alignment.fromLog2Units(
	15587	std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
15330	15588	)),
15331	15589	),
15332	15590	.lea_frame => .{ .move = .{ ._, .lea } },
15333	15591	else => unreachable,
15334	15592	}).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {
15335	15593	.register_offset, .indirect => \|reg_off\| .{
15336		.base = .{ .reg = reg_off.reg },
	15594	.base = .{ .reg = reg_off.reg.to64() },
15337	15595	.mod = .{ .rm = .{
15338	15596	.size = self.memSize(ty),
15339	15597	.disp = reg_off.off,
...	...	@@ -15350,7 +15608,7 @@ fn genSetReg(
15350	15608	}),
15351	15609	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
15352	15610	switch (src_mcv) {
15353		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
	15611	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
15354	15612	return (try self.moveStrategy(
15355	15613	ty,
15356	15614	dst_reg.class(),
...	...	@@ -15400,14 +15658,10 @@ fn genSetReg(
15400	15658	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
15401	15659	defer self.register_manager.unlockReg(addr_lock);
15402	15660
15403		try (try self.moveStrategy(ty, dst_reg.class(), false)).read(
15404		self,
15405		registerAlias(dst_reg, abi_size),
15406		.{
15407		.base = .{ .reg = addr_reg },
15408		.mod = .{ .rm = .{ .size = self.memSize(ty) } },
15409		},
15410		);
	15661	try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, registerAlias(dst_reg, abi_size), .{
	15662	.base = .{ .reg = addr_reg.to64() },
	15663	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
	15664	});
15411	15665	},
15412	15666	.lea_symbol => \|sym_off\| switch (self.bin_file.tag) {
15413	15667	.elf, .macho => try self.asmRegisterMemory(
...	...	@@ -15478,12 +15732,12 @@ fn genSetMem(
15478	15732	),
15479	15733	.immediate => \|imm\| switch (abi_size) {
15480	15734	1, 2, 4 => {
15481		const immediate = switch (if (ty.isAbiInt(zcu))
	15735	const immediate: Immediate = switch (if (ty.isAbiInt(zcu))
15482	15736	ty.intInfo(zcu).signedness
15483	15737	else
15484	15738	.unsigned) {
15485		.signed => Immediate.s(@truncate(@as(i64, @bitCast(imm)))),
15486		.unsigned => Immediate.u(@as(u32, @intCast(imm))),
	15739	.signed => .s(@truncate(@as(i64, @bitCast(imm)))),
	15740	.unsigned => .u(@as(u32, @intCast(imm))),
15487	15741	};
15488	15742	try self.asmMemoryImmediate(
15489	15743	.{ ._, .mov },
...	...	@@ -15495,14 +15749,14 @@ fn genSetMem(
15495	15749	);
15496	15750	},
15497	15751	3, 5...7 => unreachable,
15498		else => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {
	15752	else => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {
15499	15753	try self.asmMemoryImmediate(
15500	15754	.{ ._, .mov },
15501	15755	.{ .base = base, .mod = .{ .rm = .{
15502	15756	.size = Memory.Size.fromSize(abi_size),
15503	15757	.disp = disp,
15504	15758	} } },
15505		Immediate.s(small),
	15759	.s(small),
15506	15760	);
15507	15761	} else {
15508	15762	var offset: i32 = 0;
...	...	@@ -15512,10 +15766,10 @@ fn genSetMem(
15512	15766	.size = .dword,
15513	15767	.disp = disp + offset,
15514	15768	} } },
15515		if (ty.isSignedInt(zcu)) Immediate.s(
15516		@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),
15517		) else Immediate.u(
15518		@as(u32, @truncate(if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),
	15769	if (ty.isSignedInt(zcu)) .s(
	15770	@truncate(@as(i64, @bitCast(imm)) >> (std.math.cast(u6, offset * 8) orelse 63)),
	15771	) else .u(
	15772	@as(u32, @truncate(if (std.math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),
15519	15773	),
15520	15774	);
15521	15775	},
...	...	@@ -15542,7 +15796,9 @@ fn genSetMem(
15542	15796	.general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),
15543	15797	.mmx, .sse => abi_size,
15544	15798	});
15545		const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size));
	15799	const src_align = InternPool.Alignment.fromNonzeroByteUnits(
	15800	std.math.ceilPowerOfTwoAssert(u32, src_size),
	15801	);
15546	15802	if (src_size > mem_size) {
15547	15803	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
15548	15804	.size = src_size,
...	...	@@ -15755,7 +16011,7 @@ fn genLazySymbolRef(
15755	16011	.base = .{ .reloc = sym_index },
15756	16012	.mod = .{ .rm = .{ .size = .qword } },
15757	16013	}),
15758		.call => try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index })),
	16014	.call => try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index })),
15759	16015	else => unreachable,
15760	16016	}
15761	16017	} else if (self.bin_file.cast(.plan9)) \|p9_file\| {
...	...	@@ -15861,7 +16117,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
15861	16117	const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and
15862	16118	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
15863	16119	const dst_mcv = try self.allocRegOrMem(inst, true);
15864		try self.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
	16120	try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
15865	16121	.lt => dst_ty,
15866	16122	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
15867	16123	.gt => src_ty,
...	...	@@ -15878,7 +16134,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
15878	16134	const bit_size = dst_ty.bitSize(zcu);
15879	16135	if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
15880	16136
15881		const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
	16137	const dst_limbs_len = std.math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
15882	16138	const high_mcv: MCValue = switch (dst_mcv) {
15883	16139	.register => \|dst_reg\| .{ .register = dst_reg },
15884	16140	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
...	...	@@ -15940,7 +16196,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
15940	16196	const src_bits: u32 = @intCast(src_ty.bitSize(zcu));
15941	16197	const src_signedness =
15942	16198	if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;
15943		const src_size = math.divCeil(u32, @max(switch (src_signedness) {
	16199	const src_size = std.math.divCeil(u32, @max(switch (src_signedness) {
15944	16200	.signed => src_bits,
15945	16201	.unsigned => src_bits + 1,
15946	16202	}, 32), 8) catch unreachable;
...	...	@@ -16017,7 +16273,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16017	16273	const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu));
16018	16274	const dst_signedness =
16019	16275	if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned;
16020		const dst_size = math.divCeil(u32, @max(switch (dst_signedness) {
	16276	const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) {
16021	16277	.signed => dst_bits,
16022	16278	.unsigned => dst_bits + 1,
16023	16279	}, 32), 8) catch unreachable;
...	...	@@ -16593,13 +16849,17 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16593	16849	const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx });
16594	16850	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
16595	16851
16596		const dst_ptr = try self.resolveInst(bin_op.lhs);
16597		const dst_ptr_ty = self.typeOf(bin_op.lhs);
16598		const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
16599		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
16600		else => null,
	16852	const dst = try self.resolveInst(bin_op.lhs);
	16853	const dst_ty = self.typeOf(bin_op.lhs);
	16854	const dst_locks: [2]?RegisterLock = switch (dst) {
	16855	.register => \|dst_reg\| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
	16856	.register_pair => \|dst_regs\| .{
	16857	self.register_manager.lockRegAssumeUnused(dst_regs[0]),
	16858	self.register_manager.lockRegAssumeUnused(dst_regs[1]),
	16859	},
	16860	else => .{ null, null },
16601	16861	};
16602		defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	16862	for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
16603	16863
16604	16864	const src_val = try self.resolveInst(bin_op.rhs);
16605	16865	const elem_ty = self.typeOf(bin_op.rhs);
...	...	@@ -16612,16 +16872,20 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16612	16872	const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));
16613	16873
16614	16874	if (elem_abi_size == 1) {
16615		const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
16616		// TODO: this only handles slices stored in the stack
16617		.slice => dst_ptr,
16618		.one => dst_ptr,
	16875	const dst_ptr: MCValue = switch (dst_ty.ptrSize(zcu)) {
	16876	.slice => switch (dst) {
	16877	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
	16878	else => dst,
	16879	},
	16880	.one => dst,
16619	16881	.c, .many => unreachable,
16620	16882	};
16621		const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
16622		// TODO: this only handles slices stored in the stack
16623		.slice => dst_ptr.address().offset(8).deref(),
16624		.one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) },
	16883	const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
	16884	.slice => switch (dst) {
	16885	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
	16886	else => dst.address().offset(8).deref(),
	16887	},
	16888	.one => .{ .immediate = dst_ty.childType(zcu).arrayLen(zcu) },
16625	16889	.c, .many => unreachable,
16626	16890	};
16627	16891	const len_lock: ?RegisterLock = switch (len) {
...	...	@@ -16630,20 +16894,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16630	16894	};
16631	16895	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
16632	16896
16633		try self.genInlineMemset(ptr, src_val, len, .{ .safety = safety });
	16897	try self.genInlineMemset(dst_ptr, src_val, len, .{ .safety = safety });
16634	16898	break :result;
16635	16899	}
16636	16900
16637	16901	// Store the first element, and then rely on memcpy copying forwards.
16638	16902	// Length zero requires a runtime check - so we handle arrays specially
16639	16903	// here to elide it.
16640		switch (dst_ptr_ty.ptrSize(zcu)) {
	16904	switch (dst_ty.ptrSize(zcu)) {
16641	16905	.slice => {
16642		const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(zcu);
	16906	const slice_ptr_ty = dst_ty.slicePtrFieldType(zcu);
16643	16907
16644		// TODO: this only handles slices stored in the stack
16645		const ptr = dst_ptr;
16646		const len = dst_ptr.address().offset(8).deref();
	16908	const dst_ptr: MCValue = switch (dst) {
	16909	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
	16910	else => dst,
	16911	};
	16912	const len: MCValue = switch (dst) {
	16913	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
	16914	else => dst.address().offset(8).deref(),
	16915	};
16647	16916
16648	16917	// Used to store the number of elements for comparison.
16649	16918	// After comparison, updated to store number of bytes needed to copy.
...	...	@@ -16656,7 +16925,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16656	16925	try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);
16657	16926
16658	16927	const skip_reloc = try self.asmJccReloc(.z, undefined);
16659		try self.store(slice_ptr_ty, ptr, src_val, .{ .safety = safety });
	16928	try self.store(slice_ptr_ty, dst_ptr, src_val, .{ .safety = safety });
16660	16929
16661	16930	const second_elem_ptr_reg =
16662	16931	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	...	@@ -16666,7 +16935,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16666	16935	defer self.register_manager.unlockReg(second_elem_ptr_lock);
16667	16936
16668	16937	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
16669		.reg = try self.copyToTmpRegister(Type.usize, ptr),
	16938	.reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
16670	16939	.off = elem_abi_size,
16671	16940	} }, .{});
16672	16941
...	...	@@ -16675,19 +16944,19 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16675	16944	.{ .i_, .mul },
16676	16945	len_reg,
16677	16946	len_reg,
16678		Immediate.s(elem_abi_size),
	16947	.s(elem_abi_size),
16679	16948	);
16680		try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
	16949	try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv);
16681	16950
16682	16951	self.performReloc(skip_reloc);
16683	16952	},
16684	16953	.one => {
16685	16954	const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);
16686	16955
16687		const len = dst_ptr_ty.childType(zcu).arrayLen(zcu);
	16956	const len = dst_ty.childType(zcu).arrayLen(zcu);
16688	16957
16689	16958	assert(len != 0); // prevented by Sema
16690		try self.store(elem_ptr_ty, dst_ptr, src_val, .{ .safety = safety });
	16959	try self.store(elem_ptr_ty, dst, src_val, .{ .safety = safety });
16691	16960
16692	16961	const second_elem_ptr_reg =
16693	16962	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	...	@@ -16697,12 +16966,12 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16697	16966	defer self.register_manager.unlockReg(second_elem_ptr_lock);
16698	16967
16699	16968	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
16700		.reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
	16969	.reg = try self.copyToTmpRegister(Type.usize, dst),
16701	16970	.off = elem_abi_size,
16702	16971	} }, .{});
16703	16972
16704	16973	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
16705		try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
	16974	try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy);
16706	16975	},
16707	16976	.c, .many => unreachable,
16708	16977	}
...	...	@@ -16719,48 +16988,72 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
16719	16988	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
16720	16989	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
16721	16990
16722		const dst_ptr = try self.resolveInst(bin_op.lhs);
16723		const dst_ptr_ty = self.typeOf(bin_op.lhs);
16724		const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
16725		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
16726		else => null,
	16991	const dst = try self.resolveInst(bin_op.lhs);
	16992	const dst_ty = self.typeOf(bin_op.lhs);
	16993	const dst_locks: [2]?RegisterLock = switch (dst) {
	16994	.register => \|dst_reg\| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
	16995	.register_pair => \|dst_regs\| .{
	16996	self.register_manager.lockRegAssumeUnused(dst_regs[0]),
	16997	self.register_manager.lockReg(dst_regs[1]),
	16998	},
	16999	else => .{ null, null },
16727	17000	};
16728		defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	17001	for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
16729	17002
16730		const src_ptr = try self.resolveInst(bin_op.rhs);
16731		const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
16732		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
16733		else => null,
	17003	const src = try self.resolveInst(bin_op.rhs);
	17004	const src_locks: [2]?RegisterLock = switch (src) {
	17005	.register => \|src_reg\| .{ self.register_manager.lockReg(src_reg), null },
	17006	.register_pair => \|src_regs\| .{
	17007	self.register_manager.lockRegAssumeUnused(src_regs[0]),
	17008	self.register_manager.lockRegAssumeUnused(src_regs[1]),
	17009	},
	17010	else => .{ null, null },
16734	17011	};
16735		defer if (src_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	17012	for (src_locks) \|src_lock\| if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
16736	17013
16737		const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
	17014	const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
16738	17015	.slice => len: {
16739	17016	const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
16740	17017	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
16741	17018	defer self.register_manager.unlockReg(len_lock);
16742	17019
16743		try self.asmRegisterMemoryImmediate(
16744		.{ .i_, .mul },
16745		len_reg,
16746		try dst_ptr.address().offset(8).deref().mem(self, .qword),
16747		Immediate.s(@intCast(dst_ptr_ty.childType(zcu).abiSize(zcu))),
16748		);
	17020	switch (dst) {
	17021	.register_pair => \|dst_regs\| try self.asmRegisterRegisterImmediate(
	17022	.{ .i_, .mul },
	17023	len_reg,
	17024	dst_regs[1],
	17025	.s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
	17026	),
	17027	else => try self.asmRegisterMemoryImmediate(
	17028	.{ .i_, .mul },
	17029	len_reg,
	17030	try dst.address().offset(8).deref().mem(self, .qword),
	17031	.s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
	17032	),
	17033	}
16749	17034	break :len .{ .register = len_reg };
16750	17035	},
16751	17036	.one => len: {
16752		const array_ty = dst_ptr_ty.childType(zcu);
	17037	const array_ty = dst_ty.childType(zcu);
16753	17038	break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };
16754	17039	},
16755	17040	.c, .many => unreachable,
16756	17041	};
16757	17042	const len_lock: ?RegisterLock = switch (len) {
16758		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	17043	.register => \|reg\| self.register_manager.lockReg(reg),
16759	17044	else => null,
16760	17045	};
16761	17046	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
16762	17047
16763		// TODO: dst_ptr and src_ptr could be slices rather than raw pointers
	17048	const dst_ptr: MCValue = switch (dst) {
	17049	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
	17050	else => dst,
	17051	};
	17052	const src_ptr: MCValue = switch (src) {
	17053	.register_pair => \|src_regs\| .{ .register = src_regs[0] },
	17054	else => src,
	17055	};
	17056
16764	17057	try self.genInlineMemcpy(dst_ptr, src_ptr, len);
16765	17058
16766	17059	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -16930,27 +17223,23 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16930	17223	try self.genSetReg(
16931	17224	regs[1],
16932	17225	vector_ty,
16933		.{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },
	17226	.{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) },
16934	17227	.{},
16935	17228	);
16936	17229	const src_mcv = try self.resolveInst(ty_op.operand);
16937		const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);
	17230	const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4);
16938	17231	try self.asmCmovccRegisterRegister(
16939	17232	switch (src_mcv) {
16940	17233	.eflags => \|cc\| cc,
16941	17234	.register => \|src_reg\| cc: {
16942		try self.asmRegisterImmediate(
16943		.{ ._, .@"test" },
16944		src_reg.to8(),
16945		Immediate.u(1),
16946		);
	17235	try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1));
16947	17236	break :cc .nz;
16948	17237	},
16949	17238	else => cc: {
16950	17239	try self.asmMemoryImmediate(
16951	17240	.{ ._, .@"test" },
16952	17241	try src_mcv.mem(self, .byte),
16953		Immediate.u(1),
	17242	.u(1),
16954	17243	);
16955	17244	break :cc .nz;
16956	17245	},
...	...	@@ -17037,7 +17326,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17037	17326	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
17038	17327	dst_alias,
17039	17328	dst_alias,
17040		Immediate.u(0b00_00_00_00),
	17329	.u(0b00_00_00_00),
17041	17330	);
17042	17331	if (switch (scalar_bits) {
17043	17332	1...8 => vector_len > 4,
...	...	@@ -17049,7 +17338,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17049	17338	.{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
17050	17339	dst_alias,
17051	17340	dst_alias,
17052		Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
	17341	.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
17053	17342	);
17054	17343	break :result .{ .register = dst_reg };
17055	17344	},
...	...	@@ -17080,7 +17369,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17080	17369	dst_reg.to128(),
17081	17370	src_reg.to128(),
17082	17371	src_reg.to128(),
17083		Immediate.u(0),
	17372	.u(0),
17084	17373	);
17085	17374	}
17086	17375	break :result .{ .register = dst_reg };
...	...	@@ -17095,7 +17384,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17095	17384	.{ ._ps, .shuf },
17096	17385	dst_reg.to128(),
17097	17386	dst_reg.to128(),
17098		Immediate.u(0),
	17387	.u(0),
17099	17388	);
17100	17389	break :result dst_mcv;
17101	17390	}
...	...	@@ -17122,14 +17411,14 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17122	17411	dst_reg.to128(),
17123	17412	src_reg.to128(),
17124	17413	src_reg.to128(),
17125		Immediate.u(0),
	17414	.u(0),
17126	17415	);
17127	17416	try self.asmRegisterRegisterRegisterImmediate(
17128	17417	.{ .v_f128, .insert },
17129	17418	dst_reg.to256(),
17130	17419	dst_reg.to256(),
17131	17420	dst_reg.to128(),
17132		Immediate.u(1),
	17421	.u(1),
17133	17422	);
17134	17423	}
17135	17424	}
...	...	@@ -17198,7 +17487,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17198	17487	dst_reg.to256(),
17199	17488	dst_reg.to256(),
17200	17489	dst_reg.to128(),
17201		Immediate.u(1),
	17490	.u(1),
17202	17491	);
17203	17492	}
17204	17493	}
...	...	@@ -17231,7 +17520,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17231	17520	dst_reg.to256(),
17232	17521	src_reg.to256(),
17233	17522	src_reg.to128(),
17234		Immediate.u(1),
	17523	.u(1),
17235	17524	);
17236	17525	}
17237	17526	break :result .{ .register = dst_reg };
...	...	@@ -17308,7 +17597,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17308	17597	mask_alias,
17309	17598	mask_alias,
17310	17599	mask_reg.to128(),
17311		Immediate.u(1),
	17600	.u(1),
17312	17601	);
17313	17602	break :broadcast;
17314	17603	},
...	...	@@ -17362,7 +17651,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17362	17651	.{ if (has_avx) .vp_w else .p_w, .shufl },
17363	17652	mask_alias,
17364	17653	mask_alias,
17365		Immediate.u(0b00_00_00_00),
	17654	.u(0b00_00_00_00),
17366	17655	);
17367	17656	if (abi_size <= 8) break :broadcast;
17368	17657	}
...	...	@@ -17370,7 +17659,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17370	17659	.{ if (has_avx) .vp_d else .p_d, .shuf },
17371	17660	mask_alias,
17372	17661	mask_alias,
17373		Immediate.u(switch (elem_abi_size) {
	17662	.u(switch (elem_abi_size) {
17374	17663	1...2, 5...8 => 0b01_00_01_00,
17375	17664	3...4 => 0b00_00_00_00,
17376	17665	else => unreachable,
...	...	@@ -17649,7 +17938,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17649	17938	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17650	17939	const mask_elem = maybe_mask_elem orelse continue;
17651	17940	const mask_elem_index =
17652		math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
	17941	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
17653	17942	const elem_byte = (elem_index >> 1) * elem_abi_size;
17654	17943	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {
17655	17944	.unpckl => 0b0000,
...	...	@@ -17746,10 +18035,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17746	18035	} else sources[(elem_index & 0b010) >> 1] = source;
17747	18036
17748	18037	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17749		const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
	18038	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17750	18039	if (elem_index & 0b100 == 0)
17751		control \|= select
17752		else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;
	18040	control \|= select_mask
	18041	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd;
17753	18042	}
17754	18043
17755	18044	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
...	...	@@ -17767,7 +18056,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17767	18056	.{ if (has_avx) .vp_d else .p_d, .shuf },
17768	18057	dst_alias,
17769	18058	try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17770		Immediate.u(control),
	18059	.u(control),
17771	18060	) else try self.asmRegisterRegisterImmediate(
17772	18061	.{ if (has_avx) .vp_d else .p_d, .shuf },
17773	18062	dst_alias,
...	...	@@ -17775,7 +18064,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17775	18064	src_mcv.getReg().?
17776	18065	else
17777	18066	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
17778		Immediate.u(control),
	18067	.u(control),
17779	18068	);
17780	18069	break :result .{ .register = dst_reg };
17781	18070	}
...	...	@@ -17797,10 +18086,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17797	18086	} else sources[(elem_index & 0b010) >> 1] = source;
17798	18087
17799	18088	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17800		const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
	18089	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17801	18090	if (elem_index & 0b100 == 0)
17802		control \|= select
17803		else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;
	18091	control \|= select_mask
	18092	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps;
17804	18093	}
17805	18094	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
17806	18095
...	...	@@ -17824,7 +18113,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17824	18113	dst_alias,
17825	18114	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17826	18115	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17827		Immediate.u(control),
	18116	.u(control),
17828	18117	) else try self.asmRegisterRegisterRegisterImmediate(
17829	18118	.{ .v_ps, .shuf },
17830	18119	dst_alias,
...	...	@@ -17833,12 +18122,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17833	18122	rhs_mcv.getReg().?
17834	18123	else
17835	18124	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17836		Immediate.u(control),
	18125	.u(control),
17837	18126	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17838	18127	.{ ._ps, .shuf },
17839	18128	dst_alias,
17840	18129	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17841		Immediate.u(control),
	18130	.u(control),
17842	18131	) else try self.asmRegisterRegisterImmediate(
17843	18132	.{ ._ps, .shuf },
17844	18133	dst_alias,
...	...	@@ -17846,7 +18135,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17846	18135	rhs_mcv.getReg().?
17847	18136	else
17848	18137	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17849		Immediate.u(control),
	18138	.u(control),
17850	18139	);
17851	18140	break :result dst_mcv;
17852	18141	}
...	...	@@ -17891,7 +18180,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17891	18180	dst_alias,
17892	18181	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17893	18182	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17894		Immediate.u(control),
	18183	.u(control),
17895	18184	) else try self.asmRegisterRegisterRegisterImmediate(
17896	18185	.{ .v_pd, .shuf },
17897	18186	dst_alias,
...	...	@@ -17900,12 +18189,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17900	18189	rhs_mcv.getReg().?
17901	18190	else
17902	18191	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17903		Immediate.u(control),
	18192	.u(control),
17904	18193	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17905	18194	.{ ._pd, .shuf },
17906	18195	dst_alias,
17907	18196	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17908		Immediate.u(control),
	18197	.u(control),
17909	18198	) else try self.asmRegisterRegisterImmediate(
17910	18199	.{ ._pd, .shuf },
17911	18200	dst_alias,
...	...	@@ -17913,7 +18202,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17913	18202	rhs_mcv.getReg().?
17914	18203	else
17915	18204	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17916		Immediate.u(control),
	18205	.u(control),
17917	18206	);
17918	18207	break :result dst_mcv;
17919	18208	}
...	...	@@ -17927,13 +18216,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17927	18216	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17928	18217	const mask_elem = maybe_mask_elem orelse continue;
17929	18218	const mask_elem_index =
17930		math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
	18219	std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
17931	18220	if (mask_elem_index != elem_index) break :blend;
17932	18221
17933		const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
	18222	const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
17934	18223	if (elem_index & 0b1000 == 0)
17935		control \|= select
17936		else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;
	18224	control \|= select_mask
	18225	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend;
17937	18226	}
17938	18227
17939	18228	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
...	...	@@ -17961,7 +18250,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17961	18250	registerAlias(dst_reg, dst_abi_size),
17962	18251	registerAlias(lhs_reg, dst_abi_size),
17963	18252	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17964		Immediate.u(expanded_control),
	18253	.u(expanded_control),
17965	18254	) else try self.asmRegisterRegisterRegisterImmediate(
17966	18255	.{ .vp_d, .blend },
17967	18256	registerAlias(dst_reg, dst_abi_size),
...	...	@@ -17970,7 +18259,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17970	18259	rhs_mcv.getReg().?
17971	18260	else
17972	18261	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17973		Immediate.u(expanded_control),
	18262	.u(expanded_control),
17974	18263	);
17975	18264	break :result .{ .register = dst_reg };
17976	18265	}
...	...	@@ -18016,7 +18305,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18016	18305	else
18017	18306	dst_reg, dst_abi_size),
18018	18307	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18019		Immediate.u(expanded_control),
	18308	.u(expanded_control),
18020	18309	) else try self.asmRegisterRegisterRegisterImmediate(
18021	18310	.{ .vp_w, .blend },
18022	18311	registerAlias(dst_reg, dst_abi_size),
...	...	@@ -18028,12 +18317,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18028	18317	rhs_mcv.getReg().?
18029	18318	else
18030	18319	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18031		Immediate.u(expanded_control),
	18320	.u(expanded_control),
18032	18321	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18033	18322	.{ .p_w, .blend },
18034	18323	registerAlias(dst_reg, dst_abi_size),
18035	18324	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18036		Immediate.u(expanded_control),
	18325	.u(expanded_control),
18037	18326	) else try self.asmRegisterRegisterImmediate(
18038	18327	.{ .p_w, .blend },
18039	18328	registerAlias(dst_reg, dst_abi_size),
...	...	@@ -18041,7 +18330,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18041	18330	rhs_mcv.getReg().?
18042	18331	else
18043	18332	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18044		Immediate.u(expanded_control),
	18333	.u(expanded_control),
18045	18334	);
18046	18335	break :result .{ .register = dst_reg };
18047	18336	}
...	...	@@ -18077,7 +18366,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18077	18366	else
18078	18367	dst_reg, dst_abi_size),
18079	18368	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18080		Immediate.u(expanded_control),
	18369	.u(expanded_control),
18081	18370	) else try self.asmRegisterRegisterRegisterImmediate(
18082	18371	switch (elem_abi_size) {
18083	18372	4 => .{ .v_ps, .blend },
...	...	@@ -18093,7 +18382,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18093	18382	rhs_mcv.getReg().?
18094	18383	else
18095	18384	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18096		Immediate.u(expanded_control),
	18385	.u(expanded_control),
18097	18386	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18098	18387	switch (elem_abi_size) {
18099	18388	4 => .{ ._ps, .blend },
...	...	@@ -18102,7 +18391,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18102	18391	},
18103	18392	registerAlias(dst_reg, dst_abi_size),
18104	18393	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18105		Immediate.u(expanded_control),
	18394	.u(expanded_control),
18106	18395	) else try self.asmRegisterRegisterImmediate(
18107	18396	switch (elem_abi_size) {
18108	18397	4 => .{ ._ps, .blend },
...	...	@@ -18114,7 +18403,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18114	18403	rhs_mcv.getReg().?
18115	18404	else
18116	18405	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18117		Immediate.u(expanded_control),
	18406	.u(expanded_control),
18118	18407	);
18119	18408	break :result .{ .register = dst_reg };
18120	18409	}
...	...	@@ -18138,7 +18427,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18138	18427	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {
18139	18428	const mask_elem = maybe_mask_elem orelse continue;
18140	18429	const mask_elem_index =
18141		math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
	18430	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
18142	18431	if (mask_elem_index != elem_index) break :blendv;
18143	18432
18144	18433	select_mask_elem.* = (if (mask_elem < 0)
...	...	@@ -18380,7 +18669,9 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18380	18669
18381	18670	break :result null;
18382	18671	}) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
18383		lhs_ty.fmt(pt), rhs_ty.fmt(pt), dst_ty.fmt(pt),
	18672	lhs_ty.fmt(pt),
	18673	rhs_ty.fmt(pt),
	18674	dst_ty.fmt(pt),
18384	18675	Value.fromInterned(extra.mask).fmtValue(pt),
18385	18676	});
18386	18677	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
...	...	@@ -18397,7 +18688,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18397	18688	try self.spillEflagsIfOccupied();
18398	18689
18399	18690	const operand_mcv = try self.resolveInst(reduce.operand);
18400		const mask_len = (math.cast(u6, operand_ty.vectorLen(zcu)) orelse
	18691	const mask_len = (std.math.cast(u6, operand_ty.vectorLen(zcu)) orelse
18401	18692	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}));
18402	18693	const mask = (@as(u64, 1) << mask_len) - 1;
18403	18694	const abi_size: u32 = @intCast(operand_ty.abiSize(zcu));
...	...	@@ -18406,7 +18697,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18406	18697	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
18407	18698	.{ ._, .@"test" },
18408	18699	try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
18409		Immediate.u(mask),
	18700	.u(mask),
18410	18701	) else {
18411	18702	const operand_reg = registerAlias(if (operand_mcv.isRegister())
18412	18703	operand_mcv.getReg().?
...	...	@@ -18415,7 +18706,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18415	18706	if (mask_len < abi_size * 8) try self.asmRegisterImmediate(
18416	18707	.{ ._, .@"test" },
18417	18708	operand_reg,
18418		Immediate.u(mask),
	18709	.u(mask),
18419	18710	) else try self.asmRegisterRegister(
18420	18711	.{ ._, .@"test" },
18421	18712	operand_reg,
...	...	@@ -18431,7 +18722,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18431	18722
18432	18723	try self.asmRegister(.{ ._, .not }, tmp_reg);
18433	18724	if (mask_len < abi_size * 8)
18434		try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, Immediate.u(mask))
	18725	try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, .u(mask))
18435	18726	else
18436	18727	try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg);
18437	18728	break :result .{ .eflags = .z };
...	...	@@ -18579,12 +18870,12 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
18579	18870	try self.asmRegisterImmediate(
18580	18871	.{ ._, .@"and" },
18581	18872	registerAlias(elem_reg, @min(result_size, 4)),
18582		Immediate.u(1),
	18873	.u(1),
18583	18874	);
18584	18875	if (elem_i > 0) try self.asmRegisterImmediate(
18585	18876	.{ ._l, .sh },
18586	18877	registerAlias(elem_reg, result_size),
18587		Immediate.u(@intCast(elem_i)),
	18878	.u(@intCast(elem_i)),
18588	18879	);
18589	18880	try self.asmRegisterRegister(
18590	18881	.{ ._, .@"or" },
...	...	@@ -18748,8 +19039,8 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
18748	19039	lock.* = self.register_manager.lockRegAssumeUnused(reg);
18749	19040	}
18750	19041
18751		const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or
18752		mem.eql(u2, &order, &.{ 3, 1, 2 }))
	19042	const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or
	19043	std.mem.eql(u2, &order, &.{ 3, 1, 2 }))
18753	19044	switch (ty.zigTypeTag(zcu)) {
18754	19045	.float => switch (ty.floatBits(self.target.*)) {
18755	19046	32 => .{ .v_ss, .fmadd132 },
...	...	@@ -18776,7 +19067,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
18776	19067	},
18777	19068	else => unreachable,
18778	19069	}
18779		else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
	19070	else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 }))
18780	19071	switch (ty.zigTypeTag(zcu)) {
18781	19072	.float => switch (ty.floatBits(self.target.*)) {
18782	19073	32 => .{ .v_ss, .fmadd213 },
...	...	@@ -18803,7 +19094,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
18803	19094	},
18804	19095	else => unreachable,
18805	19096	}
18806		else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
	19097	else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 }))
18807	19098	switch (ty.zigTypeTag(zcu)) {
18808	19099	.float => switch (ty.floatBits(self.target.*)) {
18809	19100	32 => .{ .v_ss, .fmadd231 },
...	...	@@ -18953,13 +19244,13 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
18953	19244	const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
18954	19245	const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
18955	19246
18956		const classes = mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
	19247	const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
18957	19248	switch (classes[0]) {
18958	19249	.integer => {
18959	19250	assert(classes.len == 1);
18960	19251
18961	19252	try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{});
18962		try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
	19253	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
18963	19254	abi.SysV.c_abi_int_param_regs.len * 8,
18964	19255	));
18965	19256	const mem_reloc = try self.asmJccReloc(.ae, undefined);
...	...	@@ -19007,7 +19298,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19007	19298	assert(classes.len == 1);
19008	19299
19009	19300	try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{});
19010		try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
	19301	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
19011	19302	abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
19012	19303	));
19013	19304	const mem_reloc = try self.asmJccReloc(.ae, undefined);
...	...	@@ -19055,9 +19346,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19055	19346	assert(classes.len == 1);
19056	19347	unreachable;
19057	19348	},
19058		else => return self.fail("TODO implement c_va_arg for {} on SysV", .{
19059		promote_ty.fmt(pt),
19060		}),
	19349	else => return self.fail("TODO implement c_va_arg for {} on SysV", .{promote_ty.fmt(pt)}),
19061	19350	}
19062	19351
19063	19352	if (unused) break :result .unreach;
...	...	@@ -19194,7 +19483,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV
19194	19483	.immediate => \|imm\| {
19195	19484	// This immediate is unsigned.
19196	19485	const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));
19197		if (imm >= math.maxInt(U)) {
	19486	if (imm >= std.math.maxInt(U)) {
19198	19487	return MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) };
19199	19488	}
19200	19489	},
...	...	@@ -19226,7 +19515,7 @@ const CallMCValues = struct {
19226	19515	args: []MCValue,
19227	19516	return_value: InstTracking,
19228	19517	stack_byte_count: u31,
19229		stack_align: Alignment,
	19518	stack_align: InternPool.Alignment,
19230	19519	gp_count: u32,
19231	19520	fp_count: u32,
19232	19521
...	...	@@ -19303,7 +19592,7 @@ fn resolveCallingConventionValues(
19303	19592	var ret_tracking_i: usize = 0;
19304	19593
19305	19594	const classes = switch (resolved_cc) {
19306		.x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),
	19595	.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),
19307	19596	.x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},
19308	19597	else => unreachable,
19309	19598	};
...	...	@@ -19378,7 +19667,7 @@ fn resolveCallingConventionValues(
19378	19667	var arg_mcv_i: usize = 0;
19379	19668
19380	19669	const classes = switch (resolved_cc) {
19381		.x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),
	19670	.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),
19382	19671	.x86_64_win => &.{abi.classifyWindows(ty, zcu)},
19383	19672	else => unreachable,
19384	19673	};
...	...	@@ -19435,7 +19724,7 @@ fn resolveCallingConventionValues(
19435	19724
19436	19725	const frame_elem_align = 8;
19437	19726	const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs;
19438		const frame_elem_size = mem.alignForward(
	19727	const frame_elem_size = std.mem.alignForward(
19439	19728	u64,
19440	19729	ty.childType(zcu).abiSize(zcu),
19441	19730	frame_elem_align,
...	...	@@ -19443,7 +19732,7 @@ fn resolveCallingConventionValues(
19443	19732	const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size);
19444	19733
19445	19734	result.stack_byte_count =
19446		mem.alignForward(u31, result.stack_byte_count, frame_elem_align);
	19735	std.mem.alignForward(u31, result.stack_byte_count, frame_elem_align);
19447	19736	arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{
19448	19737	.regs = remaining_param_int_regs,
19449	19738	.frame_off = @intCast(result.stack_byte_count),
...	...	@@ -19461,19 +19750,14 @@ fn resolveCallingConventionValues(
19461	19750	continue;
19462	19751	}
19463	19752
19464		const param_size: u31 = @intCast(ty.abiSize(zcu));
19465	19753	const param_align = ty.abiAlignment(zcu).max(.@"8");
19466		result.stack_byte_count = mem.alignForward(
19467		u31,
19468		result.stack_byte_count,
19469		@intCast(param_align.toByteUnits().?),
19470		);
	19754	result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
19471	19755	result.stack_align = result.stack_align.max(param_align);
19472	19756	arg.* = .{ .load_frame = .{
19473	19757	.index = stack_frame_base,
19474	19758	.off = result.stack_byte_count,
19475	19759	} };
19476		result.stack_byte_count += param_size;
	19760	result.stack_byte_count += @intCast(ty.abiSize(zcu));
19477	19761	}
19478	19762	assert(param_int_reg_i <= 6);
19479	19763	result.gp_count = param_int_reg_i;
...	...	@@ -19509,19 +19793,14 @@ fn resolveCallingConventionValues(
19509	19793	arg.* = .none;
19510	19794	continue;
19511	19795	}
19512		const param_size: u31 = @intCast(ty.abiSize(zcu));
19513	19796	const param_align = ty.abiAlignment(zcu);
19514		result.stack_byte_count = mem.alignForward(
19515		u31,
19516		result.stack_byte_count,
19517		@intCast(param_align.toByteUnits().?),
19518		);
	19797	result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
19519	19798	result.stack_align = result.stack_align.max(param_align);
19520	19799	arg.* = .{ .load_frame = .{
19521	19800	.index = stack_frame_base,
19522	19801	.off = result.stack_byte_count,
19523	19802	} };
19524		result.stack_byte_count += param_size;
	19803	result.stack_byte_count += @intCast(ty.abiSize(zcu));
19525	19804	}
19526	19805	},
19527	19806	else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
...	...	@@ -19541,7 +19820,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMem
19541	19820	return error.CodegenFail;
19542	19821	}
19543	19822
19544		fn failMsg(self: Self, msg: ErrorMsg) error{ OutOfMemory, CodegenFail } {
	19823	fn failMsg(self: Self, msg: Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
19545	19824	@branchHint(.cold);
19546	19825	const zcu = self.pt.zcu;
19547	19826	switch (self.owner) {
...	...	@@ -19603,8 +19882,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
19603	19882	}
19604	19883
19605	19884	fn memSize(self: *Self, ty: Type) Memory.Size {
19606		const pt = self.pt;
19607		const zcu = pt.zcu;
	19885	const zcu = self.pt.zcu;
19608	19886	return switch (ty.zigTypeTag(zcu)) {
19609	19887	.float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),
19610	19888	else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))),
...	...	@@ -19614,7 +19892,7 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
19614	19892	fn splitType(self: *Self, ty: Type) ![2]Type {
19615	19893	const pt = self.pt;
19616	19894	const zcu = pt.zcu;
19617		const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
	19895	const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
19618	19896	var parts: [2]Type = undefined;
19619	19897	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {
19620	19898	part.* = switch (class) {
...	...	@@ -19648,7 +19926,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
19648	19926	.signedness = .unsigned,
19649	19927	.bits = @intCast(ty.bitSize(zcu)),
19650	19928	};
19651		const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return;
	19929	const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
19652	19930	try self.spillEflagsIfOccupied();
19653	19931	switch (int_info.signedness) {
19654	19932	.signed => {
...	...	@@ -19690,8 +19968,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
19690	19968	}
19691	19969
19692	19970	fn regBitSize(self: *Self, ty: Type) u64 {
19693		const pt = self.pt;
19694		const zcu = pt.zcu;
	19971	const zcu = self.pt.zcu;
19695	19972	const abi_size = ty.abiSize(zcu);
19696	19973	return switch (ty.zigTypeTag(zcu)) {
19697	19974	else => switch (abi_size) {
...	...	@@ -19713,14 +19990,14 @@ fn regExtraBits(self: *Self, ty: Type) u64 {
19713	19990	return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
19714	19991	}
19715	19992
19716		fn hasFeature(self: *Self, feature: Target.x86.Feature) bool {
19717		return Target.x86.featureSetHas(self.target.cpu.features, feature);
	19993	fn hasFeature(self: *Self, feature: std.Target.x86.Feature) bool {
	19994	return std.Target.x86.featureSetHas(self.target.cpu.features, feature);
19718	19995	}
19719	19996	fn hasAnyFeatures(self: *Self, features: anytype) bool {
19720		return Target.x86.featureSetHasAny(self.target.cpu.features, features);
	19997	return std.Target.x86.featureSetHasAny(self.target.cpu.features, features);
19721	19998	}
19722	19999	fn hasAllFeatures(self: *Self, features: anytype) bool {
19723		return Target.x86.featureSetHasAll(self.target.cpu.features, features);
	20000	return std.Target.x86.featureSetHasAll(self.target.cpu.features, features);
19724	20001	}
19725	20002
19726	20003	fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
...	...	@@ -19732,9 +20009,13 @@ fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
19732	20009	fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
19733	20010	const pt = self.pt;
19734	20011	const zcu = pt.zcu;
19735		return switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {
19736		.loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),
19737		else => self.air.typeOfIndex(inst, &zcu.intern_pool),
	20012	const temp: Temp = .{ .index = inst };
	20013	return switch (temp.unwrap(self)) {
	20014	.ref => switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {
	20015	.loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),
	20016	else => self.air.typeOfIndex(inst, &zcu.intern_pool),
	20017	},
	20018	.temp => temp.typeOf(self),
19738	20019	};
19739	20020	}
19740	20021
...	...	@@ -19815,3 +20096,613 @@ fn promoteVarArg(self: *Self, ty: Type) Type {
19815	20096	},
19816	20097	}
19817	20098	}
	20099
	20100	// ====================================== rewrite starts here ======================================
	20101
	20102	const Temp = struct {
	20103	index: Air.Inst.Index,
	20104
	20105	fn unwrap(temp: Temp, self: *Self) union(enum) {
	20106	ref: Air.Inst.Ref,
	20107	temp: Index,
	20108	} {
	20109	switch (temp.index.unwrap()) {
	20110	.ref => \|ref\| return .{ .ref = ref },
	20111	.target => \|target_index\| {
	20112	const temp_index: Index = @enumFromInt(target_index);
	20113	assert(temp_index.isValid(self));
	20114	return .{ .temp = temp_index };
	20115	},
	20116	}
	20117	}
	20118
	20119	fn typeOf(temp: Temp, self: *Self) Type {
	20120	return switch (temp.unwrap(self)) {
	20121	.ref => \|ref\| self.typeOf(ref),
	20122	.temp => \|temp_index\| temp_index.typeOf(self),
	20123	};
	20124	}
	20125
	20126	fn isMut(temp: Temp, self: *Self) bool {
	20127	return temp.unwrap(self) == .temp;
	20128	}
	20129
	20130	fn tracking(temp: Temp, self: *Self) InstTracking {
	20131	return self.inst_tracking.get(temp.index).?;
	20132	}
	20133
	20134	fn getOffset(temp: Temp, off: i32, self: *Self) !Temp {
	20135	const new_temp_index = self.next_temp_index;
	20136	self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
	20137	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
	20138	switch (temp.tracking(self).short) {
	20139	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
	20140	.register => \|reg\| {
	20141	const new_reg =
	20142	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20143	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20144	try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
	20145	.base = .{ .reg = reg.to64() },
	20146	.mod = .{ .rm = .{
	20147	.size = .qword,
	20148	.disp = off,
	20149	} },
	20150	});
	20151	},
	20152	.register_offset => \|reg_off\| {
	20153	const new_reg =
	20154	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20155	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20156	try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
	20157	.base = .{ .reg = reg_off.reg.to64() },
	20158	.mod = .{ .rm = .{
	20159	.size = .qword,
	20160	.disp = reg_off.off + off,
	20161	} },
	20162	});
	20163	},
	20164	.lea_symbol => \|sym_off\| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = .{
	20165	.sym_index = sym_off.sym_index,
	20166	.off = sym_off.off + off,
	20167	} }),
	20168	.load_frame => \|frame_addr\| {
	20169	const new_reg =
	20170	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20171	new_temp_index.tracking(self).* = InstTracking.init(.{ .register_offset = .{
	20172	.reg = new_reg,
	20173	.off = off,
	20174	} });
	20175	try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
	20176	.base = .{ .frame = frame_addr.index },
	20177	.mod = .{ .rm = .{
	20178	.size = .qword,
	20179	.disp = frame_addr.off,
	20180	} },
	20181	});
	20182	},
	20183	.lea_frame => \|frame_addr\| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = .{
	20184	.index = frame_addr.index,
	20185	.off = frame_addr.off + off,
	20186	} }),
	20187	}
	20188	return .{ .index = new_temp_index.toIndex() };
	20189	}
	20190
	20191	fn toOffset(temp: Temp, off: i32, self: Self) !void {
	20192	if (off == 0) return;
	20193	switch (temp.unwrap(self)) {
	20194	.ref => {},
	20195	.temp => \|temp_index\| {
	20196	const temp_tracking = temp_index.tracking(self);
	20197	switch (temp_tracking.short) {
	20198	else => {},
	20199	.register => \|reg\| {
	20200	try self.freeValue(temp_tracking.long);
	20201	temp_tracking.* = InstTracking.init(.{ .register_offset = .{
	20202	.reg = reg,
	20203	.off = off,
	20204	} });
	20205	return;
	20206	},
	20207	.register_offset => \|reg_off\| {
	20208	try self.freeValue(temp_tracking.long);
	20209	temp_tracking.* = InstTracking.init(.{ .register_offset = .{
	20210	.reg = reg_off.reg,
	20211	.off = reg_off.off + off,
	20212	} });
	20213	return;
	20214	},
	20215	.lea_symbol => \|sym_off\| {
	20216	assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
	20217	temp_tracking.* = InstTracking.init(.{ .lea_symbol = .{
	20218	.sym_index = sym_off.sym_index,
	20219	.off = sym_off.off + off,
	20220	} });
	20221	return;
	20222	},
	20223	.lea_frame => \|frame_addr\| {
	20224	assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
	20225	temp_tracking.* = InstTracking.init(.{ .lea_frame = .{
	20226	.index = frame_addr.index,
	20227	.off = frame_addr.off + off,
	20228	} });
	20229	return;
	20230	},
	20231	}
	20232	},
	20233	}
	20234	const new_temp = try temp.getOffset(off, self);
	20235	try temp.die(self);
	20236	temp.* = new_temp;
	20237	}
	20238
	20239	fn getLimb(temp: Temp, limb_index: u28, self: *Self) !Temp {
	20240	const new_temp_index = self.next_temp_index;
	20241	self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
	20242	switch (temp.tracking(self).short) {
	20243	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
	20244	.immediate => \|imm\| {
	20245	assert(limb_index == 0);
	20246	new_temp_index.tracking(self).* = InstTracking.init(.{ .immediate = imm });
	20247	},
	20248	.register => \|reg\| {
	20249	assert(limb_index == 0);
	20250	const new_reg =
	20251	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20252	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20253	try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
	20254	},
	20255	.register_pair => \|regs\| {
	20256	const new_reg =
	20257	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20258	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20259	try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
	20260	},
	20261	.register_offset => \|reg_off\| {
	20262	assert(limb_index == 0);
	20263	const new_reg =
	20264	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20265	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20266	try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
	20267	.base = .{ .reg = reg_off.reg.to64() },
	20268	.mod = .{ .rm = .{
	20269	.size = .qword,
	20270	.disp = reg_off.off + @as(u31, limb_index) * 8,
	20271	} },
	20272	});
	20273	},
	20274	.load_symbol => \|sym_off\| {
	20275	const new_reg =
	20276	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20277	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20278	try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
	20279	.base = .{ .reloc = sym_off.sym_index },
	20280	.mod = .{ .rm = .{
	20281	.size = .qword,
	20282	.disp = sym_off.off + @as(u31, limb_index) * 8,
	20283	} },
	20284	});
	20285	},
	20286	.lea_symbol => \|sym_off\| {
	20287	assert(limb_index == 0);
	20288	new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = sym_off });
	20289	},
	20290	.load_frame => \|frame_addr\| {
	20291	const new_reg =
	20292	try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
	20293	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20294	try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
	20295	.base = .{ .frame = frame_addr.index },
	20296	.mod = .{ .rm = .{
	20297	.size = .qword,
	20298	.disp = frame_addr.off + @as(u31, limb_index) * 8,
	20299	} },
	20300	});
	20301	},
	20302	.lea_frame => \|frame_addr\| {
	20303	assert(limb_index == 0);
	20304	new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = frame_addr });
	20305	},
	20306	}
	20307	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
	20308	return .{ .index = new_temp_index.toIndex() };
	20309	}
	20310
	20311	fn toLimb(temp: Temp, limb_index: u28, self: Self) !void {
	20312	switch (temp.unwrap(self)) {
	20313	.ref => {},
	20314	.temp => \|temp_index\| {
	20315	const temp_tracking = temp_index.tracking(self);
	20316	switch (temp_tracking.short) {
	20317	else => {},
	20318	.register, .lea_symbol, .lea_frame => {
	20319	assert(limb_index == 0);
	20320	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
	20321	return;
	20322	},
	20323	.register_pair => \|regs\| {
	20324	switch (temp_tracking.long) {
	20325	.none, .reserved_frame => {},
	20326	else => temp_tracking.long =
	20327	temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
	20328	}
	20329	for (regs, 0..) \|reg, reg_index\| if (reg_index != limb_index)
	20330	self.register_manager.freeReg(reg);
	20331	temp_tracking.* = InstTracking.init(.{ .register = regs[limb_index] });
	20332	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
	20333	return;
	20334	},
	20335	.load_symbol => \|sym_off\| {
	20336	assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
	20337	temp_tracking.* = InstTracking.init(.{ .load_symbol = .{
	20338	.sym_index = sym_off.sym_index,
	20339	.off = sym_off.off + @as(u31, limb_index) * 8,
	20340	} });
	20341	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
	20342	return;
	20343	},
	20344	.load_frame => \|frame_addr\| if (!frame_addr.index.isNamed()) {
	20345	assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
	20346	temp_tracking.* = InstTracking.init(.{ .load_frame = .{
	20347	.index = frame_addr.index,
	20348	.off = frame_addr.off + @as(u31, limb_index) * 8,
	20349	} });
	20350	self.temp_type[@intFromEnum(temp_index)] = Type.usize;
	20351	return;
	20352	},
	20353	}
	20354	},
	20355	}
	20356	const new_temp = try temp.getLimb(limb_index, self);
	20357	try temp.die(self);
	20358	temp.* = new_temp;
	20359	}
	20360
	20361	fn toReg(temp: Temp, new_reg: Register, self: Self) !bool {
	20362	const val, const ty = switch (temp.unwrap(self)) {
	20363	.ref => \|ref\| .{ temp.tracking(self).short, self.typeOf(ref) },
	20364	.temp => \|temp_index\| val: {
	20365	const temp_tracking = temp_index.tracking(self);
	20366	if (temp_tracking.short == .register and
	20367	temp_tracking.short.register == new_reg) return false;
	20368	break :val .{ temp_tracking.short, temp_index.typeOf(self) };
	20369	},
	20370	};
	20371	const new_temp_index = self.next_temp_index;
	20372	self.temp_type[@intFromEnum(new_temp_index)] = ty;
	20373	try self.genSetReg(new_reg, ty, val, .{});
	20374	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20375	try temp.die(self);
	20376	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
	20377	temp.* = .{ .index = new_temp_index.toIndex() };
	20378	return true;
	20379	}
	20380
	20381	fn toAnyReg(temp: Temp, self: Self) !bool {
	20382	const val, const ty = switch (temp.unwrap(self)) {
	20383	.ref => \|ref\| .{ temp.tracking(self).short, self.typeOf(ref) },
	20384	.temp => \|temp_index\| val: {
	20385	const temp_tracking = temp_index.tracking(self);
	20386	if (temp_tracking.short == .register) return false;
	20387	break :val .{ temp_tracking.short, temp_index.typeOf(self) };
	20388	},
	20389	};
	20390	const new_temp_index = self.next_temp_index;
	20391	self.temp_type[@intFromEnum(new_temp_index)] = ty;
	20392	const new_reg =
	20393	try self.register_manager.allocReg(new_temp_index.toIndex(), self.regClassForType(ty));
	20394	try self.genSetReg(new_reg, ty, val, .{});
	20395	new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg });
	20396	try temp.die(self);
	20397	self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
	20398	temp.* = .{ .index = new_temp_index.toIndex() };
	20399	return true;
	20400	}
	20401
	20402	fn toPair(first_temp: Temp, second_temp: Temp, self: *Self) !void {
	20403	while (true) for ([_]*Temp{ first_temp, second_temp }) \|part_temp\| {
	20404	if (try part_temp.toAnyReg(self)) break;
	20405	} else break;
	20406	const first_temp_tracking = first_temp.unwrap(self).temp.tracking(self);
	20407	const second_temp_tracking = second_temp.unwrap(self).temp.tracking(self);
	20408	const result: MCValue = .{ .register_pair = .{
	20409	first_temp_tracking.short.register,
	20410	second_temp_tracking.short.register,
	20411	} };
	20412	const result_temp_index = self.next_temp_index;
	20413	const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
	20414	assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
	20415	assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
	20416	self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
	20417	result_temp_index.tracking(self).* = InstTracking.init(result);
	20418	first_temp.* = result_temp;
	20419	}
	20420
	20421	fn toLea(temp: Temp, self: Self) !bool {
	20422	switch (temp.tracking(self).short) {
	20423	.none,
	20424	.unreach,
	20425	.dead,
	20426	.undef,
	20427	.eflags,
	20428	.register_pair,
	20429	.register_overflow,
	20430	.elementwise_regs_then_frame,
	20431	.reserved_frame,
	20432	.air_ref,
	20433	=> unreachable, // not a valid pointer
	20434	.immediate,
	20435	.register,
	20436	.register_offset,
	20437	.lea_direct,
	20438	.lea_got,
	20439	.lea_tlv,
	20440	.lea_frame,
	20441	=> return false,
	20442	.memory,
	20443	.indirect,
	20444	.load_symbol,
	20445	.load_direct,
	20446	.load_got,
	20447	.load_tlv,
	20448	.load_frame,
	20449	=> return temp.toAnyReg(self),
	20450	.lea_symbol => \|sym_off\| {
	20451	const off = sym_off.off;
	20452	if (off == 0) return false;
	20453	try temp.toOffset(-off, self);
	20454	while (try temp.toAnyReg(self)) {}
	20455	try temp.toOffset(off, self);
	20456	return true;
	20457	},
	20458	}
	20459	}
	20460
	20461	fn load(ptr: Temp, val_ty: Type, self: Self) !Temp {
	20462	const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));
	20463	const val = try self.tempAlloc(val_ty);
	20464	switch (val.tracking(self).short) {
	20465	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
	20466	.register => \|val_reg\| {
	20467	while (try ptr.toLea(self)) {}
	20468	switch (val_reg.class()) {
	20469	.general_purpose => try self.asmRegisterMemory(
	20470	.{ ._, .mov },
	20471	registerAlias(val_reg, val_abi_size),
	20472	try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
	20473	),
	20474	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
	20475	}
	20476	},
	20477	.load_frame => \|val_frame_addr\| {
	20478	var val_ptr = try self.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr });
	20479	var len = try self.tempFromValue(Type.usize, .{ .immediate = val_abi_size });
	20480	try val_ptr.memcpy(ptr, &len, self);
	20481	try val_ptr.die(self);
	20482	try len.die(self);
	20483	},
	20484	}
	20485	return val;
	20486	}
	20487
	20488	fn store(ptr: Temp, val: Temp, self: *Self) !void {
	20489	const val_ty = val.typeOf(self);
	20490	const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));
	20491	val: switch (val.tracking(self).short) {
	20492	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
	20493	.immediate => \|imm\| if (std.math.cast(i32, imm)) \|s\| {
	20494	while (try ptr.toLea(self)) {}
	20495	try self.asmMemoryImmediate(
	20496	.{ ._, .mov },
	20497	try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
	20498	.s(s),
	20499	);
	20500	} else continue :val .{ .register = undefined },
	20501	.register => {
	20502	while (try ptr.toLea(self) or try val.toAnyReg(self)) {}
	20503	const val_reg = val.tracking(self).short.register;
	20504	switch (val_reg.class()) {
	20505	.general_purpose => try self.asmMemoryRegister(
	20506	.{ ._, .mov },
	20507	try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),
	20508	registerAlias(val_reg, val_abi_size),
	20509	),
	20510	else => \|tag\| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
	20511	}
	20512	},
	20513	}
	20514	}
	20515
	20516	fn memcpy(dst: Temp, src: Temp, len: Temp, self: Self) !void {
	20517	while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) \|temp, reg\| {
	20518	if (try temp.toReg(reg, self)) break;
	20519	} else break;
	20520	try self.asmOpOnly(.{ .@"rep _sb", .mov });
	20521	}
	20522
	20523	fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void {
	20524	if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) {
	20525	.ref => {
	20526	const result = try self.allocRegOrMem(inst, true);
	20527	try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{});
	20528	tracking_log.debug("{} => {} (birth)", .{ inst, result });
	20529	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
	20530	},
	20531	.temp => \|temp_index\| {
	20532	const temp_tracking = temp_index.tracking(self);
	20533	tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });
	20534	self.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
	20535	assert(self.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
	20536	},
	20537	}
	20538	}
	20539
	20540	fn die(temp: Temp, self: *Self) !void {
	20541	switch (temp.unwrap(self)) {
	20542	.ref => {},
	20543	.temp => \|temp_index\| try temp_index.tracking(self).die(self, temp_index.toIndex()),
	20544	}
	20545	}
	20546
	20547	const Index = enum(u4) {
	20548	_,
	20549
	20550	fn toIndex(index: Index) Air.Inst.Index {
	20551	return Air.Inst.Index.fromTargetIndex(@intFromEnum(index));
	20552	}
	20553
	20554	fn fromIndex(index: Air.Inst.Index) Index {
	20555	return @enumFromInt(index.toTargetIndex());
	20556	}
	20557
	20558	fn tracking(index: Index, self: Self) InstTracking {
	20559	return &self.inst_tracking.values()[@intFromEnum(index)];
	20560	}
	20561
	20562	fn isValid(index: Index, self: *Self) bool {
	20563	return index.tracking(self).short != .dead;
	20564	}
	20565
	20566	fn typeOf(index: Index, self: *Self) Type {
	20567	assert(index.isValid(self));
	20568	return self.temp_type[@intFromEnum(index)];
	20569	}
	20570
	20571	const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type);
	20572	const Set = std.StaticBitSet(max);
	20573	const SafetySet = if (std.debug.runtime_safety) Set else struct {
	20574	inline fn initEmpty() @This() {
	20575	return .{};
	20576	}
	20577
	20578	inline fn isSet(_: @This(), index: usize) bool {
	20579	assert(index < max);
	20580	return true;
	20581	}
	20582
	20583	inline fn set(_: @This(), index: usize) void {
	20584	assert(index < max);
	20585	}
	20586
	20587	inline fn eql(_: @This(), _: @This()) bool {
	20588	return true;
	20589	}
	20590	};
	20591	};
	20592	};
	20593
	20594	fn resetTemps(self: *Self) void {
	20595	for (0..@intFromEnum(self.next_temp_index)) \|temp_index\| {
	20596	const temp: Temp.Index = @enumFromInt(temp_index);
	20597	assert(!temp.isValid(self));
	20598	self.temp_type[temp_index] = undefined;
	20599	}
	20600	self.next_temp_index = @enumFromInt(0);
	20601	}
	20602
	20603	fn reuseTemp(
	20604	self: *Self,
	20605	new_inst: Air.Inst.Index,
	20606	old_inst: Air.Inst.Index,
	20607	tracking: *InstTracking,
	20608	) bool {
	20609	switch (tracking.short) {
	20610	.register,
	20611	.register_pair,
	20612	.register_offset,
	20613	.register_overflow,
	20614	=> for (tracking.short.getRegs()) \|tracked_reg\| {
	20615	if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) \|tracked_index\| {
	20616	self.register_manager.registers[tracked_index] = new_inst;
	20617	}
	20618	},
	20619	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
	20620	else => {},
	20621	}
	20622	switch (tracking.short) {
	20623	.eflags, .register_overflow => self.eflags_inst = new_inst,
	20624	else => {},
	20625	}
	20626	tracking.reuse(self, new_inst, old_inst);
	20627	return true;
	20628	}
	20629
	20630	fn tempAlloc(self: *Self, ty: Type) !Temp {
	20631	const temp_index = self.next_temp_index;
	20632	temp_index.tracking(self).* = InstTracking.init(try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true));
	20633	self.temp_type[@intFromEnum(temp_index)] = ty;
	20634	self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
	20635	return .{ .index = temp_index.toIndex() };
	20636	}
	20637
	20638	fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp {
	20639	const temp_index = self.next_temp_index;
	20640	temp_index.tracking(self).* = InstTracking.init(value);
	20641	self.temp_type[@intFromEnum(temp_index)] = ty;
	20642	try self.getValue(value, temp_index.toIndex());
	20643	self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
	20644	return .{ .index = temp_index.toIndex() };
	20645	}
	20646
	20647	fn tempFromOperand(
	20648	self: *Self,
	20649	inst: Air.Inst.Index,
	20650	op_index: Liveness.OperandInt,
	20651	op_ref: Air.Inst.Ref,
	20652	) !Temp {
	20653	const zcu = self.pt.zcu;
	20654	const ip = &zcu.intern_pool;
	20655
	20656	if (!self.liveness.operandDies(inst, op_index)) {
	20657	if (op_ref.toIndex()) \|op_inst\| return .{ .index = op_inst };
	20658	const val = op_ref.toInterned().?;
	20659	const gop = try self.const_tracking.getOrPut(self.gpa, val);
	20660	if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {
	20661	const const_mcv = try self.genTypedValue(Value.fromInterned(val));
	20662	switch (const_mcv) {
	20663	.lea_tlv => \|tlv_sym\| switch (self.bin_file.tag) {
	20664	.elf, .macho => {
	20665	if (self.mod.pic) {
	20666	try self.spillRegisters(&.{ .rdi, .rax });
	20667	} else {
	20668	try self.spillRegisters(&.{.rax});
	20669	}
	20670	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
	20671	.size = 8,
	20672	.alignment = .@"8",
	20673	}));
	20674	try self.genSetMem(
	20675	.{ .frame = frame_index },
	20676	0,
	20677	Type.usize,
	20678	.{ .lea_symbol = .{ .sym_index = tlv_sym } },
	20679	.{},
	20680	);
	20681	break :init .{ .load_frame = .{ .index = frame_index } };
	20682	},
	20683	else => break :init const_mcv,
	20684	},
	20685	else => break :init const_mcv,
	20686	}
	20687	});
	20688	return self.tempFromValue(Type.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
	20689	}
	20690
	20691	const temp_index = self.next_temp_index;
	20692	const temp: Temp = .{ .index = temp_index.toIndex() };
	20693	const op_inst = op_ref.toIndex().?;
	20694	const tracking = self.getResolvedInstValue(op_inst);
	20695	temp_index.tracking(self).* = tracking.*;
	20696	if (!self.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
	20697	self.temp_type[@intFromEnum(temp_index)] = self.typeOf(op_ref);
	20698	self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
	20699	return temp;
	20700	}
	20701
	20702	inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
	20703	var temps: [op_refs.len]Temp = undefined;
	20704	inline for (&temps, 0.., op_refs) \|*temp, op_index, op_ref\| {
	20705	temp.* = try self.tempFromOperand(inst, op_index, op_ref);
	20706	}
	20707	return temps;
	20708	}

src/arch/x86_64/abi.zig+2-3

...	...	@@ -250,9 +250,8 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8
250	250	return memory_class;
251	251	},
252	252	.optional => {
253		if (ty.isPtrLikeOptional(zcu)) {
254		result[0] = .integer;
255		return result;
	253	if (ty.optionalReprIsPayload(zcu)) {
	254	return classifySystemV(ty.optionalChild(zcu), zcu, target, ctx);
256	255	}
257	256	return memory_class;
258	257	},

src/arch/x86_64/bits.zig+33-1

...	...	@@ -547,7 +547,39 @@ pub const Memory = struct {
547	547	}
548	548	};
549	549
550		pub const Scale = enum(u2) { @"1", @"2", @"4", @"8" };
	550	pub const Scale = enum(u2) {
	551	@"1",
	552	@"2",
	553	@"4",
	554	@"8",
	555
	556	pub fn fromFactor(factor: u4) Scale {
	557	return switch (factor) {
	558	else => unreachable,
	559	1 => .@"1",
	560	2 => .@"2",
	561	4 => .@"4",
	562	8 => .@"8",
	563	};
	564	}
	565
	566	pub fn toFactor(scale: Scale) u4 {
	567	return switch (scale) {
	568	.@"1" => 1,
	569	.@"2" => 2,
	570	.@"4" => 4,
	571	.@"8" => 8,
	572	};
	573	}
	574
	575	pub fn fromLog2(log2: u2) Scale {
	576	return @enumFromInt(log2);
	577	}
	578
	579	pub fn toLog2(scale: Scale) u2 {
	580	return @intFromEnum(scale);
	581	}
	582	};
551	583	};
552	584
553	585	pub const Immediate = union(enum) {

src/print_air.zig+12-12

...	...	@@ -96,8 +96,8 @@ const Writer = struct {
96	96	fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
97	97	const tag = w.air.instructions.items(.tag)[@intFromEnum(inst)];
98	98	try s.writeByteNTimes(' ', w.indent);
99		try s.print("%{d}{c}= {s}(", .{
100		@intFromEnum(inst),
	99	try s.print("{}{c}= {s}(", .{
	100	inst,
101	101	@as(u8, if (if (w.liveness) \|liveness\| liveness.isUnused(inst) else false) '!' else ' '),
102	102	@tagName(tag),
103	103	});
...	...	@@ -409,7 +409,7 @@ const Writer = struct {
409	409	try s.writeAll("}");
410	410
411	411	for (liveness_block.deaths) \|operand\| {
412		try s.print(" %{d}!", .{@intFromEnum(operand)});
	412	try s.print(" {}!", .{operand});
413	413	}
414	414	}
415	415
...	...	@@ -728,7 +728,7 @@ const Writer = struct {
728	728	try s.writeByteNTimes(' ', w.indent);
729	729	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
730	730	if (i != 0) try s.writeAll(" ");
731		try s.print("%{d}!", .{@intFromEnum(operand)});
	731	try s.print("{}!", .{operand});
732	732	}
733	733	try s.writeAll("\n");
734	734	}
...	...	@@ -739,7 +739,7 @@ const Writer = struct {
739	739	try s.writeAll("}");
740	740
741	741	for (liveness_condbr.then_deaths) \|operand\| {
742		try s.print(" %{d}!", .{@intFromEnum(operand)});
	742	try s.print(" {}!", .{operand});
743	743	}
744	744	}
745	745
...	...	@@ -765,7 +765,7 @@ const Writer = struct {
765	765	try s.writeByteNTimes(' ', w.indent);
766	766	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
767	767	if (i != 0) try s.writeAll(" ");
768		try s.print("%{d}!", .{@intFromEnum(operand)});
	768	try s.print("{}!", .{operand});
769	769	}
770	770	try s.writeAll("\n");
771	771	}
...	...	@@ -776,7 +776,7 @@ const Writer = struct {
776	776	try s.writeAll("}");
777	777
778	778	for (liveness_condbr.then_deaths) \|operand\| {
779		try s.print(" %{d}!", .{@intFromEnum(operand)});
	779	try s.print(" {}!", .{operand});
780	780	}
781	781	}
782	782
...	...	@@ -807,7 +807,7 @@ const Writer = struct {
807	807	try s.writeByteNTimes(' ', w.indent);
808	808	for (liveness_condbr.then_deaths, 0..) \|operand, i\| {
809	809	if (i != 0) try s.writeAll(" ");
810		try s.print("%{d}!", .{@intFromEnum(operand)});
	810	try s.print("{}!", .{operand});
811	811	}
812	812	try s.writeAll("\n");
813	813	}
...	...	@@ -827,7 +827,7 @@ const Writer = struct {
827	827	try s.writeByteNTimes(' ', w.indent);
828	828	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
829	829	if (i != 0) try s.writeAll(" ");
830		try s.print("%{d}!", .{@intFromEnum(operand)});
	830	try s.print("{}!", .{operand});
831	831	}
832	832	try s.writeAll("\n");
833	833	}
...	...	@@ -884,7 +884,7 @@ const Writer = struct {
884	884	try s.writeByteNTimes(' ', w.indent);
885	885	for (deaths, 0..) \|operand, i\| {
886	886	if (i != 0) try s.writeAll(" ");
887		try s.print("%{d}!", .{@intFromEnum(operand)});
	887	try s.print("{}!", .{operand});
888	888	}
889	889	try s.writeAll("\n");
890	890	}
...	...	@@ -910,7 +910,7 @@ const Writer = struct {
910	910	try s.writeByteNTimes(' ', w.indent);
911	911	for (deaths, 0..) \|operand, i\| {
912	912	if (i != 0) try s.writeAll(" ");
913		try s.print("%{d}!", .{@intFromEnum(operand)});
	913	try s.print("{}!", .{operand});
914	914	}
915	915	try s.writeAll("\n");
916	916	}
...	...	@@ -994,7 +994,7 @@ const Writer = struct {
994	994	dies: bool,
995	995	) @TypeOf(s).Error!void {
996	996	_ = w;
997		try s.print("%{d}", .{@intFromEnum(inst)});
	997	try s.print("{}", .{inst});
998	998	if (dies) try s.writeByte('!');
999	999	}
1000	1000

tools/lldb_pretty_printers.py+1-1

...	...	@@ -383,7 +383,7 @@ def InstRef_SummaryProvider(value, _=None):
383	383	'InternPool.Index(%d)' % value.unsigned if value.unsigned < 0x80000000 else 'instructions[%d]' % (value.unsigned - 0x80000000))
384	384
385	385	def InstIndex_SummaryProvider(value, _=None):
386		return 'instructions[%d]' % value.unsigned
	386	return 'instructions[%d]' % value.unsigned if value.unsigned < 0x80000000 else 'temps[%d]' % (value.unsigned - 0x80000000)
387	387
388	388	class zig_DeclIndex_SynthProvider:
389	389	def __init__(self, value, _=None): self.value = value