Commit 4bace0f6212a3007247c42b0effcc40c6cfe61a8 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-01-17 01:20:11-05:00
committer	GitHub <noreply@github.com> 2025-01-17 01:20:11-05:00
log	4bace0f6212a3007247c42b0effcc40c6cfe61a8
tree	a5c1fee87a9266319d8ceffa7717ca5c70b61f7b
parent	257054a1467b2612725bd66852d84496024cf66c
parent	8c8dfb35f398407319764f0f8998de34c5247ed6
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #22386 from jacobly0/x86_64-rewrite

x86_64: begin rewriting instruction selection

75 files changed, 16973 insertions(+), 4863 deletions(-)

lib
- std
  - Target
    Query.zig
    x86.zig
  - Thread.zig
  - Thread
    Condition.zig
    Mutex.zig
    Pool.zig
  - crypto
    aes
    aesni.zig
    chacha20.zig
    tls
    Client.zig
  - debug.zig
  - debug
    NoPanic.zig
    SelfInfo.zig
  - heap.zig
  - http
    protocol.zig
  - math
    big
    int.zig
  - mem.zig
  - os.zig
  - os
    windows.zig
  - posix.zig
  - posix
    test.zig
  - simd.zig
  - zig
    system
    NativePaths.zig
    x86.zig
src
- Air.zig
- Compilation.zig
- Liveness.zig
- Type.zig
- arch
  - aarch64
    CodeGen.zig
  - arm
    CodeGen.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - x86_64
    CodeGen.zig
    Disassembler.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    abi.zig
    bits.zig
    encoder.zig
    encodings.zig
- codegen
  - c
    Type.zig
  - llvm.zig
- dev.zig
- link
  - C.zig
  - Elf
    Atom.zig
  - MachO.zig
  - MachO
    Atom.zig
- main.zig
- print_air.zig
- register_manager.zig
test
- behavior.zig
- behavior
tools
- lldb_pretty_printers.py
- update_cpu_features.zig

lib/std/Target/Query.zig+4-4

lib/std/Target/x86.zig+65

...	@@ -47,6 +47,7 @@ pub const Feature = enum {	...	@@ -47,6 +47,7 @@ pub const Feature = enum {
47	bmi2,	47	bmi2,
48	branch_hint,	48	branch_hint,
49	branchfusion,	49	branchfusion,
		50	bsf_bsr_0_clobbers_result,
50	ccmp,	51	ccmp,
51	cf,	52	cf,
52	cldemote,	53	cldemote,
...	@@ -167,6 +168,8 @@ pub const Feature = enum {	...	@@ -167,6 +168,8 @@ pub const Feature = enum {
167	slow_unaligned_mem_32,	168	slow_unaligned_mem_32,
168	sm3,	169	sm3,
169	sm4,	170	sm4,
		171	smap,
		172	smep,
170	soft_float,	173	soft_float,
171	sse,	174	sse,
172	sse2,	175	sse2,
...	@@ -497,6 +500,11 @@ pub const all_features = blk: {	...	@@ -497,6 +500,11 @@ pub const all_features = blk: {
497	.description = "CMP/TEST can be fused with conditional branches",	500	.description = "CMP/TEST can be fused with conditional branches",
498	.dependencies = featureSet(&[_]Feature{}),	501	.dependencies = featureSet(&[_]Feature{}),
499	};	502	};
		503	result[@intFromEnum(Feature.bsf_bsr_0_clobbers_result)] = .{
		504	.llvm_name = null,
		505	.description = "BSF/BSR may clobber the lower 32-bits of the result register when the source is zero",
		506	.dependencies = featureSet(&[_]Feature{}),
		507	};
500	result[@intFromEnum(Feature.ccmp)] = .{	508	result[@intFromEnum(Feature.ccmp)] = .{
501	.llvm_name = "ccmp",	509	.llvm_name = "ccmp",
502	.description = "Support conditional cmp & test instructions",	510	.description = "Support conditional cmp & test instructions",
...	@@ -1127,6 +1135,16 @@ pub const all_features = blk: {	...	@@ -1127,6 +1135,16 @@ pub const all_features = blk: {
1127	.avx2,	1135	.avx2,
1128	}),	1136	}),
1129	};	1137	};
		1138	result[@intFromEnum(Feature.smap)] = .{
		1139	.llvm_name = null,
		1140	.description = "Enable Supervisor Mode Access Prevention",
		1141	.dependencies = featureSet(&[_]Feature{}),
		1142	};
		1143	result[@intFromEnum(Feature.smep)] = .{
		1144	.llvm_name = null,
		1145	.description = "Enable Supervisor Mode Execution Prevention",
		1146	.dependencies = featureSet(&[_]Feature{}),
		1147	};
1130	result[@intFromEnum(Feature.soft_float)] = .{	1148	result[@intFromEnum(Feature.soft_float)] = .{
1131	.llvm_name = "soft-float",	1149	.llvm_name = "soft-float",
1132	.description = "Use software floating point features",	1150	.description = "Use software floating point features",
...	@@ -1371,6 +1389,8 @@ pub const cpu = struct {	...	@@ -1371,6 +1389,8 @@ pub const cpu = struct {
1371	.sha,	1389	.sha,
1372	.shstk,	1390	.shstk,
1373	.slow_3ops_lea,	1391	.slow_3ops_lea,
		1392	.smap,
		1393	.smep,
1374	.tuning_fast_imm_vector_shift,	1394	.tuning_fast_imm_vector_shift,
1375	.vaes,	1395	.vaes,
1376	.vpclmulqdq,	1396	.vpclmulqdq,
...	@@ -1467,6 +1487,8 @@ pub const cpu = struct {	...	@@ -1467,6 +1487,8 @@ pub const cpu = struct {
1467	.sha,	1487	.sha,
1468	.shstk,	1488	.shstk,
1469	.slow_3ops_lea,	1489	.slow_3ops_lea,
		1490	.smap,
		1491	.smep,
1470	.tuning_fast_imm_vector_shift,	1492	.tuning_fast_imm_vector_shift,
1471	.uintr,	1493	.uintr,
1472	.vaes,	1494	.vaes,
...	@@ -1545,6 +1567,8 @@ pub const cpu = struct {	...	@@ -1545,6 +1567,8 @@ pub const cpu = struct {
1545	.slow_3ops_lea,	1567	.slow_3ops_lea,
1546	.sm3,	1568	.sm3,
1547	.sm4,	1569	.sm4,
		1570	.smap,
		1571	.smep,
1548	.tuning_fast_imm_vector_shift,	1572	.tuning_fast_imm_vector_shift,
1549	.uintr,	1573	.uintr,
1550	.vaes,	1574	.vaes,
...	@@ -1783,6 +1807,8 @@ pub const cpu = struct {	...	@@ -1783,6 +1807,8 @@ pub const cpu = struct {
1783	.sahf,	1807	.sahf,
1784	.sbb_dep_breaking,	1808	.sbb_dep_breaking,
1785	.slow_shld,	1809	.slow_shld,
		1810	.smap,
		1811	.smep,
1786	.sse4a,	1812	.sse4a,
1787	.vzeroupper,	1813	.vzeroupper,
1788	.x87,	1814	.x87,
...	@@ -1995,6 +2021,8 @@ pub const cpu = struct {	...	@@ -1995,6 +2021,8 @@ pub const cpu = struct {
1995	.rdseed,	2021	.rdseed,
1996	.sahf,	2022	.sahf,
1997	.slow_3ops_lea,	2023	.slow_3ops_lea,
		2024	.smap,
		2025	.smep,
1998	.vzeroupper,	2026	.vzeroupper,
1999	.x87,	2027	.x87,
2000	.xsaveopt,	2028	.xsaveopt,
...	@@ -2136,6 +2164,8 @@ pub const cpu = struct {	...	@@ -2136,6 +2164,8 @@ pub const cpu = struct {
2136	.sahf,	2164	.sahf,
2137	.sha,	2165	.sha,
2138	.slow_3ops_lea,	2166	.slow_3ops_lea,
		2167	.smap,
		2168	.smep,
2139	.tuning_fast_imm_vector_shift,	2169	.tuning_fast_imm_vector_shift,
2140	.vzeroupper,	2170	.vzeroupper,
2141	.x87,	2171	.x87,
...	@@ -2195,6 +2225,8 @@ pub const cpu = struct {	...	@@ -2195,6 +2225,8 @@ pub const cpu = struct {
2195	.rdseed,	2225	.rdseed,
2196	.sahf,	2226	.sahf,
2197	.slow_3ops_lea,	2227	.slow_3ops_lea,
		2228	.smap,
		2229	.smep,
2198	.tuning_fast_imm_vector_shift,	2230	.tuning_fast_imm_vector_shift,
2199	.vzeroupper,	2231	.vzeroupper,
2200	.x87,	2232	.x87,
...	@@ -2450,6 +2482,8 @@ pub const cpu = struct {	...	@@ -2450,6 +2482,8 @@ pub const cpu = struct {
2450	.serialize,	2482	.serialize,
2451	.sha,	2483	.sha,
2452	.shstk,	2484	.shstk,
		2485	.smap,
		2486	.smep,
2453	.tsxldtrk,	2487	.tsxldtrk,
2454	.tuning_fast_imm_vector_shift,	2488	.tuning_fast_imm_vector_shift,
2455	.uintr,	2489	.uintr,
...	@@ -2519,6 +2553,8 @@ pub const cpu = struct {	...	@@ -2519,6 +2553,8 @@ pub const cpu = struct {
2519	.slow_incdec,	2553	.slow_incdec,
2520	.slow_lea,	2554	.slow_lea,
2521	.slow_two_mem_ops,	2555	.slow_two_mem_ops,
		2556	.smap,
		2557	.smep,
2522	.sse4_2,	2558	.sse4_2,
2523	.use_glm_div_sqrt_costs,	2559	.use_glm_div_sqrt_costs,
2524	.vzeroupper,	2560	.vzeroupper,
...	@@ -2898,6 +2934,7 @@ pub const cpu = struct {	...	@@ -2898,6 +2934,7 @@ pub const cpu = struct {
2898	.rdrnd,	2934	.rdrnd,
2899	.sahf,	2935	.sahf,
2900	.slow_3ops_lea,	2936	.slow_3ops_lea,
		2937	.smep,
2901	.vzeroupper,	2938	.vzeroupper,
2902	.x87,	2939	.x87,
2903	.xsaveopt,	2940	.xsaveopt,
...	@@ -2907,6 +2944,7 @@ pub const cpu = struct {	...	@@ -2907,6 +2944,7 @@ pub const cpu = struct {
2907	.name = "i386",	2944	.name = "i386",
2908	.llvm_name = "i386",	2945	.llvm_name = "i386",
2909	.features = featureSet(&[_]Feature{	2946	.features = featureSet(&[_]Feature{
		2947	.bsf_bsr_0_clobbers_result,
2910	.slow_unaligned_mem_16,	2948	.slow_unaligned_mem_16,
2911	.vzeroupper,	2949	.vzeroupper,
2912	.x87,	2950	.x87,
...	@@ -2916,6 +2954,7 @@ pub const cpu = struct {	...	@@ -2916,6 +2954,7 @@ pub const cpu = struct {
2916	.name = "i486",	2954	.name = "i486",
2917	.llvm_name = "i486",	2955	.llvm_name = "i486",
2918	.features = featureSet(&[_]Feature{	2956	.features = featureSet(&[_]Feature{
		2957	.bsf_bsr_0_clobbers_result,
2919	.slow_unaligned_mem_16,	2958	.slow_unaligned_mem_16,
2920	.vzeroupper,	2959	.vzeroupper,
2921	.x87,	2960	.x87,
...	@@ -3096,6 +3135,7 @@ pub const cpu = struct {	...	@@ -3096,6 +3135,7 @@ pub const cpu = struct {
3096	.sahf,	3135	.sahf,
3097	.slow_3ops_lea,	3136	.slow_3ops_lea,
3098	.slow_unaligned_mem_32,	3137	.slow_unaligned_mem_32,
		3138	.smep,
3099	.vzeroupper,	3139	.vzeroupper,
3100	.x87,	3140	.x87,
3101	.xsaveopt,	3141	.xsaveopt,
...	@@ -3403,6 +3443,8 @@ pub const cpu = struct {	...	@@ -3403,6 +3443,8 @@ pub const cpu = struct {
3403	.sha,	3443	.sha,
3404	.shstk,	3444	.shstk,
3405	.slow_3ops_lea,	3445	.slow_3ops_lea,
		3446	.smap,
		3447	.smep,
3406	.tuning_fast_imm_vector_shift,	3448	.tuning_fast_imm_vector_shift,
3407	.vaes,	3449	.vaes,
3408	.vpclmulqdq,	3450	.vpclmulqdq,
...	@@ -3766,6 +3808,8 @@ pub const cpu = struct {	...	@@ -3766,6 +3808,8 @@ pub const cpu = struct {
3766	.sha,	3808	.sha,
3767	.shstk,	3809	.shstk,
3768	.slow_3ops_lea,	3810	.slow_3ops_lea,
		3811	.smap,
		3812	.smep,
3769	.tuning_fast_imm_vector_shift,	3813	.tuning_fast_imm_vector_shift,
3770	.vaes,	3814	.vaes,
3771	.vpclmulqdq,	3815	.vpclmulqdq,
...	@@ -3831,6 +3875,8 @@ pub const cpu = struct {	...	@@ -3831,6 +3875,8 @@ pub const cpu = struct {
3831	.rdseed,	3875	.rdseed,
3832	.sahf,	3876	.sahf,
3833	.sha,	3877	.sha,
		3878	.smap,
		3879	.smep,
3834	.tuning_fast_imm_vector_shift,	3880	.tuning_fast_imm_vector_shift,
3835	.vaes,	3881	.vaes,
3836	.vpclmulqdq,	3882	.vpclmulqdq,
...	@@ -3939,6 +3985,8 @@ pub const cpu = struct {	...	@@ -3939,6 +3985,8 @@ pub const cpu = struct {
3939	.serialize,	3985	.serialize,
3940	.sha,	3986	.sha,
3941	.shstk,	3987	.shstk,
		3988	.smap,
		3989	.smep,
3942	.tsxldtrk,	3990	.tsxldtrk,
3943	.tuning_fast_imm_vector_shift,	3991	.tuning_fast_imm_vector_shift,
3944	.uintr,	3992	.uintr,
...	@@ -4042,6 +4090,7 @@ pub const cpu = struct {	...	@@ -4042,6 +4090,7 @@ pub const cpu = struct {
4042	.slow_lea,	4090	.slow_lea,
4043	.slow_pmulld,	4091	.slow_pmulld,
4044	.slow_two_mem_ops,	4092	.slow_two_mem_ops,
		4093	.smep,
4045	.sse4_2,	4094	.sse4_2,
4046	.use_slm_arith_costs,	4095	.use_slm_arith_costs,
4047	.vzeroupper,	4096	.vzeroupper,
...	@@ -4098,6 +4147,8 @@ pub const cpu = struct {	...	@@ -4098,6 +4147,8 @@ pub const cpu = struct {
4098	.rdseed,	4147	.rdseed,
4099	.sahf,	4148	.sahf,
4100	.slow_3ops_lea,	4149	.slow_3ops_lea,
		4150	.smap,
		4151	.smep,
4101	.tuning_fast_imm_vector_shift,	4152	.tuning_fast_imm_vector_shift,
4102	.vzeroupper,	4153	.vzeroupper,
4103	.x87,	4154	.x87,
...	@@ -4150,6 +4201,8 @@ pub const cpu = struct {	...	@@ -4150,6 +4201,8 @@ pub const cpu = struct {
4150	.rdseed,	4201	.rdseed,
4151	.sahf,	4202	.sahf,
4152	.slow_3ops_lea,	4203	.slow_3ops_lea,
		4204	.smap,
		4205	.smep,
4153	.vzeroupper,	4206	.vzeroupper,
4154	.x87,	4207	.x87,
4155	.xsavec,	4208	.xsavec,
...	@@ -4305,6 +4358,8 @@ pub const cpu = struct {	...	@@ -4305,6 +4358,8 @@ pub const cpu = struct {
4305	.sahf,	4358	.sahf,
4306	.sha,	4359	.sha,
4307	.shstk,	4360	.shstk,
		4361	.smap,
		4362	.smep,
4308	.tuning_fast_imm_vector_shift,	4363	.tuning_fast_imm_vector_shift,
4309	.vaes,	4364	.vaes,
4310	.vpclmulqdq,	4365	.vpclmulqdq,
...	@@ -4574,6 +4629,8 @@ pub const cpu = struct {	...	@@ -4574,6 +4629,8 @@ pub const cpu = struct {
4574	.sbb_dep_breaking,	4629	.sbb_dep_breaking,
4575	.sha,	4630	.sha,
4576	.slow_shld,	4631	.slow_shld,
		4632	.smap,
		4633	.smep,
4577	.sse4a,	4634	.sse4a,
4578	.vzeroupper,	4635	.vzeroupper,
4579	.x87,	4636	.x87,
...	@@ -4629,6 +4686,8 @@ pub const cpu = struct {	...	@@ -4629,6 +4686,8 @@ pub const cpu = struct {
4629	.sbb_dep_breaking,	4686	.sbb_dep_breaking,
4630	.sha,	4687	.sha,
4631	.slow_shld,	4688	.slow_shld,
		4689	.smap,
		4690	.smep,
4632	.sse4a,	4691	.sse4a,
4633	.vzeroupper,	4692	.vzeroupper,
4634	.wbnoinvd,	4693	.wbnoinvd,
...	@@ -4686,6 +4745,8 @@ pub const cpu = struct {	...	@@ -4686,6 +4745,8 @@ pub const cpu = struct {
4686	.sbb_dep_breaking,	4745	.sbb_dep_breaking,
4687	.sha,	4746	.sha,
4688	.slow_shld,	4747	.slow_shld,
		4748	.smap,
		4749	.smep,
4689	.sse4a,	4750	.sse4a,
4690	.vaes,	4751	.vaes,
4691	.vpclmulqdq,	4752	.vpclmulqdq,
...	@@ -4757,6 +4818,8 @@ pub const cpu = struct {	...	@@ -4757,6 +4818,8 @@ pub const cpu = struct {
4757	.sha,	4818	.sha,
4758	.shstk,	4819	.shstk,
4759	.slow_shld,	4820	.slow_shld,
		4821	.smap,
		4822	.smep,
4760	.sse4a,	4823	.sse4a,
4761	.vaes,	4824	.vaes,
4762	.vpclmulqdq,	4825	.vpclmulqdq,
...	@@ -4833,6 +4896,8 @@ pub const cpu = struct {	...	@@ -4833,6 +4896,8 @@ pub const cpu = struct {
4833	.sha,	4896	.sha,
4834	.shstk,	4897	.shstk,
4835	.slow_shld,	4898	.slow_shld,
		4899	.smap,
		4900	.smep,
4836	.sse4a,	4901	.sse4a,
4837	.vaes,	4902	.vaes,
4838	.vpclmulqdq,	4903	.vpclmulqdq,

lib/std/Thread.zig+3-1

 @@ -372,9 +372,11 @@ pub const SpawnConfig = struct {
     // https://github.com/ziglang/zig/issues/157
     /// Size in bytes of the Thread's stack
-    stack_size: usize = 16 * 1024 * 1024,
+    stack_size: usize = default_stack_size,
     /// The allocator to be used to allocate memory for the to-be-spawned thread
     allocator: ?std.mem.Allocator = null,
+    pub const default_stack_size = 16 * 1024 * 1024;
 };
 pub const SpawnError = error{

lib/std/Thread/Condition.zig+3-3

...	@@ -161,17 +161,17 @@ const WindowsImpl = struct {	...	@@ -161,17 +161,17 @@ const WindowsImpl = struct {
161	}	161	}
162	}	162	}
163		163
164	if (comptime builtin.mode == .Debug) {	164	if (builtin.mode == .Debug) {
165	// The internal state of the DebugMutex needs to be handled here as well.	165	// The internal state of the DebugMutex needs to be handled here as well.
166	mutex.impl.locking_thread.store(0, .unordered);	166	mutex.impl.locking_thread.store(0, .unordered);
167	}	167	}
168	const rc = os.windows.kernel32.SleepConditionVariableSRW(	168	const rc = os.windows.kernel32.SleepConditionVariableSRW(
169	&self.condition,	169	&self.condition,
170	if (comptime builtin.mode == .Debug) &mutex.impl.impl.srwlock else &mutex.impl.srwlock,	170	if (builtin.mode == .Debug) &mutex.impl.impl.srwlock else &mutex.impl.srwlock,
171	timeout_ms,	171	timeout_ms,
172	0, // the srwlock was assumed to acquired in exclusive mode not shared	172	0, // the srwlock was assumed to acquired in exclusive mode not shared
173	);	173	);
174	if (comptime builtin.mode == .Debug) {	174	if (builtin.mode == .Debug) {
175	// The internal state of the DebugMutex needs to be handled here as well.	175	// The internal state of the DebugMutex needs to be handled here as well.
176	mutex.impl.locking_thread.store(std.Thread.getCurrentId(), .unordered);	176	mutex.impl.locking_thread.store(std.Thread.getCurrentId(), .unordered);
177	}	177	}

lib/std/Thread/Mutex.zig+1-1

...	@@ -158,7 +158,7 @@ const FutexImpl = struct {	...	@@ -158,7 +158,7 @@ const FutexImpl = struct {
158	// On x86, use `lock bts` instead of `lock cmpxchg` as:	158	// On x86, use `lock bts` instead of `lock cmpxchg` as:
159	// - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048	159	// - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048
160	// - `lock bts` is smaller instruction-wise which makes it better for inlining	160	// - `lock bts` is smaller instruction-wise which makes it better for inlining
161	if (comptime builtin.target.cpu.arch.isX86()) {	161	if (builtin.target.cpu.arch.isX86()) {
162	const locked_bit = @ctz(locked);	162	const locked_bit = @ctz(locked);
163	return self.state.bitSet(locked_bit, .acquire) == 0;	163	return self.state.bitSet(locked_bit, .acquire) == 0;
164	}	164	}

lib/std/Thread/Pool.zig+5-1

...	@@ -27,6 +27,7 @@ pub const Options = struct {	...	@@ -27,6 +27,7 @@ pub const Options = struct {
27	allocator: std.mem.Allocator,	27	allocator: std.mem.Allocator,
28	n_jobs: ?usize = null,	28	n_jobs: ?usize = null,
29	track_ids: bool = false,	29	track_ids: bool = false,
		30	stack_size: usize = std.Thread.SpawnConfig.default_stack_size,
30	};	31	};
31		32
32	pub fn init(pool: *Pool, options: Options) !void {	33	pub fn init(pool: *Pool, options: Options) !void {
...	@@ -54,7 +55,10 @@ pub fn init(pool: *Pool, options: Options) !void {	...	@@ -54,7 +55,10 @@ pub fn init(pool: *Pool, options: Options) !void {
54	errdefer pool.join(spawned);	55	errdefer pool.join(spawned);
55		56
56	for (pool.threads) \|*thread\| {	57	for (pool.threads) \|*thread\| {
57	thread.* = try std.Thread.spawn(.{}, worker, .{pool});	58	thread.* = try std.Thread.spawn(.{
		59	.stack_size = options.stack_size,
		60	.allocator = allocator,
		61	}, worker, .{pool});
58	spawned += 1;	62	spawned += 1;
59	}	63	}
60	}	64	}

lib/std/crypto/aes/aesni.zig+1-1

...	@@ -4,7 +4,7 @@ const mem = std.mem;	...	@@ -4,7 +4,7 @@ const mem = std.mem;
4	const debug = std.debug;	4	const debug = std.debug;
5		5
6	const has_vaes = builtin.cpu.arch == .x86_64 and std.Target.x86.featureSetHas(builtin.cpu.features, .vaes);	6	const has_vaes = builtin.cpu.arch == .x86_64 and std.Target.x86.featureSetHas(builtin.cpu.features, .vaes);
7	const has_avx512f = builtin.cpu.arch == .x86_64 and std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);	7	const has_avx512f = builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64 and std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);
8		8
9	/// A single AES block.	9	/// A single AES block.
10	pub const Block = struct {	10	pub const Block = struct {

lib/std/crypto/chacha20.zig+1-3

 @@ -499,11 +499,9 @@ fn ChaChaNonVecImpl(comptime rounds_nb: usize) type {
 fn ChaChaImpl(comptime rounds_nb: usize) type {
     switch (builtin.cpu.arch) {
         .x86_64 => {
-            if (builtin.zig_backend == .stage2_x86_64) return ChaChaNonVecImpl(rounds_nb);
             const has_avx2 = std.Target.x86.featureSetHas(builtin.cpu.features, .avx2);
             const has_avx512f = std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);
-            if (has_avx512f) return ChaChaVecImpl(rounds_nb, 4);
+            if (builtin.zig_backend != .stage2_x86_64 and has_avx512f) return ChaChaVecImpl(rounds_nb, 4);
             if (has_avx2) return ChaChaVecImpl(rounds_nb, 2);
             return ChaChaVecImpl(rounds_nb, 1);
         },

lib/std/crypto/tls/Client.zig+7-56

...	@@ -356,14 +356,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client	...	@@ -356,14 +356,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
356	if (ciphertext.len > cleartext_fragment_buf.len) return error.TlsRecordOverflow;	356	if (ciphertext.len > cleartext_fragment_buf.len) return error.TlsRecordOverflow;
357	const cleartext = cleartext_fragment_buf[0..ciphertext.len];	357	const cleartext = cleartext_fragment_buf[0..ciphertext.len];
358	const auth_tag = record_decoder.array(P.AEAD.tag_length).*;	358	const auth_tag = record_decoder.array(P.AEAD.tag_length).*;
359	const nonce = if (builtin.zig_backend == .stage2_x86_64 and	359	const nonce = nonce: {
360	P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
361	nonce: {
362	var nonce = pv.server_handshake_iv;
363	const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
364	std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ read_seq, .big);
365	break :nonce nonce;
366	} else nonce: {
367	const V = @Vector(P.AEAD.nonce_length, u8);	360	const V = @Vector(P.AEAD.nonce_length, u8);
368	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);	361	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
369	const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));	362	const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));
...	@@ -400,14 +393,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client	...	@@ -400,14 +393,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
400	const record_iv = record_decoder.array(P.record_iv_length).*;	393	const record_iv = record_decoder.array(P.record_iv_length).*;
401	const masked_read_seq = read_seq &	394	const masked_read_seq = read_seq &
402	comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);	395	comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
403	const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and	396	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
404	P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
405	nonce: {
406	var nonce = pv.app_cipher.server_write_IV ++ record_iv;
407	const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
408	std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
409	break :nonce nonce;
410	} else nonce: {
411	const V = @Vector(P.AEAD.nonce_length, u8);	397	const V = @Vector(P.AEAD.nonce_length, u8);
412	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);	398	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
413	const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));	399	const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
...	@@ -750,14 +736,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client	...	@@ -750,14 +736,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
750	.app_cipher = std.mem.bytesToValue(P.Tls_1_2, &key_block),	736	.app_cipher = std.mem.bytesToValue(P.Tls_1_2, &key_block),
751	} };	737	} };
752	const pv = &p.version.tls_1_2;	738	const pv = &p.version.tls_1_2;
753	const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and	739	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
754	P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
755	nonce: {
756	var nonce = pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt;
757	const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
758	std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ write_seq, .big);
759	break :nonce nonce;
760	} else nonce: {
761	const V = @Vector(P.AEAD.nonce_length, u8);	740	const V = @Vector(P.AEAD.nonce_length, u8);
762	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);	741	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
763	const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));	742	const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));
...	@@ -1043,14 +1022,7 @@ fn prepareCiphertextRecord(	...	@@ -1043,14 +1022,7 @@ fn prepareCiphertextRecord(
1043	ciphertext_end += ciphertext_len;	1022	ciphertext_end += ciphertext_len;
1044	const auth_tag = ciphertext_buf[ciphertext_end..][0..P.AEAD.tag_length];	1023	const auth_tag = ciphertext_buf[ciphertext_end..][0..P.AEAD.tag_length];
1045	ciphertext_end += auth_tag.len;	1024	ciphertext_end += auth_tag.len;
1046	const nonce = if (builtin.zig_backend == .stage2_x86_64 and	1025	const nonce = nonce: {
1047	P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1048	nonce: {
1049	var nonce = pv.client_iv;
1050	const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1051	std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1052	break :nonce nonce;
1053	} else nonce: {
1054	const V = @Vector(P.AEAD.nonce_length, u8);	1026	const V = @Vector(P.AEAD.nonce_length, u8);
1055	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);	1027	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1056	const operand: V = pad ++ std.mem.toBytes(big(c.write_seq));	1028	const operand: V = pad ++ std.mem.toBytes(big(c.write_seq));
...	@@ -1098,14 +1070,7 @@ fn prepareCiphertextRecord(	...	@@ -1098,14 +1070,7 @@ fn prepareCiphertextRecord(
1098	const ad = std.mem.toBytes(big(c.write_seq)) ++ record_header[0 .. 1 + 2] ++ int(u16, message_len);	1070	const ad = std.mem.toBytes(big(c.write_seq)) ++ record_header[0 .. 1 + 2] ++ int(u16, message_len);
1099	const record_iv = ciphertext_buf[ciphertext_end..][0..P.record_iv_length];	1071	const record_iv = ciphertext_buf[ciphertext_end..][0..P.record_iv_length];
1100	ciphertext_end += P.record_iv_length;	1072	ciphertext_end += P.record_iv_length;
1101	const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and	1073	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1102	P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1103	nonce: {
1104	var nonce = pv.client_write_IV ++ pv.client_salt;
1105	const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1106	std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1107	break :nonce nonce;
1108	} else nonce: {
1109	const V = @Vector(P.AEAD.nonce_length, u8);	1074	const V = @Vector(P.AEAD.nonce_length, u8);
1110	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);	1075	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1111	const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));	1076	const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));
...	@@ -1374,14 +1339,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove	...	@@ -1374,14 +1339,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
1374	const ciphertext = frag[in..][0..ciphertext_len];	1339	const ciphertext = frag[in..][0..ciphertext_len];
1375	in += ciphertext_len;	1340	in += ciphertext_len;
1376	const auth_tag = frag[in..][0..P.AEAD.tag_length].*;	1341	const auth_tag = frag[in..][0..P.AEAD.tag_length].*;
1377	const nonce = if (builtin.zig_backend == .stage2_x86_64 and	1342	const nonce = nonce: {
1378	P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1379	nonce: {
1380	var nonce = pv.server_iv;
1381	const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1382	std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.read_seq, .big);
1383	break :nonce nonce;
1384	} else nonce: {
1385	const V = @Vector(P.AEAD.nonce_length, u8);	1343	const V = @Vector(P.AEAD.nonce_length, u8);
1386	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);	1344	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1387	const operand: V = pad ++ std.mem.toBytes(big(c.read_seq));	1345	const operand: V = pad ++ std.mem.toBytes(big(c.read_seq));
...	@@ -1409,14 +1367,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove	...	@@ -1409,14 +1367,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
1409	in += P.record_iv_length;	1367	in += P.record_iv_length;
1410	const masked_read_seq = c.read_seq &	1368	const masked_read_seq = c.read_seq &
1411	comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);	1369	comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
1412	const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and	1370	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1413	P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1414	nonce: {
1415	var nonce = pv.server_write_IV ++ record_iv;
1416	const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1417	std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
1418	break :nonce nonce;
1419	} else nonce: {
1420	const V = @Vector(P.AEAD.nonce_length, u8);	1371	const V = @Vector(P.AEAD.nonce_length, u8);
1421	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);	1372	const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1422	const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));	1373	const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));

lib/std/debug.zig+4-3

lib/std/debug/NoPanic.zig created+59

...	@@ -0,0 +1,59 @@
1	//! This namespace can be used with `pub const Panic = std.debug.NoPanic;` in the root file.
2	//! It emits as little code as possible, for testing purposes.
3	//!
4	//! For a functional alternative, see `std.debug.FormattedPanic`.
5
6	const std = @import("../std.zig");
7
8	pub fn call(_: []const u8, _: ?*std.builtin.StackTrace, _: ?usize) noreturn {
9	@branchHint(.cold);
10	@trap();
11	}
12
13	pub inline fn sentinelMismatch(_: anytype, _: anytype) noreturn {
14	@branchHint(.cold);
15	@trap();
16	}
17
18	pub inline fn unwrapError(_: ?*std.builtin.StackTrace, _: anyerror) noreturn {
19	@branchHint(.cold);
20	@trap();
21	}
22
23	pub inline fn outOfBounds(_: usize, _: usize) noreturn {
24	@branchHint(.cold);
25	@trap();
26	}
27
28	pub inline fn startGreaterThanEnd(_: usize, _: usize) noreturn {
29	@branchHint(.cold);
30	@trap();
31	}
32
33	pub inline fn inactiveUnionField(_: anytype, _: anytype) noreturn {
34	@branchHint(.cold);
35	@trap();
36	}
37
38	pub const messages = struct {
39	pub const reached_unreachable = "";
40	pub const unwrap_null = "";
41	pub const cast_to_null = "";
42	pub const incorrect_alignment = "";
43	pub const invalid_error_code = "";
44	pub const cast_truncated_data = "";
45	pub const negative_to_unsigned = "";
46	pub const integer_overflow = "";
47	pub const shl_overflow = "";
48	pub const shr_overflow = "";
49	pub const divide_by_zero = "";
50	pub const exact_division_remainder = "";
51	pub const integer_part_out_of_bounds = "";
52	pub const corrupt_switch = "";
53	pub const shift_rhs_too_big = "";
54	pub const invalid_enum_value = "";
55	pub const for_len_mismatch = "";
56	pub const memcpy_len_mismatch = "";
57	pub const memcpy_alias = "";
58	pub const noreturn_returned = "";
59	};

lib/std/debug/SelfInfo.zig+4-4

lib/std/heap.zig+1-1

...	@@ -890,7 +890,7 @@ test {	...	@@ -890,7 +890,7 @@ test {
890	_ = @import("heap/memory_pool.zig");	890	_ = @import("heap/memory_pool.zig");
891	_ = ArenaAllocator;	891	_ = ArenaAllocator;
892	_ = GeneralPurposeAllocator;	892	_ = GeneralPurposeAllocator;
893	if (comptime builtin.target.isWasm()) {	893	if (builtin.target.isWasm()) {
894	_ = WasmAllocator;	894	_ = WasmAllocator;
895	_ = WasmPageAllocator;	895	_ = WasmPageAllocator;
896	}	896	}

lib/std/http/protocol.zig-1

...	@@ -4,7 +4,6 @@ const testing = std.testing;	...	@@ -4,7 +4,6 @@ const testing = std.testing;
4	const mem = std.mem;	4	const mem = std.mem;
5		5
6	const assert = std.debug.assert;	6	const assert = std.debug.assert;
7	const use_vectors = builtin.zig_backend != .stage2_x86_64;
8		7
9	pub const State = enum {	8	pub const State = enum {
10	invalid,	9	invalid,

lib/std/math/big/int.zig+8-5

 @@ -2520,12 +2520,13 @@ pub const Const = struct {
         return order(a, b) == .eq;
     }
+    /// Returns the number of leading zeros in twos-complement form.
     pub fn clz(a: Const, bits: Limb) Limb {
-        // Limbs are stored in little-endian order but we need
+        // Limbs are stored in little-endian order but we need to iterate big-endian.
-        // to iterate big-endian.
+        if (!a.positive and !a.eqlZero()) return 0;
         var total_limb_lz: Limb = 0;
         var i: usize = a.limbs.len;
-        const bits_per_limb = @sizeOf(Limb) * 8;
+        const bits_per_limb = @bitSizeOf(Limb);
         while (i != 0) {
             i -= 1;
             const limb = a.limbs[i];
 @@ -2537,13 +2538,15 @@ pub const Const = struct {
         return total_limb_lz + bits - total_limb_bits;
     }
+    /// Returns the number of trailing zeros in twos-complement form.
     pub fn ctz(a: Const, bits: Limb) Limb {
-        // Limbs are stored in little-endian order.
+        // Limbs are stored in little-endian order. Converting a negative number to twos-complement
+        // flips all bits above the lowest set bit, which does not affect the trailing zero count.
         var result: Limb = 0;
         for (a.limbs) |limb| {
             const limb_tz = @ctz(limb);
             result += limb_tz;
-            if (limb_tz != @sizeOf(Limb) * 8) break;
+            if (limb_tz != @bitSizeOf(Limb)) break;
         }
         return @min(result, bits);
     }

lib/std/mem.zig+2-2

...	@@ -883,8 +883,8 @@ fn SliceTo(comptime T: type, comptime end: std.meta.Elem(T)) type {	...	@@ -883,8 +883,8 @@ fn SliceTo(comptime T: type, comptime end: std.meta.Elem(T)) type {
883	@compileError("invalid type given to std.mem.sliceTo: " ++ @typeName(T));	883	@compileError("invalid type given to std.mem.sliceTo: " ++ @typeName(T));
884	}	884	}
885		885
886	/// Takes an array, a pointer to an array, a sentinel-terminated pointer, or a slice and	886	/// Takes a pointer to an array, a sentinel-terminated pointer, or a slice and iterates searching for
887	/// iterates searching for the first occurrence of `end`, returning the scanned slice.	887	/// the first occurrence of `end`, returning the scanned slice.
888	/// If `end` is not found, the full length of the array/slice/sentinel terminated pointer is returned.	888	/// If `end` is not found, the full length of the array/slice/sentinel terminated pointer is returned.
889	/// If the pointer type is sentinel terminated and `end` matches that terminator, the	889	/// If the pointer type is sentinel terminated and `end` matches that terminator, the
890	/// resulting slice is also sentinel terminated.	890	/// resulting slice is also sentinel terminated.

lib/std/os.zig+1-1

...	@@ -157,7 +157,7 @@ pub fn getFdPath(fd: std.posix.fd_t, out_buffer: *[max_path_bytes]u8) std.posix.	...	@@ -157,7 +157,7 @@ pub fn getFdPath(fd: std.posix.fd_t, out_buffer: *[max_path_bytes]u8) std.posix.
157	return target;	157	return target;
158	},	158	},
159	.freebsd => {	159	.freebsd => {
160	if (comptime builtin.os.isAtLeast(.freebsd, .{ .major = 13, .minor = 0, .patch = 0 }) orelse false) {	160	if (builtin.os.isAtLeast(.freebsd, .{ .major = 13, .minor = 0, .patch = 0 }) orelse false) {
161	var kfile: std.c.kinfo_file = undefined;	161	var kfile: std.c.kinfo_file = undefined;
162	kfile.structsize = std.c.KINFO_FILE_SIZE;	162	kfile.structsize = std.c.KINFO_FILE_SIZE;
163	switch (posix.errno(std.c.fcntl(fd, std.c.F.KINFO, @intFromPtr(&kfile)))) {	163	switch (posix.errno(std.c.fcntl(fd, std.c.F.KINFO, @intFromPtr(&kfile)))) {

lib/std/os/windows.zig+1-1

...	@@ -1061,7 +1061,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil	...	@@ -1061,7 +1061,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
1061	// us INVALID_PARAMETER.	1061	// us INVALID_PARAMETER.
1062	// The same reasoning for win10_rs5 as in os.renameatW() applies (FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE requires >= win10_rs5).	1062	// The same reasoning for win10_rs5 as in os.renameatW() applies (FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE requires >= win10_rs5).
1063	var need_fallback = true;	1063	var need_fallback = true;
1064	if (comptime builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs5)) {	1064	if (builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs5)) {
1065	// Deletion with posix semantics if the filesystem supports it.	1065	// Deletion with posix semantics if the filesystem supports it.
1066	var info = FILE_DISPOSITION_INFORMATION_EX{	1066	var info = FILE_DISPOSITION_INFORMATION_EX{
1067	.Flags = FILE_DISPOSITION_DELETE \|	1067	.Flags = FILE_DISPOSITION_DELETE \|

lib/std/posix.zig+1-1

...	@@ -6819,7 +6819,7 @@ pub fn memfd_createZ(name: [*:0]const u8, flags: u32) MemFdCreateError!fd_t {	...	@@ -6819,7 +6819,7 @@ pub fn memfd_createZ(name: [*:0]const u8, flags: u32) MemFdCreateError!fd_t {
6819	}	6819	}
6820	},	6820	},
6821	.freebsd => {	6821	.freebsd => {
6822	if (comptime builtin.os.version_range.semver.max.order(.{ .major = 13, .minor = 0, .patch = 0 }) == .lt)	6822	if (builtin.os.version_range.semver.max.order(.{ .major = 13, .minor = 0, .patch = 0 }) == .lt)
6823	@compileError("memfd_create is unavailable on FreeBSD < 13.0");	6823	@compileError("memfd_create is unavailable on FreeBSD < 13.0");
6824	const rc = system.memfd_create(name, flags);	6824	const rc = system.memfd_create(name, flags);
6825	switch (errno(rc)) {	6825	switch (errno(rc)) {

lib/std/posix/test.zig+1-1

...	@@ -804,7 +804,7 @@ test "getrlimit and setrlimit" {	...	@@ -804,7 +804,7 @@ test "getrlimit and setrlimit" {
804	//	804	//
805	// This happens for example if RLIMIT_MEMLOCK is bigger than ~2GiB.	805	// This happens for example if RLIMIT_MEMLOCK is bigger than ~2GiB.
806	// In that case the following the limit would be RLIM_INFINITY and the following setrlimit fails with EPERM.	806	// In that case the following the limit would be RLIM_INFINITY and the following setrlimit fails with EPERM.
807	if (comptime builtin.cpu.arch.isMIPS() and builtin.link_libc) {	807	if (builtin.cpu.arch.isMIPS() and builtin.link_libc) {
808	if (limit.cur != linux.RLIM.INFINITY) {	808	if (limit.cur != linux.RLIM.INFINITY) {
809	try posix.setrlimit(resource, limit);	809	try posix.setrlimit(resource, limit);
810	}	810	}

lib/std/simd.zig+4-6

lib/std/zig/system/NativePaths.zig+1-1

...	@@ -83,7 +83,7 @@ pub fn detect(arena: Allocator, native_target: std.Target) !NativePaths {	...	@@ -83,7 +83,7 @@ pub fn detect(arena: Allocator, native_target: std.Target) !NativePaths {
83		83
84	// TODO: consider also adding homebrew paths	84	// TODO: consider also adding homebrew paths
85	// TODO: consider also adding macports paths	85	// TODO: consider also adding macports paths
86	if (comptime builtin.target.isDarwin()) {	86	if (builtin.target.isDarwin()) {
87	if (std.zig.system.darwin.isSdkInstalled(arena)) sdk: {	87	if (std.zig.system.darwin.isSdkInstalled(arena)) sdk: {
88	const sdk = std.zig.system.darwin.getSdk(arena, native_target) orelse break :sdk;	88	const sdk = std.zig.system.darwin.getSdk(arena, native_target) orelse break :sdk;
89	try self.addLibDir(try std.fs.path.join(arena, &.{ sdk, "usr/lib" }));	89	try self.addLibDir(try std.fs.path.join(arena, &.{ sdk, "usr/lib" }));

lib/std/zig/system/x86.zig+2

src/Air.zig+35-11

 @@ -893,14 +893,38 @@ pub const Inst = struct {
     pub const Index = enum(u32) {
         _,
-        pub fn toRef(i: Index) Inst.Ref {
+        pub fn unwrap(index: Index) union(enum) { ref: Inst.Ref, target: u31 } {
-            assert(@intFromEnum(i) >> 31 == 0);
+            const low_index: u31 = @truncate(@intFromEnum(index));
-            return @enumFromInt((1 << 31) | @intFromEnum(i));
+            return switch (@as(u1, @intCast(@intFromEnum(index) >> 31))) {
+                0 => .{ .ref = @enumFromInt(@as(u32, 1 << 31) | low_index) },
+                1 => .{ .target = low_index },
+            };
+        }
+        pub fn toRef(index: Index) Inst.Ref {
+            return index.unwrap().ref;
+        }
+        pub fn fromTargetIndex(index: u31) Index {
+            return @enumFromInt((1 << 31) | @as(u32, index));
+        }
+        pub fn toTargetIndex(index: Index) u31 {
+            return index.unwrap().target;
         }
-        pub fn toTargetIndex(i: Index) u31 {
+        pub fn format(
-            assert(@intFromEnum(i) >> 31 == 1);
+            index: Index,
-            return @truncate(@intFromEnum(i));
+            comptime _: []const u8,
+            _: std.fmt.FormatOptions,
+            writer: anytype,
+        ) @TypeOf(writer).Error!void {
+            try writer.writeByte('%');
+            switch (index.unwrap()) {
+                .ref => {},
+                .target => try writer.writeByte('t'),
+            }
+            try writer.print("{d}", .{@as(u31, @truncate(@intFromEnum(index)))});
         }
     };
 @@ -1205,7 +1229,7 @@ pub const VectorCmp = struct {
     op: u32,
     pub fn compareOperator(self: VectorCmp) std.math.CompareOperator {
-        return @as(std.math.CompareOperator, @enumFromInt(@as(u3, @truncate(self.op))));
+        return @enumFromInt(@as(u3, @intCast(self.op)));
     }
     pub fn encodeOp(compare_operator: std.math.CompareOperator) u32 {
 @@ -1250,11 +1274,11 @@ pub const Cmpxchg = struct {
     flags: u32,
     pub fn successOrder(self: Cmpxchg) std.builtin.AtomicOrder {
-        return @as(std.builtin.AtomicOrder, @enumFromInt(@as(u3, @truncate(self.flags))));
+        return @enumFromInt(@as(u3, @truncate(self.flags)));
     }
     pub fn failureOrder(self: Cmpxchg) std.builtin.AtomicOrder {
-        return @as(std.builtin.AtomicOrder, @enumFromInt(@as(u3, @truncate(self.flags >> 3))));
+        return @enumFromInt(@as(u3, @intCast(self.flags >> 3)));
     }
 };
 @@ -1265,11 +1289,11 @@ pub const AtomicRmw = struct {
     flags: u32,
     pub fn ordering(self: AtomicRmw) std.builtin.AtomicOrder {
-        return @as(std.builtin.AtomicOrder, @enumFromInt(@as(u3, @truncate(self.flags))));
+        return @enumFromInt(@as(u3, @truncate(self.flags)));
     }
     pub fn op(self: AtomicRmw) std.builtin.AtomicRmwOp {
-        return @as(std.builtin.AtomicRmwOp, @enumFromInt(@as(u4, @truncate(self.flags >> 3))));
+        return @enumFromInt(@as(u4, @intCast(self.flags >> 3)));
     }
 };

src/Compilation.zig+1

...	@@ -3067,6 +3067,7 @@ pub fn saveState(comp: *Compilation) !void {	...	@@ -3067,6 +3067,7 @@ pub fn saveState(comp: *Compilation) !void {
3067	// linker state	3067	// linker state
3068	switch (lf.tag) {	3068	switch (lf.tag) {
3069	.wasm => {	3069	.wasm => {
		3070	dev.check(link.File.Tag.wasm.devFeature());
3070	const wasm = lf.cast(.wasm).?;	3071	const wasm = lf.cast(.wasm).?;
3071	const is_obj = comp.config.output_mode == .Obj;	3072	const is_obj = comp.config.output_mode == .Obj;
3072	try bufs.ensureUnusedCapacity(85);	3073	try bufs.ensureUnusedCapacity(85);

src/Liveness.zig+6-27

 @@ -202,14 +202,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool
     return (l.tomb_bits[usize_index] & mask) != 0;
 }
-pub fn clearOperandDeath(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) void {
-    assert(operand < bpi - 1);
-    const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
-    const mask = @as(usize, 1) <<
-        @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
-    l.tomb_bits[usize_index] &= ~mask;
-}
 const OperandCategory = enum {
     /// The operand lives on, but this instruction cannot possibly mutate memory.
     none,
 @@ -727,32 +719,25 @@ pub const SwitchBrTable = struct {
 /// Caller owns the memory.
 pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable {
-    var index: usize = l.special.get(inst) orelse return SwitchBrTable{
+    var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} };
-        .deaths = &.{},
-    };
     const else_death_count = l.extra[index];
     index += 1;
-    var deaths = std.ArrayList([]const Air.Inst.Index).init(gpa);
+    var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len);
-    defer deaths.deinit();
+    errdefer gpa.free(deaths);
-    try deaths.ensureTotalCapacity(cases_len + 1);
     var case_i: u32 = 0;
     while (case_i < cases_len - 1) : (case_i += 1) {
         const case_death_count: u32 = l.extra[index];
         index += 1;
-        const case_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..case_death_count]);
+        deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]);
         index += case_death_count;
-        deaths.appendAssumeCapacity(case_deaths);
     }
     {
         // Else
-        const else_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..else_death_count]);
+        deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]);
-        deaths.appendAssumeCapacity(else_deaths);
     }
-    return SwitchBrTable{
+    return .{ .deaths = deaths };
-        .deaths = try deaths.toOwnedSlice(),
-    };
 }
 /// Note that this information is technically redundant, but is useful for
 @@ -844,12 +829,6 @@ const Analysis = struct {
     special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
     extra: std.ArrayListUnmanaged(u32),
-    fn storeTombBits(a: *Analysis, inst: Air.Inst.Index, tomb_bits: Bpi) void {
-        const usize_index = (inst * bpi) / @bitSizeOf(usize);
-        a.tomb_bits[usize_index] |= @as(usize, tomb_bits) <<
-            @as(Log2Int(usize), @intCast((inst % (@bitSizeOf(usize) / bpi)) * bpi));
-    }
     fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
         const fields = std.meta.fields(@TypeOf(extra));
         try a.extra.ensureUnusedCapacity(a.gpa, fields.len);

src/Type.zig+25-22

 @@ -962,7 +962,6 @@ pub fn abiAlignmentInner(
 ) SemaError!AbiAlignmentInner {
     const pt = strat.pt(zcu, tid);
     const target = zcu.getTarget();
-    const use_llvm = zcu.comp.config.use_llvm;
     const ip = &zcu.intern_pool;
     switch (ty.toIntern()) {
 @@ -970,7 +969,7 @@ pub fn abiAlignmentInner(
         else => switch (ip.indexToKey(ty.toIntern())) {
             .int_type => |int_type| {
                 if (int_type.bits == 0) return .{ .scalar = .@"1" };
-                return .{ .scalar = intAbiAlignment(int_type.bits, target, use_llvm) };
+                return .{ .scalar = intAbiAlignment(int_type.bits, target) };
             },
             .ptr_type, .anyframe_type => {
                 return .{ .scalar = ptrAbiAlignment(target) };
 @@ -1023,7 +1022,7 @@ pub fn abiAlignmentInner(
             .error_set_type, .inferred_error_set_type => {
                 const bits = zcu.errorSetBits();
                 if (bits == 0) return .{ .scalar = .@"1" };
-                return .{ .scalar = intAbiAlignment(bits, target, use_llvm) };
+                return .{ .scalar = intAbiAlignment(bits, target) };
             },
             // represents machine code; not a pointer
 @@ -1036,7 +1035,7 @@ pub fn abiAlignmentInner(
                 .usize,
                 .isize,
-                => return .{ .scalar = intAbiAlignment(target.ptrBitWidth(), target, use_llvm) },
+                => return .{ .scalar = intAbiAlignment(target.ptrBitWidth(), target) },
                 .c_char => return .{ .scalar = cTypeAlign(target, .char) },
                 .c_short => return .{ .scalar = cTypeAlign(target, .short) },
 @@ -1067,7 +1066,7 @@ pub fn abiAlignmentInner(
                 .anyerror, .adhoc_inferred_error_set => {
                     const bits = zcu.errorSetBits();
                     if (bits == 0) return .{ .scalar = .@"1" };
-                    return .{ .scalar = intAbiAlignment(bits, target, use_llvm) };
+                    return .{ .scalar = intAbiAlignment(bits, target) };
                 },
                 .void,
 @@ -1291,7 +1290,6 @@ pub fn abiSizeInner(
     tid: strat.Tid(),
 ) SemaError!AbiSizeInner {
     const target = zcu.getTarget();
-    const use_llvm = zcu.comp.config.use_llvm;
     const ip = &zcu.intern_pool;
     switch (ty.toIntern()) {
 @@ -1300,7 +1298,7 @@ pub fn abiSizeInner(
         else => switch (ip.indexToKey(ty.toIntern())) {
             .int_type => |int_type| {
                 if (int_type.bits == 0) return .{ .scalar = 0 };
-                return .{ .scalar = intAbiSize(int_type.bits, target, use_llvm) };
+                return .{ .scalar = intAbiSize(int_type.bits, target) };
             },
             .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
                 .slice => return .{ .scalar = @divExact(target.ptrBitWidth(), 8) * 2 },
 @@ -1362,7 +1360,7 @@ pub fn abiSizeInner(
             .error_set_type, .inferred_error_set_type => {
                 const bits = zcu.errorSetBits();
                 if (bits == 0) return .{ .scalar = 0 };
-                return .{ .scalar = intAbiSize(bits, target, use_llvm) };
+                return .{ .scalar = intAbiSize(bits, target) };
             },
             .error_union_type => |error_union_type| {
 @@ -1455,7 +1453,7 @@ pub fn abiSizeInner(
                 .anyerror, .adhoc_inferred_error_set => {
                     const bits = zcu.errorSetBits();
                     if (bits == 0) return .{ .scalar = 0 };
-                    return .{ .scalar = intAbiSize(bits, target, use_llvm) };
+                    return .{ .scalar = intAbiSize(bits, target) };
                 },
                 .noreturn => unreachable,
 @@ -1609,11 +1607,11 @@ pub fn ptrAbiAlignment(target: Target) Alignment {
     return Alignment.fromNonzeroByteUnits(@divExact(target.ptrBitWidth(), 8));
 }
-pub fn intAbiSize(bits: u16, target: Target, use_llvm: bool) u64 {
+pub fn intAbiSize(bits: u16, target: Target) u64 {
-    return intAbiAlignment(bits, target, use_llvm).forward(@as(u16, @intCast((@as(u17, bits) + 7) / 8)));
+    return intAbiAlignment(bits, target).forward(@as(u16, @intCast((@as(u17, bits) + 7) / 8)));
 }
-pub fn intAbiAlignment(bits: u16, target: Target, use_llvm: bool) Alignment {
+pub fn intAbiAlignment(bits: u16, target: Target) Alignment {
     return switch (target.cpu.arch) {
         .x86 => switch (bits) {
             0 => .none,
 @@ -1632,19 +1630,16 @@ pub fn intAbiAlignment(bits: u16, target: Target, use_llvm: bool) Alignment {
             9...16 => .@"2",
             17...32 => .@"4",
             33...64 => .@"8",
-            else => switch (target_util.zigBackend(target, use_llvm)) {
+            else => .@"16",
-                .stage2_x86_64 => .@"8",
-                else => .@"16",
-            },
         },
         else => return Alignment.fromByteUnits(@min(
             std.math.ceilPowerOfTwoPromote(u16, @as(u16, @intCast((@as(u17, bits) + 7) / 8))),
-            maxIntAlignment(target, use_llvm),
+            maxIntAlignment(target),
         )),
     };
 }
-pub fn maxIntAlignment(target: std.Target, use_llvm: bool) u16 {
+pub fn maxIntAlignment(target: std.Target) u16 {
     return switch (target.cpu.arch) {
         .avr => 1,
         .msp430 => 2,
 @@ -1685,10 +1680,7 @@ pub fn maxIntAlignment(target: std.Target, use_llvm: bool) u16 {
             else => 8,
         },
-        .x86_64 => switch (target_util.zigBackend(target, use_llvm)) {
+        .x86_64 => 16,
-            .stage2_x86_64 => 8,
-            else => 16,
-        },
         // Even LLVMABIAlignmentOfType(i128) agrees on these targets.
         .x86,
 @@ -1928,6 +1920,17 @@ pub fn isSlice(ty: Type, zcu: *const Zcu) bool {
     };
 }
+pub fn isSliceAtRuntime(ty: Type, zcu: *const Zcu) bool {
+    return switch (zcu.intern_pool.indexToKey(ty.toIntern())) {
+        .ptr_type => |ptr_type| ptr_type.flags.size == .slice,
+        .opt_type => |child| switch (zcu.intern_pool.indexToKey(child)) {
+            .ptr_type => |ptr_type| !ptr_type.flags.is_allowzero and ptr_type.flags.size == .slice,
+            else => false,
+        },
+        else => false,
+    };
+}
 pub fn slicePtrFieldType(ty: Type, zcu: *const Zcu) Type {
     return Type.fromInterned(zcu.intern_pool.slicePtrType(ty.toIntern()));
 }

src/arch/aarch64/CodeGen.zig+10-10

 @@ -71,6 +71,8 @@ end_di_column: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// We postpone the creation of debug info for function args and locals
 /// until after all Mir instructions have been generated. Only then we
 /// will know saved_regs_stack_space which is necessary in order to
 @@ -646,6 +648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         const old_air_bookkeeping = self.air_bookkeeping;
         try self.ensureProcessDeathCapacity(Liveness.bpi);
+        self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
         switch (air_tags[@intFromEnum(inst)]) {
             // zig fmt: off
             .add             => try self.airBinOp(inst, .add),
 @@ -927,16 +930,13 @@ fn finishAirBookkeeping(self: *Self) void {
 }
 fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
-    var tomb_bits = self.liveness.getTombBits(inst);
+    const tomb_bits = self.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (self.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
+        self.processDeath(op.toIndexAllowNone() orelse continue);
-        const op_index = op.toIndex() orelse continue;
-        self.processDeath(op_index);
     }
-    const is_used = @as(u1, @truncate(tomb_bits)) == 0;
+    if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
-    if (is_used) {
         log.debug("%{d} => {}", .{ inst, result });
         const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
         branch.inst_table.putAssumeCapacityNoClobber(inst, result);
 @@ -3614,7 +3614,7 @@ fn reuseOperand(
     }
     // Prevent the operand deaths processing code from deallocating it.
-    self.liveness.clearOperandDeath(inst, op_index);
+    self.reused_operands.set(op_index);
     // That makes us responsible for doing the rest of the stuff that processDeath would have done.
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/arm/CodeGen.zig+10-10

 @@ -72,6 +72,8 @@ end_di_column: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// We postpone the creation of debug info for function args and locals
 /// until after all Mir instructions have been generated. Only then we
 /// will know saved_regs_stack_space which is necessary in order to
 @@ -635,6 +637,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         const old_air_bookkeeping = self.air_bookkeeping;
         try self.ensureProcessDeathCapacity(Liveness.bpi);
+        self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
         switch (air_tags[@intFromEnum(inst)]) {
             // zig fmt: off
             .add,            => try self.airBinOp(inst, .add),
 @@ -918,16 +921,13 @@ fn finishAirBookkeeping(self: *Self) void {
 }
 fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
-    var tomb_bits = self.liveness.getTombBits(inst);
+    const tomb_bits = self.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (self.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
+        self.processDeath(op.toIndexAllowNone() orelse continue);
-        const op_index = op.toIndex() orelse continue;
-        self.processDeath(op_index);
     }
-    const is_used = @as(u1, @truncate(tomb_bits)) == 0;
+    if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
-    if (is_used) {
         log.debug("%{d} => {}", .{ inst, result });
         const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
         branch.inst_table.putAssumeCapacityNoClobber(inst, result);
 @@ -2650,7 +2650,7 @@ fn reuseOperand(
     }
     // Prevent the operand deaths processing code from deallocating it.
-    self.liveness.clearOperandDeath(inst, op_index);
+    self.reused_operands.set(op_index);
     // That makes us responsible for doing the rest of the stuff that processDeath would have done.
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/riscv64/CodeGen.zig+11-7

 @@ -82,6 +82,8 @@ scope_generation: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// Whenever there is a runtime branch, we push a Branch onto this stack,
 /// and pop it off when the runtime branch joins. This provides an "overlay"
 /// of the table of mappings from instructions to `MCValue` from within the branch.
 @@ -1443,8 +1445,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
         verbose_tracking_log.debug("{}", .{func.fmtTracking()});
         const old_air_bookkeeping = func.air_bookkeeping;
+        try func.ensureProcessDeathCapacity(Liveness.bpi);
+        func.reused_operands = @TypeOf(func.reused_operands).initEmpty();
         try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
-        const tag: Air.Inst.Tag = air_tags[@intFromEnum(inst)];
+        const tag = air_tags[@intFromEnum(inst)];
         switch (tag) {
             // zig fmt: off
             .add,
 @@ -1783,11 +1788,10 @@ fn finishAir(
     result: MCValue,
     operands: [Liveness.bpi - 1]Air.Inst.Ref,
 ) !void {
-    var tomb_bits = func.liveness.getTombBits(inst);
+    const tomb_bits = func.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (func.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
         try func.processDeath(op.toIndexAllowNone() orelse continue);
     }
     func.finishAirResult(inst, result);
 @@ -4424,7 +4428,7 @@ fn reuseOperandAdvanced(
     }
     // Prevent the operand deaths processing code from deallocating it.
-    func.liveness.clearOperandDeath(inst, op_index);
+    func.reused_operands.set(op_index);
     const op_inst = operand.toIndex().?;
     func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst);

src/arch/sparc64/CodeGen.zig+10-10

 @@ -78,6 +78,8 @@ end_di_column: u32,
 /// which is a relative jump, based on the address following the reloc.
 exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
+reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
 /// Whenever there is a runtime branch, we push a Branch onto this stack,
 /// and pop it off when the runtime branch joins. This provides an "overlay"
 /// of the table of mappings from instructions to `MCValue` from within the branch.
 @@ -493,6 +495,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         const old_air_bookkeeping = self.air_bookkeeping;
         try self.ensureProcessDeathCapacity(Liveness.bpi);
+        self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
         switch (air_tags[@intFromEnum(inst)]) {
             // zig fmt: off
             .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
 @@ -3523,16 +3526,13 @@ fn finishAirBookkeeping(self: *Self) void {
 }
 fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
-    var tomb_bits = self.liveness.getTombBits(inst);
+    const tomb_bits = self.liveness.getTombBits(inst);
-    for (operands) |op| {
+    for (0.., operands) |op_index, op| {
-        const dies = @as(u1, @truncate(tomb_bits)) != 0;
+        if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
-        tomb_bits >>= 1;
+        if (self.reused_operands.isSet(op_index)) continue;
-        if (!dies) continue;
+        self.processDeath(op.toIndexAllowNone() orelse continue);
-        const op_index = op.toIndex() orelse continue;
-        self.processDeath(op_index);
     }
-    const is_used = @as(u1, @truncate(tomb_bits)) == 0;
+    if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
-    if (is_used) {
         log.debug("%{d} => {}", .{ inst, result });
         const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
         branch.inst_table.putAssumeCapacityNoClobber(inst, result);
 @@ -4568,7 +4568,7 @@ fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_ind
     }
     // Prevent the operand deaths processing code from deallocating it.
-    self.liveness.clearOperandDeath(inst, op_index);
+    self.reused_operands.set(op_index);
     // That makes us responsible for doing the rest of the stuff that processDeath would have done.
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];

src/arch/x86_64/CodeGen.zig+14357-4049

...	@@ -1,41 +1,26 @@	...	@@ -1,41 +1,26 @@
1	const std = @import("std");	1	const std = @import("std");
2	const build_options = @import("build_options");
3	const builtin = @import("builtin");
4	const assert = std.debug.assert;	2	const assert = std.debug.assert;
5	const codegen = @import("../../codegen.zig");	3	const codegen = @import("../../codegen.zig");
6	const leb128 = std.leb;
7	const link = @import("../../link.zig");	4	const link = @import("../../link.zig");
8	const log = std.log.scoped(.codegen);	5	const log = std.log.scoped(.codegen);
9	const tracking_log = std.log.scoped(.tracking);	6	const tracking_log = std.log.scoped(.tracking);
10	const verbose_tracking_log = std.log.scoped(.verbose_tracking);	7	const verbose_tracking_log = std.log.scoped(.verbose_tracking);
11	const wip_mir_log = std.log.scoped(.wip_mir);	8	const wip_mir_log = std.log.scoped(.wip_mir);
12	const math = std.math;
13	const mem = std.mem;
14	const target_util = @import("../../target.zig");
15	const trace = @import("../../tracy.zig").trace;
16		9
17	const Air = @import("../../Air.zig");	10	const Air = @import("../../Air.zig");
18	const Allocator = mem.Allocator;	11	const Allocator = std.mem.Allocator;
19	const CodeGenError = codegen.CodeGenError;
20	const Compilation = @import("../../Compilation.zig");
21	const ErrorMsg = Zcu.ErrorMsg;
22	const Emit = @import("Emit.zig");	12	const Emit = @import("Emit.zig");
23	const Liveness = @import("../../Liveness.zig");	13	const Liveness = @import("../../Liveness.zig");
24	const Lower = @import("Lower.zig");	14	const Lower = @import("Lower.zig");
25	const Mir = @import("Mir.zig");	15	const Mir = @import("Mir.zig");
26	const Package = @import("../../Package.zig");
27	const Zcu = @import("../../Zcu.zig");	16	const Zcu = @import("../../Zcu.zig");
		17	const Module = @import("../../Package/Module.zig");
28	const InternPool = @import("../../InternPool.zig");	18	const InternPool = @import("../../InternPool.zig");
29	const Alignment = InternPool.Alignment;
30	const Target = std.Target;
31	const Type = @import("../../Type.zig");	19	const Type = @import("../../Type.zig");
32	const Value = @import("../../Value.zig");	20	const Value = @import("../../Value.zig");
33	const Instruction = @import("encoder.zig").Instruction;
34		21
35	const abi = @import("abi.zig");	22	const abi = @import("abi.zig");
36	const bits = @import("bits.zig");	23	const bits = @import("bits.zig");
37	const errUnionErrorOffset = codegen.errUnionErrorOffset;
38	const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
39	const encoder = @import("encoder.zig");	24	const encoder = @import("encoder.zig");
40		25
41	const Condition = bits.Condition;	26	const Condition = bits.Condition;
...	@@ -46,7 +31,7 @@ const RegisterManager = abi.RegisterManager;	...	@@ -46,7 +31,7 @@ const RegisterManager = abi.RegisterManager;
46	const RegisterLock = RegisterManager.RegisterLock;	31	const RegisterLock = RegisterManager.RegisterLock;
47	const FrameIndex = bits.FrameIndex;	32	const FrameIndex = bits.FrameIndex;
48		33
49	const InnerError = CodeGenError \|\| error{OutOfRegisters};	34	const InnerError = codegen.CodeGenError \|\| error{OutOfRegisters};
50		35
51	gpa: Allocator,	36	gpa: Allocator,
52	pt: Zcu.PerThread,	37	pt: Zcu.PerThread,
...	@@ -57,7 +42,7 @@ debug_output: link.File.DebugInfoOutput,	...	@@ -57,7 +42,7 @@ debug_output: link.File.DebugInfoOutput,
57	target: *const std.Target,	42	target: *const std.Target,
58	owner: Owner,	43	owner: Owner,
59	inline_func: InternPool.Index,	44	inline_func: InternPool.Index,
60	mod: *Package.Module,	45	mod: *Module,
61	arg_index: u32,	46	arg_index: u32,
62	args: []MCValue,	47	args: []MCValue,
63	va_info: union {	48	va_info: union {
...	@@ -76,9 +61,10 @@ src_loc: Zcu.LazySrcLoc,	...	@@ -76,9 +61,10 @@ src_loc: Zcu.LazySrcLoc,
76	eflags_inst: ?Air.Inst.Index = null,	61	eflags_inst: ?Air.Inst.Index = null,
77		62
78	/// MIR Instructions	63	/// MIR Instructions
79	mir_instructions: std.MultiArrayList(Mir.Inst) = .{},	64	mir_instructions: std.MultiArrayList(Mir.Inst) = .empty,
80	/// MIR extra data	65	/// MIR extra data
81	mir_extra: std.ArrayListUnmanaged(u32) = .empty,	66	mir_extra: std.ArrayListUnmanaged(u32) = .empty,
		67	mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
82		68
83	/// Byte offset within the source file of the ending curly.	69	/// Byte offset within the source file of the ending curly.
84	end_di_line: u32,	70	end_di_line: u32,
...	@@ -87,10 +73,11 @@ end_di_column: u32,	...	@@ -87,10 +73,11 @@ end_di_column: u32,
87	/// The value is an offset into the `Function` `code` from the beginning.	73	/// The value is an offset into the `Function` `code` from the beginning.
88	/// To perform the reloc, write 32-bit signed little-endian integer	74	/// To perform the reloc, write 32-bit signed little-endian integer
89	/// which is a relative jump, based on the address following the reloc.	75	/// which is a relative jump, based on the address following the reloc.
90	exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,	76	epilogue_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
91		77
92	const_tracking: ConstTrackingMap = .{},	78	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
93	inst_tracking: InstTrackingMap = .{},	79	const_tracking: ConstTrackingMap = .empty,
		80	inst_tracking: InstTrackingMap = .empty,
94		81
95	// Key is the block instruction	82	// Key is the block instruction
96	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,	83	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
...	@@ -100,27 +87,35 @@ register_manager: RegisterManager = .{},	...	@@ -100,27 +87,35 @@ register_manager: RegisterManager = .{},
100	/// Generation of the current scope, increments by 1 for every entered scope.	87	/// Generation of the current scope, increments by 1 for every entered scope.
101	scope_generation: u32 = 0,	88	scope_generation: u32 = 0,
102		89
103	frame_allocs: std.MultiArrayList(FrameAlloc) = .{},	90	frame_allocs: std.MultiArrayList(FrameAlloc) = .empty,
104	free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty,	91	free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty,
105	frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},	92	frame_locs: std.MultiArrayList(Mir.FrameLoc) = .empty,
106		93
107	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {	94	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
108	/// The state to restore before branching.	95	/// The state to restore before branching.
109	state: State,	96	state: State,
110	/// The branch target.	97	/// The branch target.
111	jmp_target: Mir.Inst.Index,	98	target: Mir.Inst.Index,
112	}) = .{},	99	}) = .empty,
113		100	loop_switches: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
114	/// Debug field, used to find bugs in the compiler.	101	start: u31,
115	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,	102	len: u11,
		103	min: Value,
		104	else_relocs: union(enum) {
		105	@"unreachable",
		106	forward: std.ArrayListUnmanaged(Mir.Inst.Index),
		107	backward: Mir.Inst.Index,
		108	},
		109	}) = .empty,
116		110
117	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};	111	next_temp_index: Temp.Index = @enumFromInt(0),
		112	temp_type: [Temp.Index.max]Type = undefined,
118		113
119	const Owner = union(enum) {	114	const Owner = union(enum) {
120	nav_index: InternPool.Nav.Index,	115	nav_index: InternPool.Nav.Index,
121	lazy_sym: link.File.LazySymbol,	116	lazy_sym: link.File.LazySymbol,
122		117
123	fn getSymbolIndex(owner: Owner, ctx: *Self) !u32 {	118	fn getSymbolIndex(owner: Owner, ctx: *CodeGen) !u32 {
124	const pt = ctx.pt;	119	const pt = ctx.pt;
125	switch (owner) {	120	switch (owner) {
126	.nav_index => \|nav_index\| if (ctx.bin_file.cast(.elf)) \|elf_file\| {	121	.nav_index => \|nav_index\| if (ctx.bin_file.cast(.elf)) \|elf_file\| {
...	@@ -151,6 +146,9 @@ const Owner = union(enum) {	...	@@ -151,6 +146,9 @@ const Owner = union(enum) {
151	}	146	}
152	};	147	};
153		148
		149	const MaskKind = enum(u1) { sign, all };
		150	const MaskInfo = packed struct { kind: MaskKind, inverted: bool, scalar: Memory.Size };
		151
154	pub const MCValue = union(enum) {	152	pub const MCValue = union(enum) {
155	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.	153	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
156	/// TODO Look into deleting this tag and using `dead` instead, since every use	154	/// TODO Look into deleting this tag and using `dead` instead, since every use
...	@@ -172,10 +170,16 @@ pub const MCValue = union(enum) {	...	@@ -172,10 +170,16 @@ pub const MCValue = union(enum) {
172	register: Register,	170	register: Register,
173	/// The value is split across two registers.	171	/// The value is split across two registers.
174	register_pair: [2]Register,	172	register_pair: [2]Register,
		173	/// The value is split across three registers.
		174	register_triple: [3]Register,
		175	/// The value is split across four registers.
		176	register_quadruple: [4]Register,
175	/// The value is a constant offset from the value in a register.	177	/// The value is a constant offset from the value in a register.
176	register_offset: bits.RegisterOffset,	178	register_offset: bits.RegisterOffset,
177	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.	179	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
178	register_overflow: struct { reg: Register, eflags: Condition },	180	register_overflow: struct { reg: Register, eflags: Condition },
		181	/// The value is a bool vector stored in a vector register with a different scalar type.
		182	register_mask: struct { reg: Register, info: MaskInfo },
179	/// The value is in memory at a hard-coded address.	183	/// The value is in memory at a hard-coded address.
180	/// If the type is a pointer, it means the pointer address is stored at this memory location.	184	/// If the type is a pointer, it means the pointer address is stored at this memory location.
181	memory: u64,	185	memory: u64,
...	@@ -211,7 +215,7 @@ pub const MCValue = union(enum) {	...	@@ -211,7 +215,7 @@ pub const MCValue = union(enum) {
211	/// Payload is a frame address.	215	/// Payload is a frame address.
212	lea_frame: bits.FrameAddr,	216	lea_frame: bits.FrameAddr,
213	/// Supports integer_per_element abi	217	/// Supports integer_per_element abi
214	elementwise_regs_then_frame: packed struct { regs: u3 = 0, frame_off: i29 = 0, frame_index: FrameIndex },	218	elementwise_regs_then_frame: packed struct { regs: u3, frame_off: i29, frame_index: FrameIndex },
215	/// This indicates that we have already allocated a frame index for this instruction,	219	/// This indicates that we have already allocated a frame index for this instruction,
216	/// but it has not been spilled there yet in the current control flow.	220	/// but it has not been spilled there yet in the current control flow.
217	/// Payload is a frame index.	221	/// Payload is a frame index.
...	@@ -226,6 +230,7 @@ pub const MCValue = union(enum) {	...	@@ -226,6 +230,7 @@ pub const MCValue = union(enum) {
226	.undef,	230	.undef,
227	.immediate,	231	.immediate,
228	.register_offset,	232	.register_offset,
		233	.register_mask,
229	.eflags,	234	.eflags,
230	.register_overflow,	235	.register_overflow,
231	.lea_symbol,	236	.lea_symbol,
...	@@ -239,6 +244,8 @@ pub const MCValue = union(enum) {	...	@@ -239,6 +244,8 @@ pub const MCValue = union(enum) {
239	=> false,	244	=> false,
240	.register,	245	.register,
241	.register_pair,	246	.register_pair,
		247	.register_triple,
		248	.register_quadruple,
242	.memory,	249	.memory,
243	.load_symbol,	250	.load_symbol,
244	.load_got,	251	.load_got,
...	@@ -250,13 +257,21 @@ pub const MCValue = union(enum) {	...	@@ -250,13 +257,21 @@ pub const MCValue = union(enum) {
250	};	257	};
251	}	258	}
252		259
253	fn isMemory(mcv: MCValue) bool {	260	// hack around linker relocation bugs
		261	fn isBase(mcv: MCValue) bool {
254	return switch (mcv) {	262	return switch (mcv) {
255	.memory, .indirect, .load_frame => true,	263	.memory, .indirect, .load_frame => true,
256	else => false,	264	else => false,
257	};	265	};
258	}	266	}
259		267
		268	fn isMemory(mcv: MCValue) bool {
		269	return switch (mcv) {
		270	.memory, .indirect, .load_frame, .load_symbol => true,
		271	else => false,
		272	};
		273	}
		274
260	fn isImmediate(mcv: MCValue) bool {	275	fn isImmediate(mcv: MCValue) bool {
261	return switch (mcv) {	276	return switch (mcv) {
262	.immediate => true,	277	.immediate => true,
...	@@ -284,16 +299,23 @@ pub const MCValue = union(enum) {	...	@@ -284,16 +299,23 @@ pub const MCValue = union(enum) {
284	.register => \|reg\| reg,	299	.register => \|reg\| reg,
285	.register_offset, .indirect => \|ro\| ro.reg,	300	.register_offset, .indirect => \|ro\| ro.reg,
286	.register_overflow => \|ro\| ro.reg,	301	.register_overflow => \|ro\| ro.reg,
		302	.register_mask => \|rm\| rm.reg,
287	else => null,	303	else => null,
288	};	304	};
289	}	305	}
290		306
291	fn getRegs(mcv: *const MCValue) []const Register {	307	fn getRegs(mcv: *const MCValue) []const Register {
292	return switch (mcv.*) {	308	return switch (mcv.*) {
293	.register => \|reg\| @as(const [1]Register, reg),	309	.register => \|*reg\| reg[0..1],
294	.register_pair => \|*regs\| regs,	310	inline .register_pair,
295	.register_offset, .indirect => \|ro\| @as(const [1]Register, &ro.reg),	311	.register_triple,
296	.register_overflow => \|ro\| @as(const [1]Register, &ro.reg),	312	.register_quadruple,
		313	=> \|*regs\| regs,
		314	inline .register_offset,
		315	.indirect,
		316	.register_overflow,
		317	.register_mask,
		318	=> \|*pl\| (&pl.reg)[0..1],
297	else => &.{},	319	else => &.{},
298	};	320	};
299	}	321	}
...	@@ -306,6 +328,13 @@ pub const MCValue = union(enum) {	...	@@ -306,6 +328,13 @@ pub const MCValue = union(enum) {
306	};	328	};
307	}	329	}
308		330
		331	fn isAddress(mcv: MCValue) bool {
		332	return switch (mcv) {
		333	.immediate, .register, .register_offset, .lea_frame => true,
		334	else => false,
		335	};
		336	}
		337
309	fn address(mcv: MCValue) MCValue {	338	fn address(mcv: MCValue) MCValue {
310	return switch (mcv) {	339	return switch (mcv) {
311	.none,	340	.none,
...	@@ -316,8 +345,11 @@ pub const MCValue = union(enum) {	...	@@ -316,8 +345,11 @@ pub const MCValue = union(enum) {
316	.eflags,	345	.eflags,
317	.register,	346	.register,
318	.register_pair,	347	.register_pair,
		348	.register_triple,
		349	.register_quadruple,
319	.register_offset,	350	.register_offset,
320	.register_overflow,	351	.register_overflow,
		352	.register_mask,
321	.lea_symbol,	353	.lea_symbol,
322	.lea_direct,	354	.lea_direct,
323	.lea_got,	355	.lea_got,
...	@@ -348,7 +380,10 @@ pub const MCValue = union(enum) {	...	@@ -348,7 +380,10 @@ pub const MCValue = union(enum) {
348	.undef,	380	.undef,
349	.eflags,	381	.eflags,
350	.register_pair,	382	.register_pair,
		383	.register_triple,
		384	.register_quadruple,
351	.register_overflow,	385	.register_overflow,
		386	.register_mask,
352	.memory,	387	.memory,
353	.indirect,	388	.indirect,
354	.load_direct,	389	.load_direct,
...	@@ -383,7 +418,10 @@ pub const MCValue = union(enum) {	...	@@ -383,7 +418,10 @@ pub const MCValue = union(enum) {
383	=> unreachable, // not valid	418	=> unreachable, // not valid
384	.eflags,	419	.eflags,
385	.register_pair,	420	.register_pair,
		421	.register_triple,
		422	.register_quadruple,
386	.register_overflow,	423	.register_overflow,
		424	.register_mask,
387	.memory,	425	.memory,
388	.indirect,	426	.indirect,
389	.load_direct,	427	.load_direct,
...	@@ -410,7 +448,7 @@ pub const MCValue = union(enum) {	...	@@ -410,7 +448,7 @@ pub const MCValue = union(enum) {
410	};	448	};
411	}	449	}
412		450
413	fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {	451	fn mem(mcv: MCValue, function: *CodeGen, mod_rm: Memory.Mod.Rm) !Memory {
414	return switch (mcv) {	452	return switch (mcv) {
415	.none,	453	.none,
416	.unreach,	454	.unreach,
...	@@ -420,8 +458,11 @@ pub const MCValue = union(enum) {	...	@@ -420,8 +458,11 @@ pub const MCValue = union(enum) {
420	.eflags,	458	.eflags,
421	.register,	459	.register,
422	.register_pair,	460	.register_pair,
		461	.register_triple,
		462	.register_quadruple,
423	.register_offset,	463	.register_offset,
424	.register_overflow,	464	.register_overflow,
		465	.register_mask,
425	.load_direct,	466	.load_direct,
426	.lea_direct,	467	.lea_direct,
427	.load_got,	468	.load_got,
...	@@ -433,25 +474,31 @@ pub const MCValue = union(enum) {	...	@@ -433,25 +474,31 @@ pub const MCValue = union(enum) {
433	.reserved_frame,	474	.reserved_frame,
434	.lea_symbol,	475	.lea_symbol,
435	=> unreachable,	476	=> unreachable,
436	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{	477	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
437	.base = .{ .reg = .ds },	478	.base = .{ .reg = .ds },
438	.mod = .{ .rm = .{	479	.mod = .{ .rm = .{
439	.size = size,	480	.size = mod_rm.size,
440	.disp = small_addr,	481	.index = mod_rm.index,
		482	.scale = mod_rm.scale,
		483	.disp = small_addr + mod_rm.disp,
441	} },	484	} },
442	} else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },	485	} else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
443	.indirect => \|reg_off\| .{	486	.indirect => \|reg_off\| .{
444	.base = .{ .reg = reg_off.reg },	487	.base = .{ .reg = registerAlias(reg_off.reg, @divExact(function.target.ptrBitWidth(), 8)) },
445	.mod = .{ .rm = .{	488	.mod = .{ .rm = .{
446	.size = size,	489	.size = mod_rm.size,
447	.disp = reg_off.off,	490	.index = mod_rm.index,
		491	.scale = mod_rm.scale,
		492	.disp = reg_off.off + mod_rm.disp,
448	} },	493	} },
449	},	494	},
450	.load_frame => \|frame_addr\| .{	495	.load_frame => \|frame_addr\| .{
451	.base = .{ .frame = frame_addr.index },	496	.base = .{ .frame = frame_addr.index },
452	.mod = .{ .rm = .{	497	.mod = .{ .rm = .{
453	.size = size,	498	.size = mod_rm.size,
454	.disp = frame_addr.off,	499	.index = mod_rm.index,
		500	.scale = mod_rm.scale,
		501	.disp = frame_addr.off + mod_rm.disp,
455	} },	502	} },
456	},	503	},
457	.load_symbol => \|sym_off\| {	504	.load_symbol => \|sym_off\| {
...	@@ -459,12 +506,14 @@ pub const MCValue = union(enum) {	...	@@ -459,12 +506,14 @@ pub const MCValue = union(enum) {
459	return .{	506	return .{
460	.base = .{ .reloc = sym_off.sym_index },	507	.base = .{ .reloc = sym_off.sym_index },
461	.mod = .{ .rm = .{	508	.mod = .{ .rm = .{
462	.size = size,	509	.size = mod_rm.size,
463	.disp = sym_off.off,	510	.index = mod_rm.index,
		511	.scale = mod_rm.scale,
		512	.disp = sym_off.off + mod_rm.disp,
464	} },	513	} },
465	};	514	};
466	},	515	},
467	.air_ref => \|ref\| (try function.resolveInst(ref)).mem(function, size),	516	.air_ref => \|ref\| (try function.resolveInst(ref)).mem(function, mod_rm),
468	};	517	};
469	}	518	}
470		519
...	@@ -480,12 +529,25 @@ pub const MCValue = union(enum) {	...	@@ -480,12 +529,25 @@ pub const MCValue = union(enum) {
480	.memory => \|pl\| try writer.print("[ds:0x{x}]", .{pl}),	529	.memory => \|pl\| try writer.print("[ds:0x{x}]", .{pl}),
481	inline .eflags, .register => \|pl\| try writer.print("{s}", .{@tagName(pl)}),	530	inline .eflags, .register => \|pl\| try writer.print("{s}", .{@tagName(pl)}),
482	.register_pair => \|pl\| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }),	531	.register_pair => \|pl\| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }),
		532	.register_triple => \|pl\| try writer.print("{s}:{s}:{s}", .{
		533	@tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]),
		534	}),
		535	.register_quadruple => \|pl\| try writer.print("{s}:{s}:{s}:{s}", .{
		536	@tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]),
		537	}),
483	.register_offset => \|pl\| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }),	538	.register_offset => \|pl\| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }),
484	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{	539	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{
485	@tagName(pl.eflags), @tagName(pl.reg),	540	@tagName(pl.eflags),
		541	@tagName(pl.reg),
486	}),	542	}),
487	.load_symbol => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.sym_index, pl.off }),	543	.register_mask => \|pl\| try writer.print("mask({s},{}):{c}{s}", .{
488	.lea_symbol => \|pl\| try writer.print("{} + 0x{x}", .{ pl.sym_index, pl.off }),	544	@tagName(pl.info.kind),
		545	pl.info.scalar,
		546	@as(u8, if (pl.info.inverted) '!' else ' '),
		547	@tagName(pl.reg),
		548	}),
		549	.load_symbol => \|pl\| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }),
		550	.lea_symbol => \|pl\| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }),
489	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),	551	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
490	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),	552	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),
491	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),	553	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),
...	@@ -494,7 +556,9 @@ pub const MCValue = union(enum) {	...	@@ -494,7 +556,9 @@ pub const MCValue = union(enum) {
494	.load_tlv => \|pl\| try writer.print("[tlv:{d}]", .{pl}),	556	.load_tlv => \|pl\| try writer.print("[tlv:{d}]", .{pl}),
495	.lea_tlv => \|pl\| try writer.print("tlv:{d}", .{pl}),	557	.lea_tlv => \|pl\| try writer.print("tlv:{d}", .{pl}),
496	.load_frame => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }),	558	.load_frame => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }),
497	.elementwise_regs_then_frame => \|pl\| try writer.print("elementwise:{d}:[{} + 0x{x}]", .{ pl.regs, pl.frame_index, pl.frame_off }),	559	.elementwise_regs_then_frame => \|pl\| try writer.print("elementwise:{d}:[{} + 0x{x}]", .{
		560	pl.regs, pl.frame_index, pl.frame_off,
		561	}),
498	.lea_frame => \|pl\| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),	562	.lea_frame => \|pl\| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
499	.reserved_frame => \|pl\| try writer.print("(dead:{})", .{pl}),	563	.reserved_frame => \|pl\| try writer.print("(dead:{})", .{pl}),
500	.air_ref => \|pl\| try writer.print("(air:0x{x})", .{@intFromEnum(pl)}),	564	.air_ref => \|pl\| try writer.print("(air:0x{x})", .{@intFromEnum(pl)}),
...	@@ -534,8 +598,11 @@ const InstTracking = struct {	...	@@ -534,8 +598,11 @@ const InstTracking = struct {
534	.eflags,	598	.eflags,
535	.register,	599	.register,
536	.register_pair,	600	.register_pair,
		601	.register_triple,
		602	.register_quadruple,
537	.register_offset,	603	.register_offset,
538	.register_overflow,	604	.register_overflow,
		605	.register_mask,
539	.indirect,	606	.indirect,
540	=> .none,	607	=> .none,
541	}, .short = result };	608	}, .short = result };
...	@@ -553,64 +620,48 @@ const InstTracking = struct {	...	@@ -553,64 +620,48 @@ const InstTracking = struct {
553	return self.short.getCondition();	620	return self.short.getCondition();
554	}	621	}
555		622
556	fn spill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {	623	fn spill(self: InstTracking, cg: CodeGen, inst: Air.Inst.Index) !void {
557	if (std.meta.eql(self.long, self.short)) return; // Already spilled	624	if (std.meta.eql(self.long, self.short)) return; // Already spilled
558	// Allocate or reuse frame index	625	// Allocate or reuse frame index
559	switch (self.long) {	626	switch (self.long) {
560	.none => self.long = try function.allocRegOrMem(inst, false),	627	.none => self.long = try cg.allocRegOrMem(inst, false),
561	.load_frame => {},	628	.load_frame => {},
562	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },	629	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
563	else => unreachable,	630	else => unreachable,
564	}	631	}
565	tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });	632	tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long });
566	try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{});	633	try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{});
567	}	634	}
568		635
569	fn reuseFrame(self: *InstTracking) void {	636	fn reuseFrame(self: *InstTracking) void {
570	switch (self.long) {	637	self.* = .init(switch (self.long) {
571	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },	638	.none => switch (self.short) {
572	else => {},	639	.dead => .none,
573	}	640	else => \|short\| short,
574	self.short = switch (self.long) {	641	},
575	.none,	642	.reserved_frame => \|index\| .{ .load_frame = .{ .index = index } },
576	.unreach,	643	else => \|long\| long,
577	.undef,	644	});
578	.immediate,
579	.memory,
580	.load_direct,
581	.lea_direct,
582	.load_got,
583	.lea_got,
584	.load_tlv,
585	.lea_tlv,
586	.load_frame,
587	.lea_frame,
588	.load_symbol,
589	.lea_symbol,
590	=> self.long,
591	.dead,
592	.eflags,
593	.register,
594	.register_pair,
595	.register_offset,
596	.register_overflow,
597	.indirect,
598	.elementwise_regs_then_frame,
599	.reserved_frame,
600	.air_ref,
601	=> unreachable,
602	};
603	}	645	}
604		646
605	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {	647	fn trackSpill(self: InstTracking, function: CodeGen, inst: Air.Inst.Index) !void {
606	try function.freeValue(self.short);	648	try function.freeValue(self.short);
607	self.reuseFrame();	649	self.reuseFrame();
608	tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });	650	tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });
609	}	651	}
610		652
611	fn verifyMaterialize(self: InstTracking, target: InstTracking) void {	653	fn verifyMaterialize(self: InstTracking, target: InstTracking) void {
612	switch (self.long) {	654	switch (self.long) {
613	.none,	655	.none,
		656	.load_frame,
		657	.reserved_frame,
		658	=> switch (target.long) {
		659	.none,
		660	.load_frame,
		661	.reserved_frame,
		662	=> {},
		663	else => unreachable,
		664	},
614	.unreach,	665	.unreach,
615	.undef,	666	.undef,
616	.immediate,	667	.immediate,
...	@@ -625,21 +676,15 @@ const InstTracking = struct {	...	@@ -625,21 +676,15 @@ const InstTracking = struct {
625	.load_symbol,	676	.load_symbol,
626	.lea_symbol,	677	.lea_symbol,
627	=> assert(std.meta.eql(self.long, target.long)),	678	=> assert(std.meta.eql(self.long, target.long)),
628	.load_frame,
629	.reserved_frame,
630	=> switch (target.long) {
631	.none,
632	.load_frame,
633	.reserved_frame,
634	=> {},
635	else => unreachable,
636	},
637	.dead,	679	.dead,
638	.eflags,	680	.eflags,
639	.register,	681	.register,
640	.register_pair,	682	.register_pair,
		683	.register_triple,
		684	.register_quadruple,
641	.register_offset,	685	.register_offset,
642	.register_overflow,	686	.register_overflow,
		687	.register_mask,
643	.indirect,	688	.indirect,
644	.elementwise_regs_then_frame,	689	.elementwise_regs_then_frame,
645	.air_ref,	690	.air_ref,
...	@@ -649,7 +694,7 @@ const InstTracking = struct {	...	@@ -649,7 +694,7 @@ const InstTracking = struct {
649		694
650	fn materialize(	695	fn materialize(
651	self: *InstTracking,	696	self: *InstTracking,
652	function: *Self,	697	function: *CodeGen,
653	inst: Air.Inst.Index,	698	inst: Air.Inst.Index,
654	target: InstTracking,	699	target: InstTracking,
655	) !void {	700	) !void {
...	@@ -659,7 +704,7 @@ const InstTracking = struct {	...	@@ -659,7 +704,7 @@ const InstTracking = struct {
659		704
660	fn materializeUnsafe(	705	fn materializeUnsafe(
661	self: InstTracking,	706	self: InstTracking,
662	function: *Self,	707	function: *CodeGen,
663	inst: Air.Inst.Index,	708	inst: Air.Inst.Index,
664	target: InstTracking,	709	target: InstTracking,
665	) !void {	710	) !void {
...	@@ -678,43 +723,42 @@ const InstTracking = struct {	...	@@ -678,43 +723,42 @@ const InstTracking = struct {
678	else => target.long,	723	else => target.long,
679	} else target.long;	724	} else target.long;
680	self.short = target.short;	725	self.short = target.short;
681	tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });	726	tracking_log.debug("{} => {} (materialize)", .{ inst, self.* });
682	}	727	}
683		728
684	fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {	729	fn resurrect(self: InstTracking, function: CodeGen, inst: Air.Inst.Index, scope_generation: u32) !void {
685	switch (self.short) {	730	switch (self.short) {
686	.dead => \|die_generation\| if (die_generation >= scope_generation) {	731	.dead => \|die_generation\| if (die_generation >= scope_generation) {
687	self.reuseFrame();	732	self.reuseFrame();
688	tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });	733	try function.getValue(self.short, inst);
		734	tracking_log.debug("{} => {} (resurrect)", .{ inst, self.* });
689	},	735	},
690	else => {},	736	else => {},
691	}	737	}
692	}	738	}
693		739
694	fn die(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {	740	fn die(self: InstTracking, function: CodeGen, inst: Air.Inst.Index) !void {
695	if (self.short == .dead) return;	741	if (self.short == .dead) return;
696	try function.freeValue(self.short);	742	try function.freeValue(self.short);
		743	if (self.long == .none) self.long = self.short;
697	self.short = .{ .dead = function.scope_generation };	744	self.short = .{ .dead = function.scope_generation };
698	tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });	745	tracking_log.debug("{} => {} (death)", .{ inst, self.* });
699	}	746	}
700		747
701	fn reuse(	748	fn reuse(
702	self: *InstTracking,	749	self: *InstTracking,
703	function: *Self,	750	function: *CodeGen,
704	new_inst: ?Air.Inst.Index,	751	new_inst: ?Air.Inst.Index,
705	old_inst: Air.Inst.Index,	752	old_inst: Air.Inst.Index,
706	) void {	753	) void {
707	self.short = .{ .dead = function.scope_generation };	754	self.short = .{ .dead = function.scope_generation };
708	if (new_inst) \|inst\|	755	tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst });
709	tracking_log.debug("%{d} => {} (reuse %{d})", .{ inst, self.*, old_inst })
710	else
711	tracking_log.debug("tmp => {} (reuse %{d})", .{ self.*, old_inst });
712	}	756	}
713		757
714	fn liveOut(self: InstTracking, function: Self, inst: Air.Inst.Index) void {	758	fn liveOut(self: InstTracking, function: CodeGen, inst: Air.Inst.Index) void {
715	for (self.getRegs()) \|reg\| {	759	for (self.getRegs()) \|reg\| {
716	if (function.register_manager.isRegFree(reg)) {	760	if (function.register_manager.isRegFree(reg)) {
717	tracking_log.debug("%{d} => {} (live-out)", .{ inst, self.* });	761	tracking_log.debug("{} => {} (live-out)", .{ inst, self.* });
718	continue;	762	continue;
719	}	763	}
720		764
...	@@ -741,28 +785,28 @@ const InstTracking = struct {	...	@@ -741,28 +785,28 @@ const InstTracking = struct {
741	// Perform side-effects of freeValue manually.	785	// Perform side-effects of freeValue manually.
742	function.register_manager.freeReg(reg);	786	function.register_manager.freeReg(reg);
743		787
744	tracking_log.debug("%{d} => {} (live-out %{d})", .{ inst, self.*, tracked_inst });	788	tracking_log.debug("{} => {} (live-out {})", .{ inst, self.*, tracked_inst });
745	}	789	}
746	}	790	}
747		791
748	pub fn format(	792	pub fn format(
749	self: InstTracking,	793	tracking: InstTracking,
750	comptime _: []const u8,	794	comptime _: []const u8,
751	_: std.fmt.FormatOptions,	795	_: std.fmt.FormatOptions,
752	writer: anytype,	796	writer: anytype,
753	) @TypeOf(writer).Error!void {	797	) @TypeOf(writer).Error!void {
754	if (!std.meta.eql(self.long, self.short)) try writer.print("\|{}\| ", .{self.long});	798	if (!std.meta.eql(tracking.long, tracking.short)) try writer.print("\|{}\| ", .{tracking.long});
755	try writer.print("{}", .{self.short});	799	try writer.print("{}", .{tracking.short});
756	}	800	}
757	};	801	};
758		802
759	const FrameAlloc = struct {	803	const FrameAlloc = struct {
760	abi_size: u31,	804	abi_size: u31,
761	spill_pad: u3,	805	spill_pad: u3,
762	abi_align: Alignment,	806	abi_align: InternPool.Alignment,
763	ref_count: u16,	807	ref_count: u16,
764		808
765	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {	809	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc {
766	return .{	810	return .{
767	.abi_size = @intCast(alloc_abi.size),	811	.abi_size = @intCast(alloc_abi.size),
768	.spill_pad = alloc_abi.pad,	812	.spill_pad = alloc_abi.pad,
...	@@ -779,14 +823,14 @@ const FrameAlloc = struct {	...	@@ -779,14 +823,14 @@ const FrameAlloc = struct {
779	fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {	823	fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {
780	const abi_size = ty.abiSize(zcu);	824	const abi_size = ty.abiSize(zcu);
781	const spill_size = if (abi_size < 8)	825	const spill_size = if (abi_size < 8)
782	math.ceilPowerOfTwoAssert(u64, abi_size)	826	std.math.ceilPowerOfTwoAssert(u64, abi_size)
783	else	827	else
784	std.mem.alignForward(u64, abi_size, 8);	828	std.mem.alignForward(u64, abi_size, 8);
785	return init(.{	829	return init(.{
786	.size = spill_size,	830	.size = spill_size,
787	.pad = @intCast(spill_size - abi_size),	831	.pad = @intCast(spill_size - abi_size),
788	.alignment = ty.abiAlignment(zcu).maxStrict(	832	.alignment = ty.abiAlignment(zcu).maxStrict(
789	Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),	833	.fromNonzeroByteUnits(@min(spill_size, 8)),
790	),	834	),
791	});	835	});
792	}	836	}
...	@@ -808,7 +852,7 @@ const BlockData = struct {	...	@@ -808,7 +852,7 @@ const BlockData = struct {
808	}	852	}
809	};	853	};
810		854
811	const Self = @This();	855	const CodeGen = @This();
812		856
813	pub fn generate(	857	pub fn generate(
814	bin_file: *link.File,	858	bin_file: *link.File,
...	@@ -819,16 +863,16 @@ pub fn generate(	...	@@ -819,16 +863,16 @@ pub fn generate(
819	liveness: Liveness,	863	liveness: Liveness,
820	code: *std.ArrayListUnmanaged(u8),	864	code: *std.ArrayListUnmanaged(u8),
821	debug_output: link.File.DebugInfoOutput,	865	debug_output: link.File.DebugInfoOutput,
822	) CodeGenError!void {	866	) codegen.CodeGenError!void {
823	const zcu = pt.zcu;	867	const zcu = pt.zcu;
824	const comp = zcu.comp;	868	const comp = zcu.comp;
825	const gpa = zcu.gpa;	869	const gpa = zcu.gpa;
826	const ip = &zcu.intern_pool;	870	const ip = &zcu.intern_pool;
827	const func = zcu.funcInfo(func_index);	871	const func = zcu.funcInfo(func_index);
828	const fn_type = Type.fromInterned(func.ty);	872	const fn_type: Type = .fromInterned(func.ty);
829	const mod = zcu.navFileScope(func.owner_nav).mod;	873	const mod = zcu.navFileScope(func.owner_nav).mod;
830		874
831	var function: Self = .{	875	var function: CodeGen = .{
832	.gpa = gpa,	876	.gpa = gpa,
833	.pt = pt,	877	.pt = pt,
834	.air = air,	878	.air = air,
...	@@ -853,14 +897,21 @@ pub fn generate(	...	@@ -853,14 +897,21 @@ pub fn generate(
853	function.free_frame_indices.deinit(gpa);	897	function.free_frame_indices.deinit(gpa);
854	function.frame_locs.deinit(gpa);	898	function.frame_locs.deinit(gpa);
855	function.loops.deinit(gpa);	899	function.loops.deinit(gpa);
		900	function.loop_switches.deinit(gpa);
856	var block_it = function.blocks.valueIterator();	901	var block_it = function.blocks.valueIterator();
857	while (block_it.next()) \|block\| block.deinit(gpa);	902	while (block_it.next()) \|block\| block.deinit(gpa);
858	function.blocks.deinit(gpa);	903	function.blocks.deinit(gpa);
859	function.inst_tracking.deinit(gpa);	904	function.inst_tracking.deinit(gpa);
860	function.const_tracking.deinit(gpa);	905	function.const_tracking.deinit(gpa);
861	function.exitlude_jump_relocs.deinit(gpa);	906	function.epilogue_relocs.deinit(gpa);
862	function.mir_instructions.deinit(gpa);	907	function.mir_instructions.deinit(gpa);
863	function.mir_extra.deinit(gpa);	908	function.mir_extra.deinit(gpa);
		909	function.mir_table.deinit(gpa);
		910	}
		911	try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);
		912	for (0..Temp.Index.max) \|temp_index\| {
		913	const temp: Temp.Index = @enumFromInt(temp_index);
		914	function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none));
864	}	915	}
865		916
866	wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});	917	wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)});
...	@@ -868,15 +919,14 @@ pub fn generate(	...	@@ -868,15 +919,14 @@ pub fn generate(
868	try function.frame_allocs.resize(gpa, FrameIndex.named_count);	919	try function.frame_allocs.resize(gpa, FrameIndex.named_count);
869	function.frame_allocs.set(	920	function.frame_allocs.set(
870	@intFromEnum(FrameIndex.stack_frame),	921	@intFromEnum(FrameIndex.stack_frame),
871	FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),	922	.init(.{ .size = 0, .alignment = .@"1" }),
872	);	923	);
873	function.frame_allocs.set(	924	function.frame_allocs.set(
874	@intFromEnum(FrameIndex.call_frame),	925	@intFromEnum(FrameIndex.call_frame),
875	FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),	926	.init(.{ .size = 0, .alignment = .@"1" }),
876	);	927	);
877		928
878	const fn_info = zcu.typeToFunc(fn_type).?;	929	const fn_info = zcu.typeToFunc(fn_type).?;
879	const cc = abi.resolveCallingConvention(fn_info.cc, function.target.*);
880	var call_info = function.resolveCallingConventionValues(fn_info, &.{}, .args_frame) catch \|err\| switch (err) {	930	var call_info = function.resolveCallingConventionValues(fn_info, &.{}, .args_frame) catch \|err\| switch (err) {
881	error.CodegenFail => return error.CodegenFail,	931	error.CodegenFail => return error.CodegenFail,
882	else => \|e\| return e,	932	else => \|e\| return e,
...	@@ -885,25 +935,24 @@ pub fn generate(	...	@@ -885,25 +935,24 @@ pub fn generate(
885		935
886	function.args = call_info.args;	936	function.args = call_info.args;
887	function.ret_mcv = call_info.return_value;	937	function.ret_mcv = call_info.return_value;
888	function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{	938	function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), .init(.{
889	.size = Type.usize.abiSize(zcu),	939	.size = Type.usize.abiSize(zcu),
890	.alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align),	940	.alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align),
891	}));	941	}));
892	function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{	942	function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), .init(.{
893	.size = Type.usize.abiSize(zcu),	943	.size = Type.usize.abiSize(zcu),
894	.alignment = Alignment.min(	944	.alignment = call_info.stack_align.min(
895	call_info.stack_align,	945	.fromNonzeroByteUnits(function.target.stackAlignment()),
896	Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
897	),	946	),
898	}));	947	}));
899	function.frame_allocs.set(	948	function.frame_allocs.set(
900	@intFromEnum(FrameIndex.args_frame),	949	@intFromEnum(FrameIndex.args_frame),
901	FrameAlloc.init(.{	950	.init(.{
902	.size = call_info.stack_byte_count,	951	.size = call_info.stack_byte_count,
903	.alignment = call_info.stack_align,	952	.alignment = call_info.stack_align,
904	}),	953	}),
905	);	954	);
906	function.va_info = switch (cc) {	955	function.va_info = switch (fn_info.cc) {
907	else => undefined,	956	else => undefined,
908	.x86_64_sysv => .{ .sysv = .{	957	.x86_64_sysv => .{ .sysv = .{
909	.gp_count = call_info.gp_count,	958	.gp_count = call_info.gp_count,
...	@@ -923,6 +972,7 @@ pub fn generate(	...	@@ -923,6 +972,7 @@ pub fn generate(
923	var mir: Mir = .{	972	var mir: Mir = .{
924	.instructions = function.mir_instructions.toOwnedSlice(),	973	.instructions = function.mir_instructions.toOwnedSlice(),
925	.extra = try function.mir_extra.toOwnedSlice(gpa),	974	.extra = try function.mir_extra.toOwnedSlice(gpa),
		975	.table = try function.mir_table.toOwnedSlice(gpa),
926	.frame_locs = function.frame_locs.toOwnedSlice(),	976	.frame_locs = function.frame_locs.toOwnedSlice(),
927	};	977	};
928	defer mir.deinit(gpa);	978	defer mir.deinit(gpa);
...	@@ -931,9 +981,10 @@ pub fn generate(	...	@@ -931,9 +981,10 @@ pub fn generate(
931	.air = function.air,	981	.air = function.air,
932	.lower = .{	982	.lower = .{
933	.bin_file = bin_file,	983	.bin_file = bin_file,
		984	.target = function.target,
934	.allocator = gpa,	985	.allocator = gpa,
935	.mir = mir,	986	.mir = mir,
936	.cc = cc,	987	.cc = fn_info.cc,
937	.src_loc = src_loc,	988	.src_loc = src_loc,
938	.output_mode = comp.config.output_mode,	989	.output_mode = comp.config.output_mode,
939	.link_mode = comp.config.link_mode,	990	.link_mode = comp.config.link_mode,
...	@@ -956,7 +1007,6 @@ pub fn generate(	...	@@ -956,7 +1007,6 @@ pub fn generate(
956	},	1007	},
957	.prev_di_pc = 0,	1008	.prev_di_pc = 0,
958	};	1009	};
959	defer emit.deinit();
960	emit.emitMir() catch \|err\| switch (err) {	1010	emit.emitMir() catch \|err\| switch (err) {
961	error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),	1011	error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),
962		1012
...	@@ -972,12 +1022,12 @@ pub fn generateLazy(	...	@@ -972,12 +1022,12 @@ pub fn generateLazy(
972	lazy_sym: link.File.LazySymbol,	1022	lazy_sym: link.File.LazySymbol,
973	code: *std.ArrayListUnmanaged(u8),	1023	code: *std.ArrayListUnmanaged(u8),
974	debug_output: link.File.DebugInfoOutput,	1024	debug_output: link.File.DebugInfoOutput,
975	) CodeGenError!void {	1025	) codegen.CodeGenError!void {
976	const comp = bin_file.comp;	1026	const comp = bin_file.comp;
977	const gpa = comp.gpa;	1027	const gpa = comp.gpa;
978	// This function is for generating global code, so we use the root module.	1028	// This function is for generating global code, so we use the root module.
979	const mod = comp.root_mod;	1029	const mod = comp.root_mod;
980	var function: Self = .{	1030	var function: CodeGen = .{
981	.gpa = gpa,	1031	.gpa = gpa,
982	.pt = pt,	1032	.pt = pt,
983	.air = undefined,	1033	.air = undefined,
...	@@ -1000,6 +1050,7 @@ pub fn generateLazy(	...	@@ -1000,6 +1050,7 @@ pub fn generateLazy(
1000	defer {	1050	defer {
1001	function.mir_instructions.deinit(gpa);	1051	function.mir_instructions.deinit(gpa);
1002	function.mir_extra.deinit(gpa);	1052	function.mir_extra.deinit(gpa);
		1053	function.mir_table.deinit(gpa);
1003	}	1054	}
1004		1055
1005	function.genLazy(lazy_sym) catch \|err\| switch (err) {	1056	function.genLazy(lazy_sym) catch \|err\| switch (err) {
...	@@ -1011,6 +1062,7 @@ pub fn generateLazy(	...	@@ -1011,6 +1062,7 @@ pub fn generateLazy(
1011	var mir: Mir = .{	1062	var mir: Mir = .{
1012	.instructions = function.mir_instructions.toOwnedSlice(),	1063	.instructions = function.mir_instructions.toOwnedSlice(),
1013	.extra = try function.mir_extra.toOwnedSlice(gpa),	1064	.extra = try function.mir_extra.toOwnedSlice(gpa),
		1065	.table = try function.mir_table.toOwnedSlice(gpa),
1014	.frame_locs = function.frame_locs.toOwnedSlice(),	1066	.frame_locs = function.frame_locs.toOwnedSlice(),
1015	};	1067	};
1016	defer mir.deinit(gpa);	1068	defer mir.deinit(gpa);
...	@@ -1019,9 +1071,10 @@ pub fn generateLazy(	...	@@ -1019,9 +1071,10 @@ pub fn generateLazy(
1019	.air = function.air,	1071	.air = function.air,
1020	.lower = .{	1072	.lower = .{
1021	.bin_file = bin_file,	1073	.bin_file = bin_file,
		1074	.target = function.target,
1022	.allocator = gpa,	1075	.allocator = gpa,
1023	.mir = mir,	1076	.mir = mir,
1024	.cc = abi.resolveCallingConvention(.auto, function.target.*),	1077	.cc = .auto,
1025	.src_loc = src_loc,	1078	.src_loc = src_loc,
1026	.output_mode = comp.config.output_mode,	1079	.output_mode = comp.config.output_mode,
1027	.link_mode = comp.config.link_mode,	1080	.link_mode = comp.config.link_mode,
...	@@ -1036,7 +1089,6 @@ pub fn generateLazy(	...	@@ -1036,7 +1089,6 @@ pub fn generateLazy(
1036	.prev_di_loc = undefined, // no debug info yet	1089	.prev_di_loc = undefined, // no debug info yet
1037	.prev_di_pc = undefined, // no debug info yet	1090	.prev_di_pc = undefined, // no debug info yet
1038	};	1091	};
1039	defer emit.deinit();
1040	emit.emitMir() catch \|err\| switch (err) {	1092	emit.emitMir() catch \|err\| switch (err) {
1041	error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),	1093	error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),
1042	error.InvalidInstruction => return function.fail("failed to find a viable x86 instruction (Zig compiler bug)", .{}),	1094	error.InvalidInstruction => return function.fail("failed to find a viable x86 instruction (Zig compiler bug)", .{}),
...	@@ -1065,7 +1117,7 @@ fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Format	...	@@ -1065,7 +1117,7 @@ fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Format
1065	}	1117	}
1066		1118
1067	const FormatAirData = struct {	1119	const FormatAirData = struct {
1068	self: *Self,	1120	self: *CodeGen,
1069	inst: Air.Inst.Index,	1121	inst: Air.Inst.Index,
1070	};	1122	};
1071	fn formatAir(	1123	fn formatAir(
...	@@ -1081,12 +1133,12 @@ fn formatAir(	...	@@ -1081,12 +1133,12 @@ fn formatAir(
1081	data.self.liveness,	1133	data.self.liveness,
1082	);	1134	);
1083	}	1135	}
1084	fn fmtAir(self: *Self, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {	1136	fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {
1085	return .{ .data = .{ .self = self, .inst = inst } };	1137	return .{ .data = .{ .self = self, .inst = inst } };
1086	}	1138	}
1087		1139
1088	const FormatWipMirData = struct {	1140	const FormatWipMirData = struct {
1089	self: *Self,	1141	self: *CodeGen,
1090	inst: Mir.Inst.Index,	1142	inst: Mir.Inst.Index,
1091	};	1143	};
1092	fn formatWipMir(	1144	fn formatWipMir(
...	@@ -1099,10 +1151,12 @@ fn formatWipMir(	...	@@ -1099,10 +1151,12 @@ fn formatWipMir(
1099	const mod = comp.root_mod;	1151	const mod = comp.root_mod;
1100	var lower: Lower = .{	1152	var lower: Lower = .{
1101	.bin_file = data.self.bin_file,	1153	.bin_file = data.self.bin_file,
		1154	.target = data.self.target,
1102	.allocator = data.self.gpa,	1155	.allocator = data.self.gpa,
1103	.mir = .{	1156	.mir = .{
1104	.instructions = data.self.mir_instructions.slice(),	1157	.instructions = data.self.mir_instructions.slice(),
1105	.extra = data.self.mir_extra.items,	1158	.extra = data.self.mir_extra.items,
		1159	.table = data.self.mir_table.items,
1106	.frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),	1160	.frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
1107	},	1161	},
1108	.cc = .auto,	1162	.cc = .auto,
...	@@ -1169,14 +1223,14 @@ fn formatWipMir(	...	@@ -1169,14 +1223,14 @@ fn formatWipMir(
1169	lower.mir.extraData(Mir.Imm64, mir_inst.data.ai.i).data.decode(),	1223	lower.mir.extraData(Mir.Imm64, mir_inst.data.ai.i).data.decode(),
1170	}),	1224	}),
1171	.pseudo_dbg_local_as => {	1225	.pseudo_dbg_local_as => {
1172	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1226	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1173	.base = .{ .reloc = mir_inst.data.as.sym_index },	1227	.base = .{ .reloc = mir_inst.data.as.sym_index },
1174	}) };	1228	}) };
1175	try writer.print(" {}, {}", .{ mir_inst.data.as.air_inst, mem_op.fmt(.m) });	1229	try writer.print(" {}, {}", .{ mir_inst.data.as.air_inst, mem_op.fmt(.m) });
1176	},	1230	},
1177	.pseudo_dbg_local_aso => {	1231	.pseudo_dbg_local_aso => {
1178	const sym_off = lower.mir.extraData(bits.SymbolOffset, mir_inst.data.ax.payload).data;	1232	const sym_off = lower.mir.extraData(bits.SymbolOffset, mir_inst.data.ax.payload).data;
1179	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1233	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1180	.base = .{ .reloc = sym_off.sym_index },	1234	.base = .{ .reloc = sym_off.sym_index },
1181	.disp = sym_off.off,	1235	.disp = sym_off.off,
1182	}) };	1236	}) };
...	@@ -1184,7 +1238,7 @@ fn formatWipMir(	...	@@ -1184,7 +1238,7 @@ fn formatWipMir(
1184	},	1238	},
1185	.pseudo_dbg_local_aro => {	1239	.pseudo_dbg_local_aro => {
1186	const air_off = lower.mir.extraData(Mir.AirOffset, mir_inst.data.rx.payload).data;	1240	const air_off = lower.mir.extraData(Mir.AirOffset, mir_inst.data.rx.payload).data;
1187	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1241	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1188	.base = .{ .reg = mir_inst.data.rx.r1 },	1242	.base = .{ .reg = mir_inst.data.rx.r1 },
1189	.disp = air_off.off,	1243	.disp = air_off.off,
1190	}) };	1244	}) };
...	@@ -1192,14 +1246,14 @@ fn formatWipMir(	...	@@ -1192,14 +1246,14 @@ fn formatWipMir(
1192	},	1246	},
1193	.pseudo_dbg_local_af => {	1247	.pseudo_dbg_local_af => {
1194	const frame_addr = lower.mir.extraData(bits.FrameAddr, mir_inst.data.ax.payload).data;	1248	const frame_addr = lower.mir.extraData(bits.FrameAddr, mir_inst.data.ax.payload).data;
1195	const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{	1249	const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{
1196	.base = .{ .frame = frame_addr.index },	1250	.base = .{ .frame = frame_addr.index },
1197	.disp = frame_addr.off,	1251	.disp = frame_addr.off,
1198	}) };	1252	}) };
1199	try writer.print(" {}, {d}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });	1253	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
1200	},	1254	},
1201	.pseudo_dbg_local_am => {	1255	.pseudo_dbg_local_am => {
1202	const mem_op: Instruction.Operand = .{	1256	const mem_op: encoder.Instruction.Operand = .{
1203	.mem = lower.mir.extraData(Mir.Memory, mir_inst.data.ax.payload).data.decode(),	1257	.mem = lower.mir.extraData(Mir.Memory, mir_inst.data.ax.payload).data.decode(),
1204	};	1258	};
1205	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });	1259	try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) });
...	@@ -1207,12 +1261,12 @@ fn formatWipMir(	...	@@ -1207,12 +1261,12 @@ fn formatWipMir(
1207	}	1261	}
1208	}	1262	}
1209	}	1263	}
1210	fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {	1264	fn fmtWipMir(self: *CodeGen, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
1211	return .{ .data = .{ .self = self, .inst = inst } };	1265	return .{ .data = .{ .self = self, .inst = inst } };
1212	}	1266	}
1213		1267
1214	const FormatTrackingData = struct {	1268	const FormatTrackingData = struct {
1215	self: *Self,	1269	self: *CodeGen,
1216	};	1270	};
1217	fn formatTracking(	1271	fn formatTracking(
1218	data: FormatTrackingData,	1272	data: FormatTrackingData,
...	@@ -1221,28 +1275,28 @@ fn formatTracking(	...	@@ -1221,28 +1275,28 @@ fn formatTracking(
1221	writer: anytype,	1275	writer: anytype,
1222	) @TypeOf(writer).Error!void {	1276	) @TypeOf(writer).Error!void {
1223	var it = data.self.inst_tracking.iterator();	1277	var it = data.self.inst_tracking.iterator();
1224	while (it.next()) \|entry\| try writer.print("\n%{d} = {}", .{ entry.key_ptr., entry.value_ptr. });	1278	while (it.next()) \|entry\| try writer.print("\n{} = {}", .{ entry.key_ptr., entry.value_ptr. });
1225	}	1279	}
1226	fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {	1280	fn fmtTracking(self: *CodeGen) std.fmt.Formatter(formatTracking) {
1227	return .{ .data = .{ .self = self } };	1281	return .{ .data = .{ .self = self } };
1228	}	1282	}
1229		1283
1230	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {	1284	fn addInst(self: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
1231	const gpa = self.gpa;	1285	const gpa = self.gpa;
1232	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);	1286	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
1233	const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);	1287	const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
1234	self.mir_instructions.appendAssumeCapacity(inst);	1288	self.mir_instructions.appendAssumeCapacity(inst);
1235	wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});	1289	if (inst.ops != .pseudo_dead_none) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
1236	return result_index;	1290	return result_index;
1237	}	1291	}
1238		1292
1239	fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {	1293	fn addExtra(self: *CodeGen, extra: anytype) Allocator.Error!u32 {
1240	const fields = std.meta.fields(@TypeOf(extra));	1294	const fields = std.meta.fields(@TypeOf(extra));
1241	try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);	1295	try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
1242	return self.addExtraAssumeCapacity(extra);	1296	return self.addExtraAssumeCapacity(extra);
1243	}	1297	}
1244		1298
1245	fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {	1299	fn addExtraAssumeCapacity(self: *CodeGen, extra: anytype) u32 {
1246	const fields = std.meta.fields(@TypeOf(extra));	1300	const fields = std.meta.fields(@TypeOf(extra));
1247	const result: u32 = @intCast(self.mir_extra.items.len);	1301	const result: u32 = @intCast(self.mir_extra.items.len);
1248	inline for (fields) \|field\| {	1302	inline for (fields) \|field\| {
...	@@ -1256,9 +1310,83 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {	...	@@ -1256,9 +1310,83 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1256	return result;	1310	return result;
1257	}	1311	}
1258		1312
		1313	fn asmOps(self: *CodeGen, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
		1314	return switch (ops[0]) {
		1315	.none => self.asmOpOnly(tag),
		1316	.reg => \|reg0\| switch (ops[1]) {
		1317	.none => self.asmRegister(tag, reg0),
		1318	.reg => \|reg1\| switch (ops[2]) {
		1319	.none => self.asmRegisterRegister(tag, reg0, reg1),
		1320	.reg => \|reg2\| switch (ops[3]) {
		1321	.none => self.asmRegisterRegisterRegister(tag, reg0, reg1, reg2),
		1322	.reg => \|reg3\| self.asmRegisterRegisterRegisterRegister(tag, reg0, reg1, reg2, reg3),
		1323	.imm => \|imm3\| self.asmRegisterRegisterRegisterImmediate(tag, reg0, reg1, reg2, imm3),
		1324	else => error.InvalidInstruction,
		1325	},
		1326	.mem => \|mem2\| switch (ops[3]) {
		1327	.none => self.asmRegisterRegisterMemory(tag, reg0, reg1, mem2),
		1328	.reg => \|reg3\| self.asmRegisterRegisterMemoryRegister(tag, reg0, reg1, mem2, reg3),
		1329	.imm => \|imm3\| self.asmRegisterRegisterMemoryImmediate(tag, reg0, reg1, mem2, imm3),
		1330	else => error.InvalidInstruction,
		1331	},
		1332	.imm => \|imm2\| switch (ops[3]) {
		1333	.none => self.asmRegisterRegisterImmediate(tag, reg0, reg1, imm2),
		1334	else => error.InvalidInstruction,
		1335	},
		1336	else => error.InvalidInstruction,
		1337	},
		1338	.mem => \|mem1\| switch (ops[2]) {
		1339	.none => self.asmRegisterMemory(tag, reg0, mem1),
		1340	.reg => \|reg2\| switch (ops[3]) {
		1341	.none => self.asmRegisterMemoryRegister(tag, reg0, mem1, reg2),
		1342	else => error.InvalidInstruction,
		1343	},
		1344	.imm => \|imm2\| switch (ops[3]) {
		1345	.none => self.asmRegisterMemoryImmediate(tag, reg0, mem1, imm2),
		1346	else => error.InvalidInstruction,
		1347	},
		1348	else => error.InvalidInstruction,
		1349	},
		1350	.imm => \|imm1\| switch (ops[2]) {
		1351	.none => self.asmRegisterImmediate(tag, reg0, imm1),
		1352	else => error.InvalidInstruction,
		1353	},
		1354	else => error.InvalidInstruction,
		1355	},
		1356	.mem => \|mem0\| switch (ops[1]) {
		1357	.none => self.asmMemory(tag, mem0),
		1358	.reg => \|reg1\| switch (ops[2]) {
		1359	.none => self.asmMemoryRegister(tag, mem0, reg1),
		1360	.reg => \|reg2\| switch (ops[3]) {
		1361	.none => self.asmMemoryRegisterRegister(tag, mem0, reg1, reg2),
		1362	else => error.InvalidInstruction,
		1363	},
		1364	.imm => \|imm2\| switch (ops[3]) {
		1365	.none => self.asmMemoryRegisterImmediate(tag, mem0, reg1, imm2),
		1366	else => error.InvalidInstruction,
		1367	},
		1368	else => error.InvalidInstruction,
		1369	},
		1370	.imm => \|imm1\| switch (ops[2]) {
		1371	.none => self.asmMemoryImmediate(tag, mem0, imm1),
		1372	else => error.InvalidInstruction,
		1373	},
		1374	else => error.InvalidInstruction,
		1375	},
		1376	.imm => \|imm0\| switch (ops[1]) {
		1377	.none => self.asmImmediate(tag, imm0),
		1378	else => error.InvalidInstruction,
		1379	},
		1380	.inst => \|inst0\| switch (ops[1]) {
		1381	.none => self.asmReloc(tag, inst0),
		1382	else => error.InvalidInstruction,
		1383	},
		1384	};
		1385	}
		1386
1259	/// A `cc` of `.z_and_np` clobbers `reg2`!	1387	/// A `cc` of `.z_and_np` clobbers `reg2`!
1260	fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: Register) !void {	1388	fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2: Register) !void {
1261	_ = try self.addInst(.{	1389	if (self.hasFeature(.cmov)) _ = try self.addInst(.{
1262	.tag = switch (cc) {	1390	.tag = switch (cc) {
1263	else => .cmov,	1391	else => .cmov,
1264	.z_and_np, .nz_or_p => .pseudo,	1392	.z_and_np, .nz_or_p => .pseudo,
...	@@ -1270,18 +1398,22 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R	...	@@ -1270,18 +1398,22 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R
1270	},	1398	},
1271	.data = .{ .rr = .{	1399	.data = .{ .rr = .{
1272	.fixes = switch (cc) {	1400	.fixes = switch (cc) {
1273	else => Mir.Inst.Fixes.fromCondition(cc),	1401	else => .fromCondition(cc),
1274	.z_and_np, .nz_or_p => ._,	1402	.z_and_np, .nz_or_p => ._,
1275	},	1403	},
1276	.r1 = reg1,	1404	.r1 = reg1,
1277	.r2 = reg2,	1405	.r2 = reg2,
1278	} },	1406	} },
1279	});	1407	}) else {
		1408	const reloc = try self.asmJccReloc(cc.negate(), undefined);
		1409	try self.asmRegisterRegister(.{ ._, .mov }, reg1, reg2);
		1410	self.performReloc(reloc);
		1411	}
1280	}	1412	}
1281		1413
1282	/// A `cc` of `.z_and_np` is not supported by this encoding!	1414	/// A `cc` of `.z_and_np` is not supported by this encoding!
1283	fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory) !void {	1415	fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memory) !void {
1284	_ = try self.addInst(.{	1416	if (self.hasFeature(.cmov)) _ = try self.addInst(.{
1285	.tag = switch (cc) {	1417	.tag = switch (cc) {
1286	else => .cmov,	1418	else => .cmov,
1287	.z_and_np => unreachable,	1419	.z_and_np => unreachable,
...	@@ -1294,17 +1426,21 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)	...	@@ -1294,17 +1426,21 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
1294	},	1426	},
1295	.data = .{ .rx = .{	1427	.data = .{ .rx = .{
1296	.fixes = switch (cc) {	1428	.fixes = switch (cc) {
1297	else => Mir.Inst.Fixes.fromCondition(cc),	1429	else => .fromCondition(cc),
1298	.z_and_np => unreachable,	1430	.z_and_np => unreachable,
1299	.nz_or_p => ._,	1431	.nz_or_p => ._,
1300	},	1432	},
1301	.r1 = reg,	1433	.r1 = reg,
1302	.payload = try self.addExtra(Mir.Memory.encode(m)),	1434	.payload = try self.addExtra(Mir.Memory.encode(m)),
1303	} },	1435	} },
1304	});	1436	}) else {
		1437	const reloc = try self.asmJccReloc(cc.negate(), undefined);
		1438	try self.asmRegisterMemory(.{ ._, .mov }, reg, m);
		1439	self.performReloc(reloc);
		1440	}
1305	}	1441	}
1306		1442
1307	fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {	1443	fn asmSetccRegister(self: *CodeGen, cc: Condition, reg: Register) !void {
1308	_ = try self.addInst(.{	1444	_ = try self.addInst(.{
1309	.tag = switch (cc) {	1445	.tag = switch (cc) {
1310	else => .set,	1446	else => .set,
...	@@ -1317,7 +1453,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {	...	@@ -1317,7 +1453,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1317	},	1453	},
1318	.data = switch (cc) {	1454	.data = switch (cc) {
1319	else => .{ .r = .{	1455	else => .{ .r = .{
1320	.fixes = Mir.Inst.Fixes.fromCondition(cc),	1456	.fixes = .fromCondition(cc),
1321	.r1 = reg,	1457	.r1 = reg,
1322	} },	1458	} },
1323	.z_and_np, .nz_or_p => .{ .rr = .{	1459	.z_and_np, .nz_or_p => .{ .rr = .{
...	@@ -1328,7 +1464,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {	...	@@ -1328,7 +1464,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1328	});	1464	});
1329	}	1465	}
1330		1466
1331	fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {	1467	fn asmSetccMemory(self: *CodeGen, cc: Condition, m: Memory) !void {
1332	const payload = try self.addExtra(Mir.Memory.encode(m));	1468	const payload = try self.addExtra(Mir.Memory.encode(m));
1333	_ = try self.addInst(.{	1469	_ = try self.addInst(.{
1334	.tag = switch (cc) {	1470	.tag = switch (cc) {
...	@@ -1342,7 +1478,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {	...	@@ -1342,7 +1478,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1342	},	1478	},
1343	.data = switch (cc) {	1479	.data = switch (cc) {
1344	else => .{ .x = .{	1480	else => .{ .x = .{
1345	.fixes = Mir.Inst.Fixes.fromCondition(cc),	1481	.fixes = .fromCondition(cc),
1346	.payload = payload,	1482	.payload = payload,
1347	} },	1483	} },
1348	.z_and_np, .nz_or_p => .{ .rx = .{	1484	.z_and_np, .nz_or_p => .{ .rx = .{
...	@@ -1353,7 +1489,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {	...	@@ -1353,7 +1489,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1353	});	1489	});
1354	}	1490	}
1355		1491
1356	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {	1492	fn asmJmpReloc(self: *CodeGen, target: Mir.Inst.Index) !Mir.Inst.Index {
1357	return self.addInst(.{	1493	return self.addInst(.{
1358	.tag = .jmp,	1494	.tag = .jmp,
1359	.ops = .inst,	1495	.ops = .inst,
...	@@ -1363,7 +1499,7 @@ fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {	...	@@ -1363,7 +1499,7 @@ fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1363	});	1499	});
1364	}	1500	}
1365		1501
1366	fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {	1502	fn asmJccReloc(self: *CodeGen, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {
1367	return self.addInst(.{	1503	return self.addInst(.{
1368	.tag = switch (cc) {	1504	.tag = switch (cc) {
1369	else => .j,	1505	else => .j,
...	@@ -1376,7 +1512,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind	...	@@ -1376,7 +1512,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
1376	},	1512	},
1377	.data = .{ .inst = .{	1513	.data = .{ .inst = .{
1378	.fixes = switch (cc) {	1514	.fixes = switch (cc) {
1379	else => Mir.Inst.Fixes.fromCondition(cc),	1515	else => .fromCondition(cc),
1380	.z_and_np, .nz_or_p => ._,	1516	.z_and_np, .nz_or_p => ._,
1381	},	1517	},
1382	.inst = target,	1518	.inst = target,
...	@@ -1384,7 +1520,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind	...	@@ -1384,7 +1520,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
1384	});	1520	});
1385	}	1521	}
1386		1522
1387	fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {	1523	fn asmReloc(self: *CodeGen, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
1388	_ = try self.addInst(.{	1524	_ = try self.addInst(.{
1389	.tag = tag[1],	1525	.tag = tag[1],
1390	.ops = .inst,	1526	.ops = .inst,
...	@@ -1395,7 +1531,7 @@ fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {	...	@@ -1395,7 +1531,7 @@ fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
1395	});	1531	});
1396	}	1532	}
1397		1533
1398	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {	1534	fn asmPlaceholder(self: *CodeGen) !Mir.Inst.Index {
1399	return self.addInst(.{	1535	return self.addInst(.{
1400	.tag = .pseudo,	1536	.tag = .pseudo,
1401	.ops = .pseudo_dead_none,	1537	.ops = .pseudo_dead_none,
...	@@ -1405,7 +1541,7 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {	...	@@ -1405,7 +1541,7 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1405		1541
1406	const MirTagAir = enum { dbg_local };	1542	const MirTagAir = enum { dbg_local };
1407		1543
1408	fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {	1544	fn asmAir(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index) !void {
1409	_ = try self.addInst(.{	1545	_ = try self.addInst(.{
1410	.tag = .pseudo,	1546	.tag = .pseudo,
1411	.ops = switch (tag) {	1547	.ops = switch (tag) {
...	@@ -1415,7 +1551,7 @@ fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {	...	@@ -1415,7 +1551,7 @@ fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {
1415	});	1551	});
1416	}	1552	}
1417		1553
1418	fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {	1554	fn asmAirImmediate(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {
1419	switch (imm) {	1555	switch (imm) {
1420	.signed => \|s\| _ = try self.addInst(.{	1556	.signed => \|s\| _ = try self.addInst(.{
1421	.tag = .pseudo,	1557	.tag = .pseudo,
...	@@ -1427,7 +1563,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed	...	@@ -1427,7 +1563,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
1427	.i = @bitCast(s),	1563	.i = @bitCast(s),
1428	} },	1564	} },
1429	}),	1565	}),
1430	.unsigned => \|u\| _ = if (math.cast(u32, u)) \|small\| try self.addInst(.{	1566	.unsigned => \|u\| _ = if (std.math.cast(u32, u)) \|small\| try self.addInst(.{
1431	.tag = .pseudo,	1567	.tag = .pseudo,
1432	.ops = switch (tag) {	1568	.ops = switch (tag) {
1433	.dbg_local => .pseudo_dbg_local_ai_u,	1569	.dbg_local => .pseudo_dbg_local_ai_u,
...	@@ -1469,7 +1605,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed	...	@@ -1469,7 +1605,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
1469	}	1605	}
1470		1606
1471	fn asmAirRegisterImmediate(	1607	fn asmAirRegisterImmediate(
1472	self: *Self,	1608	self: *CodeGen,
1473	tag: MirTagAir,	1609	tag: MirTagAir,
1474	inst: Air.Inst.Index,	1610	inst: Air.Inst.Index,
1475	reg: Register,	1611	reg: Register,
...	@@ -1491,7 +1627,7 @@ fn asmAirRegisterImmediate(	...	@@ -1491,7 +1627,7 @@ fn asmAirRegisterImmediate(
1491	}	1627	}
1492		1628
1493	fn asmAirFrameAddress(	1629	fn asmAirFrameAddress(
1494	self: *Self,	1630	self: *CodeGen,
1495	tag: MirTagAir,	1631	tag: MirTagAir,
1496	inst: Air.Inst.Index,	1632	inst: Air.Inst.Index,
1497	frame_addr: bits.FrameAddr,	1633	frame_addr: bits.FrameAddr,
...	@@ -1508,7 +1644,7 @@ fn asmAirFrameAddress(	...	@@ -1508,7 +1644,7 @@ fn asmAirFrameAddress(
1508	});	1644	});
1509	}	1645	}
1510		1646
1511	fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {	1647	fn asmAirMemory(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {
1512	_ = try self.addInst(.{	1648	_ = try self.addInst(.{
1513	.tag = .pseudo,	1649	.tag = .pseudo,
1514	.ops = switch (tag) {	1650	.ops = switch (tag) {
...	@@ -1521,7 +1657,7 @@ fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !v	...	@@ -1521,7 +1657,7 @@ fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !v
1521	});	1657	});
1522	}	1658	}
1523		1659
1524	fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {	1660	fn asmOpOnly(self: *CodeGen, tag: Mir.Inst.FixedTag) !void {
1525	_ = try self.addInst(.{	1661	_ = try self.addInst(.{
1526	.tag = tag[1],	1662	.tag = tag[1],
1527	.ops = .none,	1663	.ops = .none,
...	@@ -1531,7 +1667,7 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {	...	@@ -1531,7 +1667,7 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1531	});	1667	});
1532	}	1668	}
1533		1669
1534	fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {	1670	fn asmPseudo(self: *CodeGen, ops: Mir.Inst.Ops) !void {
1535	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and	1671	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1536	std.mem.endsWith(u8, @tagName(ops), "_none"));	1672	std.mem.endsWith(u8, @tagName(ops), "_none"));
1537	_ = try self.addInst(.{	1673	_ = try self.addInst(.{
...	@@ -1541,7 +1677,7 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {	...	@@ -1541,7 +1677,7 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
1541	});	1677	});
1542	}	1678	}
1543		1679
1544	fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {	1680	fn asmPseudoRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register) !void {
1545	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and	1681	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1546	std.mem.endsWith(u8, @tagName(ops), "_r"));	1682	std.mem.endsWith(u8, @tagName(ops), "_r"));
1547	_ = try self.addInst(.{	1683	_ = try self.addInst(.{
...	@@ -1551,7 +1687,7 @@ fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {	...	@@ -1551,7 +1687,7 @@ fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {
1551	});	1687	});
1552	}	1688	}
1553		1689
1554	fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {	1690	fn asmPseudoImmediate(self: *CodeGen, ops: Mir.Inst.Ops, imm: Immediate) !void {
1555	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and	1691	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1556	std.mem.endsWith(u8, @tagName(ops), "_i_s"));	1692	std.mem.endsWith(u8, @tagName(ops), "_i_s"));
1557	_ = try self.addInst(.{	1693	_ = try self.addInst(.{
...	@@ -1561,7 +1697,7 @@ fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {	...	@@ -1561,7 +1697,7 @@ fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {
1561	});	1697	});
1562	}	1698	}
1563		1699
1564	fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {	1700	fn asmPseudoRegisterRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {
1565	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and	1701	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1566	std.mem.endsWith(u8, @tagName(ops), "_rr"));	1702	std.mem.endsWith(u8, @tagName(ops), "_rr"));
1567	_ = try self.addInst(.{	1703	_ = try self.addInst(.{
...	@@ -1571,7 +1707,7 @@ fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg	...	@@ -1571,7 +1707,7 @@ fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg
1571	});	1707	});
1572	}	1708	}
1573		1709
1574	fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {	1710	fn asmPseudoRegisterImmediate(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {
1575	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and	1711	assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1576	std.mem.endsWith(u8, @tagName(ops), "_ri_s"));	1712	std.mem.endsWith(u8, @tagName(ops), "_ri_s"));
1577	_ = try self.addInst(.{	1713	_ = try self.addInst(.{
...	@@ -1581,7 +1717,7 @@ fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm	...	@@ -1581,7 +1717,7 @@ fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm
1581	});	1717	});
1582	}	1718	}
1583		1719
1584	fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {	1720	fn asmRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register) !void {
1585	_ = try self.addInst(.{	1721	_ = try self.addInst(.{
1586	.tag = tag[1],	1722	.tag = tag[1],
1587	.ops = .r,	1723	.ops = .r,
...	@@ -1592,7 +1728,7 @@ fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {	...	@@ -1592,7 +1728,7 @@ fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1592	});	1728	});
1593	}	1729	}
1594		1730
1595	fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {	1731	fn asmImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1596	_ = try self.addInst(.{	1732	_ = try self.addInst(.{
1597	.tag = tag[1],	1733	.tag = tag[1],
1598	.ops = switch (imm) {	1734	.ops = switch (imm) {
...	@@ -1617,7 +1753,7 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {	...	@@ -1617,7 +1753,7 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1617	});	1753	});
1618	}	1754	}
1619		1755
1620	fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {	1756	fn asmRegisterRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1621	_ = try self.addInst(.{	1757	_ = try self.addInst(.{
1622	.tag = tag[1],	1758	.tag = tag[1],
1623	.ops = .rr,	1759	.ops = .rr,
...	@@ -1629,10 +1765,10 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2	...	@@ -1629,10 +1765,10 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2
1629	});	1765	});
1630	}	1766	}
1631		1767
1632	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {	1768	fn asmRegisterImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1633	const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {	1769	const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {
1634	.signed => \|s\| .{ .ri_s, @bitCast(s) },	1770	.signed => \|s\| .{ .ri_s, @bitCast(s) },
1635	.unsigned => \|u\| if (math.cast(u32, u)) \|small\|	1771	.unsigned => \|u\| if (std.math.cast(u32, u)) \|small\|
1636	.{ .ri_u, small }	1772	.{ .ri_u, small }
1637	else	1773	else
1638	.{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) },	1774	.{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) },
...	@@ -1650,7 +1786,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm:	...	@@ -1650,7 +1786,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm:
1650	}	1786	}
1651		1787
1652	fn asmRegisterRegisterRegister(	1788	fn asmRegisterRegisterRegister(
1653	self: *Self,	1789	self: *CodeGen,
1654	tag: Mir.Inst.FixedTag,	1790	tag: Mir.Inst.FixedTag,
1655	reg1: Register,	1791	reg1: Register,
1656	reg2: Register,	1792	reg2: Register,
...	@@ -1669,7 +1805,7 @@ fn asmRegisterRegisterRegister(	...	@@ -1669,7 +1805,7 @@ fn asmRegisterRegisterRegister(
1669	}	1805	}
1670		1806
1671	fn asmRegisterRegisterRegisterRegister(	1807	fn asmRegisterRegisterRegisterRegister(
1672	self: *Self,	1808	self: *CodeGen,
1673	tag: Mir.Inst.FixedTag,	1809	tag: Mir.Inst.FixedTag,
1674	reg1: Register,	1810	reg1: Register,
1675	reg2: Register,	1811	reg2: Register,
...	@@ -1690,7 +1826,7 @@ fn asmRegisterRegisterRegisterRegister(	...	@@ -1690,7 +1826,7 @@ fn asmRegisterRegisterRegisterRegister(
1690	}	1826	}
1691		1827
1692	fn asmRegisterRegisterRegisterImmediate(	1828	fn asmRegisterRegisterRegisterImmediate(
1693	self: *Self,	1829	self: *CodeGen,
1694	tag: Mir.Inst.FixedTag,	1830	tag: Mir.Inst.FixedTag,
1695	reg1: Register,	1831	reg1: Register,
1696	reg2: Register,	1832	reg2: Register,
...	@@ -1715,7 +1851,7 @@ fn asmRegisterRegisterRegisterImmediate(	...	@@ -1715,7 +1851,7 @@ fn asmRegisterRegisterRegisterImmediate(
1715	}	1851	}
1716		1852
1717	fn asmRegisterRegisterImmediate(	1853	fn asmRegisterRegisterImmediate(
1718	self: *Self,	1854	self: *CodeGen,
1719	tag: Mir.Inst.FixedTag,	1855	tag: Mir.Inst.FixedTag,
1720	reg1: Register,	1856	reg1: Register,
1721	reg2: Register,	1857	reg2: Register,
...	@@ -1742,7 +1878,7 @@ fn asmRegisterRegisterImmediate(	...	@@ -1742,7 +1878,7 @@ fn asmRegisterRegisterImmediate(
1742	}	1878	}
1743		1879
1744	fn asmRegisterRegisterMemory(	1880	fn asmRegisterRegisterMemory(
1745	self: *Self,	1881	self: *CodeGen,
1746	tag: Mir.Inst.FixedTag,	1882	tag: Mir.Inst.FixedTag,
1747	reg1: Register,	1883	reg1: Register,
1748	reg2: Register,	1884	reg2: Register,
...	@@ -1761,7 +1897,7 @@ fn asmRegisterRegisterMemory(	...	@@ -1761,7 +1897,7 @@ fn asmRegisterRegisterMemory(
1761	}	1897	}
1762		1898
1763	fn asmRegisterRegisterMemoryRegister(	1899	fn asmRegisterRegisterMemoryRegister(
1764	self: *Self,	1900	self: *CodeGen,
1765	tag: Mir.Inst.FixedTag,	1901	tag: Mir.Inst.FixedTag,
1766	reg1: Register,	1902	reg1: Register,
1767	reg2: Register,	1903	reg2: Register,
...	@@ -1781,7 +1917,7 @@ fn asmRegisterRegisterMemoryRegister(	...	@@ -1781,7 +1917,7 @@ fn asmRegisterRegisterMemoryRegister(
1781	});	1917	});
1782	}	1918	}
1783		1919
1784	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {	1920	fn asmMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory) !void {
1785	_ = try self.addInst(.{	1921	_ = try self.addInst(.{
1786	.tag = tag[1],	1922	.tag = tag[1],
1787	.ops = .m,	1923	.ops = .m,
...	@@ -1792,7 +1928,7 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {	...	@@ -1792,7 +1928,7 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1792	});	1928	});
1793	}	1929	}
1794		1930
1795	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {	1931	fn asmRegisterMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1796	_ = try self.addInst(.{	1932	_ = try self.addInst(.{
1797	.tag = tag[1],	1933	.tag = tag[1],
1798	.ops = .rm,	1934	.ops = .rm,
...	@@ -1805,7 +1941,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo	...	@@ -1805,7 +1941,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
1805	}	1941	}
1806		1942
1807	fn asmRegisterMemoryRegister(	1943	fn asmRegisterMemoryRegister(
1808	self: *Self,	1944	self: *CodeGen,
1809	tag: Mir.Inst.FixedTag,	1945	tag: Mir.Inst.FixedTag,
1810	reg1: Register,	1946	reg1: Register,
1811	m: Memory,	1947	m: Memory,
...	@@ -1824,15 +1960,15 @@ fn asmRegisterMemoryRegister(	...	@@ -1824,15 +1960,15 @@ fn asmRegisterMemoryRegister(
1824	}	1960	}
1825		1961
1826	fn asmRegisterMemoryImmediate(	1962	fn asmRegisterMemoryImmediate(
1827	self: *Self,	1963	self: *CodeGen,
1828	tag: Mir.Inst.FixedTag,	1964	tag: Mir.Inst.FixedTag,
1829	reg: Register,	1965	reg: Register,
1830	m: Memory,	1966	m: Memory,
1831	imm: Immediate,	1967	imm: Immediate,
1832	) !void {	1968	) !void {
1833	if (switch (imm) {	1969	if (switch (imm) {
1834	.signed => \|s\| if (math.cast(i16, s)) \|x\| @as(u16, @bitCast(x)) else null,	1970	.signed => \|s\| if (std.math.cast(i16, s)) \|x\| @as(u16, @bitCast(x)) else null,
1835	.unsigned => \|u\| math.cast(u16, u),	1971	.unsigned => \|u\| std.math.cast(u16, u),
1836	.reloc => unreachable,	1972	.reloc => unreachable,
1837	}) \|small_imm\| {	1973	}) \|small_imm\| {
1838	_ = try self.addInst(.{	1974	_ = try self.addInst(.{
...	@@ -1869,7 +2005,7 @@ fn asmRegisterMemoryImmediate(	...	@@ -1869,7 +2005,7 @@ fn asmRegisterMemoryImmediate(
1869	}	2005	}
1870		2006
1871	fn asmRegisterRegisterMemoryImmediate(	2007	fn asmRegisterRegisterMemoryImmediate(
1872	self: *Self,	2008	self: *CodeGen,
1873	tag: Mir.Inst.FixedTag,	2009	tag: Mir.Inst.FixedTag,
1874	reg1: Register,	2010	reg1: Register,
1875	reg2: Register,	2011	reg2: Register,
...	@@ -1889,7 +2025,7 @@ fn asmRegisterRegisterMemoryImmediate(	...	@@ -1889,7 +2025,7 @@ fn asmRegisterRegisterMemoryImmediate(
1889	});	2025	});
1890	}	2026	}
1891		2027
1892	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {	2028	fn asmMemoryRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1893	_ = try self.addInst(.{	2029	_ = try self.addInst(.{
1894	.tag = tag[1],	2030	.tag = tag[1],
1895	.ops = .mr,	2031	.ops = .mr,
...	@@ -1901,7 +2037,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist	...	@@ -1901,7 +2037,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist
1901	});	2037	});
1902	}	2038	}
1903		2039
1904	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {	2040	fn asmMemoryImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1905	const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {	2041	const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
1906	.signed => \|s\| @bitCast(s),	2042	.signed => \|s\| @bitCast(s),
1907	.unsigned => \|u\| @intCast(u),	2043	.unsigned => \|u\| @intCast(u),
...	@@ -1923,7 +2059,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed	...	@@ -1923,7 +2059,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed
1923	}	2059	}
1924		2060
1925	fn asmMemoryRegisterRegister(	2061	fn asmMemoryRegisterRegister(
1926	self: *Self,	2062	self: *CodeGen,
1927	tag: Mir.Inst.FixedTag,	2063	tag: Mir.Inst.FixedTag,
1928	m: Memory,	2064	m: Memory,
1929	reg1: Register,	2065	reg1: Register,
...	@@ -1942,7 +2078,7 @@ fn asmMemoryRegisterRegister(	...	@@ -1942,7 +2078,7 @@ fn asmMemoryRegisterRegister(
1942	}	2078	}
1943		2079
1944	fn asmMemoryRegisterImmediate(	2080	fn asmMemoryRegisterImmediate(
1945	self: *Self,	2081	self: *CodeGen,
1946	tag: Mir.Inst.FixedTag,	2082	tag: Mir.Inst.FixedTag,
1947	m: Memory,	2083	m: Memory,
1948	reg: Register,	2084	reg: Register,
...	@@ -1960,15 +2096,14 @@ fn asmMemoryRegisterImmediate(	...	@@ -1960,15 +2096,14 @@ fn asmMemoryRegisterImmediate(
1960	});	2096	});
1961	}	2097	}
1962		2098
1963	fn gen(self: *Self) InnerError!void {	2099	fn gen(self: *CodeGen) InnerError!void {
1964	const pt = self.pt;	2100	const pt = self.pt;
1965	const zcu = pt.zcu;	2101	const zcu = pt.zcu;
1966	const fn_info = zcu.typeToFunc(self.fn_type).?;	2102	const fn_info = zcu.typeToFunc(self.fn_type).?;
1967	const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);	2103	if (fn_info.cc != .naked) {
1968	if (cc != .naked) {
1969	try self.asmRegister(.{ ._, .push }, .rbp);	2104	try self.asmRegister(.{ ._, .push }, .rbp);
1970	try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, Immediate.s(8));	2105	try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, .s(8));
1971	try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, Immediate.s(0));	2106	try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, .s(0));
1972	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);	2107	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1973	try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp);	2108	try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp);
1974	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();	2109	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
...	@@ -1983,11 +2118,11 @@ fn gen(self: *Self) InnerError!void {	...	@@ -1983,11 +2118,11 @@ fn gen(self: *Self) InnerError!void {
1983	// The address where to store the return value for the caller is in a	2118	// The address where to store the return value for the caller is in a
1984	// register which the callee is free to clobber. Therefore, we purposely	2119	// register which the callee is free to clobber. Therefore, we purposely
1985	// spill it to stack immediately.	2120	// spill it to stack immediately.
1986	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, zcu));	2121	const frame_index = try self.allocFrameIndex(.initSpill(.usize, zcu));
1987	try self.genSetMem(	2122	try self.genSetMem(
1988	.{ .frame = frame_index },	2123	.{ .frame = frame_index },
1989	0,	2124	0,
1990	Type.usize,	2125	.usize,
1991	self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),	2126	self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
1992	.{},	2127	.{},
1993	);	2128	);
...	@@ -1997,10 +2132,10 @@ fn gen(self: *Self) InnerError!void {	...	@@ -1997,10 +2132,10 @@ fn gen(self: *Self) InnerError!void {
1997	else => unreachable,	2132	else => unreachable,
1998	}	2133	}
1999		2134
2000	if (fn_info.is_var_args) switch (cc) {	2135	if (fn_info.is_var_args) switch (fn_info.cc) {
2001	.x86_64_sysv => {	2136	.x86_64_sysv => {
2002	const info = &self.va_info.sysv;	2137	const info = &self.va_info.sysv;
2003	const reg_save_area_fi = try self.allocFrameIndex(FrameAlloc.init(.{	2138	const reg_save_area_fi = try self.allocFrameIndex(.init(.{
2004	.size = abi.SysV.c_abi_int_param_regs.len * 8 +	2139	.size = abi.SysV.c_abi_int_param_regs.len * 8 +
2005	abi.SysV.c_abi_sse_param_regs.len * 16,	2140	abi.SysV.c_abi_sse_param_regs.len * 16,
2006	.alignment = .@"16",	2141	.alignment = .@"16",
...	@@ -2008,15 +2143,9 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2008,15 +2143,9 @@ fn gen(self: *Self) InnerError!void {
2008	info.reg_save_area = .{ .index = reg_save_area_fi };	2143	info.reg_save_area = .{ .index = reg_save_area_fi };
2009		2144
2010	for (abi.SysV.c_abi_int_param_regs[info.gp_count..], info.gp_count..) \|reg, reg_i\|	2145	for (abi.SysV.c_abi_int_param_regs[info.gp_count..], info.gp_count..) \|reg, reg_i\|
2011	try self.genSetMem(	2146	try self.genSetMem(.{ .frame = reg_save_area_fi }, @intCast(reg_i * 8), .usize, .{ .register = reg }, .{});
2012	.{ .frame = reg_save_area_fi },
2013	@intCast(reg_i * 8),
2014	Type.usize,
2015	.{ .register = reg },
2016	.{},
2017	);
2018		2147
2019	try self.asmRegisterImmediate(.{ ._, .cmp }, .al, Immediate.u(info.fp_count));	2148	try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count));
2020	const skip_sse_reloc = try self.asmJccReloc(.na, undefined);	2149	const skip_sse_reloc = try self.asmJccReloc(.na, undefined);
2021		2150
2022	const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type });	2151	const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type });
...	@@ -2032,38 +2161,38 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2032,38 +2161,38 @@ fn gen(self: *Self) InnerError!void {
2032	self.performReloc(skip_sse_reloc);	2161	self.performReloc(skip_sse_reloc);
2033	},	2162	},
2034	.x86_64_win => return self.fail("TODO implement gen var arg function for Win64", .{}),	2163	.x86_64_win => return self.fail("TODO implement gen var arg function for Win64", .{}),
2035	else => unreachable,	2164	else => \|cc\| return self.fail("{s} does not support var args", .{@tagName(cc)}),
2036	};	2165	};
2037		2166
2038	try self.asmPseudo(.pseudo_dbg_prologue_end_none);	2167	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
2039		2168
2040	try self.genBody(self.air.getMainBody());	2169	try self.genBody(self.air.getMainBody());
2041		2170
2042	// TODO can single exitlude jump reloc be elided? What if it is not at the end of the code?	2171	const epilogue = if (self.epilogue_relocs.items.len > 0) epilogue: {
2043	// Example:	2172	const epilogue_relocs_last_index = self.epilogue_relocs.items.len - 1;
2044	// pub fn main() void {	2173	for (if (self.epilogue_relocs.items[epilogue_relocs_last_index] == self.mir_instructions.len - 1) epilogue_relocs: {
2045	// maybeErr() catch return;	2174	_ = self.mir_instructions.pop();
2046	// unreachable;	2175	break :epilogue_relocs self.epilogue_relocs.items[0..epilogue_relocs_last_index];
2047	// }	2176	} else self.epilogue_relocs.items) \|epilogue_reloc\| self.performReloc(epilogue_reloc);
2048	// Eliding the reloc will cause a miscompilation in this case.	2177
2049	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {	2178	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
2050	self.mir_instructions.items(.data)[jmp_reloc].inst.inst =	2179	const backpatch_stack_dealloc = try self.asmPlaceholder();
2051	@intCast(self.mir_instructions.len);	2180	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
2052	}	2181	try self.asmRegister(.{ ._, .pop }, .rbp);
		2182	try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, .s(8));
		2183	try self.asmOpOnly(.{ ._, .ret });
		2184	break :epilogue .{
		2185	.backpatch_stack_dealloc = backpatch_stack_dealloc,
		2186	.backpatch_pop_callee_preserved_regs = backpatch_pop_callee_preserved_regs,
		2187	};
		2188	} else null;
2053		2189
2054	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);	2190	const frame_layout = try self.computeFrameLayout(fn_info.cc);
2055	const backpatch_stack_dealloc = try self.asmPlaceholder();	2191	const need_frame_align = frame_layout.stack_mask != std.math.maxInt(u32);
2056	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
2057	try self.asmRegister(.{ ._, .pop }, .rbp);
2058	try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, Immediate.s(8));
2059	try self.asmOpOnly(.{ ._, .ret });
2060
2061	const frame_layout = try self.computeFrameLayout(cc);
2062	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
2063	const need_stack_adjust = frame_layout.stack_adjust > 0;	2192	const need_stack_adjust = frame_layout.stack_adjust > 0;
2064	const need_save_reg = frame_layout.save_reg_list.count() > 0;	2193	const need_save_reg = frame_layout.save_reg_list.count() > 0;
2065	if (need_frame_align) {	2194	if (need_frame_align) {
2066	const page_align = @as(u32, math.maxInt(u32)) << 12;	2195	const page_align = @as(u32, std.math.maxInt(u32)) << 12;
2067	self.mir_instructions.set(backpatch_frame_align, .{	2196	self.mir_instructions.set(backpatch_frame_align, .{
2068	.tag = .@"and",	2197	.tag = .@"and",
2069	.ops = .ri_s,	2198	.ops = .ri_s,
...	@@ -2106,12 +2235,13 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2106,12 +2235,13 @@ fn gen(self: *Self) InnerError!void {
2106	} },	2235	} },
2107	});	2236	});
2108	} else {	2237	} else {
		2238	const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc);
2109	self.mir_instructions.set(backpatch_stack_alloc, .{	2239	self.mir_instructions.set(backpatch_stack_alloc, .{
2110	.tag = .pseudo,	2240	.tag = .pseudo,
2111	.ops = .pseudo_probe_adjust_setup_rri_s,	2241	.ops = .pseudo_probe_adjust_setup_rri_s,
2112	.data = .{ .rri = .{	2242	.data = .{ .rri = .{
2113	.r1 = .rsp,	2243	.r1 = .rsp,
2114	.r2 = .rax,	2244	.r2 = scratch_reg,
2115	.i = frame_layout.stack_adjust,	2245	.i = frame_layout.stack_adjust,
2116	} },	2246	} },
2117	});	2247	});
...	@@ -2120,34 +2250,44 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2120,34 +2250,44 @@ fn gen(self: *Self) InnerError!void {
2120	.ops = .pseudo_probe_adjust_loop_rr,	2250	.ops = .pseudo_probe_adjust_loop_rr,
2121	.data = .{ .rr = .{	2251	.data = .{ .rr = .{
2122	.r1 = .rsp,	2252	.r1 = .rsp,
2123	.r2 = .rax,	2253	.r2 = scratch_reg,
2124	} },	2254	} },
2125	});	2255	});
2126	}	2256	}
2127	}	2257	}
2128	if (need_frame_align or need_stack_adjust) {	2258	if (epilogue) \|e\| if (need_frame_align or need_stack_adjust) {
2129	self.mir_instructions.set(backpatch_stack_dealloc, .{	2259	self.mir_instructions.set(e.backpatch_stack_dealloc, switch (-frame_layout.save_reg_list.size(self.target)) {
2130	.tag = .lea,	2260	0 => .{
2131	.ops = .rm,	2261	.tag = .mov,
2132	.data = .{ .rx = .{	2262	.ops = .rr,
2133	.r1 = .rsp,	2263	.data = .{ .rr = .{
2134	.payload = try self.addExtra(Mir.Memory.encode(.{	2264	.r1 = .rsp,
2135	.base = .{ .reg = .rbp },	2265	.r2 = .rbp,
2136	.mod = .{ .rm = .{	2266	} },
2137	.size = .qword,	2267	},
2138	.disp = -frame_layout.save_reg_list.size(),	2268	else => \|disp\| .{
2139	} },	2269	.tag = .lea,
2140	})),	2270	.ops = .rm,
2141	} },	2271	.data = .{ .rx = .{
		2272	.r1 = .rsp,
		2273	.payload = try self.addExtra(Mir.Memory.encode(.{
		2274	.base = .{ .reg = .rbp },
		2275	.mod = .{ .rm = .{
		2276	.size = .qword,
		2277	.disp = disp,
		2278	} },
		2279	})),
		2280	} },
		2281	},
2142	});	2282	});
2143	}	2283	};
2144	if (need_save_reg) {	2284	if (need_save_reg) {
2145	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{	2285	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
2146	.tag = .pseudo,	2286	.tag = .pseudo,
2147	.ops = .pseudo_push_reg_list,	2287	.ops = .pseudo_push_reg_list,
2148	.data = .{ .reg_list = frame_layout.save_reg_list },	2288	.data = .{ .reg_list = frame_layout.save_reg_list },
2149	});	2289	});
2150	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{	2290	if (epilogue) \|e\| self.mir_instructions.set(e.backpatch_pop_callee_preserved_regs, .{
2151	.tag = .pseudo,	2291	.tag = .pseudo,
2152	.ops = .pseudo_pop_reg_list,	2292	.ops = .pseudo_pop_reg_list,
2153	.data = .{ .reg_list = frame_layout.save_reg_list },	2293	.data = .{ .reg_list = frame_layout.save_reg_list },
...	@@ -2170,97 +2310,85 @@ fn gen(self: *Self) InnerError!void {	...	@@ -2170,97 +2310,85 @@ fn gen(self: *Self) InnerError!void {
2170	});	2310	});
2171	}	2311	}
2172		2312
2173	fn checkInvariantsAfterAirInst(self: *Self, inst: Air.Inst.Index, old_air_bookkeeping: @TypeOf(air_bookkeeping_init)) void {	2313	fn checkInvariantsAfterAirInst(self: *CodeGen) void {
2174	assert(!self.register_manager.lockedRegsExist());	2314	assert(!self.register_manager.lockedRegsExist());
2175		2315
2176	if (std.debug.runtime_safety) {	2316	if (std.debug.runtime_safety) {
2177	if (self.air_bookkeeping < old_air_bookkeeping + 1) {	2317	// check consistency of tracked registers
2178	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, self.air.instructions.items(.tag)[@intFromEnum(inst)] });	2318	var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
2179	}	2319	while (it.next()) \|index\| {
2180		2320	const tracked_inst = self.register_manager.registers[index];
2181	{ // check consistency of tracked registers	2321	const tracking = self.getResolvedInstValue(tracked_inst);
2182	var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });	2322	for (tracking.getRegs()) \|reg\| {
2183	while (it.next()) \|index\| {	2323	if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
2184	const tracked_inst = self.register_manager.registers[index];	2324	} else unreachable; // tracked register not in use
2185	const tracking = self.getResolvedInstValue(tracked_inst);
2186	for (tracking.getRegs()) \|reg\| {
2187	if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
2188	} else unreachable; // tracked register not in use
2189	}
2190	}	2325	}
2191	}	2326	}
2192	}	2327	}
2193		2328
2194	fn genBodyBlock(self: *Self, body: []const Air.Inst.Index) InnerError!void {	2329	fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2195	try self.asmPseudo(.pseudo_dbg_enter_block_none);	2330	try self.asmPseudo(.pseudo_dbg_enter_block_none);
2196	try self.genBody(body);	2331	try self.genBody(body);
2197	try self.asmPseudo(.pseudo_dbg_leave_block_none);	2332	try self.asmPseudo(.pseudo_dbg_leave_block_none);
2198	}	2333	}
2199		2334
2200	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	2335	fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2201	const pt = self.pt;	2336	@setEvalBranchQuota(1_700);
		2337	const pt = cg.pt;
2202	const zcu = pt.zcu;	2338	const zcu = pt.zcu;
2203	const ip = &zcu.intern_pool;	2339	const ip = &zcu.intern_pool;
2204	const air_tags = self.air.instructions.items(.tag);	2340	const air_tags = cg.air.instructions.items(.tag);
		2341	const air_datas = cg.air.instructions.items(.data);
		2342	const use_old = cg.target.ofmt == .coff;
2205		2343
2206	self.arg_index = 0;	2344	cg.arg_index = 0;
2207	for (body) \|inst\| switch (air_tags[@intFromEnum(inst)]) {	2345	for (body) \|inst\| switch (air_tags[@intFromEnum(inst)]) {
2208	.arg => {	2346	.arg => {
2209	wip_mir_log.debug("{}", .{self.fmtAir(inst)});	2347	wip_mir_log.debug("{}", .{cg.fmtAir(inst)});
2210	verbose_tracking_log.debug("{}", .{self.fmtTracking()});	2348	verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
2211		2349
2212	const old_air_bookkeeping = self.air_bookkeeping;	2350	cg.reused_operands = .initEmpty();
2213	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);	2351	try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1);
2214		2352
2215	try self.airArg(inst);	2353	try cg.airArg(inst);
2216		2354
2217	self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);	2355	cg.resetTemps();
		2356	cg.checkInvariantsAfterAirInst();
2218	},	2357	},
2219	else => break,	2358	else => break,
2220	};	2359	};
2221		2360
2222	if (self.arg_index == 0) try self.airDbgVarArgs();	2361	if (cg.arg_index == 0) try cg.airDbgVarArgs();
2223	self.arg_index = 0;	2362	cg.arg_index = 0;
2224	for (body) \|inst\| {	2363	for (body) \|inst\| {
2225	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;	2364	if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) continue;
2226	wip_mir_log.debug("{}", .{self.fmtAir(inst)});	2365	wip_mir_log.debug("{}", .{cg.fmtAir(inst)});
2227	verbose_tracking_log.debug("{}", .{self.fmtTracking()});	2366	verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
2228		2367
2229	const old_air_bookkeeping = self.air_bookkeeping;	2368	cg.reused_operands = .initEmpty();
2230	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);	2369	try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1);
2231	switch (air_tags[@intFromEnum(inst)]) {	2370	switch (air_tags[@intFromEnum(inst)]) {
2232	// zig fmt: off	2371	// zig fmt: off
2233	.not,
2234	=> \|tag\| try self.airUnOp(inst, tag),
2235
2236	.add,	2372	.add,
2237	.add_wrap,	2373	.add_wrap,
2238	.sub,	2374	.sub,
2239	.sub_wrap,	2375	.sub_wrap,
2240	.bool_and,
2241	.bool_or,
2242	.bit_and,
2243	.bit_or,
2244	.xor,
2245	.min,	2376	.min,
2246	.max,	2377	.max,
2247	=> \|tag\| try self.airBinOp(inst, tag),	2378	=> \|air_tag\| try cg.airBinOp(inst, air_tag),
2248		2379
2249	.ptr_add, .ptr_sub => \|tag\| try self.airPtrArithmetic(inst, tag),	2380	.shr, .shr_exact => try cg.airShlShrBinOp(inst),
		2381	.shl, .shl_exact => try cg.airShlShrBinOp(inst),
2250		2382
2251	.shr, .shr_exact => try self.airShlShrBinOp(inst),	2383	.mul => try cg.airMulDivBinOp(inst),
2252	.shl, .shl_exact => try self.airShlShrBinOp(inst),	2384	.mul_wrap => try cg.airMulDivBinOp(inst),
		2385	.rem => try cg.airMulDivBinOp(inst),
		2386	.mod => try cg.airMulDivBinOp(inst),
2253		2387
2254	.mul => try self.airMulDivBinOp(inst),	2388	.add_sat => try cg.airAddSat(inst),
2255	.mul_wrap => try self.airMulDivBinOp(inst),	2389	.sub_sat => try cg.airSubSat(inst),
2256	.rem => try self.airMulDivBinOp(inst),	2390	.mul_sat => try cg.airMulSat(inst),
2257	.mod => try self.airMulDivBinOp(inst),	2391	.shl_sat => try cg.airShlSat(inst),
2258
2259	.add_sat => try self.airAddSat(inst),
2260	.sub_sat => try self.airSubSat(inst),
2261	.mul_sat => try self.airMulSat(inst),
2262	.shl_sat => try self.airShlSat(inst),
2263	.slice => try self.airSlice(inst),
2264		2392
2265	.sin,	2393	.sin,
2266	.cos,	2394	.cos,
...	@@ -2271,162 +2399,80 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2271,162 +2399,80 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2271	.log2,	2399	.log2,
2272	.log10,	2400	.log10,
2273	.round,	2401	.round,
2274	=> \|tag\| try self.airUnaryMath(inst, tag),	2402	=> \|air_tag\| try cg.airUnaryMath(inst, air_tag),
2275		2403
2276	.floor => try self.airRound(inst, .{ .mode = .down, .precision = .inexact }),	2404	.floor => try cg.airRound(inst, .{ .mode = .down, .precision = .inexact }),
2277	.ceil => try self.airRound(inst, .{ .mode = .up, .precision = .inexact }),	2405	.ceil => try cg.airRound(inst, .{ .mode = .up, .precision = .inexact }),
2278	.trunc_float => try self.airRound(inst, .{ .mode = .zero, .precision = .inexact }),	2406	.trunc_float => try cg.airRound(inst, .{ .mode = .zero, .precision = .inexact }),
2279	.sqrt => try self.airSqrt(inst),	2407	.sqrt => try cg.airSqrt(inst),
2280	.neg => try self.airFloatSign(inst),	2408	.neg => try cg.airFloatSign(inst),
2281		2409
2282	.abs => try self.airAbs(inst),	2410	.abs => try cg.airAbs(inst),
2283		2411
2284	.add_with_overflow => try self.airAddSubWithOverflow(inst),	2412	.add_with_overflow => try cg.airAddSubWithOverflow(inst),
2285	.sub_with_overflow => try self.airAddSubWithOverflow(inst),	2413	.sub_with_overflow => try cg.airAddSubWithOverflow(inst),
2286	.mul_with_overflow => try self.airMulWithOverflow(inst),	2414	.mul_with_overflow => try cg.airMulWithOverflow(inst),
2287	.shl_with_overflow => try self.airShlWithOverflow(inst),	2415	.shl_with_overflow => try cg.airShlWithOverflow(inst),
2288		2416
2289	.div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst),	2417	.div_float, .div_trunc, .div_floor, .div_exact => try cg.airMulDivBinOp(inst),
2290		2418
2291	.cmp_lt => try self.airCmp(inst, .lt),	2419	.cmp_lt_errors_len => try cg.airCmpLtErrorsLen(inst),
2292	.cmp_lte => try self.airCmp(inst, .lte),	2420
2293	.cmp_eq => try self.airCmp(inst, .eq),	2421	.bitcast => try cg.airBitCast(inst),
2294	.cmp_gte => try self.airCmp(inst, .gte),	2422	.fptrunc => try cg.airFptrunc(inst),
2295	.cmp_gt => try self.airCmp(inst, .gt),	2423	.fpext => try cg.airFpext(inst),
2296	.cmp_neq => try self.airCmp(inst, .neq),	2424	.intcast => try cg.airIntCast(inst),
2297		2425	.trunc => try cg.airTrunc(inst),
2298	.cmp_vector => try self.airCmpVector(inst),	2426	.is_non_null => try cg.airIsNonNull(inst),
2299	.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),	2427	.is_null => try cg.airIsNull(inst),
2300		2428	.is_non_err => try cg.airIsNonErr(inst),
2301	.alloc => try self.airAlloc(inst),	2429	.is_err => try cg.airIsErr(inst),
2302	.ret_ptr => try self.airRetPtr(inst),	2430	.float_from_int => try cg.airFloatFromInt(inst),
2303	.arg => try self.airDbgArg(inst),	2431	.int_from_float => try cg.airIntFromFloat(inst),
2304	.assembly => try self.airAsm(inst),	2432	.cmpxchg_strong => try cg.airCmpxchg(inst),
2305	.bitcast => try self.airBitCast(inst),	2433	.cmpxchg_weak => try cg.airCmpxchg(inst),
2306	.block => try self.airBlock(inst),	2434	.atomic_rmw => try cg.airAtomicRmw(inst),
2307	.br => try self.airBr(inst),	2435	.atomic_load => try cg.airAtomicLoad(inst),
2308	.repeat => try self.airRepeat(inst),	2436	.memcpy => try cg.airMemcpy(inst),
2309	.switch_dispatch => try self.airSwitchDispatch(inst),	2437	.memset => try cg.airMemset(inst, false),
2310	.trap => try self.airTrap(),	2438	.memset_safe => try cg.airMemset(inst, true),
2311	.breakpoint => try self.airBreakpoint(),	2439	.ctz => try cg.airCtz(inst),
2312	.ret_addr => try self.airRetAddr(inst),	2440	.popcount => try cg.airPopCount(inst),
2313	.frame_addr => try self.airFrameAddress(inst),	2441	.byte_swap => try cg.airByteSwap(inst),
2314	.cond_br => try self.airCondBr(inst),	2442	.bit_reverse => try cg.airBitReverse(inst),
2315	.fptrunc => try self.airFptrunc(inst),	2443	.tag_name => try cg.airTagName(inst),
2316	.fpext => try self.airFpext(inst),	2444	.error_name => try cg.airErrorName(inst),
2317	.intcast => try self.airIntCast(inst),	2445	.splat => try cg.airSplat(inst),
2318	.trunc => try self.airTrunc(inst),	2446	.select => try cg.airSelect(inst),
2319	.int_from_bool => try self.airIntFromBool(inst),	2447	.shuffle => try cg.airShuffle(inst),
2320	.is_non_null => try self.airIsNonNull(inst),	2448	.reduce => try cg.airReduce(inst),
2321	.is_non_null_ptr => try self.airIsNonNullPtr(inst),	2449	.aggregate_init => try cg.airAggregateInit(inst),
2322	.is_null => try self.airIsNull(inst),	2450	.prefetch => try cg.airPrefetch(inst),
2323	.is_null_ptr => try self.airIsNullPtr(inst),	2451	.mul_add => try cg.airMulAdd(inst),
2324	.is_non_err => try self.airIsNonErr(inst),	2452
2325	.is_non_err_ptr => try self.airIsNonErrPtr(inst),	2453	.atomic_store_unordered => try cg.airAtomicStore(inst, .unordered),
2326	.is_err => try self.airIsErr(inst),	2454	.atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic),
2327	.is_err_ptr => try self.airIsErrPtr(inst),	2455	.atomic_store_release => try cg.airAtomicStore(inst, .release),
2328	.load => try self.airLoad(inst),	2456	.atomic_store_seq_cst => try cg.airAtomicStore(inst, .seq_cst),
2329	.loop => try self.airLoop(inst),	2457
2330	.int_from_ptr => try self.airIntFromPtr(inst),	2458	.array_elem_val => try cg.airArrayElemVal(inst),
2331	.ret => try self.airRet(inst, false),	2459
2332	.ret_safe => try self.airRet(inst, true),	2460	.optional_payload => try cg.airOptionalPayload(inst),
2333	.ret_load => try self.airRetLoad(inst),	2461	.unwrap_errunion_err => try cg.airUnwrapErrUnionErr(inst),
2334	.store => try self.airStore(inst, false),	2462	.unwrap_errunion_payload => try cg.airUnwrapErrUnionPayload(inst),
2335	.store_safe => try self.airStore(inst, true),	2463	.err_return_trace => try cg.airErrReturnTrace(inst),
2336	.struct_field_ptr=> try self.airStructFieldPtr(inst),	2464	.set_err_return_trace => try cg.airSetErrReturnTrace(inst),
2337	.struct_field_val=> try self.airStructFieldVal(inst),	2465	.save_err_return_trace_index=> try cg.airSaveErrReturnTraceIndex(inst),
2338	.array_to_slice => try self.airArrayToSlice(inst),	2466
2339	.float_from_int => try self.airFloatFromInt(inst),	2467	.wrap_optional => try cg.airWrapOptional(inst),
2340	.int_from_float => try self.airIntFromFloat(inst),	2468	.wrap_errunion_payload => try cg.airWrapErrUnionPayload(inst),
2341	.cmpxchg_strong => try self.airCmpxchg(inst),	2469	.wrap_errunion_err => try cg.airWrapErrUnionErr(inst),
2342	.cmpxchg_weak => try self.airCmpxchg(inst),	2470	// zig fmt: on
2343	.atomic_rmw => try self.airAtomicRmw(inst),
2344	.atomic_load => try self.airAtomicLoad(inst),
2345	.memcpy => try self.airMemcpy(inst),
2346	.memset => try self.airMemset(inst, false),
2347	.memset_safe => try self.airMemset(inst, true),
2348	.set_union_tag => try self.airSetUnionTag(inst),
2349	.get_union_tag => try self.airGetUnionTag(inst),
2350	.clz => try self.airClz(inst),
2351	.ctz => try self.airCtz(inst),
2352	.popcount => try self.airPopCount(inst),
2353	.byte_swap => try self.airByteSwap(inst),
2354	.bit_reverse => try self.airBitReverse(inst),
2355	.tag_name => try self.airTagName(inst),
2356	.error_name => try self.airErrorName(inst),
2357	.splat => try self.airSplat(inst),
2358	.select => try self.airSelect(inst),
2359	.shuffle => try self.airShuffle(inst),
2360	.reduce => try self.airReduce(inst),
2361	.aggregate_init => try self.airAggregateInit(inst),
2362	.union_init => try self.airUnionInit(inst),
2363	.prefetch => try self.airPrefetch(inst),
2364	.mul_add => try self.airMulAdd(inst),
2365	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
2366
2367	.@"try" => try self.airTry(inst),
2368	.try_cold => try self.airTry(inst), // TODO
2369	.try_ptr => try self.airTryPtr(inst),
2370	.try_ptr_cold => try self.airTryPtr(inst), // TODO
2371
2372	.dbg_stmt => try self.airDbgStmt(inst),
2373	.dbg_empty_stmt => try self.airDbgEmptyStmt(),
2374	.dbg_inline_block => try self.airDbgInlineBlock(inst),
2375	.dbg_var_ptr,
2376	.dbg_var_val,
2377	.dbg_arg_inline,
2378	=> try self.airDbgVar(inst),
2379
2380	.call => try self.airCall(inst, .auto),
2381	.call_always_tail => try self.airCall(inst, .always_tail),
2382	.call_never_tail => try self.airCall(inst, .never_tail),
2383	.call_never_inline => try self.airCall(inst, .never_inline),
2384
2385	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
2386	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
2387	.atomic_store_release => try self.airAtomicStore(inst, .release),
2388	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
2389
2390	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
2391	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
2392	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
2393	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
2394
2395	.field_parent_ptr => try self.airFieldParentPtr(inst),
2396
2397	.switch_br => try self.airSwitchBr(inst),
2398	.loop_switch_br => try self.airLoopSwitchBr(inst),
2399	.slice_ptr => try self.airSlicePtr(inst),
2400	.slice_len => try self.airSliceLen(inst),
2401
2402	.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
2403	.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
2404
2405	.array_elem_val => try self.airArrayElemVal(inst),
2406	.slice_elem_val => try self.airSliceElemVal(inst),
2407	.slice_elem_ptr => try self.airSliceElemPtr(inst),
2408	.ptr_elem_val => try self.airPtrElemVal(inst),
2409	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
2410
2411	.inferred_alloc, .inferred_alloc_comptime => unreachable,
2412	.unreach => self.finishAirBookkeeping(),
2413
2414	.optional_payload => try self.airOptionalPayload(inst),
2415	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
2416	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
2417	.unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst),
2418	.unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst),
2419	.unwrap_errunion_err_ptr => try self.airUnwrapErrUnionErrPtr(inst),
2420	.unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst),
2421	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
2422	.err_return_trace => try self.airErrReturnTrace(inst),
2423	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
2424	.save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst),
2425
2426	.wrap_optional => try self.airWrapOptional(inst),
2427	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
2428	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
2429		2471
		2472	.add_safe,
		2473	.sub_safe,
		2474	.mul_safe,
		2475	=> return cg.fail("TODO implement safety_checked_instructions", .{}),
2430	.add_optimized,	2476	.add_optimized,
2431	.sub_optimized,	2477	.sub_optimized,
2432	.mul_optimized,	2478	.mul_optimized,
...	@@ -2437,179 +2483,7698 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2437,179 +2483,7698 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2437	.rem_optimized,	2483	.rem_optimized,
2438	.mod_optimized,	2484	.mod_optimized,
2439	.neg_optimized,	2485	.neg_optimized,
2440	.cmp_lt_optimized,
2441	.cmp_lte_optimized,
2442	.cmp_eq_optimized,
2443	.cmp_gte_optimized,
2444	.cmp_gt_optimized,
2445	.cmp_neq_optimized,
2446	.cmp_vector_optimized,
2447	.reduce_optimized,	2486	.reduce_optimized,
2448	.int_from_float_optimized,	2487	.int_from_float_optimized,
2449	=> return self.fail("TODO implement optimized float mode", .{}),	2488	=> return cg.fail("TODO implement optimized float mode", .{}),
2450		2489
2451	.add_safe,	2490	.arg => try cg.airDbgArg(inst),
2452	.sub_safe,	2491	.ptr_add => \|air_tag\| if (use_old) try cg.airPtrArithmetic(inst, air_tag) else {
2453	.mul_safe,	2492	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
2454	=> return self.fail("TODO implement safety_checked_instructions", .{}),	2493	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
2455		2494	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2456	.is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),	2495	try ops[0].toSlicePtr(cg);
2457	.error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),	2496	var res: [1]Temp = undefined;
2458	.vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),	2497	if (ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{
2459		2498	.patterns = &.{
2460	.c_va_arg => try self.airVaArg(inst),	2499	.{ .src = .{ .to_gpr, .simm32 } },
2461	.c_va_copy => try self.airVaCopy(inst),	2500	},
2462	.c_va_end => try self.airVaEnd(inst),	2501	.dst_temps = .{.{ .rc = .general_purpose }},
2463	.c_va_start => try self.airVaStart(inst),	2502	.each = .{ .once = &.{
2464		2503	.{ ._, ._, .lea, .dst0p, .leaa(.none, .src0, .add_src0_elem_size_times_src1), ._, ._ },
2465	.wasm_memory_size => unreachable,	2504	} },
2466	.wasm_memory_grow => unreachable,	2505	}, .{
2467		2506	.dst_constraints = .{.{ .elem_size_is = 1 }},
2468	.work_item_id => unreachable,	2507	.patterns = &.{
2469	.work_group_size => unreachable,	2508	.{ .src = .{ .to_gpr, .to_gpr } },
2470	.work_group_id => unreachable,	2509	},
2471	// zig fmt: on	2510	.dst_temps = .{.{ .rc = .general_purpose }},
2472	}	2511	.each = .{ .once = &.{
2473	self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping);	2512	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
2474	}	2513	} },
2475	verbose_tracking_log.debug("{}", .{self.fmtTracking()});	2514	}, .{
2476	}	2515	.dst_constraints = .{.{ .elem_size_is = 2 }},
2477		2516	.patterns = &.{
2478	fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {	2517	.{ .src = .{ .to_gpr, .to_gpr } },
2479	const pt = self.pt;	2518	},
2480	const zcu = pt.zcu;	2519	.dst_temps = .{.{ .rc = .general_purpose }},
2481	const ip = &zcu.intern_pool;	2520	.each = .{ .once = &.{
2482	switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {	2521	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"2", .src1), ._, ._ },
2483	.@"enum" => {	2522	} },
2484	const enum_ty = Type.fromInterned(lazy_sym.ty);	2523	}, .{
2485	wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});	2524	.dst_constraints = .{.{ .elem_size_is = 2 + 1 }},
2486		2525	.patterns = &.{
2487	const resolved_cc = abi.resolveCallingConvention(.auto, self.target.*);	2526	.{ .src = .{ .to_gpr, .to_gpr } },
2488	const param_regs = abi.getCAbiIntParamRegs(resolved_cc);	2527	},
2489	const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);	2528	.dst_temps = .{.{ .rc = .general_purpose }},
2490	defer for (param_locks) \|lock\| self.register_manager.unlockReg(lock);	2529	.each = .{ .once = &.{
2491		2530	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"2", .src1), ._, ._ },
2492	const ret_reg = param_regs[0];	2531	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2493	const enum_mcv = MCValue{ .register = param_regs[1] };	2532	} },
2494		2533	}, .{
2495	const exitlude_jump_relocs = try self.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu));	2534	.dst_constraints = .{.{ .elem_size_is = 4 }},
2496	defer self.gpa.free(exitlude_jump_relocs);	2535	.patterns = &.{
2497		2536	.{ .src = .{ .to_gpr, .to_gpr } },
2498	const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	2537	},
2499	const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);	2538	.dst_temps = .{.{ .rc = .general_purpose }},
2500	defer self.register_manager.unlockReg(data_lock);	2539	.each = .{ .once = &.{
2501	try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = enum_ty.toIntern() });	2540	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"4", .src1), ._, ._ },
2502		2541	} },
2503	var data_off: i32 = 0;	2542	}, .{
2504	const tag_names = enum_ty.enumFields(zcu);	2543	.dst_constraints = .{.{ .elem_size_is = 4 + 1 }},
2505	for (exitlude_jump_relocs, 0..) \|*exitlude_jump_reloc, tag_index\| {	2544	.patterns = &.{
2506	const tag_name_len = tag_names.get(ip)[tag_index].length(ip);	2545	.{ .src = .{ .to_gpr, .to_gpr } },
2507	const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index));	2546	},
2508	const tag_mcv = try self.genTypedValue(tag_val);	2547	.dst_temps = .{.{ .ref = .src1 }},
2509	try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);	2548	.each = .{ .once = &.{
2510	const skip_reloc = try self.asmJccReloc(.ne, undefined);	2549	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"4", .src1), ._, ._ },
2511		2550	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2512	try self.genSetMem(	2551	} },
2513	.{ .reg = ret_reg },	2552	}, .{
2514	0,	2553	.required_features = .{ .@"64bit", null, null, null },
2515	Type.usize,	2554	.dst_constraints = .{.{ .elem_size_is = 8 }},
2516	.{ .register_offset = .{ .reg = data_reg, .off = data_off } },	2555	.patterns = &.{
2517	.{},	2556	.{ .src = .{ .to_gpr, .to_gpr } },
2518	);	2557	},
2519	try self.genSetMem(	2558	.dst_temps = .{.{ .rc = .general_purpose }},
2520	.{ .reg = ret_reg },	2559	.each = .{ .once = &.{
2521	8,	2560	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"8", .src1), ._, ._ },
2522	Type.usize,	2561	} },
2523	.{ .immediate = tag_name_len },	2562	}, .{
2524	.{},	2563	.required_features = .{ .@"64bit", null, null, null },
2525	);	2564	.dst_constraints = .{.{ .elem_size_is = 8 + 1 }},
2526		2565	.patterns = &.{
2527	exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);	2566	.{ .src = .{ .to_gpr, .to_gpr } },
2528	self.performReloc(skip_reloc);	2567	},
2529		2568	.dst_temps = .{.{ .ref = .src1 }},
2530	data_off += @intCast(tag_name_len + 1);	2569	.each = .{ .once = &.{
2531	}	2570	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"8", .src1), ._, ._ },
2532		2571	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2533	try self.airTrap();	2572	} },
2534		2573	}, .{
2535	for (exitlude_jump_relocs) \|reloc\| self.performReloc(reloc);	2574	.dst_constraints = .{.po2_elem_size},
2536	try self.asmOpOnly(.{ ._, .ret });	2575	.patterns = &.{
2537	},	2576	.{ .src = .{ .to_gpr, .to_mut_gpr } },
2538	else => return self.fail(	2577	},
2539	"TODO implement {s} for {}",	2578	.dst_temps = .{.{ .ref = .src1 }},
2540	.{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },	2579	.clobbers = .{ .eflags = true },
2541	),	2580	.each = .{ .once = &.{
2542	}	2581	.{ ._, ._l, .sh, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ },
2543	}	2582	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
2544		2583	} },
2545	fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {	2584	}, .{
2546	for (value.getRegs()) \|reg\| try self.register_manager.getReg(reg, inst);	2585	.patterns = &.{
2547	}	2586	.{ .src = .{ .to_gpr, .to_gpr } },
2548		2587	},
2549	fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {	2588	.dst_temps = .{.{ .rc = .general_purpose }},
2550	for (value.getRegs()) \|reg\| if (self.register_manager.isRegFree(reg))	2589	.clobbers = .{ .eflags = true },
2551	self.register_manager.getRegAssumeFree(reg, inst);	2590	.each = .{ .once = &.{
2552	}	2591	.{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .add_src0_elem_size), ._ },
2553		2592	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
2554	fn freeValue(self: *Self, value: MCValue) !void {	2593	} },
2555	switch (value) {	2594	} }) catch \|err\| switch (err) {
2556	.register => \|reg\| {	2595	error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
2557	self.register_manager.freeReg(reg);	2596	@tagName(air_tag),
2558	if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);	2597	cg.typeOf(bin_op.lhs).fmt(pt),
2559	},	2598	ops[0].tracking(cg),
2560	.register_pair => \|regs\| for (regs) \|reg\| self.register_manager.freeReg(reg),	2599	ops[1].tracking(cg),
2561	.register_offset => \|reg_off\| self.register_manager.freeReg(reg_off.reg),	2600	}),
2562	.register_overflow => \|reg_ov\| {	2601	else => \|e\| return e,
2563	self.register_manager.freeReg(reg_ov.reg);	2602	} else { // hack around Sema OPV bugs
2564	self.eflags_inst = null;	2603	res[0] = ops[0];
2565	},	2604	}
2566	.eflags => self.eflags_inst = null,	2605	for (ops) \|op\| for (res) \|r\| {
2567	else => {}, // TODO process stack allocation death	2606	if (op.index == r.index) break;
2568	}	2607	} else try op.die(cg);
2569	}	2608	try res[0].moveTo(inst, cg);
2570		2609	},
2571	fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) !void {	2610	.ptr_sub => \|air_tag\| if (use_old) try cg.airPtrArithmetic(inst, air_tag) else {
2572	if (bt.feed()) if (operand.toIndex()) \|inst\| try self.processDeath(inst);	2611	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		2612	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
		2613	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		2614	try ops[0].toSlicePtr(cg);
		2615	var res: [1]Temp = undefined;
		2616	if (ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{
		2617	.patterns = &.{
		2618	.{ .src = .{ .to_gpr, .simm32 } },
		2619	},
		2620	.dst_temps = .{.{ .rc = .general_purpose }},
		2621	.each = .{ .once = &.{
		2622	.{ ._, ._, .lea, .dst0p, .leaa(.none, .src0, .sub_src0_elem_size_times_src1), ._, ._ },
		2623	} },
		2624	}, .{
		2625	.dst_constraints = .{.{ .elem_size_is = 1 }},
		2626	.patterns = &.{
		2627	.{ .src = .{ .to_gpr, .to_mut_gpr } },
		2628	},
		2629	.dst_temps = .{.{ .ref = .src1 }},
		2630	.clobbers = .{ .eflags = true },
		2631	.each = .{ .once = &.{
		2632	.{ ._, ._, .neg, .src1p, ._, ._, ._ },
		2633	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
		2634	} },
		2635	}, .{
		2636	.dst_constraints = .{.{ .elem_size_is = 2 }},
		2637	.patterns = &.{
		2638	.{ .src = .{ .to_gpr, .to_mut_gpr } },
		2639	},
		2640	.dst_temps = .{.{ .ref = .src1 }},
		2641	.clobbers = .{ .eflags = true },
		2642	.each = .{ .once = &.{
		2643	.{ ._, ._, .neg, .src1p, ._, ._, ._ },
		2644	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"2", .src1), ._, ._ },
		2645	} },
		2646	}, .{
		2647	.dst_constraints = .{.{ .elem_size_is = 2 + 1 }},
		2648	.patterns = &.{
		2649	.{ .src = .{ .to_gpr, .to_gpr } },
		2650	},
		2651	.dst_temps = .{.{ .rc = .general_purpose }},
		2652	.clobbers = .{ .eflags = true },
		2653	.each = .{ .once = &.{
		2654	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"2", .src1), ._, ._ },
		2655	.{ ._, ._, .neg, .dst0p, ._, ._, ._ },
		2656	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
		2657	} },
		2658	}, .{
		2659	.dst_constraints = .{.{ .elem_size_is = 4 }},
		2660	.patterns = &.{
		2661	.{ .src = .{ .to_gpr, .to_mut_gpr } },
		2662	},
		2663	.dst_temps = .{.{ .ref = .src1 }},
		2664	.clobbers = .{ .eflags = true },
		2665	.each = .{ .once = &.{
		2666	.{ ._, ._, .neg, .src1p, ._, ._, ._ },
		2667	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"4", .src1), ._, ._ },
		2668	} },
		2669	}, .{
		2670	.dst_constraints = .{.{ .elem_size_is = 4 + 1 }},
		2671	.patterns = &.{
		2672	.{ .src = .{ .to_gpr, .to_gpr } },
		2673	},
		2674	.dst_temps = .{.{ .rc = .general_purpose }},
		2675	.clobbers = .{ .eflags = true },
		2676	.each = .{ .once = &.{
		2677	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"4", .src1), ._, ._ },
		2678	.{ ._, ._, .neg, .dst0p, ._, ._, ._ },
		2679	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
		2680	} },
		2681	}, .{
		2682	.required_features = .{ .@"64bit", null, null, null },
		2683	.dst_constraints = .{.{ .elem_size_is = 8 }},
		2684	.patterns = &.{
		2685	.{ .src = .{ .to_gpr, .to_mut_gpr } },
		2686	},
		2687	.dst_temps = .{.{ .ref = .src1 }},
		2688	.clobbers = .{ .eflags = true },
		2689	.each = .{ .once = &.{
		2690	.{ ._, ._, .neg, .src1p, ._, ._, ._ },
		2691	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src0, .@"8", .src1), ._, ._ },
		2692	} },
		2693	}, .{
		2694	.required_features = .{ .@"64bit", null, null, null },
		2695	.dst_constraints = .{.{ .elem_size_is = 8 + 1 }},
		2696	.patterns = &.{
		2697	.{ .src = .{ .to_gpr, .to_gpr } },
		2698	},
		2699	.dst_temps = .{.{ .rc = .general_purpose }},
		2700	.clobbers = .{ .eflags = true },
		2701	.each = .{ .once = &.{
		2702	.{ ._, ._, .lea, .dst0p, .leasi(.none, .src1, .@"8", .src1), ._, ._ },
		2703	.{ ._, ._, .neg, .dst0p, ._, ._, ._ },
		2704	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
		2705	} },
		2706	}, .{
		2707	.dst_constraints = .{.po2_elem_size},
		2708	.patterns = &.{
		2709	.{ .src = .{ .to_gpr, .to_mut_gpr } },
		2710	},
		2711	.dst_temps = .{.{ .ref = .src1 }},
		2712	.clobbers = .{ .eflags = true },
		2713	.each = .{ .once = &.{
		2714	.{ ._, ._l, .sa, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ },
		2715	.{ ._, ._, .neg, .src1p, ._, ._, ._ },
		2716	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .src1), ._, ._ },
		2717	} },
		2718	}, .{
		2719	.patterns = &.{
		2720	.{ .src = .{ .to_gpr, .to_gpr } },
		2721	},
		2722	.dst_temps = .{.{ .rc = .general_purpose }},
		2723	.clobbers = .{ .eflags = true },
		2724	.each = .{ .once = &.{
		2725	.{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .sub_src0_elem_size), ._ },
		2726	.{ ._, ._, .lea, .dst0p, .leai(.none, .src0, .dst0), ._, ._ },
		2727	} },
		2728	} }) catch \|err\| switch (err) {
		2729	error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
		2730	@tagName(air_tag),
		2731	cg.typeOf(bin_op.lhs).fmt(pt),
		2732	ops[0].tracking(cg),
		2733	ops[1].tracking(cg),
		2734	}),
		2735	else => \|e\| return e,
		2736	} else {
		2737	// hack around Sema OPV bugs
		2738	res[0] = ops[0];
		2739	}
		2740	for (ops) \|op\| for (res) \|r\| {
		2741	if (op.index == r.index) break;
		2742	} else try op.die(cg);
		2743	try res[0].moveTo(inst, cg);
		2744	},
		2745	.alloc => if (use_old) try cg.airAlloc(inst) else {
		2746	const ty = air_datas[@intFromEnum(inst)].ty;
		2747	var slot = try cg.tempInit(ty, .{ .lea_frame = .{
		2748	.index = try cg.allocMemPtr(inst),
		2749	} });
		2750	try slot.moveTo(inst, cg);
		2751	},
		2752	.inferred_alloc, .inferred_alloc_comptime => unreachable,
		2753	.ret_ptr => if (use_old) try cg.airRetPtr(inst) else {
		2754	const ty = air_datas[@intFromEnum(inst)].ty;
		2755	var slot = switch (cg.ret_mcv.long) {
		2756	else => unreachable,
		2757	.none => try cg.tempInit(ty, .{ .lea_frame = .{
		2758	.index = try cg.allocMemPtr(inst),
		2759	} }),
		2760	.load_frame => slot: {
		2761	var slot = try cg.tempInit(ty, cg.ret_mcv.long);
		2762	try slot.toOffset(cg.ret_mcv.short.indirect.off, cg);
		2763	break :slot slot;
		2764	},
		2765	};
		2766	try slot.moveTo(inst, cg);
		2767	},
		2768	.assembly => try cg.airAsm(inst),
		2769	.bit_and, .bit_or, .xor, .bool_and, .bool_or => \|air_tag\| if (use_old) try cg.airBinOp(inst, air_tag) else {
		2770	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
		2771	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		2772	var res: [1]Temp = undefined;
		2773	cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) {
		2774	else => unreachable,
		2775	.bit_and, .bool_and => .@"and",
		2776	.bit_or, .bool_or => .@"or",
		2777	.xor => .xor,
		2778	})) {
		2779	else => unreachable,
		2780	inline .@"and", .@"or", .xor => \|mir_tag\| comptime &.{ .{
		2781	.src_constraints = .{ .{ .size = .byte }, .{ .size = .byte } },
		2782	.patterns = &.{
		2783	.{ .src = .{ .mut_mem, .imm8 } },
		2784	.{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
		2785	.{ .src = .{ .to_mut_gpr, .imm8 } },
		2786	.{ .src = .{ .imm8, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2787	.{ .src = .{ .mut_mem, .to_gpr } },
		2788	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		2789	.{ .src = .{ .to_mut_gpr, .mem } },
		2790	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2791	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		2792	},
		2793	.dst_temps = .{.{ .ref = .src0 }},
		2794	.clobbers = .{ .eflags = true },
		2795	.each = .{ .once = &.{
		2796	.{ ._, ._, mir_tag, .dst0b, .src1b, ._, ._ },
		2797	} },
		2798	}, .{
		2799	.src_constraints = .{ .{ .size = .word }, .{ .size = .word } },
		2800	.patterns = &.{
		2801	.{ .src = .{ .mut_mem, .imm16 } },
		2802	.{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
		2803	.{ .src = .{ .to_mut_gpr, .imm16 } },
		2804	.{ .src = .{ .imm16, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2805	.{ .src = .{ .mut_mem, .to_gpr } },
		2806	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		2807	.{ .src = .{ .to_mut_gpr, .mem } },
		2808	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2809	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		2810	},
		2811	.dst_temps = .{.{ .ref = .src0 }},
		2812	.clobbers = .{ .eflags = true },
		2813	.each = .{ .once = &.{
		2814	.{ ._, ._, mir_tag, .dst0w, .src1w, ._, ._ },
		2815	} },
		2816	}, .{
		2817	.src_constraints = .{ .{ .size = .dword }, .{ .size = .dword } },
		2818	.patterns = &.{
		2819	.{ .src = .{ .mut_mem, .imm32 } },
		2820	.{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
		2821	.{ .src = .{ .to_mut_gpr, .imm32 } },
		2822	.{ .src = .{ .imm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2823	.{ .src = .{ .mut_mem, .to_gpr } },
		2824	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		2825	.{ .src = .{ .to_mut_gpr, .mem } },
		2826	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2827	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		2828	},
		2829	.dst_temps = .{.{ .ref = .src0 }},
		2830	.clobbers = .{ .eflags = true },
		2831	.each = .{ .once = &.{
		2832	.{ ._, ._, mir_tag, .dst0d, .src1d, ._, ._ },
		2833	} },
		2834	}, .{
		2835	.required_features = .{ .@"64bit", null, null, null },
		2836	.src_constraints = .{ .{ .size = .qword }, .{ .size = .qword } },
		2837	.patterns = &.{
		2838	.{ .src = .{ .mut_mem, .simm32 } },
		2839	.{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
		2840	.{ .src = .{ .to_mut_gpr, .simm32 } },
		2841	.{ .src = .{ .simm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2842	.{ .src = .{ .mut_mem, .to_gpr } },
		2843	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		2844	.{ .src = .{ .to_mut_gpr, .mem } },
		2845	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		2846	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		2847	},
		2848	.dst_temps = .{.{ .ref = .src0 }},
		2849	.clobbers = .{ .eflags = true },
		2850	.each = .{ .once = &.{
		2851	.{ ._, ._, mir_tag, .dst0q, .src1q, ._, ._ },
		2852	} },
		2853	}, .{
		2854	.required_features = .{ .mmx, null, null, null },
		2855	.src_constraints = .{ .{ .size = .qword }, .{ .size = .qword } },
		2856	.patterns = &.{
		2857	.{ .src = .{ .to_mut_mm, .mem } },
		2858	.{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
		2859	.{ .src = .{ .to_mut_mm, .to_mm } },
		2860	},
		2861	.dst_temps = .{.{ .ref = .src0 }},
		2862	.each = .{ .once = &.{
		2863	.{ ._, .p_, mir_tag, .dst0q, .src1q, ._, ._ },
		2864	} },
		2865	}, .{
		2866	.required_features = .{ .avx, null, null, null },
		2867	.src_constraints = .{ .{ .size = .xword }, .{ .size = .xword } },
		2868	.patterns = &.{
		2869	.{ .src = .{ .to_xmm, .mem } },
		2870	.{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
		2871	.{ .src = .{ .to_xmm, .to_xmm } },
		2872	},
		2873	.dst_temps = .{.{ .rc = .sse }},
		2874	.each = .{ .once = &.{
		2875	.{ ._, .vp_, mir_tag, .dst0x, .src0x, .src1x, ._ },
		2876	} },
		2877	}, .{
		2878	.required_features = .{ .sse2, null, null, null },
		2879	.src_constraints = .{ .{ .size = .xword }, .{ .size = .xword } },
		2880	.patterns = &.{
		2881	.{ .src = .{ .to_mut_xmm, .mem } },
		2882	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		2883	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		2884	},
		2885	.dst_temps = .{.{ .ref = .src0 }},
		2886	.each = .{ .once = &.{
		2887	.{ ._, .p_, mir_tag, .dst0x, .src1x, ._, ._ },
		2888	} },
		2889	}, .{
		2890	.required_features = .{ .sse, null, null, null },
		2891	.src_constraints = .{ .{ .size = .xword }, .{ .size = .xword } },
		2892	.patterns = &.{
		2893	.{ .src = .{ .to_mut_xmm, .mem } },
		2894	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		2895	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		2896	},
		2897	.dst_temps = .{.{ .ref = .src0 }},
		2898	.each = .{ .once = &.{
		2899	.{ ._, ._ps, mir_tag, .dst0x, .src1x, ._, ._ },
		2900	} },
		2901	}, .{
		2902	.required_features = .{ .avx2, null, null, null },
		2903	.src_constraints = .{ .{ .size = .yword }, .{ .size = .yword } },
		2904	.patterns = &.{
		2905	.{ .src = .{ .to_ymm, .mem } },
		2906	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		2907	.{ .src = .{ .to_ymm, .to_ymm } },
		2908	},
		2909	.dst_temps = .{.{ .rc = .sse }},
		2910	.each = .{ .once = &.{
		2911	.{ ._, .vp_, mir_tag, .dst0y, .src0y, .src1y, ._ },
		2912	} },
		2913	}, .{
		2914	.required_features = .{ .avx, null, null, null },
		2915	.src_constraints = .{ .{ .size = .yword }, .{ .size = .yword } },
		2916	.patterns = &.{
		2917	.{ .src = .{ .to_ymm, .mem } },
		2918	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		2919	.{ .src = .{ .to_ymm, .to_ymm } },
		2920	},
		2921	.dst_temps = .{.{ .rc = .sse }},
		2922	.each = .{ .once = &.{
		2923	.{ ._, .v_pd, mir_tag, .dst0y, .src0y, .src1y, ._ },
		2924	} },
		2925	}, .{
		2926	.required_features = .{ .avx2, null, null, null },
		2927	.src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword } },
		2928	.patterns = &.{
		2929	.{ .src = .{ .to_mem, .to_mem } },
		2930	},
		2931	.extra_temps = .{
		2932	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		2933	.{ .kind = .{ .rc = .sse } },
		2934	.unused,
		2935	.unused,
		2936	.unused,
		2937	.unused,
		2938	},
		2939	.dst_temps = .{.mem},
		2940	.clobbers = .{ .eflags = true },
		2941	.each = .{ .once = &.{
		2942	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		2943	.{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		2944	.{ ._, .vp_, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
		2945	.{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ },
		2946	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		2947	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		2948	} },
		2949	}, .{
		2950	.required_features = .{ .avx, null, null, null },
		2951	.src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword } },
		2952	.patterns = &.{
		2953	.{ .src = .{ .to_mem, .to_mem } },
		2954	},
		2955	.extra_temps = .{
		2956	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		2957	.{ .kind = .{ .rc = .sse } },
		2958	.unused,
		2959	.unused,
		2960	.unused,
		2961	.unused,
		2962	},
		2963	.dst_temps = .{.mem},
		2964	.clobbers = .{ .eflags = true },
		2965	.each = .{ .once = &.{
		2966	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		2967	.{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		2968	.{ ._, .v_pd, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
		2969	.{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ },
		2970	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		2971	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		2972	} },
		2973	}, .{
		2974	.required_features = .{ .avx, null, null, null },
		2975	.src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword } },
		2976	.patterns = &.{
		2977	.{ .src = .{ .to_mem, .to_mem } },
		2978	},
		2979	.extra_temps = .{
		2980	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		2981	.{ .kind = .{ .rc = .sse } },
		2982	.unused,
		2983	.unused,
		2984	.unused,
		2985	.unused,
		2986	},
		2987	.dst_temps = .{.mem},
		2988	.clobbers = .{ .eflags = true },
		2989	.each = .{ .once = &.{
		2990	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		2991	.{ .@"0:", .v_dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		2992	.{ ._, .vp_, mir_tag, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ },
		2993	.{ ._, .v_dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
		2994	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		2995	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		2996	} },
		2997	}, .{
		2998	.required_features = .{ .sse2, null, null, null },
		2999	.src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword } },
		3000	.patterns = &.{
		3001	.{ .src = .{ .to_mem, .to_mem } },
		3002	},
		3003	.extra_temps = .{
		3004	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3005	.{ .kind = .{ .rc = .sse } },
		3006	.unused,
		3007	.unused,
		3008	.unused,
		3009	.unused,
		3010	},
		3011	.dst_temps = .{.mem},
		3012	.clobbers = .{ .eflags = true },
		3013	.each = .{ .once = &.{
		3014	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3015	.{ .@"0:", ._dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		3016	.{ ._, .p_, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		3017	.{ ._, ._dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
		3018	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3019	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3020	} },
		3021	}, .{
		3022	.required_features = .{ .sse, null, null, null },
		3023	.src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword } },
		3024	.patterns = &.{
		3025	.{ .src = .{ .to_mem, .to_mem } },
		3026	},
		3027	.extra_temps = .{
		3028	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3029	.{ .kind = .{ .rc = .sse } },
		3030	.unused,
		3031	.unused,
		3032	.unused,
		3033	.unused,
		3034	},
		3035	.dst_temps = .{.mem},
		3036	.clobbers = .{ .eflags = true },
		3037	.each = .{ .once = &.{
		3038	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3039	.{ .@"0:", ._ps, .movu, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		3040	.{ ._, ._ps, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		3041	.{ ._, ._ps, .movu, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
		3042	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3043	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3044	} },
		3045	}, .{
		3046	.required_features = .{ .mmx, null, null, null },
		3047	.src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword } },
		3048	.patterns = &.{
		3049	.{ .src = .{ .to_mem, .to_mem } },
		3050	},
		3051	.extra_temps = .{
		3052	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3053	.{ .kind = .{ .rc = .mmx } },
		3054	.unused,
		3055	.unused,
		3056	.unused,
		3057	.unused,
		3058	},
		3059	.dst_temps = .{.mem},
		3060	.clobbers = .{ .eflags = true },
		3061	.each = .{ .once = &.{
		3062	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3063	.{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		3064	.{ ._, .p_, mir_tag, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		3065	.{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ },
		3066	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3067	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3068	} },
		3069	}, .{
		3070	.src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword } },
		3071	.patterns = &.{
		3072	.{ .src = .{ .to_mem, .to_mem } },
		3073	},
		3074	.extra_temps = .{
		3075	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3076	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3077	.unused,
		3078	.unused,
		3079	.unused,
		3080	.unused,
		3081	},
		3082	.dst_temps = .{.mem},
		3083	.clobbers = .{ .eflags = true },
		3084	.each = .{ .once = &.{
		3085	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3086	.{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
		3087	.{ ._, ._, mir_tag, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
		3088	.{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
		3089	.{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
		3090	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3091	} },
		3092	} },
		3093	}) catch \|err\| switch (err) {
		3094	error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
		3095	@tagName(air_tag),
		3096	cg.typeOf(bin_op.lhs).fmt(pt),
		3097	ops[0].tracking(cg),
		3098	ops[1].tracking(cg),
		3099	}),
		3100	else => \|e\| return e,
		3101	};
		3102	for (ops) \|op\| for (res) \|r\| {
		3103	if (op.index == r.index) break;
		3104	} else try op.die(cg);
		3105	try res[0].moveTo(inst, cg);
		3106	},
		3107	.not => \|air_tag\| if (use_old) try cg.airUnOp(inst, air_tag) else {
		3108	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		3109	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		3110	var res: [1]Temp = undefined;
		3111	cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
		3112	.src_constraints = .{ .{ .signed_or_exact_int = .byte }, .any },
		3113	.patterns = &.{
		3114	.{ .src = .{ .mut_mem, .none } },
		3115	.{ .src = .{ .to_mut_gpr, .none } },
		3116	},
		3117	.dst_temps = .{.{ .ref = .src0 }},
		3118	.each = .{ .once = &.{
		3119	.{ ._, ._, .not, .dst0b, ._, ._, ._ },
		3120	} },
		3121	}, .{
		3122	.src_constraints = .{ .{ .unsigned_int = .byte }, .any },
		3123	.patterns = &.{
		3124	.{ .src = .{ .mut_mem, .none } },
		3125	.{ .src = .{ .to_mut_gpr, .none } },
		3126	},
		3127	.dst_temps = .{.{ .ref = .src0 }},
		3128	.clobbers = .{ .eflags = true },
		3129	.each = .{ .once = &.{
		3130	.{ ._, ._, .xor, .dst0b, .sa(.src0, .add_umax), ._, ._ },
		3131	} },
		3132	}, .{
		3133	.src_constraints = .{ .{ .signed_or_exact_int = .word }, .any },
		3134	.patterns = &.{
		3135	.{ .src = .{ .mut_mem, .none } },
		3136	.{ .src = .{ .to_mut_gpr, .none } },
		3137	},
		3138	.dst_temps = .{.{ .ref = .src0 }},
		3139	.each = .{ .once = &.{
		3140	.{ ._, ._, .not, .dst0w, ._, ._, ._ },
		3141	} },
		3142	}, .{
		3143	.src_constraints = .{ .{ .unsigned_int = .word }, .any },
		3144	.patterns = &.{
		3145	.{ .src = .{ .mut_mem, .none } },
		3146	.{ .src = .{ .to_mut_gpr, .none } },
		3147	},
		3148	.dst_temps = .{.{ .ref = .src0 }},
		3149	.clobbers = .{ .eflags = true },
		3150	.each = .{ .once = &.{
		3151	.{ ._, ._, .xor, .dst0w, .sa(.src0, .add_umax), ._, ._ },
		3152	} },
		3153	}, .{
		3154	.src_constraints = .{ .{ .signed_or_exact_int = .dword }, .any },
		3155	.patterns = &.{
		3156	.{ .src = .{ .mut_mem, .none } },
		3157	.{ .src = .{ .to_mut_gpr, .none } },
		3158	},
		3159	.dst_temps = .{.{ .ref = .src0 }},
		3160	.each = .{ .once = &.{
		3161	.{ ._, ._, .not, .dst0d, ._, ._, ._ },
		3162	} },
		3163	}, .{
		3164	.src_constraints = .{ .{ .unsigned_int = .dword }, .any },
		3165	.patterns = &.{
		3166	.{ .src = .{ .mut_mem, .none } },
		3167	.{ .src = .{ .to_mut_gpr, .none } },
		3168	},
		3169	.dst_temps = .{.{ .ref = .src0 }},
		3170	.clobbers = .{ .eflags = true },
		3171	.each = .{ .once = &.{
		3172	.{ ._, ._, .xor, .dst0d, .sa(.src0, .add_umax), ._, ._ },
		3173	} },
		3174	}, .{
		3175	.required_features = .{ .@"64bit", null, null, null },
		3176	.src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any },
		3177	.patterns = &.{
		3178	.{ .src = .{ .mut_mem, .none } },
		3179	.{ .src = .{ .to_mut_gpr, .none } },
		3180	},
		3181	.dst_temps = .{.{ .ref = .src0 }},
		3182	.each = .{ .once = &.{
		3183	.{ ._, ._, .not, .dst0q, ._, ._, ._ },
		3184	} },
		3185	}, .{
		3186	.required_features = .{ .@"64bit", null, null, null },
		3187	.src_constraints = .{ .{ .unsigned_int = .qword }, .any },
		3188	.patterns = &.{
		3189	.{ .src = .{ .mem, .none } },
		3190	.{ .src = .{ .to_gpr, .none } },
		3191	},
		3192	.dst_temps = .{.{ .rc = .general_purpose }},
		3193	.each = .{ .once = &.{
		3194	.{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
		3195	.{ ._, ._, .xor, .dst0q, .src0q, ._, ._ },
		3196	} },
		3197	}, .{
		3198	.required_features = .{ .mmx, null, null, null },
		3199	.src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any },
		3200	.patterns = &.{
		3201	.{ .src = .{ .mem, .none } },
		3202	.{ .src = .{ .to_mm, .none } },
		3203	},
		3204	.dst_temps = .{.{ .rc = .mmx }},
		3205	.each = .{ .once = &.{
		3206	.{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ },
		3207	.{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ },
		3208	} },
		3209	}, .{
		3210	.required_features = .{ .mmx, null, null, null },
		3211	.src_constraints = .{ .{ .unsigned_int = .qword }, .any },
		3212	.patterns = &.{
		3213	.{ .src = .{ .to_mut_mm, .none } },
		3214	},
		3215	.extra_temps = .{
		3216	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3217	.{ .kind = .{ .umax_mem = .src0 } },
		3218	.unused,
		3219	.unused,
		3220	.unused,
		3221	.unused,
		3222	},
		3223	.dst_temps = .{.{ .ref = .src0 }},
		3224	.each = .{ .once = &.{
		3225	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3226	.{ ._, .p_, .xor, .dst0q, .lea(.qword, .tmp0), ._, ._ },
		3227	} },
		3228	}, .{
		3229	.required_features = .{ .avx, null, null, null },
		3230	.src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any },
		3231	.patterns = &.{
		3232	.{ .src = .{ .mem, .none } },
		3233	.{ .src = .{ .to_xmm, .none } },
		3234	},
		3235	.dst_temps = .{.{ .rc = .sse }},
		3236	.each = .{ .once = &.{
		3237	.{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ },
		3238	.{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ },
		3239	} },
		3240	}, .{
		3241	.required_features = .{ .avx, null, null, null },
		3242	.src_constraints = .{ .{ .unsigned_int = .xword }, .any },
		3243	.patterns = &.{
		3244	.{ .src = .{ .to_xmm, .none } },
		3245	},
		3246	.extra_temps = .{
		3247	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3248	.{ .kind = .{ .umax_mem = .src0 } },
		3249	.unused,
		3250	.unused,
		3251	.unused,
		3252	.unused,
		3253	},
		3254	.dst_temps = .{.{ .rc = .sse }},
		3255	.each = .{ .once = &.{
		3256	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3257	.{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
		3258	} },
		3259	}, .{
		3260	.required_features = .{ .sse2, null, null, null },
		3261	.src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any },
		3262	.patterns = &.{
		3263	.{ .src = .{ .mem, .none } },
		3264	.{ .src = .{ .to_xmm, .none } },
		3265	},
		3266	.dst_temps = .{.{ .rc = .sse }},
		3267	.each = .{ .once = &.{
		3268	.{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ },
		3269	.{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ },
		3270	} },
		3271	}, .{
		3272	.required_features = .{ .sse2, null, null, null },
		3273	.src_constraints = .{ .{ .unsigned_int = .xword }, .any },
		3274	.patterns = &.{
		3275	.{ .src = .{ .to_mut_xmm, .none } },
		3276	},
		3277	.extra_temps = .{
		3278	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3279	.{ .kind = .{ .umax_mem = .src0 } },
		3280	.unused,
		3281	.unused,
		3282	.unused,
		3283	.unused,
		3284	},
		3285	.dst_temps = .{.{ .ref = .src0 }},
		3286	.each = .{ .once = &.{
		3287	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3288	.{ ._, .p_, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
		3289	} },
		3290	}, .{
		3291	.required_features = .{ .sse, null, null, null },
		3292	.src_constraints = .{ .{ .int = .xword }, .any },
		3293	.patterns = &.{
		3294	.{ .src = .{ .to_mut_xmm, .none } },
		3295	},
		3296	.extra_temps = .{
		3297	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3298	.{ .kind = .{ .umax_mem = .src0 } },
		3299	.unused,
		3300	.unused,
		3301	.unused,
		3302	.unused,
		3303	},
		3304	.dst_temps = .{.{ .ref = .src0 }},
		3305	.each = .{ .once = &.{
		3306	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3307	.{ ._, ._ps, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
		3308	} },
		3309	}, .{
		3310	.required_features = .{ .avx2, null, null, null },
		3311	.src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any },
		3312	.patterns = &.{
		3313	.{ .src = .{ .mem, .none } },
		3314	.{ .src = .{ .to_ymm, .none } },
		3315	},
		3316	.dst_temps = .{.{ .rc = .sse }},
		3317	.each = .{ .once = &.{
		3318	.{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ },
		3319	.{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ },
		3320	} },
		3321	}, .{
		3322	.required_features = .{ .avx2, null, null, null },
		3323	.src_constraints = .{ .{ .unsigned_int = .yword }, .any },
		3324	.patterns = &.{
		3325	.{ .src = .{ .to_ymm, .none } },
		3326	},
		3327	.extra_temps = .{
		3328	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3329	.{ .kind = .{ .umax_mem = .src0 } },
		3330	.unused,
		3331	.unused,
		3332	.unused,
		3333	.unused,
		3334	},
		3335	.dst_temps = .{.{ .rc = .sse }},
		3336	.each = .{ .once = &.{
		3337	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3338	.{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
		3339	} },
		3340	}, .{
		3341	.required_features = .{ .avx, null, null, null },
		3342	.src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any },
		3343	.patterns = &.{
		3344	.{ .src = .{ .mem, .none } },
		3345	.{ .src = .{ .to_ymm, .none } },
		3346	},
		3347	.dst_temps = .{.{ .rc = .sse }},
		3348	.each = .{ .once = &.{
		3349	.{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) },
		3350	.{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ },
		3351	} },
		3352	}, .{
		3353	.required_features = .{ .avx, null, null, null },
		3354	.src_constraints = .{ .{ .unsigned_int = .yword }, .any },
		3355	.patterns = &.{
		3356	.{ .src = .{ .to_ymm, .none } },
		3357	},
		3358	.extra_temps = .{
		3359	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3360	.{ .kind = .{ .umax_mem = .src0 } },
		3361	.unused,
		3362	.unused,
		3363	.unused,
		3364	.unused,
		3365	},
		3366	.dst_temps = .{.{ .rc = .sse }},
		3367	.each = .{ .once = &.{
		3368	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3369	.{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
		3370	} },
		3371	}, .{
		3372	.required_features = .{ .avx2, null, null, null },
		3373	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any },
		3374	.patterns = &.{
		3375	.{ .src = .{ .to_mem, .none } },
		3376	},
		3377	.extra_temps = .{
		3378	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3379	.{ .kind = .{ .rc = .sse } },
		3380	.{ .kind = .{ .rc = .sse } },
		3381	.unused,
		3382	.unused,
		3383	.unused,
		3384	},
		3385	.dst_temps = .{.mem},
		3386	.clobbers = .{ .eflags = true },
		3387	.each = .{ .once = &.{
		3388	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3389	.{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
		3390	.{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
		3391	.{ ._, .v_dqu, .mov, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
		3392	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		3393	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3394	.{ .@"0:", .vp_, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
		3395	.{ ._, .v_dqa, .mov, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
		3396	} },
		3397	}, .{
		3398	.required_features = .{ .avx2, null, null, null },
		3399	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any },
		3400	.patterns = &.{
		3401	.{ .src = .{ .to_mem, .none } },
		3402	},
		3403	.extra_temps = .{
		3404	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3405	.{ .kind = .{ .rc = .sse } },
		3406	.{ .kind = .{ .rc = .sse } },
		3407	.unused,
		3408	.unused,
		3409	.unused,
		3410	},
		3411	.dst_temps = .{.mem},
		3412	.clobbers = .{ .eflags = true },
		3413	.each = .{ .once = &.{
		3414	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3415	.{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
		3416	.{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
		3417	.{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
		3418	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		3419	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3420	} },
		3421	}, .{
		3422	.required_features = .{ .avx, null, null, null },
		3423	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any },
		3424	.patterns = &.{
		3425	.{ .src = .{ .to_mem, .none } },
		3426	},
		3427	.extra_temps = .{
		3428	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3429	.{ .kind = .{ .rc = .sse } },
		3430	.{ .kind = .{ .rc = .sse } },
		3431	.unused,
		3432	.unused,
		3433	.unused,
		3434	},
		3435	.dst_temps = .{.mem},
		3436	.clobbers = .{ .eflags = true },
		3437	.each = .{ .once = &.{
		3438	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3439	.{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) },
		3440	.{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
		3441	.{ ._, .v_pd, .movu, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
		3442	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		3443	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3444	.{ .@"0:", .v_pd, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
		3445	.{ ._, .v_pd, .mova, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
		3446	} },
		3447	}, .{
		3448	.required_features = .{ .avx, null, null, null },
		3449	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any },
		3450	.patterns = &.{
		3451	.{ .src = .{ .to_mem, .none } },
		3452	},
		3453	.extra_temps = .{
		3454	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3455	.{ .kind = .{ .rc = .sse } },
		3456	.{ .kind = .{ .rc = .sse } },
		3457	.unused,
		3458	.unused,
		3459	.unused,
		3460	},
		3461	.dst_temps = .{.mem},
		3462	.clobbers = .{ .eflags = true },
		3463	.each = .{ .once = &.{
		3464	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3465	.{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) },
		3466	.{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
		3467	.{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
		3468	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		3469	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3470	} },
		3471	}, .{
		3472	.required_features = .{ .avx, null, null, null },
		3473	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		3474	.patterns = &.{
		3475	.{ .src = .{ .to_mem, .none } },
		3476	},
		3477	.extra_temps = .{
		3478	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3479	.{ .kind = .{ .rc = .sse } },
		3480	.{ .kind = .{ .rc = .sse } },
		3481	.unused,
		3482	.unused,
		3483	.unused,
		3484	},
		3485	.dst_temps = .{.mem},
		3486	.clobbers = .{ .eflags = true },
		3487	.each = .{ .once = &.{
		3488	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3489	.{ ._, .vp_q, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ },
		3490	.{ .@"0:", .v_, .xor, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._ },
		3491	.{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
		3492	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3493	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3494	} },
		3495	}, .{
		3496	.required_features = .{ .sse2, null, null, null },
		3497	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		3498	.patterns = &.{
		3499	.{ .src = .{ .to_mem, .none } },
		3500	},
		3501	.extra_temps = .{
		3502	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3503	.{ .kind = .{ .rc = .sse } },
		3504	.{ .kind = .{ .rc = .sse } },
		3505	.unused,
		3506	.unused,
		3507	.unused,
		3508	},
		3509	.dst_temps = .{.mem},
		3510	.clobbers = .{ .eflags = true },
		3511	.each = .{ .once = &.{
		3512	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3513	.{ ._, .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ },
		3514	.{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		3515	.{ ._, .p_, .xor, .tmp2x, .tmp1x, ._, ._ },
		3516	.{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
		3517	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3518	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3519	} },
		3520	}, .{
		3521	.required_features = .{ .@"64bit", null, null, null },
		3522	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		3523	.patterns = &.{
		3524	.{ .src = .{ .mut_mem, .none } },
		3525	},
		3526	.extra_temps = .{
		3527	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3528	.unused,
		3529	.unused,
		3530	.unused,
		3531	.unused,
		3532	.unused,
		3533	},
		3534	.dst_temps = .{.{ .ref = .src0 }},
		3535	.clobbers = .{ .eflags = true },
		3536	.each = .{ .once = &.{
		3537	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3538	.{ .@"0:", ._, .not, .memia(.dst0q, .tmp0, .add_size), ._, ._, ._ },
		3539	.{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, 8), ._, ._, ._ },
		3540	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3541	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3542	} },
		3543	}, .{
		3544	.required_features = .{ .@"64bit", null, null, null },
		3545	.src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		3546	.patterns = &.{
		3547	.{ .src = .{ .to_mem, .none } },
		3548	},
		3549	.extra_temps = .{
		3550	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3551	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3552	.unused,
		3553	.unused,
		3554	.unused,
		3555	.unused,
		3556	},
		3557	.dst_temps = .{.mem},
		3558	.clobbers = .{ .eflags = true },
		3559	.each = .{ .once = &.{
		3560	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		3561	.{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		3562	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3563	.{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ },
		3564	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3565	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3566	} },
		3567	}, .{
		3568	.required_features = .{ .@"64bit", null, null, null },
		3569	.src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = .dword } }, .any },
		3570	.patterns = &.{
		3571	.{ .src = .{ .mut_mem, .none } },
		3572	},
		3573	.extra_temps = .{
		3574	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3575	.unused,
		3576	.unused,
		3577	.unused,
		3578	.unused,
		3579	.unused,
		3580	},
		3581	.dst_temps = .{.{ .ref = .src0 }},
		3582	.clobbers = .{ .eflags = true },
		3583	.each = .{ .once = &.{
		3584	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3585	.{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
		3586	.{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
		3587	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3588	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3589	.{ ._, ._, .not, .memad(.dst0d, .add_size, -16), ._, ._, ._ },
		3590	} },
		3591	}, .{
		3592	.required_features = .{ .@"64bit", null, null, null },
		3593	.src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = .dword } }, .any },
		3594	.patterns = &.{
		3595	.{ .src = .{ .to_mem, .none } },
		3596	},
		3597	.extra_temps = .{
		3598	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3599	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3600	.unused,
		3601	.unused,
		3602	.unused,
		3603	.unused,
		3604	},
		3605	.dst_temps = .{.mem},
		3606	.clobbers = .{ .eflags = true },
		3607	.each = .{ .once = &.{
		3608	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3609	.{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
		3610	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3611	.{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
		3612	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3613	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3614	.{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ },
		3615	.{ ._, ._, .not, .tmp0d, ._, ._, ._ },
		3616	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ },
		3617	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ },
		3618	.{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ },
		3619	} },
		3620	}, .{
		3621	.required_features = .{ .@"64bit", null, null, null },
		3622	.src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
		3623	.patterns = &.{
		3624	.{ .src = .{ .mut_mem, .none } },
		3625	},
		3626	.extra_temps = .{
		3627	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3628	.unused,
		3629	.unused,
		3630	.unused,
		3631	.unused,
		3632	.unused,
		3633	},
		3634	.dst_temps = .{.{ .ref = .src0 }},
		3635	.clobbers = .{ .eflags = true },
		3636	.each = .{ .once = &.{
		3637	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3638	.{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
		3639	.{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
		3640	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3641	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3642	.{ ._, ._, .not, .memad(.dst0q, .add_size, -16), ._, ._, ._ },
		3643	} },
		3644	}, .{
		3645	.required_features = .{ .@"64bit", null, null, null },
		3646	.src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
		3647	.patterns = &.{
		3648	.{ .src = .{ .to_mem, .none } },
		3649	},
		3650	.extra_temps = .{
		3651	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3652	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3653	.unused,
		3654	.unused,
		3655	.unused,
		3656	.unused,
		3657	},
		3658	.dst_temps = .{.mem},
		3659	.clobbers = .{ .eflags = true },
		3660	.each = .{ .once = &.{
		3661	.{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
		3662	.{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
		3663	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3664	.{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
		3665	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3666	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3667	.{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ },
		3668	} },
		3669	}, .{
		3670	.required_features = .{ .@"64bit", null, null, null },
		3671	.src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = .dword } }, .any },
		3672	.patterns = &.{
		3673	.{ .src = .{ .mut_mem, .none } },
		3674	},
		3675	.extra_temps = .{
		3676	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3677	.unused,
		3678	.unused,
		3679	.unused,
		3680	.unused,
		3681	.unused,
		3682	},
		3683	.dst_temps = .{.{ .ref = .src0 }},
		3684	.clobbers = .{ .eflags = true },
		3685	.each = .{ .once = &.{
		3686	.{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
		3687	.{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
		3688	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3689	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3690	.{ ._, ._, .not, .memad(.dst0d, .add_size, -8), ._, ._, ._ },
		3691	} },
		3692	}, .{
		3693	.required_features = .{ .@"64bit", null, null, null },
		3694	.src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = .dword } }, .any },
		3695	.patterns = &.{
		3696	.{ .src = .{ .to_mem, .none } },
		3697	},
		3698	.extra_temps = .{
		3699	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3700	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3701	.unused,
		3702	.unused,
		3703	.unused,
		3704	.unused,
		3705	},
		3706	.dst_temps = .{.mem},
		3707	.clobbers = .{ .eflags = true },
		3708	.each = .{ .once = &.{
		3709	.{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
		3710	.{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
		3711	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3712	.{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
		3713	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3714	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3715	.{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ },
		3716	.{ ._, ._, .not, .tmp0d, ._, ._, ._ },
		3717	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ },
		3718	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ },
		3719	} },
		3720	}, .{
		3721	.required_features = .{ .@"64bit", null, null, null },
		3722	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any },
		3723	.patterns = &.{
		3724	.{ .src = .{ .mut_mem, .none } },
		3725	},
		3726	.extra_temps = .{
		3727	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3728	.unused,
		3729	.unused,
		3730	.unused,
		3731	.unused,
		3732	.unused,
		3733	},
		3734	.dst_temps = .{.{ .ref = .src0 }},
		3735	.clobbers = .{ .eflags = true },
		3736	.each = .{ .once = &.{
		3737	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3738	.{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
		3739	.{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
		3740	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3741	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3742	.{ ._, ._, .xor, .memad(.dst0d, .add_size, -16), .sa(.src0, .add_umax), ._, ._ },
		3743	} },
		3744	}, .{
		3745	.required_features = .{ .@"64bit", null, null, null },
		3746	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any },
		3747	.patterns = &.{
		3748	.{ .src = .{ .to_mem, .none } },
		3749	},
		3750	.extra_temps = .{
		3751	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3752	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3753	.unused,
		3754	.unused,
		3755	.unused,
		3756	.unused,
		3757	},
		3758	.dst_temps = .{.mem},
		3759	.clobbers = .{ .eflags = true },
		3760	.each = .{ .once = &.{
		3761	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3762	.{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
		3763	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3764	.{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
		3765	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3766	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3767	.{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ },
		3768	.{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ },
		3769	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ },
		3770	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ },
		3771	.{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ },
		3772	} },
		3773	}, .{
		3774	.required_features = .{ .@"64bit", null, null, null },
		3775	.src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any },
		3776	.patterns = &.{
		3777	.{ .src = .{ .mut_mem, .none } },
		3778	},
		3779	.extra_temps = .{
		3780	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3781	.unused,
		3782	.unused,
		3783	.unused,
		3784	.unused,
		3785	.unused,
		3786	},
		3787	.dst_temps = .{.{ .ref = .src0 }},
		3788	.clobbers = .{ .eflags = true },
		3789	.each = .{ .once = &.{
		3790	.{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
		3791	.{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
		3792	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3793	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3794	.{ ._, ._, .xor, .memad(.dst0d, .add_size, -8), .sa(.src0, .add_umax), ._, ._ },
		3795	} },
		3796	}, .{
		3797	.required_features = .{ .@"64bit", null, null, null },
		3798	.src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any },
		3799	.patterns = &.{
		3800	.{ .src = .{ .to_mem, .none } },
		3801	},
		3802	.extra_temps = .{
		3803	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3804	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3805	.unused,
		3806	.unused,
		3807	.unused,
		3808	.unused,
		3809	},
		3810	.dst_temps = .{.mem},
		3811	.clobbers = .{ .eflags = true },
		3812	.each = .{ .once = &.{
		3813	.{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
		3814	.{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
		3815	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3816	.{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
		3817	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3818	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3819	.{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ },
		3820	.{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ },
		3821	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ },
		3822	.{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ },
		3823	} },
		3824	}, .{
		3825	.required_features = .{ .@"64bit", null, null, null },
		3826	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		3827	.patterns = &.{
		3828	.{ .src = .{ .mut_mem, .none } },
		3829	},
		3830	.extra_temps = .{
		3831	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3832	.unused,
		3833	.unused,
		3834	.unused,
		3835	.unused,
		3836	.unused,
		3837	},
		3838	.dst_temps = .{.{ .ref = .src0 }},
		3839	.clobbers = .{ .eflags = true },
		3840	.each = .{ .once = &.{
		3841	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3842	.{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
		3843	.{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
		3844	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		3845	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3846	.{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
		3847	.{ ._, ._, .xor, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ },
		3848	} },
		3849	}, .{
		3850	.required_features = .{ .@"64bit", null, null, null },
		3851	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		3852	.patterns = &.{
		3853	.{ .src = .{ .to_mem, .none } },
		3854	},
		3855	.extra_temps = .{
		3856	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3857	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3858	.unused,
		3859	.unused,
		3860	.unused,
		3861	.unused,
		3862	},
		3863	.dst_temps = .{.mem},
		3864	.clobbers = .{ .eflags = true },
		3865	.each = .{ .once = &.{
		3866	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		3867	.{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
		3868	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3869	.{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
		3870	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3871	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3872	.{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
		3873	.{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ },
		3874	.{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ },
		3875	.{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ },
		3876	} },
		3877	}, .{
		3878	.required_features = .{ .@"64bit", null, null, null },
		3879	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		3880	.patterns = &.{
		3881	.{ .src = .{ .mut_mem, .none } },
		3882	},
		3883	.extra_temps = .{
		3884	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3885	.unused,
		3886	.unused,
		3887	.unused,
		3888	.unused,
		3889	.unused,
		3890	},
		3891	.dst_temps = .{.{ .ref = .src0 }},
		3892	.clobbers = .{ .eflags = true },
		3893	.each = .{ .once = &.{
		3894	.{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
		3895	.{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
		3896	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3897	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3898	.{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
		3899	.{ ._, ._, .xor, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ },
		3900	} },
		3901	}, .{
		3902	.required_features = .{ .@"64bit", null, null, null },
		3903	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		3904	.patterns = &.{
		3905	.{ .src = .{ .to_mem, .none } },
		3906	},
		3907	.extra_temps = .{
		3908	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		3909	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		3910	.unused,
		3911	.unused,
		3912	.unused,
		3913	.unused,
		3914	},
		3915	.dst_temps = .{.mem},
		3916	.clobbers = .{ .eflags = true },
		3917	.each = .{ .once = &.{
		3918	.{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
		3919	.{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
		3920	.{ ._, ._, .not, .tmp1q, ._, ._, ._ },
		3921	.{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
		3922	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		3923	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		3924	.{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
		3925	.{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ },
		3926	.{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ },
		3927	} },
		3928	}, .{
		3929	.required_features = .{ .mmx, null, null, null },
		3930	.src_constraints = .{ .{ .signed_int_or_full_vec = .qword }, .any },
		3931	.patterns = &.{
		3932	.{ .src = .{ .mem, .none } },
		3933	.{ .src = .{ .to_mm, .none } },
		3934	},
		3935	.dst_temps = .{.{ .rc = .mmx }},
		3936	.each = .{ .once = &.{
		3937	.{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ },
		3938	.{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ },
		3939	} },
		3940	}, .{
		3941	.required_features = .{ .mmx, null, null, null },
		3942	.src_constraints = .{ .{ .unsigned_int_vec = .qword }, .any },
		3943	.patterns = &.{
		3944	.{ .src = .{ .to_mut_mm, .none } },
		3945	},
		3946	.extra_temps = .{
		3947	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3948	.{ .kind = .{ .umax_mem = .src0 } },
		3949	.unused,
		3950	.unused,
		3951	.unused,
		3952	.unused,
		3953	},
		3954	.dst_temps = .{.{ .ref = .src0 }},
		3955	.each = .{ .once = &.{
		3956	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3957	.{ ._, .p_, .xor, .dst0q, .lea(.qword, .tmp0), ._, ._ },
		3958	} },
		3959	}, .{
		3960	.required_features = .{ .avx, null, null, null },
		3961	.src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any },
		3962	.patterns = &.{
		3963	.{ .src = .{ .mem, .none } },
		3964	.{ .src = .{ .to_xmm, .none } },
		3965	},
		3966	.dst_temps = .{.{ .rc = .sse }},
		3967	.each = .{ .once = &.{
		3968	.{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ },
		3969	.{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ },
		3970	} },
		3971	}, .{
		3972	.required_features = .{ .avx, null, null, null },
		3973	.src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any },
		3974	.patterns = &.{
		3975	.{ .src = .{ .to_xmm, .none } },
		3976	},
		3977	.extra_temps = .{
		3978	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		3979	.{ .kind = .{ .umax_mem = .src0 } },
		3980	.unused,
		3981	.unused,
		3982	.unused,
		3983	.unused,
		3984	},
		3985	.dst_temps = .{.{ .rc = .sse }},
		3986	.each = .{ .once = &.{
		3987	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		3988	.{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
		3989	} },
		3990	}, .{
		3991	.required_features = .{ .sse2, null, null, null },
		3992	.src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any },
		3993	.patterns = &.{
		3994	.{ .src = .{ .mem, .none } },
		3995	.{ .src = .{ .to_xmm, .none } },
		3996	},
		3997	.dst_temps = .{.{ .rc = .sse }},
		3998	.each = .{ .once = &.{
		3999	.{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ },
		4000	.{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ },
		4001	} },
		4002	}, .{
		4003	.required_features = .{ .sse2, null, null, null },
		4004	.src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any },
		4005	.patterns = &.{
		4006	.{ .src = .{ .to_mut_xmm, .none } },
		4007	},
		4008	.extra_temps = .{
		4009	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		4010	.{ .kind = .{ .umax_mem = .src0 } },
		4011	.unused,
		4012	.unused,
		4013	.unused,
		4014	.unused,
		4015	},
		4016	.dst_temps = .{.{ .ref = .src0 }},
		4017	.each = .{ .once = &.{
		4018	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		4019	.{ ._, .p_, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
		4020	} },
		4021	}, .{
		4022	.required_features = .{ .sse, null, null, null },
		4023	.src_constraints = .{ .{ .vec = .xword }, .any },
		4024	.patterns = &.{
		4025	.{ .src = .{ .to_mut_xmm, .none } },
		4026	},
		4027	.extra_temps = .{
		4028	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		4029	.{ .kind = .{ .umax_mem = .src0 } },
		4030	.unused,
		4031	.unused,
		4032	.unused,
		4033	.unused,
		4034	},
		4035	.dst_temps = .{.{ .ref = .src0 }},
		4036	.each = .{ .once = &.{
		4037	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		4038	.{ ._, ._ps, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
		4039	} },
		4040	}, .{
		4041	.required_features = .{ .avx2, null, null, null },
		4042	.src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any },
		4043	.patterns = &.{
		4044	.{ .src = .{ .mem, .none } },
		4045	.{ .src = .{ .to_ymm, .none } },
		4046	},
		4047	.dst_temps = .{.{ .rc = .sse }},
		4048	.each = .{ .once = &.{
		4049	.{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ },
		4050	.{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ },
		4051	} },
		4052	}, .{
		4053	.required_features = .{ .avx2, null, null, null },
		4054	.src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any },
		4055	.patterns = &.{
		4056	.{ .src = .{ .to_ymm, .none } },
		4057	},
		4058	.extra_temps = .{
		4059	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		4060	.{ .kind = .{ .umax_mem = .src0 } },
		4061	.unused,
		4062	.unused,
		4063	.unused,
		4064	.unused,
		4065	},
		4066	.dst_temps = .{.{ .rc = .sse }},
		4067	.each = .{ .once = &.{
		4068	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		4069	.{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
		4070	} },
		4071	}, .{
		4072	.required_features = .{ .avx, null, null, null },
		4073	.src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any },
		4074	.patterns = &.{
		4075	.{ .src = .{ .mem, .none } },
		4076	.{ .src = .{ .to_ymm, .none } },
		4077	},
		4078	.dst_temps = .{.{ .rc = .sse }},
		4079	.each = .{ .once = &.{
		4080	.{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) },
		4081	.{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ },
		4082	} },
		4083	}, .{
		4084	.required_features = .{ .avx, null, null, null },
		4085	.src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any },
		4086	.patterns = &.{
		4087	.{ .src = .{ .to_ymm, .none } },
		4088	},
		4089	.extra_temps = .{
		4090	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		4091	.{ .kind = .{ .umax_mem = .src0 } },
		4092	.unused,
		4093	.unused,
		4094	.unused,
		4095	.unused,
		4096	},
		4097	.dst_temps = .{.{ .rc = .sse }},
		4098	.each = .{ .once = &.{
		4099	.{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
		4100	.{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
		4101	} },
		4102	}, .{
		4103	.required_features = .{ .@"64bit", null, null, null },
		4104	.patterns = &.{
		4105	.{ .src = .{ .to_mem, .none } },
		4106	},
		4107	.extra_temps = .{
		4108	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		4109	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		4110	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		4111	.{ .kind = .{ .umax_mem = .src0 } },
		4112	.unused,
		4113	.unused,
		4114	},
		4115	.dst_temps = .{.mem},
		4116	.each = .{ .once = &.{
		4117	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_src0_size), ._, ._ },
		4118	.{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
		4119	.{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_src0_size), ._, ._ },
		4120	.{ ._, ._, .xor, .tmp2q, .leaia(.qword, .tmp1, .tmp0, .add_src0_size), ._, ._ },
		4121	.{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_src0_size), .tmp2q, ._, ._ },
		4122	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		4123	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		4124	} },
		4125	}, .{
		4126	.patterns = &.{
		4127	.{ .src = .{ .to_mem, .none } },
		4128	},
		4129	.extra_temps = .{
		4130	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		4131	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		4132	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4133	.{ .kind = .{ .umax_mem = .src0 } },
		4134	.unused,
		4135	.unused,
		4136	},
		4137	.dst_temps = .{.mem},
		4138	.each = .{ .once = &.{
		4139	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_src0_size), ._, ._ },
		4140	.{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
		4141	.{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_src0_size), ._, ._ },
		4142	.{ ._, ._, .xor, .tmp2d, .leaia(.dword, .tmp1, .tmp0, .add_src0_size), ._, ._ },
		4143	.{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_src0_size), .tmp2d, ._, ._ },
		4144	.{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
		4145	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		4146	} },
		4147	} }) catch \|err\| switch (err) {
		4148	error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
		4149	@tagName(air_tag),
		4150	cg.typeOf(ty_op.operand).fmt(pt),
		4151	ops[0].tracking(cg),
		4152	}),
		4153	else => \|e\| return e,
		4154	};
		4155	for (ops) \|op\| for (res) \|r\| {
		4156	if (op.index == r.index) break;
		4157	} else try op.die(cg);
		4158	try res[0].moveTo(inst, cg);
		4159	},
		4160
		4161	.block => if (use_old) try cg.airBlock(inst) else {
		4162	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		4163	const extra = cg.air.extraData(Air.Block, ty_pl.payload);
		4164	try cg.asmPseudo(.pseudo_dbg_enter_block_none);
		4165	try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
		4166	try cg.asmPseudo(.pseudo_dbg_leave_block_none);
		4167	},
		4168	.loop => if (use_old) try cg.airLoop(inst) else {
		4169	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		4170	const extra = cg.air.extraData(Air.Block, ty_pl.payload);
		4171	cg.scope_generation += 1;
		4172	try cg.loops.putNoClobber(cg.gpa, inst, .{
		4173	.state = try cg.saveState(),
		4174	.target = @intCast(cg.mir_instructions.len),
		4175	});
		4176	defer assert(cg.loops.remove(inst));
		4177	try cg.genBodyBlock(@ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
		4178	},
		4179	.repeat => if (use_old) try cg.airRepeat(inst) else {
		4180	const repeat = air_datas[@intFromEnum(inst)].repeat;
		4181	const loop = cg.loops.get(repeat.loop_inst).?;
		4182	try cg.restoreState(loop.state, &.{}, .{
		4183	.emit_instructions = true,
		4184	.update_tracking = false,
		4185	.resurrect = false,
		4186	.close_scope = true,
		4187	});
		4188	_ = try cg.asmJmpReloc(loop.target);
		4189	},
		4190	.br => try cg.airBr(inst),
		4191	.trap => try cg.asmOpOnly(.{ ._, .ud2 }),
		4192	.breakpoint => try cg.asmOpOnly(.{ ._, .int3 }),
		4193	.ret_addr => if (use_old) try cg.airRetAddr(inst) else {
		4194	var slot = try cg.tempInit(.usize, .{ .load_frame = .{
		4195	.index = .ret_addr,
		4196	} });
		4197	while (try slot.toRegClass(true, .general_purpose, cg)) {}
		4198	try slot.moveTo(inst, cg);
		4199	},
		4200	.frame_addr => if (use_old) try cg.airFrameAddress(inst) else {
		4201	var slot = try cg.tempInit(.usize, .{ .lea_frame = .{
		4202	.index = .base_ptr,
		4203	} });
		4204	try slot.moveTo(inst, cg);
		4205	},
		4206	.call => try cg.airCall(inst, .auto, .{ .safety = true }),
		4207	.call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }),
		4208	.call_never_tail => try cg.airCall(inst, .never_tail, .{ .safety = true }),
		4209	.call_never_inline => try cg.airCall(inst, .never_inline, .{ .safety = true }),
		4210
		4211	.clz => \|air_tag\| if (use_old) try cg.airClz(inst) else {
		4212	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		4213	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		4214	var res: [1]Temp = undefined;
		4215	cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
		4216	.required_features = .{ .slow_incdec, null, null, null },
		4217	.src_constraints = .{ .{ .exact_signed_int = 1 }, .any },
		4218	.patterns = &.{
		4219	.{ .src = .{ .mut_mem, .none } },
		4220	.{ .src = .{ .to_mut_gpr, .none } },
		4221	},
		4222	.dst_temps = .{.{ .ref = .src0 }},
		4223	.clobbers = .{ .eflags = true },
		4224	.each = .{ .once = &.{
		4225	.{ ._, ._, .add, .dst0b, .si(1), ._, ._ },
		4226	} },
		4227	}, .{
		4228	.src_constraints = .{ .{ .exact_signed_int = 1 }, .any },
		4229	.patterns = &.{
		4230	.{ .src = .{ .mut_mem, .none } },
		4231	.{ .src = .{ .to_mut_gpr, .none } },
		4232	},
		4233	.dst_temps = .{.{ .ref = .src0 }},
		4234	.clobbers = .{ .eflags = true },
		4235	.each = .{ .once = &.{
		4236	.{ ._, ._, .inc, .dst0b, ._, ._, ._ },
		4237	} },
		4238	}, .{
		4239	.src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any },
		4240	.patterns = &.{
		4241	.{ .src = .{ .mut_mem, .none } },
		4242	.{ .src = .{ .to_mut_gpr, .none } },
		4243	},
		4244	.dst_temps = .{.{ .ref = .src0 }},
		4245	.clobbers = .{ .eflags = true },
		4246	.each = .{ .once = &.{
		4247	.{ ._, ._, .xor, .dst0b, .si(1), ._, ._ },
		4248	} },
		4249	}, .{
		4250	.required_features = .{ .lzcnt, null, null, null },
		4251	.src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any },
		4252	.patterns = &.{
		4253	.{ .src = .{ .mem, .none } },
		4254	.{ .src = .{ .to_gpr, .none } },
		4255	},
		4256	.dst_temps = .{.{ .rc = .general_purpose }},
		4257	.clobbers = .{ .eflags = true },
		4258	.each = .{ .once = &.{
		4259	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4260	.{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ },
		4261	.{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		4262	} },
		4263	}, .{
		4264	.required_features = .{ .lzcnt, null, null, null },
		4265	.src_constraints = .{ .{ .signed_int = .byte }, .any },
		4266	.patterns = &.{
		4267	.{ .src = .{ .mem, .none } },
		4268	.{ .src = .{ .to_gpr, .none } },
		4269	},
		4270	.dst_temps = .{.{ .rc = .general_purpose }},
		4271	.clobbers = .{ .eflags = true },
		4272	.each = .{ .once = &.{
		4273	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4274	.{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
		4275	.{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ },
		4276	.{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		4277	} },
		4278	}, .{
		4279	.required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
		4280	.src_constraints = .{ .{ .exact_int = 16 }, .any },
		4281	.patterns = &.{
		4282	.{ .src = .{ .to_mut_gpr, .none } },
		4283	},
		4284	.dst_temps = .{.{ .ref = .src0 }},
		4285	.clobbers = .{ .eflags = true },
		4286	.each = .{ .once = &.{
		4287	.{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
		4288	} },
		4289	}, .{
		4290	.required_features = .{ .lzcnt, null, null, null },
		4291	.src_constraints = .{ .{ .exact_int = 16 }, .any },
		4292	.patterns = &.{
		4293	.{ .src = .{ .mem, .none } },
		4294	.{ .src = .{ .to_gpr, .none } },
		4295	},
		4296	.dst_temps = .{.{ .rc = .general_purpose }},
		4297	.clobbers = .{ .eflags = true },
		4298	.each = .{ .once = &.{
		4299	.{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
		4300	} },
		4301	}, .{
		4302	.required_features = .{ .lzcnt, null, null, null },
		4303	.src_constraints = .{ .{ .signed_int = .word }, .any },
		4304	.patterns = &.{
		4305	.{ .src = .{ .to_mut_gpr, .none } },
		4306	},
		4307	.dst_temps = .{.{ .ref = .src0 }},
		4308	.clobbers = .{ .eflags = true },
		4309	.each = .{ .once = &.{
		4310	.{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
		4311	.{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
		4312	.{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
		4313	} },
		4314	}, .{
		4315	.required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
		4316	.src_constraints = .{ .{ .unsigned_int = .word }, .any },
		4317	.patterns = &.{
		4318	.{ .src = .{ .to_mut_gpr, .none } },
		4319	},
		4320	.dst_temps = .{.{ .ref = .src0 }},
		4321	.clobbers = .{ .eflags = true },
		4322	.each = .{ .once = &.{
		4323	.{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
		4324	.{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
		4325	} },
		4326	}, .{
		4327	.required_features = .{ .lzcnt, null, null, null },
		4328	.src_constraints = .{ .{ .unsigned_int = .word }, .any },
		4329	.patterns = &.{
		4330	.{ .src = .{ .mem, .none } },
		4331	.{ .src = .{ .to_gpr, .none } },
		4332	},
		4333	.dst_temps = .{.{ .rc = .general_purpose }},
		4334	.clobbers = .{ .eflags = true },
		4335	.each = .{ .once = &.{
		4336	.{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
		4337	.{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
		4338	} },
		4339	}, .{
		4340	.required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
		4341	.src_constraints = .{ .{ .exact_int = 32 }, .any },
		4342	.patterns = &.{
		4343	.{ .src = .{ .to_mut_gpr, .none } },
		4344	},
		4345	.dst_temps = .{.{ .ref = .src0 }},
		4346	.clobbers = .{ .eflags = true },
		4347	.each = .{ .once = &.{
		4348	.{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
		4349	} },
		4350	}, .{
		4351	.required_features = .{ .lzcnt, null, null, null },
		4352	.src_constraints = .{ .{ .exact_int = 32 }, .any },
		4353	.patterns = &.{
		4354	.{ .src = .{ .mem, .none } },
		4355	.{ .src = .{ .to_gpr, .none } },
		4356	},
		4357	.dst_temps = .{.{ .rc = .general_purpose }},
		4358	.clobbers = .{ .eflags = true },
		4359	.each = .{ .once = &.{
		4360	.{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
		4361	} },
		4362	}, .{
		4363	.required_features = .{ .lzcnt, null, null, null },
		4364	.src_constraints = .{ .{ .signed_int = .dword }, .any },
		4365	.patterns = &.{
		4366	.{ .src = .{ .to_mut_gpr, .none } },
		4367	},
		4368	.dst_temps = .{.{ .ref = .src0 }},
		4369	.clobbers = .{ .eflags = true },
		4370	.each = .{ .once = &.{
		4371	.{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
		4372	.{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
		4373	.{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		4374	} },
		4375	}, .{
		4376	.required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
		4377	.src_constraints = .{ .{ .unsigned_int = .dword }, .any },
		4378	.patterns = &.{
		4379	.{ .src = .{ .to_mut_gpr, .none } },
		4380	},
		4381	.dst_temps = .{.{ .ref = .src0 }},
		4382	.clobbers = .{ .eflags = true },
		4383	.each = .{ .once = &.{
		4384	.{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
		4385	.{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		4386	} },
		4387	}, .{
		4388	.required_features = .{ .lzcnt, null, null, null },
		4389	.src_constraints = .{ .{ .unsigned_int = .dword }, .any },
		4390	.patterns = &.{
		4391	.{ .src = .{ .mem, .none } },
		4392	.{ .src = .{ .to_gpr, .none } },
		4393	},
		4394	.dst_temps = .{.{ .rc = .general_purpose }},
		4395	.clobbers = .{ .eflags = true },
		4396	.each = .{ .once = &.{
		4397	.{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
		4398	.{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		4399	} },
		4400	}, .{
		4401	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		4402	.src_constraints = .{ .{ .exact_int = 64 }, .any },
		4403	.patterns = &.{
		4404	.{ .src = .{ .to_mut_gpr, .none } },
		4405	},
		4406	.dst_temps = .{.{ .ref = .src0 }},
		4407	.clobbers = .{ .eflags = true },
		4408	.each = .{ .once = &.{
		4409	.{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
		4410	} },
		4411	}, .{
		4412	.required_features = .{ .@"64bit", .lzcnt, null, null },
		4413	.src_constraints = .{ .{ .exact_int = 64 }, .any },
		4414	.patterns = &.{
		4415	.{ .src = .{ .mem, .none } },
		4416	.{ .src = .{ .to_gpr, .none } },
		4417	},
		4418	.dst_temps = .{.{ .rc = .general_purpose }},
		4419	.clobbers = .{ .eflags = true },
		4420	.each = .{ .once = &.{
		4421	.{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
		4422	} },
		4423	}, .{
		4424	.required_features = .{ .@"64bit", .lzcnt, null, null },
		4425	.src_constraints = .{ .{ .signed_int = .qword }, .any },
		4426	.patterns = &.{
		4427	.{ .src = .{ .mem, .none } },
		4428	.{ .src = .{ .to_gpr, .none } },
		4429	},
		4430	.dst_temps = .{.{ .rc = .general_purpose }},
		4431	.clobbers = .{ .eflags = true },
		4432	.each = .{ .once = &.{
		4433	.{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
		4434	.{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ },
		4435	.{ ._, ._, .lzcnt, .dst0q, .dst0q, ._, ._ },
		4436	.{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
		4437	} },
		4438	}, .{
		4439	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		4440	.src_constraints = .{ .{ .unsigned_int = .qword }, .any },
		4441	.patterns = &.{
		4442	.{ .src = .{ .to_mut_gpr, .none } },
		4443	},
		4444	.dst_temps = .{.{ .ref = .src0 }},
		4445	.clobbers = .{ .eflags = true },
		4446	.each = .{ .once = &.{
		4447	.{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
		4448	.{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
		4449	} },
		4450	}, .{
		4451	.required_features = .{ .@"64bit", .lzcnt, null, null },
		4452	.src_constraints = .{ .{ .unsigned_int = .qword }, .any },
		4453	.patterns = &.{
		4454	.{ .src = .{ .mem, .none } },
		4455	.{ .src = .{ .to_gpr, .none } },
		4456	},
		4457	.dst_temps = .{.{ .rc = .general_purpose }},
		4458	.clobbers = .{ .eflags = true },
		4459	.each = .{ .once = &.{
		4460	.{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
		4461	.{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
		4462	} },
		4463	}, .{
		4464	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4465	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
		4466	.patterns = &.{
		4467	.{ .src = .{ .mem, .none } },
		4468	.{ .src = .{ .to_gpr, .none } },
		4469	},
		4470	.extra_temps = .{
		4471	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4472	.unused,
		4473	.unused,
		4474	.unused,
		4475	.unused,
		4476	.unused,
		4477	},
		4478	.dst_temps = .{.{ .rc = .general_purpose }},
		4479	.clobbers = .{ .eflags = true },
		4480	.each = .{ .once = &.{
		4481	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4482	.{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
		4483	.{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4484	.{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
		4485	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4486	} },
		4487	}, .{
		4488	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4489	.src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
		4490	.patterns = &.{
		4491	.{ .src = .{ .mem, .none } },
		4492	.{ .src = .{ .to_gpr, .none } },
		4493	},
		4494	.extra_temps = .{
		4495	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4496	.unused,
		4497	.unused,
		4498	.unused,
		4499	.unused,
		4500	.unused,
		4501	},
		4502	.dst_temps = .{.{ .rc = .general_purpose }},
		4503	.clobbers = .{ .eflags = true },
		4504	.each = .{ .once = &.{
		4505	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4506	.{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
		4507	.{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
		4508	.{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4509	.{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
		4510	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4511	} },
		4512	}, .{
		4513	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4514	.src_constraints = .{ .{ .signed_int = .byte }, .any },
		4515	.patterns = &.{
		4516	.{ .src = .{ .mem, .none } },
		4517	.{ .src = .{ .to_gpr, .none } },
		4518	},
		4519	.extra_temps = .{
		4520	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4521	.unused,
		4522	.unused,
		4523	.unused,
		4524	.unused,
		4525	.unused,
		4526	},
		4527	.dst_temps = .{.{ .rc = .general_purpose }},
		4528	.clobbers = .{ .eflags = true },
		4529	.each = .{ .once = &.{
		4530	.{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
		4531	.{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
		4532	.{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
		4533	.{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
		4534	.{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
		4535	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4536	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		4537	} },
		4538	}, .{
		4539	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4540	.src_constraints = .{ .{ .unsigned_int = .byte }, .any },
		4541	.patterns = &.{
		4542	.{ .src = .{ .mem, .none } },
		4543	.{ .src = .{ .to_gpr, .none } },
		4544	},
		4545	.extra_temps = .{
		4546	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4547	.unused,
		4548	.unused,
		4549	.unused,
		4550	.unused,
		4551	.unused,
		4552	},
		4553	.dst_temps = .{.{ .rc = .general_purpose }},
		4554	.clobbers = .{ .eflags = true },
		4555	.each = .{ .once = &.{
		4556	.{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
		4557	.{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
		4558	.{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
		4559	.{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
		4560	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4561	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		4562	} },
		4563	}, .{
		4564	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4565	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
		4566	.patterns = &.{
		4567	.{ .src = .{ .mem, .none } },
		4568	.{ .src = .{ .to_gpr, .none } },
		4569	},
		4570	.dst_temps = .{.{ .rc = .general_purpose }},
		4571	.clobbers = .{ .eflags = true },
		4572	.each = .{ .once = &.{
		4573	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4574	.{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
		4575	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		4576	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4577	.{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4578	} },
		4579	}, .{
		4580	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4581	.src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
		4582	.patterns = &.{
		4583	.{ .src = .{ .mem, .none } },
		4584	.{ .src = .{ .to_gpr, .none } },
		4585	},
		4586	.dst_temps = .{.{ .rc = .general_purpose }},
		4587	.clobbers = .{ .eflags = true },
		4588	.each = .{ .once = &.{
		4589	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4590	.{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
		4591	.{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
		4592	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		4593	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4594	.{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4595	} },
		4596	}, .{
		4597	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4598	.src_constraints = .{ .{ .signed_int = .byte }, .any },
		4599	.patterns = &.{
		4600	.{ .src = .{ .mem, .none } },
		4601	.{ .src = .{ .to_gpr, .none } },
		4602	},
		4603	.extra_temps = .{
		4604	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4605	.unused,
		4606	.unused,
		4607	.unused,
		4608	.unused,
		4609	.unused,
		4610	},
		4611	.dst_temps = .{.{ .rc = .general_purpose }},
		4612	.clobbers = .{ .eflags = true },
		4613	.each = .{ .once = &.{
		4614	.{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
		4615	.{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
		4616	.{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
		4617	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		4618	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		4619	.{ ._, ._c, .st, ._, ._, ._, ._ },
		4620	.{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
		4621	} },
		4622	}, .{
		4623	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4624	.src_constraints = .{ .{ .unsigned_int = .byte }, .any },
		4625	.patterns = &.{
		4626	.{ .src = .{ .mem, .none } },
		4627	.{ .src = .{ .to_gpr, .none } },
		4628	},
		4629	.extra_temps = .{
		4630	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4631	.unused,
		4632	.unused,
		4633	.unused,
		4634	.unused,
		4635	.unused,
		4636	},
		4637	.dst_temps = .{.{ .rc = .general_purpose }},
		4638	.clobbers = .{ .eflags = true },
		4639	.each = .{ .once = &.{
		4640	.{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
		4641	.{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
		4642	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		4643	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		4644	.{ ._, ._c, .st, ._, ._, ._, ._ },
		4645	.{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
		4646	} },
		4647	}, .{
		4648	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
		4649	.patterns = &.{
		4650	.{ .src = .{ .mem, .none } },
		4651	.{ .src = .{ .to_gpr, .none } },
		4652	},
		4653	.extra_temps = .{
		4654	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4655	.unused,
		4656	.unused,
		4657	.unused,
		4658	.unused,
		4659	.unused,
		4660	},
		4661	.dst_temps = .{.{ .rc = .general_purpose }},
		4662	.clobbers = .{ .eflags = true },
		4663	.each = .{ .once = &.{
		4664	.{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
		4665	.{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4666	.{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ },
		4667	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4668	} },
		4669	}, .{
		4670	.src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
		4671	.patterns = &.{
		4672	.{ .src = .{ .mem, .none } },
		4673	.{ .src = .{ .to_gpr, .none } },
		4674	},
		4675	.extra_temps = .{
		4676	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4677	.unused,
		4678	.unused,
		4679	.unused,
		4680	.unused,
		4681	.unused,
		4682	},
		4683	.dst_temps = .{.{ .rc = .general_purpose }},
		4684	.clobbers = .{ .eflags = true },
		4685	.each = .{ .once = &.{
		4686	.{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
		4687	.{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
		4688	.{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4689	.{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ },
		4690	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4691	} },
		4692	}, .{
		4693	.src_constraints = .{ .{ .signed_int = .byte }, .any },
		4694	.patterns = &.{
		4695	.{ .src = .{ .mem, .none } },
		4696	.{ .src = .{ .to_gpr, .none } },
		4697	},
		4698	.extra_temps = .{
		4699	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4700	.unused,
		4701	.unused,
		4702	.unused,
		4703	.unused,
		4704	.unused,
		4705	},
		4706	.dst_temps = .{.{ .rc = .general_purpose }},
		4707	.clobbers = .{ .eflags = true },
		4708	.each = .{ .once = &.{
		4709	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4710	.{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
		4711	.{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
		4712	.{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ },
		4713	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4714	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		4715	} },
		4716	}, .{
		4717	.src_constraints = .{ .{ .unsigned_int = .byte }, .any },
		4718	.patterns = &.{
		4719	.{ .src = .{ .mem, .none } },
		4720	.{ .src = .{ .to_gpr, .none } },
		4721	},
		4722	.extra_temps = .{
		4723	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4724	.unused,
		4725	.unused,
		4726	.unused,
		4727	.unused,
		4728	.unused,
		4729	},
		4730	.dst_temps = .{.{ .rc = .general_purpose }},
		4731	.clobbers = .{ .eflags = true },
		4732	.each = .{ .once = &.{
		4733	.{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
		4734	.{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
		4735	.{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ },
		4736	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4737	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		4738	} },
		4739	}, .{
		4740	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4741	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
		4742	.patterns = &.{
		4743	.{ .src = .{ .to_mut_gpr, .none } },
		4744	},
		4745	.dst_temps = .{.{ .rc = .general_purpose }},
		4746	.clobbers = .{ .eflags = true },
		4747	.each = .{ .once = &.{
		4748	.{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
		4749	.{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4750	.{ ._, ._nz, .cmov, .dst0w, .src0w, ._, ._ },
		4751	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4752	} },
		4753	}, .{
		4754	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4755	.src_constraints = .{ .{ .signed_int = .word }, .any },
		4756	.patterns = &.{
		4757	.{ .src = .{ .to_mut_gpr, .none } },
		4758	},
		4759	.dst_temps = .{.{ .rc = .general_purpose }},
		4760	.clobbers = .{ .eflags = true },
		4761	.each = .{ .once = &.{
		4762	.{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
		4763	.{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
		4764	.{ ._, ._, .mov, .dst0w, .si(0xff), ._, ._ },
		4765	.{ ._, ._z, .cmov, .src0w, .dst0w, ._, ._ },
		4766	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4767	.{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
		4768	} },
		4769	}, .{
		4770	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4771	.src_constraints = .{ .{ .unsigned_int = .word }, .any },
		4772	.patterns = &.{
		4773	.{ .src = .{ .to_mut_gpr, .none } },
		4774	},
		4775	.dst_temps = .{.{ .rc = .general_purpose }},
		4776	.clobbers = .{ .eflags = true },
		4777	.each = .{ .once = &.{
		4778	.{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
		4779	.{ ._, ._, .mov, .dst0w, .si(0xff), ._, ._ },
		4780	.{ ._, ._z, .cmov, .src0w, .dst0w, ._, ._ },
		4781	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4782	.{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
		4783	} },
		4784	}, .{
		4785	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4786	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
		4787	.patterns = &.{
		4788	.{ .src = .{ .to_mut_gpr, .none } },
		4789	},
		4790	.dst_temps = .{.{ .ref = .src0 }},
		4791	.clobbers = .{ .eflags = true },
		4792	.each = .{ .once = &.{
		4793	.{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ },
		4794	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		4795	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4796	.{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4797	} },
		4798	}, .{
		4799	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4800	.src_constraints = .{ .{ .signed_int = .word }, .any },
		4801	.patterns = &.{
		4802	.{ .src = .{ .to_mut_gpr, .none } },
		4803	},
		4804	.dst_temps = .{.{ .rc = .general_purpose }},
		4805	.clobbers = .{ .eflags = true },
		4806	.each = .{ .once = &.{
		4807	.{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
		4808	.{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
		4809	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		4810	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		4811	.{ ._, ._c, .st, ._, ._, ._, ._ },
		4812	.{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
		4813	} },
		4814	}, .{
		4815	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4816	.src_constraints = .{ .{ .unsigned_int = .word }, .any },
		4817	.patterns = &.{
		4818	.{ .src = .{ .to_mut_gpr, .none } },
		4819	},
		4820	.dst_temps = .{.{ .rc = .general_purpose }},
		4821	.clobbers = .{ .eflags = true },
		4822	.each = .{ .once = &.{
		4823	.{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
		4824	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		4825	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		4826	.{ ._, ._c, .st, ._, ._, ._, ._ },
		4827	.{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
		4828	} },
		4829	}, .{
		4830	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
		4831	.patterns = &.{
		4832	.{ .src = .{ .mem, .none } },
		4833	.{ .src = .{ .to_gpr, .none } },
		4834	},
		4835	.dst_temps = .{.{ .rc = .general_purpose }},
		4836	.clobbers = .{ .eflags = true },
		4837	.each = .{ .once = &.{
		4838	.{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4839	.{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ },
		4840	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4841	} },
		4842	}, .{
		4843	.src_constraints = .{ .{ .signed_int = .word }, .any },
		4844	.patterns = &.{
		4845	.{ .src = .{ .to_mut_gpr, .none } },
		4846	},
		4847	.extra_temps = .{
		4848	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		4849	.unused,
		4850	.unused,
		4851	.unused,
		4852	.unused,
		4853	.unused,
		4854	},
		4855	.dst_temps = .{.{ .rc = .general_purpose }},
		4856	.clobbers = .{ .eflags = true },
		4857	.each = .{ .once = &.{
		4858	.{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
		4859	.{ ._, ._, .mov, .tmp0w, .si(0xff), ._, ._ },
		4860	.{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ },
		4861	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4862	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		4863	} },
		4864	}, .{
		4865	.src_constraints = .{ .{ .unsigned_int = .word }, .any },
		4866	.patterns = &.{
		4867	.{ .src = .{ .mem, .none } },
		4868	.{ .src = .{ .to_gpr, .none } },
		4869	},
		4870	.extra_temps = .{
		4871	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		4872	.unused,
		4873	.unused,
		4874	.unused,
		4875	.unused,
		4876	.unused,
		4877	},
		4878	.dst_temps = .{.{ .rc = .general_purpose }},
		4879	.clobbers = .{ .eflags = true },
		4880	.each = .{ .once = &.{
		4881	.{ ._, ._, .mov, .tmp0w, .si(0xff), ._, ._ },
		4882	.{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ },
		4883	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4884	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		4885	} },
		4886	}, .{
		4887	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4888	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
		4889	.patterns = &.{
		4890	.{ .src = .{ .to_mut_gpr, .none } },
		4891	},
		4892	.dst_temps = .{.{ .rc = .general_purpose }},
		4893	.clobbers = .{ .eflags = true },
		4894	.each = .{ .once = &.{
		4895	.{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
		4896	.{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4897	.{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ },
		4898	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4899	} },
		4900	}, .{
		4901	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4902	.src_constraints = .{ .{ .signed_int = .dword }, .any },
		4903	.patterns = &.{
		4904	.{ .src = .{ .to_mut_gpr, .none } },
		4905	},
		4906	.dst_temps = .{.{ .rc = .general_purpose }},
		4907	.clobbers = .{ .eflags = true },
		4908	.each = .{ .once = &.{
		4909	.{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
		4910	.{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
		4911	.{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
		4912	.{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
		4913	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4914	.{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
		4915	} },
		4916	}, .{
		4917	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		4918	.src_constraints = .{ .{ .unsigned_int = .dword }, .any },
		4919	.patterns = &.{
		4920	.{ .src = .{ .to_mut_gpr, .none } },
		4921	},
		4922	.dst_temps = .{.{ .rc = .general_purpose }},
		4923	.clobbers = .{ .eflags = true },
		4924	.each = .{ .once = &.{
		4925	.{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
		4926	.{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
		4927	.{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
		4928	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4929	.{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
		4930	} },
		4931	}, .{
		4932	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4933	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
		4934	.patterns = &.{
		4935	.{ .src = .{ .to_mut_gpr, .none } },
		4936	},
		4937	.dst_temps = .{.{ .ref = .src0 }},
		4938	.clobbers = .{ .eflags = true },
		4939	.each = .{ .once = &.{
		4940	.{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ },
		4941	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		4942	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4943	.{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4944	} },
		4945	}, .{
		4946	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4947	.src_constraints = .{ .{ .signed_int = .dword }, .any },
		4948	.patterns = &.{
		4949	.{ .src = .{ .to_mut_gpr, .none } },
		4950	},
		4951	.dst_temps = .{.{ .rc = .general_purpose }},
		4952	.clobbers = .{ .eflags = true },
		4953	.each = .{ .once = &.{
		4954	.{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
		4955	.{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
		4956	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		4957	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		4958	.{ ._, ._c, .st, ._, ._, ._, ._ },
		4959	.{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
		4960	} },
		4961	}, .{
		4962	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		4963	.src_constraints = .{ .{ .unsigned_int = .dword }, .any },
		4964	.patterns = &.{
		4965	.{ .src = .{ .to_mut_gpr, .none } },
		4966	},
		4967	.dst_temps = .{.{ .rc = .general_purpose }},
		4968	.clobbers = .{ .eflags = true },
		4969	.each = .{ .once = &.{
		4970	.{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
		4971	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		4972	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		4973	.{ ._, ._c, .st, ._, ._, ._, ._ },
		4974	.{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
		4975	} },
		4976	}, .{
		4977	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
		4978	.patterns = &.{
		4979	.{ .src = .{ .mem, .none } },
		4980	.{ .src = .{ .to_gpr, .none } },
		4981	},
		4982	.dst_temps = .{.{ .rc = .general_purpose }},
		4983	.clobbers = .{ .eflags = true },
		4984	.each = .{ .once = &.{
		4985	.{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		4986	.{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ },
		4987	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		4988	} },
		4989	}, .{
		4990	.src_constraints = .{ .{ .signed_int = .dword }, .any },
		4991	.patterns = &.{
		4992	.{ .src = .{ .to_mut_gpr, .none } },
		4993	},
		4994	.extra_temps = .{
		4995	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		4996	.unused,
		4997	.unused,
		4998	.unused,
		4999	.unused,
		5000	.unused,
		5001	},
		5002	.dst_temps = .{.{ .rc = .general_purpose }},
		5003	.clobbers = .{ .eflags = true },
		5004	.each = .{ .once = &.{
		5005	.{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
		5006	.{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
		5007	.{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ },
		5008	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5009	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		5010	} },
		5011	}, .{
		5012	.src_constraints = .{ .{ .unsigned_int = .dword }, .any },
		5013	.patterns = &.{
		5014	.{ .src = .{ .mem, .none } },
		5015	.{ .src = .{ .to_gpr, .none } },
		5016	},
		5017	.extra_temps = .{
		5018	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5019	.unused,
		5020	.unused,
		5021	.unused,
		5022	.unused,
		5023	.unused,
		5024	},
		5025	.dst_temps = .{.{ .rc = .general_purpose }},
		5026	.clobbers = .{ .eflags = true },
		5027	.each = .{ .once = &.{
		5028	.{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
		5029	.{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ },
		5030	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5031	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		5032	} },
		5033	}, .{
		5034	.required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
		5035	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
		5036	.patterns = &.{
		5037	.{ .src = .{ .to_mut_gpr, .none } },
		5038	},
		5039	.dst_temps = .{.{ .rc = .general_purpose }},
		5040	.clobbers = .{ .eflags = true },
		5041	.each = .{ .once = &.{
		5042	.{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
		5043	.{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		5044	.{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ },
		5045	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5046	} },
		5047	}, .{
		5048	.required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
		5049	.src_constraints = .{ .{ .signed_int = .qword }, .any },
		5050	.patterns = &.{
		5051	.{ .src = .{ .mem, .none } },
		5052	.{ .src = .{ .to_gpr, .none } },
		5053	},
		5054	.extra_temps = .{
		5055	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5056	.unused,
		5057	.unused,
		5058	.unused,
		5059	.unused,
		5060	.unused,
		5061	},
		5062	.dst_temps = .{.{ .rc = .general_purpose }},
		5063	.clobbers = .{ .eflags = true },
		5064	.each = .{ .once = &.{
		5065	.{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
		5066	.{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ },
		5067	.{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ },
		5068	.{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
		5069	.{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
		5070	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5071	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		5072	} },
		5073	}, .{
		5074	.required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
		5075	.src_constraints = .{ .{ .unsigned_int = .qword }, .any },
		5076	.patterns = &.{
		5077	.{ .src = .{ .to_mut_gpr, .none } },
		5078	},
		5079	.dst_temps = .{.{ .rc = .general_purpose }},
		5080	.clobbers = .{ .eflags = true },
		5081	.each = .{ .once = &.{
		5082	.{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
		5083	.{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
		5084	.{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
		5085	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5086	.{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
		5087	} },
		5088	}, .{
		5089	.required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
		5090	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
		5091	.patterns = &.{
		5092	.{ .src = .{ .to_mut_gpr, .none } },
		5093	},
		5094	.dst_temps = .{.{ .ref = .src0 }},
		5095	.clobbers = .{ .eflags = true },
		5096	.each = .{ .once = &.{
		5097	.{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ },
		5098	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		5099	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		5100	.{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5101	} },
		5102	}, .{
		5103	.required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
		5104	.src_constraints = .{ .{ .signed_int = .qword }, .any },
		5105	.patterns = &.{
		5106	.{ .src = .{ .mem, .none } },
		5107	.{ .src = .{ .to_gpr, .none } },
		5108	},
		5109	.extra_temps = .{
		5110	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5111	.unused,
		5112	.unused,
		5113	.unused,
		5114	.unused,
		5115	.unused,
		5116	},
		5117	.dst_temps = .{.{ .rc = .general_purpose }},
		5118	.clobbers = .{ .eflags = true },
		5119	.each = .{ .once = &.{
		5120	.{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
		5121	.{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ },
		5122	.{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ },
		5123	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		5124	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		5125	.{ ._, ._c, .st, ._, ._, ._, ._ },
		5126	.{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
		5127	} },
		5128	}, .{
		5129	.required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
		5130	.src_constraints = .{ .{ .unsigned_int = .qword }, .any },
		5131	.patterns = &.{
		5132	.{ .src = .{ .to_mut_gpr, .none } },
		5133	},
		5134	.dst_temps = .{.{ .rc = .general_purpose }},
		5135	.clobbers = .{ .eflags = true },
		5136	.each = .{ .once = &.{
		5137	.{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
		5138	.{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
		5139	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		5140	.{ ._, ._c, .st, ._, ._, ._, ._ },
		5141	.{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
		5142	} },
		5143	}, .{
		5144	.required_features = .{ .@"64bit", null, null, null },
		5145	.src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
		5146	.patterns = &.{
		5147	.{ .src = .{ .mem, .none } },
		5148	.{ .src = .{ .to_gpr, .none } },
		5149	},
		5150	.dst_temps = .{.{ .rc = .general_purpose }},
		5151	.clobbers = .{ .eflags = true },
		5152	.each = .{ .once = &.{
		5153	.{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
		5154	.{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ },
		5155	.{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5156	} },
		5157	}, .{
		5158	.required_features = .{ .@"64bit", null, null, null },
		5159	.src_constraints = .{ .{ .signed_int = .qword }, .any },
		5160	.patterns = &.{
		5161	.{ .src = .{ .mem, .none } },
		5162	.{ .src = .{ .to_gpr, .none } },
		5163	},
		5164	.extra_temps = .{
		5165	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5166	.unused,
		5167	.unused,
		5168	.unused,
		5169	.unused,
		5170	.unused,
		5171	},
		5172	.dst_temps = .{.{ .rc = .general_purpose }},
		5173	.clobbers = .{ .eflags = true },
		5174	.each = .{ .once = &.{
		5175	.{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
		5176	.{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ },
		5177	.{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
		5178	.{ ._, ._r, .bs, .tmp0q, .dst0q, ._, ._ },
		5179	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5180	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		5181	} },
		5182	}, .{
		5183	.required_features = .{ .@"64bit", null, null, null },
		5184	.src_constraints = .{ .{ .unsigned_int = .qword }, .any },
		5185	.patterns = &.{
		5186	.{ .src = .{ .mem, .none } },
		5187	.{ .src = .{ .to_gpr, .none } },
		5188	},
		5189	.extra_temps = .{
		5190	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5191	.unused,
		5192	.unused,
		5193	.unused,
		5194	.unused,
		5195	.unused,
		5196	},
		5197	.dst_temps = .{.{ .rc = .general_purpose }},
		5198	.clobbers = .{ .eflags = true },
		5199	.each = .{ .once = &.{
		5200	.{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
		5201	.{ ._, ._r, .bs, .tmp0q, .src0q, ._, ._ },
		5202	.{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5203	.{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
		5204	} },
		5205	}, .{
		5206	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		5207	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		5208	.patterns = &.{
		5209	.{ .src = .{ .to_mem, .none } },
		5210	},
		5211	.extra_temps = .{
		5212	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5213	.unused,
		5214	.unused,
		5215	.unused,
		5216	.unused,
		5217	.unused,
		5218	},
		5219	.dst_temps = .{.{ .rc = .general_purpose }},
		5220	.clobbers = .{ .eflags = true },
		5221	.each = .{ .once = &.{
		5222	.{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
		5223	.{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
		5224	.{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5225	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5226	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5227	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5228	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5229	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5230	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5231	} },
		5232	}, .{
		5233	.required_features = .{ .@"64bit", .lzcnt, null, null },
		5234	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		5235	.patterns = &.{
		5236	.{ .src = .{ .to_mem, .none } },
		5237	},
		5238	.extra_temps = .{
		5239	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5240	.unused,
		5241	.unused,
		5242	.unused,
		5243	.unused,
		5244	.unused,
		5245	},
		5246	.dst_temps = .{.{ .rc = .general_purpose }},
		5247	.clobbers = .{ .eflags = true },
		5248	.each = .{ .once = &.{
		5249	.{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
		5250	.{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5251	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5252	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5253	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5254	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5255	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5256	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5257	} },
		5258	}, .{
		5259	.required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
		5260	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		5261	.patterns = &.{
		5262	.{ .src = .{ .to_mem, .none } },
		5263	},
		5264	.extra_temps = .{
		5265	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5266	.unused,
		5267	.unused,
		5268	.unused,
		5269	.unused,
		5270	.unused,
		5271	},
		5272	.dst_temps = .{.{ .rc = .general_purpose }},
		5273	.clobbers = .{ .eflags = true },
		5274	.each = .{ .once = &.{
		5275	.{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
		5276	.{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
		5277	.{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5278	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		5279	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5280	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5281	.{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ },
		5282	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5283	.{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
		5284	.{ ._, ._, .neg, .dst0d, ._, ._, ._ },
		5285	} },
		5286	}, .{
		5287	.required_features = .{ .@"64bit", null, null, null },
		5288	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		5289	.patterns = &.{
		5290	.{ .src = .{ .to_mem, .none } },
		5291	},
		5292	.extra_temps = .{
		5293	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5294	.unused,
		5295	.unused,
		5296	.unused,
		5297	.unused,
		5298	.unused,
		5299	},
		5300	.dst_temps = .{.{ .rc = .general_purpose }},
		5301	.clobbers = .{ .eflags = true },
		5302	.each = .{ .once = &.{
		5303	.{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
		5304	.{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ },
		5305	.{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5306	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		5307	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5308	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5309	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5310	.{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
		5311	.{ ._, ._, .neg, .dst0d, ._, ._, ._ },
		5312	} },
		5313	}, .{
		5314	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		5315	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		5316	.patterns = &.{
		5317	.{ .src = .{ .to_mem, .none } },
		5318	},
		5319	.extra_temps = .{
		5320	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5321	.unused,
		5322	.unused,
		5323	.unused,
		5324	.unused,
		5325	.unused,
		5326	},
		5327	.dst_temps = .{.{ .rc = .general_purpose }},
		5328	.clobbers = .{ .eflags = true },
		5329	.each = .{ .once = &.{
		5330	.{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
		5331	.{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
		5332	.{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5333	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5334	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5335	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5336	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5337	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5338	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5339	} },
		5340	}, .{
		5341	.required_features = .{ .@"64bit", .lzcnt, null, null },
		5342	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		5343	.patterns = &.{
		5344	.{ .src = .{ .to_mem, .none } },
		5345	},
		5346	.extra_temps = .{
		5347	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5348	.unused,
		5349	.unused,
		5350	.unused,
		5351	.unused,
		5352	.unused,
		5353	},
		5354	.dst_temps = .{.{ .rc = .general_purpose }},
		5355	.clobbers = .{ .eflags = true },
		5356	.each = .{ .once = &.{
		5357	.{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
		5358	.{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5359	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5360	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5361	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5362	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5363	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5364	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5365	} },
		5366	}, .{
		5367	.required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
		5368	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		5369	.patterns = &.{
		5370	.{ .src = .{ .to_mem, .none } },
		5371	},
		5372	.extra_temps = .{
		5373	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5374	.unused,
		5375	.unused,
		5376	.unused,
		5377	.unused,
		5378	.unused,
		5379	},
		5380	.dst_temps = .{.{ .rc = .general_purpose }},
		5381	.clobbers = .{ .eflags = true },
		5382	.each = .{ .once = &.{
		5383	.{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
		5384	.{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
		5385	.{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5386	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		5387	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5388	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5389	.{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ },
		5390	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5391	.{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
		5392	.{ ._, ._, .neg, .dst0d, ._, ._, ._ },
		5393	} },
		5394	}, .{
		5395	.required_features = .{ .@"64bit", null, null, null },
		5396	.src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		5397	.patterns = &.{
		5398	.{ .src = .{ .to_mem, .none } },
		5399	},
		5400	.extra_temps = .{
		5401	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5402	.unused,
		5403	.unused,
		5404	.unused,
		5405	.unused,
		5406	.unused,
		5407	},
		5408	.dst_temps = .{.{ .rc = .general_purpose }},
		5409	.clobbers = .{ .eflags = true },
		5410	.each = .{ .once = &.{
		5411	.{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
		5412	.{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ },
		5413	.{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5414	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		5415	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5416	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5417	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5418	.{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
		5419	.{ ._, ._, .neg, .dst0d, ._, ._, ._ },
		5420	} },
		5421	}, .{
		5422	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		5423	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		5424	.patterns = &.{
		5425	.{ .src = .{ .to_mem, .none } },
		5426	},
		5427	.extra_temps = .{
		5428	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5429	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5430	.unused,
		5431	.unused,
		5432	.unused,
		5433	.unused,
		5434	},
		5435	.dst_temps = .{.{ .rc = .general_purpose }},
		5436	.clobbers = .{ .eflags = true },
		5437	.each = .{ .once = &.{
		5438	.{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
		5439	.{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		5440	.{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
		5441	.{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
		5442	.{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
		5443	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5444	.{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
		5445	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5446	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5447	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5448	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5449	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5450	} },
		5451	}, .{
		5452	.required_features = .{ .@"64bit", .lzcnt, null, null },
		5453	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		5454	.patterns = &.{
		5455	.{ .src = .{ .to_mem, .none } },
		5456	},
		5457	.extra_temps = .{
		5458	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5459	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5460	.unused,
		5461	.unused,
		5462	.unused,
		5463	.unused,
		5464	},
		5465	.dst_temps = .{.{ .rc = .general_purpose }},
		5466	.clobbers = .{ .eflags = true },
		5467	.each = .{ .once = &.{
		5468	.{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
		5469	.{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		5470	.{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
		5471	.{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
		5472	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5473	.{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
		5474	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5475	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5476	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5477	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5478	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5479	} },
		5480	}, .{
		5481	.required_features = .{ .@"64bit", null, null, null },
		5482	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		5483	.patterns = &.{
		5484	.{ .src = .{ .to_mem, .none } },
		5485	},
		5486	.extra_temps = .{
		5487	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5488	.unused,
		5489	.unused,
		5490	.unused,
		5491	.unused,
		5492	.unused,
		5493	},
		5494	.dst_temps = .{.{ .rc = .general_purpose }},
		5495	.clobbers = .{ .eflags = true },
		5496	.each = .{ .once = &.{
		5497	.{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
		5498	.{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
		5499	.{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5500	.{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ },
		5501	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		5502	.{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ },
		5503	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5504	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5505	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5506	.{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
		5507	.{ ._, ._, .neg, .dst0d, ._, ._, ._ },
		5508	} },
		5509	}, .{
		5510	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		5511	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		5512	.patterns = &.{
		5513	.{ .src = .{ .to_mem, .none } },
		5514	},
		5515	.extra_temps = .{
		5516	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5517	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5518	.unused,
		5519	.unused,
		5520	.unused,
		5521	.unused,
		5522	},
		5523	.dst_temps = .{.{ .rc = .general_purpose }},
		5524	.clobbers = .{ .eflags = true },
		5525	.each = .{ .once = &.{
		5526	.{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
		5527	.{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		5528	.{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
		5529	.{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
		5530	.{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
		5531	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5532	.{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
		5533	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5534	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5535	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5536	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5537	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5538	} },
		5539	}, .{
		5540	.required_features = .{ .@"64bit", .lzcnt, null, null },
		5541	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		5542	.patterns = &.{
		5543	.{ .src = .{ .to_mem, .none } },
		5544	},
		5545	.extra_temps = .{
		5546	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5547	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5548	.unused,
		5549	.unused,
		5550	.unused,
		5551	.unused,
		5552	},
		5553	.dst_temps = .{.{ .rc = .general_purpose }},
		5554	.clobbers = .{ .eflags = true },
		5555	.each = .{ .once = &.{
		5556	.{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
		5557	.{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		5558	.{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
		5559	.{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
		5560	.{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
		5561	.{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
		5562	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5563	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5564	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5565	.{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
		5566	.{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
		5567	} },
		5568	}, .{
		5569	.required_features = .{ .@"64bit", null, null, null },
		5570	.src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		5571	.patterns = &.{
		5572	.{ .src = .{ .to_mem, .none } },
		5573	},
		5574	.extra_temps = .{
		5575	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5576	.unused,
		5577	.unused,
		5578	.unused,
		5579	.unused,
		5580	.unused,
		5581	},
		5582	.dst_temps = .{.{ .rc = .general_purpose }},
		5583	.clobbers = .{ .eflags = true },
		5584	.each = .{ .once = &.{
		5585	.{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
		5586	.{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
		5587	.{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ },
		5588	.{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ },
		5589	.{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
		5590	.{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ },
		5591	.{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
		5592	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5593	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		5594	.{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
		5595	.{ ._, ._, .neg, .dst0d, ._, ._, ._ },
		5596	} },
		5597	}, .{
		5598	.required_features = .{ .lzcnt, .slow_incdec, null, null },
		5599	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5600	.patterns = &.{
		5601	.{ .src = .{ .to_mem, .none } },
		5602	},
		5603	.extra_temps = .{
		5604	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5605	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5606	.unused,
		5607	.unused,
		5608	.unused,
		5609	.unused,
		5610	},
		5611	.dst_temps = .{.mem},
		5612	.clobbers = .{ .eflags = true },
		5613	.each = .{ .once = &.{
		5614	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5615	.{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5616	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5617	.{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
		5618	.{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		5619	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5620	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		5621	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5622	} },
		5623	}, .{
		5624	.required_features = .{ .lzcnt, null, null, null },
		5625	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5626	.patterns = &.{
		5627	.{ .src = .{ .to_mem, .none } },
		5628	},
		5629	.extra_temps = .{
		5630	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5631	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5632	.unused,
		5633	.unused,
		5634	.unused,
		5635	.unused,
		5636	},
		5637	.dst_temps = .{.mem},
		5638	.clobbers = .{ .eflags = true },
		5639	.each = .{ .once = &.{
		5640	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5641	.{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5642	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5643	.{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
		5644	.{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		5645	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5646	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		5647	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		5648	} },
		5649	}, .{
		5650	.required_features = .{ .lzcnt, .slow_incdec, null, null },
		5651	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		5652	.patterns = &.{
		5653	.{ .src = .{ .to_mem, .none } },
		5654	},
		5655	.extra_temps = .{
		5656	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5657	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5658	.unused,
		5659	.unused,
		5660	.unused,
		5661	.unused,
		5662	},
		5663	.dst_temps = .{.mem},
		5664	.clobbers = .{ .eflags = true },
		5665	.each = .{ .once = &.{
		5666	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5667	.{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		5668	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5669	.{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
		5670	.{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		5671	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5672	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		5673	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5674	} },
		5675	}, .{
		5676	.required_features = .{ .lzcnt, null, null, null },
		5677	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		5678	.patterns = &.{
		5679	.{ .src = .{ .to_mem, .none } },
		5680	},
		5681	.extra_temps = .{
		5682	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5683	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5684	.unused,
		5685	.unused,
		5686	.unused,
		5687	.unused,
		5688	},
		5689	.dst_temps = .{.mem},
		5690	.clobbers = .{ .eflags = true },
		5691	.each = .{ .once = &.{
		5692	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5693	.{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		5694	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5695	.{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
		5696	.{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		5697	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5698	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		5699	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		5700	} },
		5701	}, .{
		5702	.required_features = .{ .lzcnt, .slow_incdec, null, null },
		5703	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		5704	.patterns = &.{
		5705	.{ .src = .{ .to_mem, .none } },
		5706	},
		5707	.extra_temps = .{
		5708	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5709	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5710	.unused,
		5711	.unused,
		5712	.unused,
		5713	.unused,
		5714	},
		5715	.dst_temps = .{.mem},
		5716	.clobbers = .{ .eflags = true },
		5717	.each = .{ .once = &.{
		5718	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5719	.{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		5720	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5721	.{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
		5722	.{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		5723	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5724	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		5725	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5726	} },
		5727	}, .{
		5728	.required_features = .{ .lzcnt, null, null, null },
		5729	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		5730	.patterns = &.{
		5731	.{ .src = .{ .to_mem, .none } },
		5732	},
		5733	.extra_temps = .{
		5734	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5735	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5736	.unused,
		5737	.unused,
		5738	.unused,
		5739	.unused,
		5740	},
		5741	.dst_temps = .{.mem},
		5742	.clobbers = .{ .eflags = true },
		5743	.each = .{ .once = &.{
		5744	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5745	.{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		5746	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5747	.{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
		5748	.{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
		5749	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5750	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		5751	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		5752	} },
		5753	}, .{
		5754	.required_features = .{ .@"64bit", .lzcnt, .slow_incdec, null },
		5755	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		5756	.patterns = &.{
		5757	.{ .src = .{ .to_mem, .none } },
		5758	},
		5759	.extra_temps = .{
		5760	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5761	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5762	.unused,
		5763	.unused,
		5764	.unused,
		5765	.unused,
		5766	},
		5767	.dst_temps = .{.mem},
		5768	.clobbers = .{ .eflags = true },
		5769	.each = .{ .once = &.{
		5770	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5771	.{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		5772	.{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		5773	.{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ },
		5774	.{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ },
		5775	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5776	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		5777	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5778	} },
		5779	}, .{
		5780	.required_features = .{ .@"64bit", .lzcnt, null, null },
		5781	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		5782	.patterns = &.{
		5783	.{ .src = .{ .to_mem, .none } },
		5784	},
		5785	.extra_temps = .{
		5786	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5787	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		5788	.unused,
		5789	.unused,
		5790	.unused,
		5791	.unused,
		5792	},
		5793	.dst_temps = .{.mem},
		5794	.clobbers = .{ .eflags = true },
		5795	.each = .{ .once = &.{
		5796	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5797	.{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		5798	.{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		5799	.{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ },
		5800	.{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ },
		5801	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5802	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		5803	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		5804	} },
		5805	}, .{
		5806	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
		5807	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5808	.patterns = &.{
		5809	.{ .src = .{ .to_mem, .none } },
		5810	},
		5811	.extra_temps = .{
		5812	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5813	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5814	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5815	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		5816	.unused,
		5817	.unused,
		5818	},
		5819	.dst_temps = .{.mem},
		5820	.clobbers = .{ .eflags = true },
		5821	.each = .{ .once = &.{
		5822	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5823	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		5824	.{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5825	.{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
		5826	.{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
		5827	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		5828	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5829	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		5830	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		5831	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		5832	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5833	} },
		5834	}, .{
		5835	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		5836	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5837	.patterns = &.{
		5838	.{ .src = .{ .to_mem, .none } },
		5839	},
		5840	.extra_temps = .{
		5841	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5842	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5843	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5844	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		5845	.unused,
		5846	.unused,
		5847	},
		5848	.dst_temps = .{.mem},
		5849	.clobbers = .{ .eflags = true },
		5850	.each = .{ .once = &.{
		5851	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5852	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		5853	.{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5854	.{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
		5855	.{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
		5856	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		5857	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5858	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		5859	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		5860	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		5861	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		5862	} },
		5863	}, .{
		5864	.required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
		5865	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5866	.patterns = &.{
		5867	.{ .src = .{ .to_mem, .none } },
		5868	},
		5869	.extra_temps = .{
		5870	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5871	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5872	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5873	.unused,
		5874	.unused,
		5875	.unused,
		5876	},
		5877	.dst_temps = .{.mem},
		5878	.clobbers = .{ .eflags = true },
		5879	.each = .{ .once = &.{
		5880	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5881	.{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5882	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5883	.{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
		5884	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		5885	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		5886	.{ ._, ._c, .st, ._, ._, ._, ._ },
		5887	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		5888	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		5889	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		5890	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5891	} },
		5892	}, .{
		5893	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		5894	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5895	.patterns = &.{
		5896	.{ .src = .{ .to_mem, .none } },
		5897	},
		5898	.extra_temps = .{
		5899	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5900	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5901	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5902	.unused,
		5903	.unused,
		5904	.unused,
		5905	},
		5906	.dst_temps = .{.mem},
		5907	.clobbers = .{ .eflags = true },
		5908	.each = .{ .once = &.{
		5909	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5910	.{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5911	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5912	.{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
		5913	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		5914	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		5915	.{ ._, ._c, .st, ._, ._, ._, ._ },
		5916	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		5917	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		5918	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		5919	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		5920	} },
		5921	}, .{
		5922	.required_features = .{ .slow_incdec, null, null, null },
		5923	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5924	.patterns = &.{
		5925	.{ .src = .{ .to_mem, .none } },
		5926	},
		5927	.extra_temps = .{
		5928	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5929	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5930	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5931	.unused,
		5932	.unused,
		5933	.unused,
		5934	},
		5935	.dst_temps = .{.mem},
		5936	.clobbers = .{ .eflags = true },
		5937	.each = .{ .once = &.{
		5938	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5939	.{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5940	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5941	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		5942	.{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
		5943	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5944	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		5945	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5946	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		5947	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		5948	} },
		5949	}, .{
		5950	.src_constraints = .{ .{ .scalar_int = .byte }, .any },
		5951	.patterns = &.{
		5952	.{ .src = .{ .to_mem, .none } },
		5953	},
		5954	.extra_temps = .{
		5955	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5956	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5957	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5958	.unused,
		5959	.unused,
		5960	.unused,
		5961	},
		5962	.dst_temps = .{.mem},
		5963	.clobbers = .{ .eflags = true },
		5964	.each = .{ .once = &.{
		5965	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5966	.{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
		5967	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		5968	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		5969	.{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
		5970	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		5971	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		5972	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		5973	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		5974	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		5975	} },
		5976	}, .{
		5977	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
		5978	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		5979	.patterns = &.{
		5980	.{ .src = .{ .to_mem, .none } },
		5981	},
		5982	.extra_temps = .{
		5983	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		5984	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5985	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		5986	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		5987	.unused,
		5988	.unused,
		5989	},
		5990	.dst_temps = .{.mem},
		5991	.clobbers = .{ .eflags = true },
		5992	.each = .{ .once = &.{
		5993	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		5994	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		5995	.{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		5996	.{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
		5997	.{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
		5998	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		5999	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6000	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		6001	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6002	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6003	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6004	} },
		6005	}, .{
		6006	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		6007	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		6008	.patterns = &.{
		6009	.{ .src = .{ .to_mem, .none } },
		6010	},
		6011	.extra_temps = .{
		6012	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6013	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6014	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6015	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		6016	.unused,
		6017	.unused,
		6018	},
		6019	.dst_temps = .{.mem},
		6020	.clobbers = .{ .eflags = true },
		6021	.each = .{ .once = &.{
		6022	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6023	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		6024	.{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		6025	.{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
		6026	.{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
		6027	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		6028	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6029	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		6030	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6031	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6032	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6033	} },
		6034	}, .{
		6035	.required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
		6036	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		6037	.patterns = &.{
		6038	.{ .src = .{ .to_mem, .none } },
		6039	},
		6040	.extra_temps = .{
		6041	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6042	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6043	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6044	.unused,
		6045	.unused,
		6046	.unused,
		6047	},
		6048	.dst_temps = .{.mem},
		6049	.clobbers = .{ .eflags = true },
		6050	.each = .{ .once = &.{
		6051	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6052	.{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		6053	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6054	.{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
		6055	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		6056	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		6057	.{ ._, ._c, .st, ._, ._, ._, ._ },
		6058	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		6059	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		6060	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6061	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6062	} },
		6063	}, .{
		6064	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		6065	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		6066	.patterns = &.{
		6067	.{ .src = .{ .to_mem, .none } },
		6068	},
		6069	.extra_temps = .{
		6070	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6071	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6072	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6073	.unused,
		6074	.unused,
		6075	.unused,
		6076	},
		6077	.dst_temps = .{.mem},
		6078	.clobbers = .{ .eflags = true },
		6079	.each = .{ .once = &.{
		6080	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6081	.{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		6082	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6083	.{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
		6084	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		6085	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		6086	.{ ._, ._c, .st, ._, ._, ._, ._ },
		6087	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		6088	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		6089	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6090	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6091	} },
		6092	}, .{
		6093	.required_features = .{ .slow_incdec, null, null, null },
		6094	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		6095	.patterns = &.{
		6096	.{ .src = .{ .to_mem, .none } },
		6097	},
		6098	.extra_temps = .{
		6099	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6100	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6101	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6102	.unused,
		6103	.unused,
		6104	.unused,
		6105	},
		6106	.dst_temps = .{.mem},
		6107	.clobbers = .{ .eflags = true },
		6108	.each = .{ .once = &.{
		6109	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6110	.{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		6111	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6112	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		6113	.{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
		6114	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6115	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		6116	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		6117	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6118	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6119	} },
		6120	}, .{
		6121	.src_constraints = .{ .{ .scalar_int = .word }, .any },
		6122	.patterns = &.{
		6123	.{ .src = .{ .to_mem, .none } },
		6124	},
		6125	.extra_temps = .{
		6126	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6127	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6128	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6129	.unused,
		6130	.unused,
		6131	.unused,
		6132	},
		6133	.dst_temps = .{.mem},
		6134	.clobbers = .{ .eflags = true },
		6135	.each = .{ .once = &.{
		6136	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6137	.{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
		6138	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6139	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		6140	.{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
		6141	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6142	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		6143	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		6144	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6145	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6146	} },
		6147	}, .{
		6148	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
		6149	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		6150	.patterns = &.{
		6151	.{ .src = .{ .to_mem, .none } },
		6152	},
		6153	.extra_temps = .{
		6154	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6155	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6156	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6157	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		6158	.unused,
		6159	.unused,
		6160	},
		6161	.dst_temps = .{.mem},
		6162	.clobbers = .{ .eflags = true },
		6163	.each = .{ .once = &.{
		6164	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6165	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		6166	.{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		6167	.{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
		6168	.{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
		6169	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		6170	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6171	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		6172	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6173	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6174	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6175	} },
		6176	}, .{
		6177	.required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
		6178	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		6179	.patterns = &.{
		6180	.{ .src = .{ .to_mem, .none } },
		6181	},
		6182	.extra_temps = .{
		6183	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6184	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6185	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6186	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		6187	.unused,
		6188	.unused,
		6189	},
		6190	.dst_temps = .{.mem},
		6191	.clobbers = .{ .eflags = true },
		6192	.each = .{ .once = &.{
		6193	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6194	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		6195	.{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		6196	.{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
		6197	.{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
		6198	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		6199	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6200	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		6201	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6202	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6203	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6204	} },
		6205	}, .{
		6206	.required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
		6207	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		6208	.patterns = &.{
		6209	.{ .src = .{ .to_mem, .none } },
		6210	},
		6211	.extra_temps = .{
		6212	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6213	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6214	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6215	.unused,
		6216	.unused,
		6217	.unused,
		6218	},
		6219	.dst_temps = .{.mem},
		6220	.clobbers = .{ .eflags = true },
		6221	.each = .{ .once = &.{
		6222	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6223	.{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		6224	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6225	.{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
		6226	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		6227	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		6228	.{ ._, ._c, .st, ._, ._, ._, ._ },
		6229	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		6230	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		6231	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6232	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6233	} },
		6234	}, .{
		6235	.required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
		6236	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		6237	.patterns = &.{
		6238	.{ .src = .{ .to_mem, .none } },
		6239	},
		6240	.extra_temps = .{
		6241	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6242	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6243	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6244	.unused,
		6245	.unused,
		6246	.unused,
		6247	},
		6248	.dst_temps = .{.mem},
		6249	.clobbers = .{ .eflags = true },
		6250	.each = .{ .once = &.{
		6251	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6252	.{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		6253	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6254	.{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
		6255	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		6256	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		6257	.{ ._, ._c, .st, ._, ._, ._, ._ },
		6258	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		6259	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		6260	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6261	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6262	} },
		6263	}, .{
		6264	.required_features = .{ .slow_incdec, null, null, null },
		6265	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		6266	.patterns = &.{
		6267	.{ .src = .{ .to_mem, .none } },
		6268	},
		6269	.extra_temps = .{
		6270	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6271	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6272	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6273	.unused,
		6274	.unused,
		6275	.unused,
		6276	},
		6277	.dst_temps = .{.mem},
		6278	.clobbers = .{ .eflags = true },
		6279	.each = .{ .once = &.{
		6280	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6281	.{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		6282	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6283	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		6284	.{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
		6285	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6286	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		6287	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		6288	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6289	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6290	} },
		6291	}, .{
		6292	.src_constraints = .{ .{ .scalar_int = .dword }, .any },
		6293	.patterns = &.{
		6294	.{ .src = .{ .to_mem, .none } },
		6295	},
		6296	.extra_temps = .{
		6297	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6298	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6299	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6300	.unused,
		6301	.unused,
		6302	.unused,
		6303	},
		6304	.dst_temps = .{.mem},
		6305	.clobbers = .{ .eflags = true },
		6306	.each = .{ .once = &.{
		6307	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6308	.{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
		6309	.{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
		6310	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		6311	.{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
		6312	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6313	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		6314	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		6315	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6316	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6317	} },
		6318	}, .{
		6319	.required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec },
		6320	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		6321	.patterns = &.{
		6322	.{ .src = .{ .to_mem, .none } },
		6323	},
		6324	.extra_temps = .{
		6325	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6326	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6327	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6328	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		6329	.unused,
		6330	.unused,
		6331	},
		6332	.dst_temps = .{.mem},
		6333	.clobbers = .{ .eflags = true },
		6334	.each = .{ .once = &.{
		6335	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6336	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		6337	.{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ },
		6338	.{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		6339	.{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ },
		6340	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		6341	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6342	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		6343	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6344	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6345	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6346	} },
		6347	}, .{
		6348	.required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
		6349	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		6350	.patterns = &.{
		6351	.{ .src = .{ .to_mem, .none } },
		6352	},
		6353	.extra_temps = .{
		6354	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6355	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6356	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6357	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		6358	.unused,
		6359	.unused,
		6360	},
		6361	.dst_temps = .{.mem},
		6362	.clobbers = .{ .eflags = true },
		6363	.each = .{ .once = &.{
		6364	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6365	.{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
		6366	.{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ },
		6367	.{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		6368	.{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ },
		6369	.{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
		6370	.{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6371	.{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
		6372	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6373	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6374	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6375	} },
		6376	}, .{
		6377	.required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, .slow_incdec, null },
		6378	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		6379	.patterns = &.{
		6380	.{ .src = .{ .to_mem, .none } },
		6381	},
		6382	.extra_temps = .{
		6383	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6384	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6385	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6386	.unused,
		6387	.unused,
		6388	.unused,
		6389	},
		6390	.dst_temps = .{.mem},
		6391	.clobbers = .{ .eflags = true },
		6392	.each = .{ .once = &.{
		6393	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6394	.{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		6395	.{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		6396	.{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ },
		6397	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		6398	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		6399	.{ ._, ._c, .st, ._, ._, ._, ._ },
		6400	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		6401	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		6402	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6403	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6404	} },
		6405	}, .{
		6406	.required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
		6407	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		6408	.patterns = &.{
		6409	.{ .src = .{ .to_mem, .none } },
		6410	},
		6411	.extra_temps = .{
		6412	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6413	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6414	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6415	.unused,
		6416	.unused,
		6417	.unused,
		6418	},
		6419	.dst_temps = .{.mem},
		6420	.clobbers = .{ .eflags = true },
		6421	.each = .{ .once = &.{
		6422	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6423	.{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		6424	.{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		6425	.{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ },
		6426	.{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
		6427	.{ ._, ._z, .j, .@"1f", ._, ._, ._ },
		6428	.{ ._, ._c, .st, ._, ._, ._, ._ },
		6429	.{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
		6430	.{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
		6431	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6432	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6433	} },
		6434	}, .{
		6435	.required_features = .{ .@"64bit", .slow_incdec, null, null },
		6436	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		6437	.patterns = &.{
		6438	.{ .src = .{ .to_mem, .none } },
		6439	},
		6440	.extra_temps = .{
		6441	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6442	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6443	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6444	.unused,
		6445	.unused,
		6446	.unused,
		6447	},
		6448	.dst_temps = .{.mem},
		6449	.clobbers = .{ .eflags = true },
		6450	.each = .{ .once = &.{
		6451	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6452	.{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		6453	.{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		6454	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		6455	.{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ },
		6456	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6457	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		6458	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		6459	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6460	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6461	} },
		6462	}, .{
		6463	.required_features = .{ .@"64bit", null, null, null },
		6464	.src_constraints = .{ .{ .scalar_int = .qword }, .any },
		6465	.patterns = &.{
		6466	.{ .src = .{ .to_mem, .none } },
		6467	},
		6468	.extra_temps = .{
		6469	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6470	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6471	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6472	.unused,
		6473	.unused,
		6474	.unused,
		6475	},
		6476	.dst_temps = .{.mem},
		6477	.clobbers = .{ .eflags = true },
		6478	.each = .{ .once = &.{
		6479	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6480	.{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
		6481	.{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
		6482	.{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
		6483	.{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ },
		6484	.{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
		6485	.{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
		6486	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
		6487	.{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
		6488	.{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
		6489	} },
		6490	}, .{
		6491	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		6492	.dst_constraints = .{.{ .scalar_int = .byte }},
		6493	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		6494	.patterns = &.{
		6495	.{ .src = .{ .to_mem, .none } },
		6496	},
		6497	.extra_temps = .{
		6498	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6499	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6500	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6501	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6502	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6503	.unused,
		6504	},
		6505	.dst_temps = .{.mem},
		6506	.clobbers = .{ .eflags = true },
		6507	.each = .{ .once = &.{
		6508	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6509	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6510	.{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
		6511	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6512	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6513	.{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
		6514	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6515	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6516	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6517	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6518	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6519	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6520	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6521	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6522	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6523	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6524	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6525	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6526	} },
		6527	}, .{
		6528	.required_features = .{ .@"64bit", .lzcnt, null, null },
		6529	.dst_constraints = .{.{ .scalar_int = .byte }},
		6530	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		6531	.patterns = &.{
		6532	.{ .src = .{ .to_mem, .none } },
		6533	},
		6534	.extra_temps = .{
		6535	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6536	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6537	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6538	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6539	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6540	.unused,
		6541	},
		6542	.dst_temps = .{.mem},
		6543	.clobbers = .{ .eflags = true },
		6544	.each = .{ .once = &.{
		6545	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6546	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6547	.{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
		6548	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6549	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6550	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6551	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6552	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6553	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6554	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6555	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6556	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6557	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6558	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6559	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6560	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6561	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6562	} },
		6563	}, .{
		6564	.required_features = .{ .@"64bit", null, null, null },
		6565	.dst_constraints = .{.{ .scalar_int = .byte }},
		6566	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		6567	.patterns = &.{
		6568	.{ .src = .{ .to_mem, .none } },
		6569	},
		6570	.extra_temps = .{
		6571	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6572	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6573	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6574	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6575	.unused,
		6576	.unused,
		6577	},
		6578	.dst_temps = .{.mem},
		6579	.clobbers = .{ .eflags = true },
		6580	.each = .{ .once = &.{
		6581	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6582	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6583	.{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
		6584	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6585	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6586	.{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
		6587	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		6588	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6589	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6590	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6591	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6592	.{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
		6593	.{ ._, ._, .neg, .tmp3b, ._, ._, ._ },
		6594	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6595	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6596	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6597	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6598	} },
		6599	}, .{
		6600	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		6601	.dst_constraints = .{.{ .scalar_int = .byte }},
		6602	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		6603	.patterns = &.{
		6604	.{ .src = .{ .to_mem, .none } },
		6605	},
		6606	.extra_temps = .{
		6607	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6608	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6609	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6610	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6611	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6612	.unused,
		6613	},
		6614	.dst_temps = .{.mem},
		6615	.clobbers = .{ .eflags = true },
		6616	.each = .{ .once = &.{
		6617	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6618	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6619	.{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
		6620	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6621	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6622	.{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
		6623	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6624	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6625	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6626	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6627	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6628	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6629	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6630	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6631	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6632	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6633	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6634	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6635	} },
		6636	}, .{
		6637	.required_features = .{ .@"64bit", .lzcnt, null, null },
		6638	.dst_constraints = .{.{ .scalar_int = .byte }},
		6639	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		6640	.patterns = &.{
		6641	.{ .src = .{ .to_mem, .none } },
		6642	},
		6643	.extra_temps = .{
		6644	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6645	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6646	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6647	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6648	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6649	.unused,
		6650	},
		6651	.dst_temps = .{.mem},
		6652	.clobbers = .{ .eflags = true },
		6653	.each = .{ .once = &.{
		6654	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6655	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6656	.{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
		6657	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6658	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6659	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6660	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6661	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6662	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6663	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6664	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6665	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6666	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6667	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6668	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6669	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6670	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6671	} },
		6672	}, .{
		6673	.required_features = .{ .@"64bit", null, null, null },
		6674	.dst_constraints = .{.{ .scalar_int = .byte }},
		6675	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		6676	.patterns = &.{
		6677	.{ .src = .{ .to_mem, .none } },
		6678	},
		6679	.extra_temps = .{
		6680	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6681	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6682	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6683	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6684	.unused,
		6685	.unused,
		6686	},
		6687	.dst_temps = .{.mem},
		6688	.clobbers = .{ .eflags = true },
		6689	.each = .{ .once = &.{
		6690	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6691	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6692	.{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
		6693	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6694	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6695	.{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
		6696	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		6697	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6698	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6699	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6700	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6701	.{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
		6702	.{ ._, ._, .neg, .tmp3b, ._, ._, ._ },
		6703	.{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
		6704	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6705	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6706	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6707	} },
		6708	}, .{
		6709	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		6710	.dst_constraints = .{.{ .scalar_int = .word }},
		6711	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		6712	.patterns = &.{
		6713	.{ .src = .{ .to_mem, .none } },
		6714	},
		6715	.extra_temps = .{
		6716	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6717	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6718	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6719	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6720	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6721	.unused,
		6722	},
		6723	.dst_temps = .{.mem},
		6724	.clobbers = .{ .eflags = true },
		6725	.each = .{ .once = &.{
		6726	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6727	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6728	.{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
		6729	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6730	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6731	.{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
		6732	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6733	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6734	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6735	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6736	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6737	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6738	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6739	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6740	.{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
		6741	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6742	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6743	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6744	} },
		6745	}, .{
		6746	.required_features = .{ .@"64bit", .lzcnt, null, null },
		6747	.dst_constraints = .{.{ .scalar_int = .word }},
		6748	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		6749	.patterns = &.{
		6750	.{ .src = .{ .to_mem, .none } },
		6751	},
		6752	.extra_temps = .{
		6753	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6754	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6755	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6756	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6757	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6758	.unused,
		6759	},
		6760	.dst_temps = .{.mem},
		6761	.clobbers = .{ .eflags = true },
		6762	.each = .{ .once = &.{
		6763	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6764	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6765	.{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
		6766	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6767	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6768	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6769	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6770	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6771	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6772	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6773	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6774	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6775	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6776	.{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
		6777	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6778	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6779	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6780	} },
		6781	}, .{
		6782	.required_features = .{ .@"64bit", null, null, null },
		6783	.dst_constraints = .{.{ .scalar_int = .word }},
		6784	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
		6785	.patterns = &.{
		6786	.{ .src = .{ .to_mem, .none } },
		6787	},
		6788	.extra_temps = .{
		6789	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6790	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6791	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6792	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6793	.unused,
		6794	.unused,
		6795	},
		6796	.dst_temps = .{.mem},
		6797	.clobbers = .{ .eflags = true },
		6798	.each = .{ .once = &.{
		6799	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6800	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6801	.{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
		6802	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6803	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6804	.{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
		6805	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		6806	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6807	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6808	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6809	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6810	.{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
		6811	.{ ._, ._, .neg, .tmp3d, ._, ._, ._ },
		6812	.{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
		6813	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6814	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6815	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6816	} },
		6817	}, .{
		6818	.required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
		6819	.dst_constraints = .{.{ .scalar_int = .word }},
		6820	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		6821	.patterns = &.{
		6822	.{ .src = .{ .to_mem, .none } },
		6823	},
		6824	.extra_temps = .{
		6825	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6826	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6827	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6828	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6829	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6830	.unused,
		6831	},
		6832	.dst_temps = .{.mem},
		6833	.clobbers = .{ .eflags = true },
		6834	.each = .{ .once = &.{
		6835	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6836	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6837	.{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
		6838	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6839	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6840	.{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
		6841	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6842	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6843	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6844	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6845	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6846	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6847	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6848	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6849	.{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
		6850	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6851	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6852	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6853	} },
		6854	}, .{
		6855	.required_features = .{ .@"64bit", .lzcnt, null, null },
		6856	.dst_constraints = .{.{ .scalar_int = .word }},
		6857	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		6858	.patterns = &.{
		6859	.{ .src = .{ .to_mem, .none } },
		6860	},
		6861	.extra_temps = .{
		6862	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6863	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6864	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6865	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6866	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6867	.unused,
		6868	},
		6869	.dst_temps = .{.mem},
		6870	.clobbers = .{ .eflags = true },
		6871	.each = .{ .once = &.{
		6872	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6873	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6874	.{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
		6875	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6876	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6877	.{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
		6878	.{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
		6879	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6880	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6881	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6882	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6883	.{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
		6884	.{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
		6885	.{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
		6886	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6887	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6888	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6889	} },
		6890	}, .{
		6891	.required_features = .{ .@"64bit", null, null, null },
		6892	.dst_constraints = .{.{ .scalar_int = .word }},
		6893	.src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
		6894	.patterns = &.{
		6895	.{ .src = .{ .to_mem, .none } },
		6896	},
		6897	.extra_temps = .{
		6898	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		6899	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		6900	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		6901	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		6902	.unused,
		6903	.unused,
		6904	},
		6905	.dst_temps = .{.mem},
		6906	.clobbers = .{ .eflags = true },
		6907	.each = .{ .once = &.{
		6908	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
		6909	.{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
		6910	.{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
		6911	.{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
		6912	.{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
		6913	.{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
		6914	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		6915	.{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
		6916	.{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
		6917	.{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
		6918	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		6919	.{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
		6920	.{ ._, ._, .neg, .tmp3d, ._, ._, ._ },
		6921	.{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
		6922	.{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
		6923	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		6924	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		6925	} },
		6926	} }) catch \|err\| switch (err) {
		6927	error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
		6928	@tagName(air_tag),
		6929	cg.typeOf(ty_op.operand).fmt(pt),
		6930	ops[0].tracking(cg),
		6931	}),
		6932	else => \|e\| return e,
		6933	};
		6934	for (ops) \|op\| for (res) \|r\| {
		6935	if (op.index == r.index) break;
		6936	} else try op.die(cg);
		6937	try res[0].moveTo(inst, cg);
		6938	},
		6939
		6940	.cmp_vector, .cmp_vector_optimized => \|air_tag\| if (use_old) try cg.airCmpVector(inst) else fallback: {
		6941	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		6942	const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
		6943	switch (extra.compareOperator()) {
		6944	.eq, .neq => if (cg.typeOf(extra.lhs).scalarType(zcu).isRuntimeFloat())
		6945	break :fallback try cg.airCmpVector(inst),
		6946	else => break :fallback try cg.airCmpVector(inst),
		6947	}
		6948	var ops = try cg.tempsFromOperands(inst, .{ extra.lhs, extra.rhs });
		6949	var res: [1]Temp = undefined;
		6950	switch (extra.compareOperator()) {
		6951	.lt => unreachable,
		6952	.lte => unreachable,
		6953	.eq, .neq => \|cmp_op\| cg.select(&res, &.{ty_pl.ty.toType()}, &ops, switch (@as(Condition, switch (cmp_op) {
		6954	else => unreachable,
		6955	.eq => .e,
		6956	.neq => .ne,
		6957	})) {
		6958	else => unreachable,
		6959	inline .e, .ne => \|cc\| comptime &.{ .{
		6960	.required_features = .{ .avx2, null, null, null },
		6961	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		6962	.patterns = &.{
		6963	.{ .src = .{ .to_ymm, .mem } },
		6964	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		6965	.{ .src = .{ .to_ymm, .to_ymm } },
		6966	},
		6967	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		6968	.kind = .all,
		6969	.inverted = switch (cc) {
		6970	else => unreachable,
		6971	.e => false,
		6972	.ne => true,
		6973	},
		6974	.scalar = .byte,
		6975	} } }},
		6976	.each = .{ .once = &.{
		6977	.{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ },
		6978	} },
		6979	}, .{
		6980	.required_features = .{ .avx2, null, null, null },
		6981	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		6982	.patterns = &.{
		6983	.{ .src = .{ .to_ymm, .mem } },
		6984	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		6985	.{ .src = .{ .to_ymm, .to_ymm } },
		6986	},
		6987	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		6988	.kind = .all,
		6989	.inverted = switch (cc) {
		6990	else => unreachable,
		6991	.e => false,
		6992	.ne => true,
		6993	},
		6994	.scalar = .word,
		6995	} } }},
		6996	.each = .{ .once = &.{
		6997	.{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ },
		6998	} },
		6999	}, .{
		7000	.required_features = .{ .avx2, null, null, null },
		7001	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		7002	.patterns = &.{
		7003	.{ .src = .{ .to_ymm, .mem } },
		7004	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		7005	.{ .src = .{ .to_ymm, .to_ymm } },
		7006	},
		7007	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		7008	.kind = .all,
		7009	.inverted = switch (cc) {
		7010	else => unreachable,
		7011	.e => false,
		7012	.ne => true,
		7013	},
		7014	.scalar = .dword,
		7015	} } }},
		7016	.each = .{ .once = &.{
		7017	.{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ },
		7018	} },
		7019	}, .{
		7020	.required_features = .{ .avx2, null, null, null },
		7021	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		7022	.patterns = &.{
		7023	.{ .src = .{ .to_ymm, .mem } },
		7024	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		7025	.{ .src = .{ .to_ymm, .to_ymm } },
		7026	},
		7027	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		7028	.kind = .all,
		7029	.inverted = switch (cc) {
		7030	else => unreachable,
		7031	.e => false,
		7032	.ne => true,
		7033	},
		7034	.scalar = .qword,
		7035	} } }},
		7036	.each = .{ .once = &.{
		7037	.{ ._, .vp_q, .cmpeq, .dst0y, .src0y, .src1y, ._ },
		7038	} },
		7039	}, .{
		7040	.required_features = .{ .avx, null, null, null },
		7041	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		7042	.patterns = &.{
		7043	.{ .src = .{ .to_xmm, .mem } },
		7044	.{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
		7045	.{ .src = .{ .to_xmm, .to_xmm } },
		7046	},
		7047	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		7048	.kind = .all,
		7049	.inverted = switch (cc) {
		7050	else => unreachable,
		7051	.e => false,
		7052	.ne => true,
		7053	},
		7054	.scalar = .byte,
		7055	} } }},
		7056	.each = .{ .once = &.{
		7057	.{ ._, .vp_b, .cmpeq, .dst0x, .src0x, .src1x, ._ },
		7058	} },
		7059	}, .{
		7060	.required_features = .{ .avx, null, null, null },
		7061	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		7062	.patterns = &.{
		7063	.{ .src = .{ .to_xmm, .mem } },
		7064	.{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
		7065	.{ .src = .{ .to_xmm, .to_xmm } },
		7066	},
		7067	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		7068	.kind = .all,
		7069	.inverted = switch (cc) {
		7070	else => unreachable,
		7071	.e => false,
		7072	.ne => true,
		7073	},
		7074	.scalar = .word,
		7075	} } }},
		7076	.each = .{ .once = &.{
		7077	.{ ._, .vp_w, .cmpeq, .dst0x, .src0x, .src1x, ._ },
		7078	} },
		7079	}, .{
		7080	.required_features = .{ .avx, null, null, null },
		7081	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		7082	.patterns = &.{
		7083	.{ .src = .{ .to_xmm, .mem } },
		7084	.{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
		7085	.{ .src = .{ .to_xmm, .to_xmm } },
		7086	},
		7087	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		7088	.kind = .all,
		7089	.inverted = switch (cc) {
		7090	else => unreachable,
		7091	.e => false,
		7092	.ne => true,
		7093	},
		7094	.scalar = .dword,
		7095	} } }},
		7096	.each = .{ .once = &.{
		7097	.{ ._, .vp_d, .cmpeq, .dst0x, .src0x, .src1x, ._ },
		7098	} },
		7099	}, .{
		7100	.required_features = .{ .avx, null, null, null },
		7101	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		7102	.patterns = &.{
		7103	.{ .src = .{ .to_xmm, .mem } },
		7104	.{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
		7105	.{ .src = .{ .to_xmm, .to_xmm } },
		7106	},
		7107	.dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
		7108	.kind = .all,
		7109	.inverted = switch (cc) {
		7110	else => unreachable,
		7111	.e => false,
		7112	.ne => true,
		7113	},
		7114	.scalar = .qword,
		7115	} } }},
		7116	.each = .{ .once = &.{
		7117	.{ ._, .vp_q, .cmpeq, .dst0x, .src0x, .src1x, ._ },
		7118	} },
		7119	}, .{
		7120	.required_features = .{ .sse2, null, null, null },
		7121	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		7122	.patterns = &.{
		7123	.{ .src = .{ .to_mut_xmm, .mem } },
		7124	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		7125	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		7126	},
		7127	.dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
		7128	.kind = .all,
		7129	.inverted = switch (cc) {
		7130	else => unreachable,
		7131	.e => false,
		7132	.ne => true,
		7133	},
		7134	.scalar = .byte,
		7135	} } }},
		7136	.each = .{ .once = &.{
		7137	.{ ._, .p_b, .cmpeq, .dst0x, .src1x, ._, ._ },
		7138	} },
		7139	}, .{
		7140	.required_features = .{ .sse2, null, null, null },
		7141	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		7142	.patterns = &.{
		7143	.{ .src = .{ .to_mut_xmm, .mem } },
		7144	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		7145	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		7146	},
		7147	.dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
		7148	.kind = .all,
		7149	.inverted = switch (cc) {
		7150	else => unreachable,
		7151	.e => false,
		7152	.ne => true,
		7153	},
		7154	.scalar = .word,
		7155	} } }},
		7156	.each = .{ .once = &.{
		7157	.{ ._, .p_w, .cmpeq, .dst0x, .src1x, ._, ._ },
		7158	} },
		7159	}, .{
		7160	.required_features = .{ .sse2, null, null, null },
		7161	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		7162	.patterns = &.{
		7163	.{ .src = .{ .to_mut_xmm, .mem } },
		7164	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		7165	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		7166	},
		7167	.dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
		7168	.kind = .all,
		7169	.inverted = switch (cc) {
		7170	else => unreachable,
		7171	.e => false,
		7172	.ne => true,
		7173	},
		7174	.scalar = .dword,
		7175	} } }},
		7176	.each = .{ .once = &.{
		7177	.{ ._, .p_d, .cmpeq, .dst0x, .src1x, ._, ._ },
		7178	} },
		7179	}, .{
		7180	.required_features = .{ .sse4_1, null, null, null },
		7181	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		7182	.patterns = &.{
		7183	.{ .src = .{ .to_mut_xmm, .mem } },
		7184	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		7185	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		7186	},
		7187	.dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
		7188	.kind = .all,
		7189	.inverted = switch (cc) {
		7190	else => unreachable,
		7191	.e => false,
		7192	.ne => true,
		7193	},
		7194	.scalar = .qword,
		7195	} } }},
		7196	.each = .{ .once = &.{
		7197	.{ ._, .p_q, .cmpeq, .dst0x, .src1x, ._, ._ },
		7198	} },
		7199	}, .{
		7200	.required_features = .{ .mmx, null, null, null },
		7201	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		7202	.patterns = &.{
		7203	.{ .src = .{ .to_mut_mm, .mem } },
		7204	.{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
		7205	.{ .src = .{ .to_mut_mm, .to_mm } },
		7206	},
		7207	.dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
		7208	.kind = .all,
		7209	.inverted = switch (cc) {
		7210	else => unreachable,
		7211	.e => false,
		7212	.ne => true,
		7213	},
		7214	.scalar = .byte,
		7215	} } }},
		7216	.each = .{ .once = &.{
		7217	.{ ._, .p_b, .cmpeq, .dst0q, .src1q, ._, ._ },
		7218	} },
		7219	}, .{
		7220	.required_features = .{ .mmx, null, null, null },
		7221	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		7222	.patterns = &.{
		7223	.{ .src = .{ .to_mut_mm, .mem } },
		7224	.{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
		7225	.{ .src = .{ .to_mut_mm, .to_mm } },
		7226	},
		7227	.dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
		7228	.kind = .all,
		7229	.inverted = switch (cc) {
		7230	else => unreachable,
		7231	.e => false,
		7232	.ne => true,
		7233	},
		7234	.scalar = .word,
		7235	} } }},
		7236	.each = .{ .once = &.{
		7237	.{ ._, .p_w, .cmpeq, .dst0q, .src1q, ._, ._ },
		7238	} },
		7239	}, .{
		7240	.required_features = .{ .mmx, null, null, null },
		7241	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		7242	.patterns = &.{
		7243	.{ .src = .{ .to_mut_mm, .mem } },
		7244	.{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
		7245	.{ .src = .{ .to_mut_mm, .to_mm } },
		7246	},
		7247	.dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
		7248	.kind = .all,
		7249	.inverted = switch (cc) {
		7250	else => unreachable,
		7251	.e => false,
		7252	.ne => true,
		7253	},
		7254	.scalar = .dword,
		7255	} } }},
		7256	.each = .{ .once = &.{
		7257	.{ ._, .p_d, .cmpeq, .dst0q, .src1q, ._, ._ },
		7258	} },
		7259	}, .{
		7260	.src_constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte } },
		7261	.patterns = &.{
		7262	.{ .src = .{ .mut_mem, .imm8 } },
		7263	.{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
		7264	.{ .src = .{ .to_mut_gpr, .imm8 } },
		7265	.{ .src = .{ .imm8, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7266	.{ .src = .{ .mut_mem, .to_gpr } },
		7267	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		7268	.{ .src = .{ .to_mut_gpr, .mem } },
		7269	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7270	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		7271	},
		7272	.dst_temps = .{.{ .ref = .src0 }},
		7273	.clobbers = .{ .eflags = true },
		7274	.each = .{ .once = switch (cc) {
		7275	else => unreachable,
		7276	.e => &.{
		7277	.{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
		7278	.{ ._, ._, .not, .dst0b, ._, ._, ._ },
		7279	},
		7280	.ne => &.{
		7281	.{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
		7282	},
		7283	} },
		7284	}, .{
		7285	.src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } },
		7286	.patterns = &.{
		7287	.{ .src = .{ .mut_mem, .imm16 } },
		7288	.{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
		7289	.{ .src = .{ .to_mut_gpr, .imm16 } },
		7290	.{ .src = .{ .imm16, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7291	.{ .src = .{ .mut_mem, .to_gpr } },
		7292	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		7293	.{ .src = .{ .to_mut_gpr, .mem } },
		7294	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7295	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		7296	},
		7297	.dst_temps = .{.{ .ref = .src0 }},
		7298	.clobbers = .{ .eflags = true },
		7299	.each = .{ .once = switch (cc) {
		7300	else => unreachable,
		7301	.e => &.{
		7302	.{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
		7303	.{ ._, ._, .not, .dst0w, ._, ._, ._ },
		7304	},
		7305	.ne => &.{
		7306	.{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
		7307	},
		7308	} },
		7309	}, .{
		7310	.src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } },
		7311	.patterns = &.{
		7312	.{ .src = .{ .mut_mem, .imm32 } },
		7313	.{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
		7314	.{ .src = .{ .to_mut_gpr, .imm32 } },
		7315	.{ .src = .{ .imm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7316	.{ .src = .{ .mut_mem, .to_gpr } },
		7317	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		7318	.{ .src = .{ .to_mut_gpr, .mem } },
		7319	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7320	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		7321	},
		7322	.dst_temps = .{.{ .ref = .src0 }},
		7323	.clobbers = .{ .eflags = true },
		7324	.each = .{ .once = switch (cc) {
		7325	else => unreachable,
		7326	.e => &.{
		7327	.{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
		7328	.{ ._, ._, .not, .dst0d, ._, ._, ._ },
		7329	},
		7330	.ne => &.{
		7331	.{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
		7332	},
		7333	} },
		7334	}, .{
		7335	.required_features = .{ .@"64bit", null, null, null },
		7336	.src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } },
		7337	.patterns = &.{
		7338	.{ .src = .{ .mut_mem, .simm32 } },
		7339	.{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
		7340	.{ .src = .{ .to_mut_gpr, .simm32 } },
		7341	.{ .src = .{ .simm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7342	.{ .src = .{ .mut_mem, .to_gpr } },
		7343	.{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
		7344	.{ .src = .{ .to_mut_gpr, .mem } },
		7345	.{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
		7346	.{ .src = .{ .to_mut_gpr, .to_gpr } },
		7347	},
		7348	.dst_temps = .{.{ .ref = .src0 }},
		7349	.clobbers = .{ .eflags = true },
		7350	.each = .{ .once = switch (cc) {
		7351	else => unreachable,
		7352	.e => &.{
		7353	.{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
		7354	.{ ._, ._, .not, .dst0q, ._, ._, ._ },
		7355	},
		7356	.ne => &.{
		7357	.{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
		7358	},
		7359	} },
		7360	}, .{
		7361	.src_constraints = .{ .any_bool_vec, .any_bool_vec },
		7362	.patterns = &.{
		7363	.{ .src = .{ .to_mem, .to_mem } },
		7364	},
		7365	.extra_temps = .{
		7366	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7367	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		7368	.unused,
		7369	.unused,
		7370	.unused,
		7371	.unused,
		7372	},
		7373	.dst_temps = .{.mem},
		7374	.clobbers = .{ .eflags = true },
		7375	.each = .{ .once = switch (cc) {
		7376	else => unreachable,
		7377	.e => &.{
		7378	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7379	.{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
		7380	.{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
		7381	.{ ._, ._, .not, .tmp1p, ._, ._, ._ },
		7382	.{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
		7383	.{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
		7384	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7385	},
		7386	.ne => &.{
		7387	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7388	.{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
		7389	.{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
		7390	.{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
		7391	.{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
		7392	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7393	},
		7394	} },
		7395	}, .{
		7396	.required_features = .{ .avx2, null, null, null },
		7397	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		7398	.patterns = &.{
		7399	.{ .src = .{ .to_mem, .to_mem } },
		7400	},
		7401	.extra_temps = .{
		7402	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7403	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7404	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7405	.{ .kind = .{ .rc = .sse } },
		7406	.unused,
		7407	.unused,
		7408	},
		7409	.dst_temps = .{.mem},
		7410	.clobbers = .{ .eflags = true },
		7411	.each = .{ .once = switch (cc) {
		7412	else => unreachable,
		7413	.e => &.{
		7414	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7415	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7416	.{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7417	.{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
		7418	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
		7419	.{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
		7420	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7421	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		7422	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7423	},
		7424	.ne => &.{
		7425	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7426	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7427	.{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7428	.{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
		7429	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
		7430	.{ ._, ._, .not, .tmp2d, ._, ._, ._ },
		7431	.{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
		7432	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7433	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		7434	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7435	},
		7436	} },
		7437	}, .{
		7438	.required_features = .{ .avx2, null, null, null },
		7439	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		7440	.patterns = &.{
		7441	.{ .src = .{ .to_mem, .to_mem } },
		7442	},
		7443	.extra_temps = .{
		7444	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7445	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7446	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		7447	.{ .kind = .{ .rc = .sse } },
		7448	.unused,
		7449	.unused,
		7450	},
		7451	.dst_temps = .{.mem},
		7452	.clobbers = .{ .eflags = true },
		7453	.each = .{ .once = switch (cc) {
		7454	else => unreachable,
		7455	.e => &.{
		7456	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7457	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7458	.{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7459	.{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
		7460	.{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
		7461	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
		7462	.{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
		7463	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7464	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		7465	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7466	},
		7467	.ne => &.{
		7468	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7469	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7470	.{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7471	.{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
		7472	.{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
		7473	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
		7474	.{ ._, ._, .not, .tmp2d, ._, ._, ._ },
		7475	.{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
		7476	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7477	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		7478	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7479	},
		7480	} },
		7481	}, .{
		7482	.required_features = .{ .avx2, null, null, null },
		7483	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		7484	.patterns = &.{
		7485	.{ .src = .{ .to_mem, .to_mem } },
		7486	},
		7487	.extra_temps = .{
		7488	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7489	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7490	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7491	.{ .kind = .{ .rc = .sse } },
		7492	.unused,
		7493	.unused,
		7494	},
		7495	.dst_temps = .{.mem},
		7496	.clobbers = .{ .eflags = true },
		7497	.each = .{ .once = switch (cc) {
		7498	else => unreachable,
		7499	.e => &.{
		7500	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7501	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7502	.{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7503	.{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
		7504	.{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
		7505	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		7506	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		7507	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		7508	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7509	},
		7510	.ne => &.{
		7511	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7512	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7513	.{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7514	.{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
		7515	.{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
		7516	.{ ._, ._, .not, .tmp2b, ._, ._, ._ },
		7517	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		7518	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		7519	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		7520	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7521	},
		7522	} },
		7523	}, .{
		7524	.required_features = .{ .avx2, null, null, null },
		7525	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		7526	.patterns = &.{
		7527	.{ .src = .{ .to_mem, .to_mem } },
		7528	},
		7529	.extra_temps = .{
		7530	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7531	.{ .type = .u32, .kind = .{ .reg = .rcx } },
		7532	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7533	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7534	.{ .kind = .{ .rc = .sse } },
		7535	.unused,
		7536	},
		7537	.dst_temps = .{.mem},
		7538	.clobbers = .{ .eflags = true },
		7539	.each = .{ .once = switch (cc) {
		7540	else => unreachable,
		7541	.e => &.{
		7542	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7543	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7544	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7545	.{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7546	.{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
		7547	.{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
		7548	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7549	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7550	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7551	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7552	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7553	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7554	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7555	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7556	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7557	.{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
		7558	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7559	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7560	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7561	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7562	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7563	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7564	},
		7565	.ne => &.{
		7566	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7567	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7568	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7569	.{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		7570	.{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
		7571	.{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
		7572	.{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
		7573	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7574	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7575	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7576	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7577	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7578	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7579	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7580	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7581	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7582	.{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
		7583	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7584	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7585	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7586	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7587	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7588	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7589	},
		7590	} },
		7591	}, .{
		7592	.required_features = .{ .avx, null, null, null },
		7593	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		7594	.patterns = &.{
		7595	.{ .src = .{ .to_mem, .to_mem } },
		7596	},
		7597	.extra_temps = .{
		7598	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7599	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7600	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		7601	.{ .kind = .{ .rc = .sse } },
		7602	.unused,
		7603	.unused,
		7604	},
		7605	.dst_temps = .{.mem},
		7606	.clobbers = .{ .eflags = true },
		7607	.each = .{ .once = switch (cc) {
		7608	else => unreachable,
		7609	.e => &.{
		7610	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7611	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7612	.{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7613	.{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
		7614	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7615	.{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
		7616	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7617	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7618	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7619	},
		7620	.ne => &.{
		7621	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7622	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7623	.{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7624	.{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
		7625	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7626	.{ ._, ._, .not, .tmp2d, ._, ._, ._ },
		7627	.{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
		7628	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7629	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7630	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7631	},
		7632	} },
		7633	}, .{
		7634	.required_features = .{ .avx, null, null, null },
		7635	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		7636	.patterns = &.{
		7637	.{ .src = .{ .to_mem, .to_mem } },
		7638	},
		7639	.extra_temps = .{
		7640	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7641	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7642	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7643	.{ .kind = .{ .rc = .sse } },
		7644	.unused,
		7645	.unused,
		7646	},
		7647	.dst_temps = .{.mem},
		7648	.clobbers = .{ .eflags = true },
		7649	.each = .{ .once = switch (cc) {
		7650	else => unreachable,
		7651	.e => &.{
		7652	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7653	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7654	.{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7655	.{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
		7656	.{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
		7657	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7658	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		7659	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		7660	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7661	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7662	},
		7663	.ne => &.{
		7664	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7665	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7666	.{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7667	.{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
		7668	.{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
		7669	.{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7670	.{ ._, ._, .not, .tmp2b, ._, ._, ._ },
		7671	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		7672	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		7673	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7674	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7675	},
		7676	} },
		7677	}, .{
		7678	.required_features = .{ .avx, null, null, null },
		7679	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		7680	.patterns = &.{
		7681	.{ .src = .{ .to_mem, .to_mem } },
		7682	},
		7683	.extra_temps = .{
		7684	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7685	.{ .type = .u32, .kind = .{ .reg = .rcx } },
		7686	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7687	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7688	.{ .kind = .{ .rc = .sse } },
		7689	.unused,
		7690	},
		7691	.dst_temps = .{.mem},
		7692	.clobbers = .{ .eflags = true },
		7693	.each = .{ .once = switch (cc) {
		7694	else => unreachable,
		7695	.e => &.{
		7696	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7697	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7698	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7699	.{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7700	.{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
		7701	.{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		7702	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7703	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7704	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7705	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7706	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7707	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7708	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7709	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7710	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7711	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		7712	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7713	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7714	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7715	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7716	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7717	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7718	},
		7719	.ne => &.{
		7720	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7721	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7722	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7723	.{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7724	.{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
		7725	.{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		7726	.{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
		7727	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7728	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7729	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7730	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7731	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7732	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7733	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7734	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7735	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7736	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		7737	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7738	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7739	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7740	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7741	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7742	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7743	},
		7744	} },
		7745	}, .{
		7746	.required_features = .{ .avx, null, null, null },
		7747	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		7748	.patterns = &.{
		7749	.{ .src = .{ .to_mem, .to_mem } },
		7750	},
		7751	.extra_temps = .{
		7752	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7753	.{ .type = .u32, .kind = .{ .reg = .rcx } },
		7754	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7755	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7756	.{ .kind = .{ .rc = .sse } },
		7757	.unused,
		7758	},
		7759	.dst_temps = .{.mem},
		7760	.clobbers = .{ .eflags = true },
		7761	.each = .{ .once = switch (cc) {
		7762	else => unreachable,
		7763	.e => &.{
		7764	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7765	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7766	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7767	.{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7768	.{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
		7769	.{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		7770	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7771	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7772	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7773	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7774	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7775	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7776	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7777	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7778	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7779	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		7780	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7781	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7782	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7783	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7784	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7785	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7786	},
		7787	.ne => &.{
		7788	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7789	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7790	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7791	.{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7792	.{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
		7793	.{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		7794	.{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ },
		7795	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7796	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7797	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7798	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7799	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7800	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7801	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7802	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7803	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7804	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		7805	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7806	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7807	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7808	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7809	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7810	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7811	},
		7812	} },
		7813	}, .{
		7814	.required_features = .{ .sse2, null, null, null },
		7815	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		7816	.patterns = &.{
		7817	.{ .src = .{ .to_mem, .to_mem } },
		7818	},
		7819	.extra_temps = .{
		7820	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7821	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7822	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		7823	.{ .kind = .{ .rc = .sse } },
		7824	.unused,
		7825	.unused,
		7826	},
		7827	.dst_temps = .{.mem},
		7828	.clobbers = .{ .eflags = true },
		7829	.each = .{ .once = switch (cc) {
		7830	else => unreachable,
		7831	.e => &.{
		7832	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7833	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7834	.{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7835	.{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		7836	.{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7837	.{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
		7838	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7839	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7840	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7841	},
		7842	.ne => &.{
		7843	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7844	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7845	.{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7846	.{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		7847	.{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7848	.{ ._, ._, .not, .tmp2d, ._, ._, ._ },
		7849	.{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
		7850	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7851	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7852	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7853	},
		7854	} },
		7855	}, .{
		7856	.required_features = .{ .sse2, null, null, null },
		7857	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		7858	.patterns = &.{
		7859	.{ .src = .{ .to_mem, .to_mem } },
		7860	},
		7861	.extra_temps = .{
		7862	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7863	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		7864	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7865	.{ .kind = .{ .rc = .sse } },
		7866	.unused,
		7867	.unused,
		7868	},
		7869	.dst_temps = .{.mem},
		7870	.clobbers = .{ .eflags = true },
		7871	.each = .{ .once = switch (cc) {
		7872	else => unreachable,
		7873	.e => &.{
		7874	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7875	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7876	.{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7877	.{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		7878	.{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
		7879	.{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7880	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		7881	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		7882	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7883	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7884	},
		7885	.ne => &.{
		7886	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7887	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7888	.{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7889	.{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		7890	.{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
		7891	.{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
		7892	.{ ._, ._, .not, .tmp2b, ._, ._, ._ },
		7893	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		7894	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		7895	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		7896	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7897	},
		7898	} },
		7899	}, .{
		7900	.required_features = .{ .sse2, null, null, null },
		7901	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		7902	.patterns = &.{
		7903	.{ .src = .{ .to_mem, .to_mem } },
		7904	},
		7905	.extra_temps = .{
		7906	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7907	.{ .type = .u32, .kind = .{ .reg = .rcx } },
		7908	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7909	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7910	.{ .kind = .{ .rc = .sse } },
		7911	.unused,
		7912	},
		7913	.dst_temps = .{.mem},
		7914	.clobbers = .{ .eflags = true },
		7915	.each = .{ .once = switch (cc) {
		7916	else => unreachable,
		7917	.e => &.{
		7918	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7919	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7920	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7921	.{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7922	.{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		7923	.{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		7924	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7925	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7926	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7927	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7928	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7929	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7930	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7931	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7932	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7933	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		7934	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7935	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7936	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7937	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7938	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7939	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7940	},
		7941	.ne => &.{
		7942	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7943	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7944	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7945	.{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7946	.{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		7947	.{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		7948	.{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
		7949	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7950	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7951	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		7952	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7953	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7954	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7955	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7956	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		7957	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7958	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		7959	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		7960	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7961	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		7962	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7963	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7964	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		7965	},
		7966	} },
		7967	}, .{
		7968	.required_features = .{ .sse4_1, null, null, null },
		7969	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		7970	.patterns = &.{
		7971	.{ .src = .{ .to_mem, .to_mem } },
		7972	},
		7973	.extra_temps = .{
		7974	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		7975	.{ .type = .u32, .kind = .{ .reg = .rcx } },
		7976	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7977	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		7978	.{ .kind = .{ .rc = .sse } },
		7979	.unused,
		7980	},
		7981	.dst_temps = .{.mem},
		7982	.clobbers = .{ .eflags = true },
		7983	.each = .{ .once = switch (cc) {
		7984	else => unreachable,
		7985	.e => &.{
		7986	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		7987	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		7988	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		7989	.{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		7990	.{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		7991	.{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		7992	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		7993	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		7994	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		7995	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		7996	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		7997	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		7998	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		7999	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		8000	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8001	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		8002	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8003	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8004	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		8005	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		8006	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		8007	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		8008	},
		8009	.ne => &.{
		8010	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8011	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8012	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8013	.{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		8014	.{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		8015	.{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
		8016	.{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ },
		8017	.{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
		8018	.{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
		8019	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		8020	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8021	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		8022	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		8023	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		8024	.{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
		8025	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8026	.{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
		8027	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8028	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8029	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		8030	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		8031	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		8032	.{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
		8033	},
		8034	} },
		8035	}, .{
		8036	.required_features = .{ .sse, .mmx, null, null },
		8037	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		8038	.patterns = &.{
		8039	.{ .src = .{ .to_mem, .to_mem } },
		8040	},
		8041	.extra_temps = .{
		8042	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8043	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8044	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8045	.{ .kind = .{ .rc = .mmx } },
		8046	.unused,
		8047	.unused,
		8048	},
		8049	.dst_temps = .{.mem},
		8050	.clobbers = .{ .eflags = true },
		8051	.each = .{ .once = switch (cc) {
		8052	else => unreachable,
		8053	.e => &.{
		8054	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8055	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8056	.{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8057	.{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8058	.{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
		8059	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		8060	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		8061	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		8062	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8063	},
		8064	.ne => &.{
		8065	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8066	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8067	.{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8068	.{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8069	.{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
		8070	.{ ._, ._, .not, .tmp2b, ._, ._, ._ },
		8071	.{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
		8072	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
		8073	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		8074	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8075	},
		8076	} },
		8077	}, .{
		8078	.required_features = .{ .sse, .mmx, null, null },
		8079	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		8080	.patterns = &.{
		8081	.{ .src = .{ .to_mem, .to_mem } },
		8082	},
		8083	.extra_temps = .{
		8084	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8085	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8086	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8087	.{ .kind = .{ .rc = .mmx } },
		8088	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8089	.{ .kind = .{ .rc = .mmx } },
		8090	},
		8091	.dst_temps = .{.mem},
		8092	.clobbers = .{ .eflags = true },
		8093	.each = .{ .once = switch (cc) {
		8094	else => unreachable,
		8095	.e => &.{
		8096	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8097	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8098	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8099	.{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
		8100	.{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8101	.{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8102	.{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
		8103	.{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
		8104	.{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
		8105	.{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
		8106	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		8107	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8108	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		8109	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8110	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8111	.{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
		8112	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8113	.{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
		8114	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8115	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8116	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		8117	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8118	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8119	.{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
		8120	},
		8121	.ne => &.{
		8122	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8123	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8124	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8125	.{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
		8126	.{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8127	.{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8128	.{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
		8129	.{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
		8130	.{ ._, ._, .xor, .tmp4b, .si(0b1111), ._, ._ },
		8131	.{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
		8132	.{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
		8133	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
		8134	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8135	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		8136	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8137	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8138	.{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
		8139	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8140	.{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
		8141	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8142	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8143	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		8144	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8145	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8146	.{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
		8147	},
		8148	} },
		8149	}, .{
		8150	.required_features = .{ .sse, .mmx, null, null },
		8151	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		8152	.patterns = &.{
		8153	.{ .src = .{ .to_mem, .to_mem } },
		8154	},
		8155	.extra_temps = .{
		8156	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8157	.{ .type = .u32, .kind = .{ .reg = .rcx } },
		8158	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8159	.{ .kind = .{ .rc = .mmx } },
		8160	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8161	.{ .kind = .{ .rc = .mmx } },
		8162	},
		8163	.dst_temps = .{.mem},
		8164	.clobbers = .{ .eflags = true },
		8165	.each = .{ .once = switch (cc) {
		8166	else => unreachable,
		8167	.e => &.{
		8168	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8169	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8170	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8171	.{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
		8172	.{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8173	.{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8174	.{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
		8175	.{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
		8176	.{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
		8177	.{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
		8178	.{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
		8179	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		8180	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8181	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		8182	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8183	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8184	.{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
		8185	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8186	.{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
		8187	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8188	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8189	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		8190	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8191	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8192	.{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
		8193	},
		8194	.ne => &.{
		8195	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8196	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8197	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8198	.{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
		8199	.{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8200	.{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8201	.{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
		8202	.{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
		8203	.{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
		8204	.{ ._, ._, .xor, .tmp4b, .si(0b11), ._, ._ },
		8205	.{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
		8206	.{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
		8207	.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
		8208	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8209	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		8210	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8211	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8212	.{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
		8213	.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
		8214	.{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
		8215	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8216	.{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
		8217	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		8218	.{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
		8219	.{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
		8220	.{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
		8221	},
		8222	} },
		8223	}, .{
		8224	.dst_constraints = .{.{ .bool_vec = .byte }},
		8225	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		8226	.patterns = &.{
		8227	.{ .src = .{ .to_mem, .to_mem } },
		8228	},
		8229	.extra_temps = .{
		8230	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8231	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8232	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8233	.unused,
		8234	.unused,
		8235	.unused,
		8236	},
		8237	.dst_temps = .{.{ .rc = .general_purpose }},
		8238	.clobbers = .{ .eflags = true },
		8239	.each = .{ .once = &.{
		8240	.{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
		8241	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8242	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8243	.{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
		8244	.{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
		8245	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8246	.{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
		8247	.{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
		8248	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8249	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		8250	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8251	} },
		8252	}, .{
		8253	.dst_constraints = .{.{ .bool_vec = .byte }},
		8254	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		8255	.patterns = &.{
		8256	.{ .src = .{ .to_mem, .to_mem } },
		8257	},
		8258	.extra_temps = .{
		8259	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8260	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8261	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		8262	.unused,
		8263	.unused,
		8264	.unused,
		8265	},
		8266	.dst_temps = .{.{ .rc = .general_purpose }},
		8267	.clobbers = .{ .eflags = true },
		8268	.each = .{ .once = &.{
		8269	.{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
		8270	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8271	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8272	.{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
		8273	.{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
		8274	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8275	.{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
		8276	.{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
		8277	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8278	.{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
		8279	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8280	} },
		8281	}, .{
		8282	.dst_constraints = .{.{ .bool_vec = .byte }},
		8283	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		8284	.patterns = &.{
		8285	.{ .src = .{ .to_mem, .to_mem } },
		8286	},
		8287	.extra_temps = .{
		8288	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8289	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8290	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8291	.unused,
		8292	.unused,
		8293	.unused,
		8294	},
		8295	.dst_temps = .{.{ .rc = .general_purpose }},
		8296	.clobbers = .{ .eflags = true },
		8297	.each = .{ .once = &.{
		8298	.{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
		8299	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8300	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8301	.{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
		8302	.{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
		8303	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8304	.{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
		8305	.{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
		8306	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8307	.{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
		8308	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8309	} },
		8310	}, .{
		8311	.required_features = .{ .@"64bit", null, null, null },
		8312	.dst_constraints = .{.{ .bool_vec = .byte }},
		8313	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		8314	.patterns = &.{
		8315	.{ .src = .{ .to_mem, .to_mem } },
		8316	},
		8317	.extra_temps = .{
		8318	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8319	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8320	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		8321	.unused,
		8322	.unused,
		8323	.unused,
		8324	},
		8325	.dst_temps = .{.{ .rc = .general_purpose }},
		8326	.clobbers = .{ .eflags = true },
		8327	.each = .{ .once = &.{
		8328	.{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
		8329	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8330	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8331	.{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8332	.{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8333	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8334	.{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
		8335	.{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
		8336	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8337	.{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
		8338	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8339	} },
		8340	}, .{
		8341	.dst_constraints = .{.{ .bool_vec = .byte }},
		8342	.patterns = &.{
		8343	.{ .src = .{ .to_mem, .to_mem } },
		8344	},
		8345	.extra_temps = .{
		8346	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8347	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8348	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8349	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8350	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8351	.unused,
		8352	},
		8353	.dst_temps = .{.{ .rc = .general_purpose }},
		8354	.clobbers = .{ .eflags = true },
		8355	.each = .{ .once = &.{
		8356	.{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
		8357	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		8358	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8359	.{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
		8360	.{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
		8361	.{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
		8362	.{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
		8363	.{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
		8364	.{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
		8365	.{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
		8366	.{ ._, ._b, .j, .@"1b", ._, ._, ._ },
		8367	.{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
		8368	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8369	.{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
		8370	.{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
		8371	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8372	.{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
		8373	.{ ._, ._b, .j, .@"0b", ._, ._, ._ },
		8374	} },
		8375	}, .{
		8376	.dst_constraints = .{.{ .bool_vec = .dword }},
		8377	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		8378	.patterns = &.{
		8379	.{ .src = .{ .to_mem, .to_mem } },
		8380	},
		8381	.extra_temps = .{
		8382	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8383	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8384	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8385	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8386	.unused,
		8387	.unused,
		8388	},
		8389	.dst_temps = .{.{ .rc = .general_purpose }},
		8390	.clobbers = .{ .eflags = true },
		8391	.each = .{ .once = &.{
		8392	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8393	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8394	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8395	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8396	.{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
		8397	.{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
		8398	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8399	.{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
		8400	.{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
		8401	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8402	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		8403	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8404	} },
		8405	}, .{
		8406	.dst_constraints = .{.{ .bool_vec = .dword }},
		8407	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		8408	.patterns = &.{
		8409	.{ .src = .{ .to_mem, .to_mem } },
		8410	},
		8411	.extra_temps = .{
		8412	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8413	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8414	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8415	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		8416	.unused,
		8417	.unused,
		8418	},
		8419	.dst_temps = .{.{ .rc = .general_purpose }},
		8420	.clobbers = .{ .eflags = true },
		8421	.each = .{ .once = &.{
		8422	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8423	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8424	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8425	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8426	.{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
		8427	.{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
		8428	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8429	.{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
		8430	.{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
		8431	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8432	.{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
		8433	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8434	} },
		8435	}, .{
		8436	.dst_constraints = .{.{ .bool_vec = .dword }},
		8437	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		8438	.patterns = &.{
		8439	.{ .src = .{ .to_mem, .to_mem } },
		8440	},
		8441	.extra_temps = .{
		8442	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8443	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8444	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8445	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8446	.unused,
		8447	.unused,
		8448	},
		8449	.dst_temps = .{.{ .rc = .general_purpose }},
		8450	.clobbers = .{ .eflags = true },
		8451	.each = .{ .once = &.{
		8452	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8453	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8454	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8455	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8456	.{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
		8457	.{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
		8458	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8459	.{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
		8460	.{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
		8461	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8462	.{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
		8463	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8464	} },
		8465	}, .{
		8466	.required_features = .{ .@"64bit", null, null, null },
		8467	.dst_constraints = .{.{ .bool_vec = .dword }},
		8468	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		8469	.patterns = &.{
		8470	.{ .src = .{ .to_mem, .to_mem } },
		8471	},
		8472	.extra_temps = .{
		8473	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8474	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8475	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8476	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		8477	.unused,
		8478	.unused,
		8479	},
		8480	.dst_temps = .{.{ .rc = .general_purpose }},
		8481	.clobbers = .{ .eflags = true },
		8482	.each = .{ .once = &.{
		8483	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8484	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8485	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8486	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8487	.{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8488	.{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8489	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8490	.{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
		8491	.{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
		8492	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8493	.{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
		8494	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8495	} },
		8496	}, .{
		8497	.dst_constraints = .{.{ .bool_vec = .dword }},
		8498	.patterns = &.{
		8499	.{ .src = .{ .to_mem, .to_mem } },
		8500	},
		8501	.extra_temps = .{
		8502	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8503	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8504	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8505	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8506	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8507	.unused,
		8508	},
		8509	.dst_temps = .{.{ .rc = .general_purpose }},
		8510	.clobbers = .{ .eflags = true },
		8511	.each = .{ .once = &.{
		8512	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8513	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		8514	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8515	.{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
		8516	.{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
		8517	.{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
		8518	.{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
		8519	.{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
		8520	.{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
		8521	.{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
		8522	.{ ._, ._b, .j, .@"1b", ._, ._, ._ },
		8523	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8524	.{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
		8525	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8526	.{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
		8527	.{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
		8528	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8529	.{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
		8530	.{ ._, ._b, .j, .@"0b", ._, ._, ._ },
		8531	} },
		8532	}, .{
		8533	.required_features = .{ .@"64bit", null, null, null },
		8534	.dst_constraints = .{.{ .bool_vec = .qword }},
		8535	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		8536	.patterns = &.{
		8537	.{ .src = .{ .to_mem, .to_mem } },
		8538	},
		8539	.extra_temps = .{
		8540	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8541	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8542	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		8543	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8544	.unused,
		8545	.unused,
		8546	},
		8547	.dst_temps = .{.{ .rc = .general_purpose }},
		8548	.clobbers = .{ .eflags = true },
		8549	.each = .{ .once = &.{
		8550	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8551	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8552	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8553	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8554	.{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
		8555	.{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
		8556	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8557	.{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
		8558	.{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
		8559	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8560	.{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
		8561	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8562	} },
		8563	}, .{
		8564	.required_features = .{ .@"64bit", null, null, null },
		8565	.dst_constraints = .{.{ .bool_vec = .qword }},
		8566	.src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
		8567	.patterns = &.{
		8568	.{ .src = .{ .to_mem, .to_mem } },
		8569	},
		8570	.extra_temps = .{
		8571	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8572	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8573	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		8574	.{ .type = .u16, .kind = .{ .rc = .general_purpose } },
		8575	.unused,
		8576	.unused,
		8577	},
		8578	.dst_temps = .{.{ .rc = .general_purpose }},
		8579	.clobbers = .{ .eflags = true },
		8580	.each = .{ .once = &.{
		8581	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8582	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8583	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8584	.{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
		8585	.{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
		8586	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8587	.{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
		8588	.{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
		8589	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8590	.{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
		8591	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8592	} },
		8593	}, .{
		8594	.required_features = .{ .@"64bit", null, null, null },
		8595	.dst_constraints = .{.{ .bool_vec = .qword }},
		8596	.src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
		8597	.patterns = &.{
		8598	.{ .src = .{ .to_mem, .to_mem } },
		8599	},
		8600	.extra_temps = .{
		8601	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8602	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8603	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		8604	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		8605	.unused,
		8606	.unused,
		8607	},
		8608	.dst_temps = .{.{ .rc = .general_purpose }},
		8609	.clobbers = .{ .eflags = true },
		8610	.each = .{ .once = &.{
		8611	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8612	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8613	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8614	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8615	.{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
		8616	.{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
		8617	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8618	.{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
		8619	.{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
		8620	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8621	.{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
		8622	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8623	} },
		8624	}, .{
		8625	.required_features = .{ .@"64bit", null, null, null },
		8626	.dst_constraints = .{.{ .bool_vec = .qword }},
		8627	.src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
		8628	.patterns = &.{
		8629	.{ .src = .{ .to_mem, .to_mem } },
		8630	},
		8631	.extra_temps = .{
		8632	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8633	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8634	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		8635	.{ .type = .u64, .kind = .{ .rc = .general_purpose } },
		8636	.unused,
		8637	.unused,
		8638	},
		8639	.dst_temps = .{.{ .rc = .general_purpose }},
		8640	.clobbers = .{ .eflags = true },
		8641	.each = .{ .once = &.{
		8642	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8643	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8644	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8645	.{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8646	.{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		8647	.{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		8648	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8649	.{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
		8650	.{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
		8651	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8652	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		8653	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8654	} },
		8655	}, .{
		8656	.required_features = .{ .@"64bit", null, null, null },
		8657	.dst_constraints = .{.{ .bool_vec = .qword }},
		8658	.patterns = &.{
		8659	.{ .src = .{ .to_mem, .to_mem } },
		8660	},
		8661	.extra_temps = .{
		8662	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8663	.{ .type = .u8, .kind = .{ .reg = .cl } },
		8664	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8665	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8666	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8667	.unused,
		8668	},
		8669	.dst_temps = .{.{ .rc = .general_purpose }},
		8670	.clobbers = .{ .eflags = true },
		8671	.each = .{ .once = &.{
		8672	.{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
		8673	.{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
		8674	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8675	.{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
		8676	.{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
		8677	.{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
		8678	.{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
		8679	.{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
		8680	.{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
		8681	.{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
		8682	.{ ._, ._b, .j, .@"1b", ._, ._, ._ },
		8683	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8684	.{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
		8685	.{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
		8686	.{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
		8687	.{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
		8688	.{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
		8689	.{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
		8690	.{ ._, ._b, .j, .@"0b", ._, ._, ._ },
		8691	} },
		8692	}, .{
		8693	.src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
		8694	.patterns = &.{
		8695	.{ .src = .{ .to_mem, .to_mem } },
		8696	},
		8697	.extra_temps = .{
		8698	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8699	.{ .type = .u32, .kind = .{ .reg = .ecx } },
		8700	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8701	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8702	.{ .type = .u8, .kind = .{ .rc = .general_purpose } },
		8703	.unused,
		8704	},
		8705	.dst_temps = .{.mem},
		8706	.clobbers = .{ .eflags = true },
		8707	.each = .{ .once = &.{
		8708	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8709	.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
		8710	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8711	.{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
		8712	.{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
		8713	.{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
		8714	.{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ },
		8715	.{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ },
		8716	.{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ },
		8717	.{ ._, ._, .add, .tmp1d, .si(1), ._, ._ },
		8718	.{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
		8719	.{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
		8720	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		8721	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		8722	.{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ },
		8723	.{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
		8724	.{ .@"1:", ._, .add, .tmp0p, .si(1), ._, ._ },
		8725	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8726	.{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
		8727	.{ ._, ._z, .j, .@"0f", ._, ._, ._ },
		8728	.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
		8729	.{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
		8730	.{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ },
		8731	} },
		8732	} },
		8733	}) catch \|err\| switch (err) {
		8734	error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
		8735	@tagName(air_tag),
		8736	cg.typeOf(extra.lhs).fmt(pt),
		8737	ops[0].tracking(cg),
		8738	ops[1].tracking(cg),
		8739	}),
		8740	else => \|e\| return e,
		8741	},
		8742	.gte => unreachable,
		8743	.gt => unreachable,
		8744	}
		8745	for (ops) \|op\| for (res) \|r\| {
		8746	if (op.index == r.index) break;
		8747	} else try op.die(cg);
		8748	try res[0].moveTo(inst, cg);
		8749	},
		8750
		8751	.cmp_lt,
		8752	.cmp_lt_optimized,
		8753	.cmp_lte,
		8754	.cmp_lte_optimized,
		8755	.cmp_gte,
		8756	.cmp_gte_optimized,
		8757	.cmp_gt,
		8758	.cmp_gt_optimized,
		8759	=> \|air_tag\| if (use_old) try cg.airCmp(inst, switch (air_tag) {
		8760	else => unreachable,
		8761	.cmp_lt, .cmp_lt_optimized => .lt,
		8762	.cmp_lte, .cmp_lte_optimized => .lte,
		8763	.cmp_gte, .cmp_gte_optimized => .gte,
		8764	.cmp_gt, .cmp_gt_optimized => .gt,
		8765	}) else fallback: {
		8766	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
		8767	const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
		8768	if (scalar_ty.isRuntimeFloat()) break :fallback try cg.airCmp(inst, switch (air_tag) {
		8769	else => unreachable,
		8770	.cmp_lt, .cmp_lt_optimized => .lt,
		8771	.cmp_lte, .cmp_lte_optimized => .lte,
		8772	.cmp_gte, .cmp_gte_optimized => .gte,
		8773	.cmp_gt, .cmp_gt_optimized => .gt,
		8774	});
		8775	const signedness = if (scalar_ty.isAbiInt(zcu))
		8776	scalar_ty.intInfo(zcu).signedness
		8777	else
		8778	.unsigned;
		8779	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		8780	var res: [1]Temp = undefined;
		8781	cg.select(&res, &.{.bool}, &ops, switch (@as(Condition, switch (signedness) {
		8782	.signed => switch (air_tag) {
		8783	else => unreachable,
		8784	.cmp_lt, .cmp_lt_optimized => .l,
		8785	.cmp_lte, .cmp_lte_optimized => .le,
		8786	.cmp_gte, .cmp_gte_optimized => .ge,
		8787	.cmp_gt, .cmp_gt_optimized => .g,
		8788	},
		8789	.unsigned => switch (air_tag) {
		8790	else => unreachable,
		8791	.cmp_lt, .cmp_lt_optimized => .b,
		8792	.cmp_lte, .cmp_lte_optimized => .be,
		8793	.cmp_gte, .cmp_gte_optimized => .ae,
		8794	.cmp_gt, .cmp_gt_optimized => .a,
		8795	},
		8796	})) {
		8797	else => unreachable,
		8798	inline .l, .le, .ge, .g, .b, .be, .ae, .a => \|cc\| comptime &.{ .{
		8799	.src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
		8800	.patterns = &.{
		8801	.{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
		8802	.{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
		8803	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		8804	},
		8805	.dst_temps = .{.{ .cc = cc.commute() }},
		8806	.clobbers = .{ .eflags = true },
		8807	.each = .{ .once = &.{
		8808	.{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
		8809	} },
		8810	}, .{
		8811	.src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
		8812	.patterns = &.{
		8813	.{ .src = .{ .mem, .imm8 } },
		8814	.{ .src = .{ .to_gpr, .imm8 } },
		8815	.{ .src = .{ .to_gpr, .mem } },
		8816	.{ .src = .{ .to_gpr, .to_gpr } },
		8817	},
		8818	.dst_temps = .{.{ .cc = cc }},
		8819	.clobbers = .{ .eflags = true },
		8820	.each = .{ .once = &.{
		8821	.{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
		8822	} },
		8823	}, .{
		8824	.src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
		8825	.patterns = &.{
		8826	.{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
		8827	.{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
		8828	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		8829	},
		8830	.dst_temps = .{.{ .cc = cc.commute() }},
		8831	.clobbers = .{ .eflags = true },
		8832	.each = .{ .once = &.{
		8833	.{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
		8834	} },
		8835	}, .{
		8836	.src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
		8837	.patterns = &.{
		8838	.{ .src = .{ .mem, .imm16 } },
		8839	.{ .src = .{ .to_gpr, .imm16 } },
		8840	.{ .src = .{ .to_gpr, .mem } },
		8841	.{ .src = .{ .to_gpr, .to_gpr } },
		8842	},
		8843	.dst_temps = .{.{ .cc = cc }},
		8844	.clobbers = .{ .eflags = true },
		8845	.each = .{ .once = &.{
		8846	.{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
		8847	} },
		8848	}, .{
		8849	.src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
		8850	.patterns = &.{
		8851	.{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
		8852	.{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
		8853	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		8854	},
		8855	.dst_temps = .{.{ .cc = cc.commute() }},
		8856	.clobbers = .{ .eflags = true },
		8857	.each = .{ .once = &.{
		8858	.{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
		8859	} },
		8860	}, .{
		8861	.src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
		8862	.patterns = &.{
		8863	.{ .src = .{ .mem, .imm32 } },
		8864	.{ .src = .{ .to_gpr, .imm32 } },
		8865	.{ .src = .{ .to_gpr, .mem } },
		8866	.{ .src = .{ .to_gpr, .to_gpr } },
		8867	},
		8868	.dst_temps = .{.{ .cc = cc }},
		8869	.clobbers = .{ .eflags = true },
		8870	.each = .{ .once = &.{
		8871	.{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
		8872	} },
		8873	}, .{
		8874	.required_features = .{ .@"64bit", null, null, null },
		8875	.src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
		8876	.patterns = &.{
		8877	.{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
		8878	.{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
		8879	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		8880	},
		8881	.dst_temps = .{.{ .cc = cc.commute() }},
		8882	.clobbers = .{ .eflags = true },
		8883	.each = .{ .once = &.{
		8884	.{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
		8885	} },
		8886	}, .{
		8887	.required_features = .{ .@"64bit", null, null, null },
		8888	.src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
		8889	.patterns = &.{
		8890	.{ .src = .{ .mem, .simm32 } },
		8891	.{ .src = .{ .to_gpr, .simm32 } },
		8892	.{ .src = .{ .to_gpr, .mem } },
		8893	.{ .src = .{ .to_gpr, .to_gpr } },
		8894	},
		8895	.dst_temps = .{.{ .cc = cc }},
		8896	.clobbers = .{ .eflags = true },
		8897	.each = .{ .once = &.{
		8898	.{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
		8899	} },
		8900	}, .{
		8901	.src_constraints = .{ .any_int, .any_int },
		8902	.patterns = &.{
		8903	.{ .src = .{ .to_mem, .to_mem }, .commute = switch (cc) {
		8904	else => unreachable,
		8905	.l, .ge, .b, .ae => .{ 0, 0 },
		8906	.le, .g, .be, .a => .{ 0, 1 },
		8907	} },
		8908	},
		8909	.extra_temps = .{
		8910	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		8911	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		8912	.unused,
		8913	.unused,
		8914	.unused,
		8915	.unused,
		8916	},
		8917	.dst_temps = .{.{ .rc = .general_purpose }},
		8918	.clobbers = .{ .eflags = true },
		8919	.each = .{ .once = &.{
		8920	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		8921	.{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
		8922	.{ .@"0:", ._r, .sh, .tmp1b, .si(1), ._, ._ },
		8923	.{ ._, ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
		8924	.{ ._, ._, .sbb, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
		8925	.{ ._, ._c, .set, .tmp1b, ._, ._, ._ },
		8926	.{ ._, .fromCondition(switch (cc) {
		8927	else => unreachable,
		8928	.l, .ge, .b, .ae => cc,
		8929	.le, .g, .be, .a => cc.commute(),
		8930	}), .set, .dst0b, ._, ._, ._ },
		8931	.{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
		8932	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		8933	} },
		8934	} },
		8935	}) catch \|err\| switch (err) {
		8936	error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
		8937	@tagName(air_tag),
		8938	cg.typeOf(bin_op.lhs).fmt(pt),
		8939	ops[0].tracking(cg),
		8940	ops[1].tracking(cg),
		8941	}),
		8942	else => \|e\| return e,
		8943	};
		8944	for (ops) \|op\| for (res) \|r\| {
		8945	if (op.index == r.index) break;
		8946	} else try op.die(cg);
		8947	try res[0].moveTo(inst, cg);
		8948	},
		8949	.cmp_eq,
		8950	.cmp_eq_optimized,
		8951	.cmp_neq,
		8952	.cmp_neq_optimized,
		8953	=> \|air_tag\| if (use_old) try cg.airCmp(inst, switch (air_tag) {
		8954	else => unreachable,
		8955	.cmp_eq, .cmp_eq_optimized => .eq,
		8956	.cmp_neq, .cmp_neq_optimized => .neq,
		8957	}) else fallback: {
		8958	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
		8959	const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
		8960	if (intInfo(scalar_ty, cg) == null) break :fallback try cg.airCmp(inst, switch (air_tag) {
		8961	else => unreachable,
		8962	.cmp_eq, .cmp_eq_optimized => .eq,
		8963	.cmp_neq, .cmp_neq_optimized => .neq,
		8964	});
		8965	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		8966	var res: [1]Temp = undefined;
		8967	cg.select(&res, &.{.bool}, &ops, switch (@as(Condition, switch (air_tag) {
		8968	else => unreachable,
		8969	.cmp_eq, .cmp_eq_optimized => .e,
		8970	.cmp_neq, .cmp_neq_optimized => .ne,
		8971	})) {
		8972	else => unreachable,
		8973	inline .e, .ne => \|cc\| comptime &.{ .{
		8974	.src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
		8975	.patterns = &.{
		8976	.{ .src = .{ .mem, .imm8 } },
		8977	.{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
		8978	.{ .src = .{ .to_gpr, .imm8 } },
		8979	.{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
		8980	.{ .src = .{ .to_gpr, .mem } },
		8981	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		8982	.{ .src = .{ .to_gpr, .to_gpr } },
		8983	},
		8984	.dst_temps = .{.{ .cc = cc }},
		8985	.clobbers = .{ .eflags = true },
		8986	.each = .{ .once = &.{
		8987	.{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
		8988	} },
		8989	}, .{
		8990	.src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
		8991	.patterns = &.{
		8992	.{ .src = .{ .mem, .imm16 } },
		8993	.{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
		8994	.{ .src = .{ .to_gpr, .imm16 } },
		8995	.{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
		8996	.{ .src = .{ .to_gpr, .mem } },
		8997	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		8998	.{ .src = .{ .to_gpr, .to_gpr } },
		8999	},
		9000	.dst_temps = .{.{ .cc = cc }},
		9001	.clobbers = .{ .eflags = true },
		9002	.each = .{ .once = &.{
		9003	.{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
		9004	} },
		9005	}, .{
		9006	.src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
		9007	.patterns = &.{
		9008	.{ .src = .{ .mem, .imm32 } },
		9009	.{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
		9010	.{ .src = .{ .to_gpr, .imm32 } },
		9011	.{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
		9012	.{ .src = .{ .to_gpr, .mem } },
		9013	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		9014	.{ .src = .{ .to_gpr, .to_gpr } },
		9015	},
		9016	.dst_temps = .{.{ .cc = cc }},
		9017	.clobbers = .{ .eflags = true },
		9018	.each = .{ .once = &.{
		9019	.{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
		9020	} },
		9021	}, .{
		9022	.required_features = .{ .@"64bit", null, null, null },
		9023	.src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
		9024	.patterns = &.{
		9025	.{ .src = .{ .mem, .simm32 } },
		9026	.{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
		9027	.{ .src = .{ .to_gpr, .simm32 } },
		9028	.{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
		9029	.{ .src = .{ .to_gpr, .mem } },
		9030	.{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
		9031	.{ .src = .{ .to_gpr, .to_gpr } },
		9032	},
		9033	.dst_temps = .{.{ .cc = cc }},
		9034	.clobbers = .{ .eflags = true },
		9035	.each = .{ .once = &.{
		9036	.{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
		9037	} },
		9038	}, .{
		9039	.required_features = .{ .sse, .mmx, null, null },
		9040	.src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
		9041	.patterns = &.{
		9042	.{ .src = .{ .to_mut_mm, .mem } },
		9043	.{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
		9044	.{ .src = .{ .to_mut_mm, .to_mm } },
		9045	},
		9046	.extra_temps = .{
		9047	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		9048	.{ .kind = .{ .rc = .mmx } },
		9049	.unused,
		9050	.unused,
		9051	.unused,
		9052	.unused,
		9053	},
		9054	.dst_temps = .{.{ .cc = cc }},
		9055	.clobbers = .{ .eflags = true },
		9056	.each = .{ .once = &.{
		9057	.{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
		9058	.{ ._, .p_, .xor, .src0q, .src1q, ._, ._ },
		9059	.{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ },
		9060	.{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
		9061	.{ ._, ._, .xor, .tmp0d, .si(0xff), ._, ._ },
		9062	} },
		9063	}, .{
		9064	.required_features = .{ .avx, null, null, null },
		9065	.src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
		9066	.patterns = &.{
		9067	.{ .src = .{ .to_xmm, .mem } },
		9068	.{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
		9069	.{ .src = .{ .to_xmm, .to_xmm } },
		9070	},
		9071	.extra_temps = .{
		9072	.{ .kind = .{ .rc = .sse } },
		9073	.unused,
		9074	.unused,
		9075	.unused,
		9076	.unused,
		9077	.unused,
		9078	},
		9079	.dst_temps = .{.{ .cc = cc }},
		9080	.clobbers = .{ .eflags = true },
		9081	.each = .{ .once = &.{
		9082	.{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ },
		9083	.{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ },
		9084	} },
		9085	}, .{
		9086	.required_features = .{ .sse4_1, null, null, null },
		9087	.src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
		9088	.patterns = &.{
		9089	.{ .src = .{ .to_mut_xmm, .mem } },
		9090	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		9091	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		9092	},
		9093	.dst_temps = .{.{ .cc = cc }},
		9094	.clobbers = .{ .eflags = true },
		9095	.each = .{ .once = &.{
		9096	.{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
		9097	.{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ },
		9098	} },
		9099	}, .{
		9100	.required_features = .{ .sse2, null, null, null },
		9101	.src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
		9102	.patterns = &.{
		9103	.{ .src = .{ .to_mut_xmm, .mem } },
		9104	.{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
		9105	.{ .src = .{ .to_mut_xmm, .to_xmm } },
		9106	},
		9107	.extra_temps = .{
		9108	.{ .type = .u32, .kind = .{ .rc = .general_purpose } },
		9109	.{ .kind = .{ .rc = .sse } },
		9110	.unused,
		9111	.unused,
		9112	.unused,
		9113	.unused,
		9114	},
		9115	.dst_temps = .{.{ .cc = cc }},
		9116	.clobbers = .{ .eflags = true },
		9117	.each = .{ .once = &.{
		9118	.{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
		9119	.{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
		9120	.{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ },
		9121	.{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
		9122	.{ ._, ._, .xor, .tmp0d, .si(0xffff), ._, ._ },
		9123	} },
		9124	}, .{
		9125	.required_features = .{ .avx2, null, null, null },
		9126	.src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
		9127	.patterns = &.{
		9128	.{ .src = .{ .to_ymm, .mem } },
		9129	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		9130	.{ .src = .{ .to_ymm, .to_ymm } },
		9131	},
		9132	.extra_temps = .{
		9133	.{ .kind = .{ .rc = .sse } },
		9134	.unused,
		9135	.unused,
		9136	.unused,
		9137	.unused,
		9138	.unused,
		9139	},
		9140	.dst_temps = .{.{ .cc = cc }},
		9141	.clobbers = .{ .eflags = true },
		9142	.each = .{ .once = &.{
		9143	.{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ },
		9144	.{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
		9145	} },
		9146	}, .{
		9147	.required_features = .{ .avx, null, null, null },
		9148	.src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
		9149	.patterns = &.{
		9150	.{ .src = .{ .to_ymm, .mem } },
		9151	.{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
		9152	.{ .src = .{ .to_ymm, .to_ymm } },
		9153	},
		9154	.extra_temps = .{
		9155	.{ .kind = .{ .rc = .sse } },
		9156	.unused,
		9157	.unused,
		9158	.unused,
		9159	.unused,
		9160	.unused,
		9161	},
		9162	.dst_temps = .{.{ .cc = cc }},
		9163	.clobbers = .{ .eflags = true },
		9164	.each = .{ .once = &.{
		9165	.{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ },
		9166	.{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
		9167	} },
		9168	}, .{
		9169	.required_features = .{ .avx2, null, null, null },
		9170	.src_constraints = .{
		9171	.{ .remainder_int = .{ .of = .yword, .is = .xword } },
		9172	.{ .remainder_int = .{ .of = .yword, .is = .xword } },
		9173	},
		9174	.patterns = &.{
		9175	.{ .src = .{ .to_mem, .to_mem } },
		9176	},
		9177	.extra_temps = .{
		9178	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9179	.{ .kind = .{ .rc = .sse } },
		9180	.{ .kind = .{ .rc = .sse } },
		9181	.unused,
		9182	.unused,
		9183	.unused,
		9184	},
		9185	.dst_temps = .{.{ .cc = cc }},
		9186	.clobbers = .{ .eflags = true },
		9187	.each = .{ .once = &.{
		9188	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		9189	.{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
		9190	.{ .@"0:", .v_dqu, .mov, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
		9191	.{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
		9192	.{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
		9193	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		9194	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9195	.{ ._, .v_dqa, .mov, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
		9196	.{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
		9197	.{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
		9198	.{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
		9199	} },
		9200	}, .{
		9201	.required_features = .{ .avx2, null, null, null },
		9202	.patterns = &.{
		9203	.{ .src = .{ .to_mem, .to_mem } },
		9204	},
		9205	.extra_temps = .{
		9206	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9207	.{ .kind = .{ .rc = .sse } },
		9208	.{ .kind = .{ .rc = .sse } },
		9209	.unused,
		9210	.unused,
		9211	.unused,
		9212	},
		9213	.dst_temps = .{.{ .cc = cc }},
		9214	.clobbers = .{ .eflags = true },
		9215	.each = .{ .once = &.{
		9216	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		9217	.{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
		9218	.{ .@"0:", .v_dqu, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		9219	.{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
		9220	.{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
		9221	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		9222	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9223	.{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
		9224	} },
		9225	}, .{
		9226	.required_features = .{ .avx, null, null, null },
		9227	.src_constraints = .{
		9228	.{ .remainder_int = .{ .of = .yword, .is = .xword } },
		9229	.{ .remainder_int = .{ .of = .yword, .is = .xword } },
		9230	},
		9231	.patterns = &.{
		9232	.{ .src = .{ .to_mem, .to_mem } },
		9233	},
		9234	.extra_temps = .{
		9235	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9236	.{ .kind = .{ .rc = .sse } },
		9237	.{ .kind = .{ .rc = .sse } },
		9238	.unused,
		9239	.unused,
		9240	.unused,
		9241	},
		9242	.dst_temps = .{.{ .cc = cc }},
		9243	.clobbers = .{ .eflags = true },
		9244	.each = .{ .once = &.{
		9245	.{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
		9246	.{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
		9247	.{ .@"0:", .v_pd, .movu, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
		9248	.{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
		9249	.{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
		9250	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		9251	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9252	.{ ._, .v_pd, .mova, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
		9253	.{ ._, .v_pd, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
		9254	.{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
		9255	.{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
		9256	} },
		9257	}, .{
		9258	.required_features = .{ .avx, null, null, null },
		9259	.patterns = &.{
		9260	.{ .src = .{ .to_mem, .to_mem } },
		9261	},
		9262	.extra_temps = .{
		9263	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9264	.{ .kind = .{ .rc = .sse } },
		9265	.{ .kind = .{ .rc = .sse } },
		9266	.unused,
		9267	.unused,
		9268	.unused,
		9269	},
		9270	.dst_temps = .{.{ .cc = cc }},
		9271	.clobbers = .{ .eflags = true },
		9272	.each = .{ .once = &.{
		9273	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		9274	.{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
		9275	.{ .@"0:", .v_pd, .movu, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
		9276	.{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
		9277	.{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
		9278	.{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
		9279	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9280	.{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
		9281	} },
		9282	}, .{
		9283	.required_features = .{ .avx, null, null, null },
		9284	.patterns = &.{
		9285	.{ .src = .{ .to_mem, .to_mem } },
		9286	},
		9287	.extra_temps = .{
		9288	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9289	.{ .kind = .{ .rc = .sse } },
		9290	.{ .kind = .{ .rc = .sse } },
		9291	.unused,
		9292	.unused,
		9293	.unused,
		9294	},
		9295	.dst_temps = .{.{ .cc = cc }},
		9296	.clobbers = .{ .eflags = true },
		9297	.each = .{ .once = &.{
		9298	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		9299	.{ ._, .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
		9300	.{ .@"0:", .v_dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		9301	.{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._ },
		9302	.{ ._, .vp_, .@"or", .tmp1x, .tmp1x, .tmp2x, ._ },
		9303	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		9304	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9305	.{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ },
		9306	} },
		9307	}, .{
		9308	.required_features = .{ .sse4_1, null, null, null },
		9309	.patterns = &.{
		9310	.{ .src = .{ .to_mem, .to_mem } },
		9311	},
		9312	.extra_temps = .{
		9313	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9314	.{ .kind = .{ .rc = .sse } },
		9315	.{ .kind = .{ .rc = .sse } },
		9316	.unused,
		9317	.unused,
		9318	.unused,
		9319	},
		9320	.dst_temps = .{.{ .cc = cc }},
		9321	.clobbers = .{ .eflags = true },
		9322	.each = .{ .once = &.{
		9323	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		9324	.{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
		9325	.{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		9326	.{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		9327	.{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
		9328	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		9329	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9330	.{ ._, .p_, .@"test", .tmp1x, .tmp1x, ._, ._ },
		9331	} },
		9332	}, .{
		9333	.required_features = .{ .sse2, null, null, null },
		9334	.patterns = &.{
		9335	.{ .src = .{ .to_mem, .to_mem } },
		9336	},
		9337	.extra_temps = .{
		9338	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9339	.{ .kind = .{ .rc = .sse } },
		9340	.{ .kind = .{ .rc = .sse } },
		9341	.unused,
		9342	.unused,
		9343	.unused,
		9344	},
		9345	.dst_temps = .{.{ .cc = cc }},
		9346	.clobbers = .{ .eflags = true },
		9347	.each = .{ .once = &.{
		9348	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		9349	.{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
		9350	.{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
		9351	.{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
		9352	.{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
		9353	.{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
		9354	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9355	.{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
		9356	.{ ._, .p_b, .cmpeq, .tmp1x, .tmp2x, ._, ._ },
		9357	.{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
		9358	.{ ._, ._, .cmp, .tmp0d, .si(0xffff), ._, ._ },
		9359	} },
		9360	}, .{
		9361	.required_features = .{ .sse, .mmx, null, null },
		9362	.patterns = &.{
		9363	.{ .src = .{ .to_mem, .to_mem } },
		9364	},
		9365	.extra_temps = .{
		9366	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9367	.{ .kind = .{ .rc = .mmx } },
		9368	.{ .kind = .{ .rc = .mmx } },
		9369	.unused,
		9370	.unused,
		9371	.unused,
		9372	},
		9373	.dst_temps = .{.{ .cc = cc }},
		9374	.clobbers = .{ .eflags = true },
		9375	.each = .{ .once = &.{
		9376	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		9377	.{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
		9378	.{ .@"0:", ._q, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
		9379	.{ ._, .p_, .xor, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
		9380	.{ ._, .p_, .@"or", .tmp1q, .tmp2q, ._, ._ },
		9381	.{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
		9382	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9383	.{ ._, .p_, .xor, .tmp2q, .tmp2q, ._, ._ },
		9384	.{ ._, .p_b, .cmpeq, .tmp1q, .tmp2q, ._, ._ },
		9385	.{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
		9386	.{ ._, ._, .cmp, .tmp0d, .si(0xff), ._, ._ },
		9387	} },
		9388	}, .{
		9389	.patterns = &.{
		9390	.{ .src = .{ .to_mem, .to_mem } },
		9391	},
		9392	.extra_temps = .{
		9393	.{ .type = .isize, .kind = .{ .rc = .general_purpose } },
		9394	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		9395	.{ .type = .usize, .kind = .{ .rc = .general_purpose } },
		9396	.unused,
		9397	.unused,
		9398	.unused,
		9399	},
		9400	.dst_temps = .{.{ .cc = cc }},
		9401	.clobbers = .{ .eflags = true },
		9402	.each = .{ .once = &.{
		9403	.{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
		9404	.{ ._, ._, .xor, .tmp1p, .tmp1p, ._, ._ },
		9405	.{ .@"0:", ._, .mov, .tmp2p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
		9406	.{ ._, ._, .xor, .tmp2p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
		9407	.{ ._, ._, .@"or", .tmp1p, .tmp2p, ._, ._ },
		9408	.{ ._, ._, .add, .tmp0p, .sa(.tmp2, .add_size), ._, ._ },
		9409	.{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
		9410	.{ ._, ._, .@"test", .tmp1p, .tmp1p, ._, ._ },
		9411	} },
		9412	} },
		9413	}) catch \|err\| switch (err) {
		9414	error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
		9415	@tagName(air_tag),
		9416	cg.typeOf(bin_op.lhs).fmt(pt),
		9417	ops[0].tracking(cg),
		9418	ops[1].tracking(cg),
		9419	}),
		9420	else => \|e\| return e,
		9421	};
		9422	for (ops) \|op\| for (res) \|r\| {
		9423	if (op.index == r.index) break;
		9424	} else try op.die(cg);
		9425	try res[0].moveTo(inst, cg);
		9426	},
		9427
		9428	.cond_br => try cg.airCondBr(inst),
		9429	.switch_br => try cg.airSwitchBr(inst),
		9430	.loop_switch_br => try cg.airLoopSwitchBr(inst),
		9431	.switch_dispatch => try cg.airSwitchDispatch(inst),
		9432	.@"try", .try_cold => try cg.airTry(inst),
		9433	.try_ptr, .try_ptr_cold => try cg.airTryPtr(inst),
		9434	.dbg_stmt => if (use_old) try cg.airDbgStmt(inst) else {
		9435	const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt;
		9436	_ = try cg.addInst(.{
		9437	.tag = .pseudo,
		9438	.ops = .pseudo_dbg_line_line_column,
		9439	.data = .{ .line_column = .{
		9440	.line = dbg_stmt.line,
		9441	.column = dbg_stmt.column,
		9442	} },
		9443	});
		9444	},
		9445	.dbg_empty_stmt => if (use_old) try cg.airDbgEmptyStmt() else {
		9446	if (cg.mir_instructions.len > 0) {
		9447	const prev_mir_op = &cg.mir_instructions.items(.ops)[cg.mir_instructions.len - 1];
		9448	if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column)
		9449	prev_mir_op.* = .pseudo_dbg_line_line_column;
		9450	}
		9451	try cg.asmOpOnly(.{ ._, .nop });
		9452	},
		9453	.dbg_inline_block => if (use_old) try cg.airDbgInlineBlock(inst) else {
		9454	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		9455	const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
		9456	const old_inline_func = cg.inline_func;
		9457	defer cg.inline_func = old_inline_func;
		9458	cg.inline_func = extra.data.func;
		9459	_ = try cg.addInst(.{
		9460	.tag = .pseudo,
		9461	.ops = .pseudo_dbg_enter_inline_func,
		9462	.data = .{ .func = extra.data.func },
		9463	});
		9464	try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
		9465	_ = try cg.addInst(.{
		9466	.tag = .pseudo,
		9467	.ops = .pseudo_dbg_leave_inline_func,
		9468	.data = .{ .func = old_inline_func },
		9469	});
		9470	},
		9471	.dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try cg.airDbgVar(inst) else {
		9472	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
		9473	var ops = try cg.tempsFromOperands(inst, .{pl_op.operand});
		9474	try cg.genLocalDebugInfo(inst, ops[0].tracking(cg).short);
		9475	try ops[0].die(cg);
		9476	},
		9477	.is_null_ptr => if (use_old) try cg.airIsNullPtr(inst) else {
		9478	const un_op = air_datas[@intFromEnum(inst)].un_op;
		9479	const opt_ty = cg.typeOf(un_op).childType(zcu);
		9480	const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
		9481	const opt_child_ty = opt_ty.optionalChild(zcu);
		9482	const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
		9483	var ops = try cg.tempsFromOperands(inst, .{un_op});
		9484	if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg);
		9485	while (try ops[0].toLea(cg)) {}
		9486	try cg.asmMemoryImmediate(
		9487	.{ ._, .cmp },
		9488	try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl)
		9489	.byte
		9490	else if (opt_child_ty.isSlice(zcu))
		9491	.qword
		9492	else
		9493	.fromSize(opt_child_abi_size) }),
		9494	.u(0),
		9495	);
		9496	var is_null = try cg.tempInit(.bool, .{ .eflags = .e });
		9497	try ops[0].die(cg);
		9498	try is_null.moveTo(inst, cg);
		9499	},
		9500	.is_non_null_ptr => if (use_old) try cg.airIsNonNullPtr(inst) else {
		9501	const un_op = air_datas[@intFromEnum(inst)].un_op;
		9502	const opt_ty = cg.typeOf(un_op).childType(zcu);
		9503	const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu);
		9504	const opt_child_ty = opt_ty.optionalChild(zcu);
		9505	const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu));
		9506	var ops = try cg.tempsFromOperands(inst, .{un_op});
		9507	if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg);
		9508	while (try ops[0].toLea(cg)) {}
		9509	try cg.asmMemoryImmediate(
		9510	.{ ._, .cmp },
		9511	try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl)
		9512	.byte
		9513	else if (opt_child_ty.isSlice(zcu))
		9514	.qword
		9515	else
		9516	.fromSize(opt_child_abi_size) }),
		9517	.u(0),
		9518	);
		9519	var is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne });
		9520	try ops[0].die(cg);
		9521	try is_non_null.moveTo(inst, cg);
		9522	},
		9523	.is_err_ptr => if (use_old) try cg.airIsErrPtr(inst) else {
		9524	const un_op = air_datas[@intFromEnum(inst)].un_op;
		9525	const eu_ty = cg.typeOf(un_op).childType(zcu);
		9526	const eu_err_ty = eu_ty.errorUnionSet(zcu);
		9527	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		9528	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		9529	var ops = try cg.tempsFromOperands(inst, .{un_op});
		9530	try ops[0].toOffset(eu_err_off, cg);
		9531	while (try ops[0].toLea(cg)) {}
		9532	try cg.asmMemoryImmediate(
		9533	.{ ._, .cmp },
		9534	try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }),
		9535	.u(0),
		9536	);
		9537	var is_err = try cg.tempInit(.bool, .{ .eflags = .ne });
		9538	try ops[0].die(cg);
		9539	try is_err.moveTo(inst, cg);
		9540	},
		9541	.is_non_err_ptr => if (use_old) try cg.airIsNonErrPtr(inst) else {
		9542	const un_op = air_datas[@intFromEnum(inst)].un_op;
		9543	const eu_ty = cg.typeOf(un_op).childType(zcu);
		9544	const eu_err_ty = eu_ty.errorUnionSet(zcu);
		9545	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		9546	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		9547	var ops = try cg.tempsFromOperands(inst, .{un_op});
		9548	try ops[0].toOffset(eu_err_off, cg);
		9549	while (try ops[0].toLea(cg)) {}
		9550	try cg.asmMemoryImmediate(
		9551	.{ ._, .cmp },
		9552	try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }),
		9553	.u(0),
		9554	);
		9555	var is_non_err = try cg.tempInit(.bool, .{ .eflags = .e });
		9556	try ops[0].die(cg);
		9557	try is_non_err.moveTo(inst, cg);
		9558	},
		9559	.load => if (use_old) try cg.airLoad(inst) else fallback: {
		9560	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9561	const val_ty = ty_op.ty.toType();
		9562	const ptr_ty = cg.typeOf(ty_op.operand);
		9563	const ptr_info = ptr_ty.ptrInfo(zcu);
		9564	if (ptr_info.packed_offset.host_size > 0 and
		9565	(ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type))
		9566	break :fallback try cg.airLoad(inst);
		9567	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9568	var res = try ops[0].load(val_ty, .{
		9569	.disp = switch (ptr_info.flags.vector_index) {
		9570	.none => 0,
		9571	.runtime => unreachable,
		9572	else => \|vector_index\| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
		9573	},
		9574	}, cg);
		9575	for (ops) \|op\| if (op.index != res.index) try op.die(cg);
		9576	try res.moveTo(inst, cg);
		9577	},
		9578	.int_from_ptr => if (use_old) try cg.airIntFromPtr(inst) else {
		9579	const un_op = air_datas[@intFromEnum(inst)].un_op;
		9580	var ops = try cg.tempsFromOperands(inst, .{un_op});
		9581	try ops[0].toSlicePtr(cg);
		9582	try ops[0].moveTo(inst, cg);
		9583	},
		9584	.int_from_bool => if (use_old) try cg.airIntFromBool(inst) else {
		9585	const un_op = air_datas[@intFromEnum(inst)].un_op;
		9586	var ops = try cg.tempsFromOperands(inst, .{un_op});
		9587	try ops[0].moveTo(inst, cg);
		9588	},
		9589	.ret => try cg.airRet(inst, false),
		9590	.ret_safe => try cg.airRet(inst, true),
		9591	.ret_load => try cg.airRetLoad(inst),
		9592	.store, .store_safe => \|air_tag\| if (use_old) try cg.airStore(inst, switch (air_tag) {
		9593	else => unreachable,
		9594	.store => false,
		9595	.store_safe => true,
		9596	}) else fallback: {
		9597	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
		9598	const ptr_ty = cg.typeOf(bin_op.lhs);
		9599	const ptr_info = ptr_ty.ptrInfo(zcu);
		9600	const val_ty = cg.typeOf(bin_op.rhs);
		9601	if (ptr_info.packed_offset.host_size > 0 and
		9602	(ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type))
		9603	break :fallback try cg.airStore(inst, switch (air_tag) {
		9604	else => unreachable,
		9605	.store => false,
		9606	.store_safe => true,
		9607	});
		9608	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		9609	try ops[0].store(&ops[1], .{
		9610	.disp = switch (ptr_info.flags.vector_index) {
		9611	.none => 0,
		9612	.runtime => unreachable,
		9613	else => \|vector_index\| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
		9614	},
		9615	.safe = switch (air_tag) {
		9616	else => unreachable,
		9617	.store => false,
		9618	.store_safe => true,
		9619	},
		9620	}, cg);
		9621	for (ops) \|op\| try op.die(cg);
		9622	},
		9623	.unreach => {},
		9624	.optional_payload_ptr => if (use_old) try cg.airOptionalPayloadPtr(inst) else {
		9625	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9626	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9627	try ops[0].moveTo(inst, cg);
		9628	},
		9629	.optional_payload_ptr_set => if (use_old) try cg.airOptionalPayloadPtrSet(inst) else {
		9630	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9631	const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
		9632	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9633	if (!opt_ty.optionalReprIsPayload(zcu)) {
		9634	const opt_child_ty = opt_ty.optionalChild(zcu);
		9635	const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu));
		9636	try ops[0].toOffset(opt_child_abi_size, cg);
		9637	var has_value = try cg.tempInit(.bool, .{ .immediate = 1 });
		9638	try ops[0].store(&has_value, .{}, cg);
		9639	try has_value.die(cg);
		9640	try ops[0].toOffset(-opt_child_abi_size, cg);
		9641	}
		9642	try ops[0].moveTo(inst, cg);
		9643	},
		9644	.unwrap_errunion_payload_ptr => if (use_old) try cg.airUnwrapErrUnionPayloadPtr(inst) else {
		9645	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9646	const eu_ty = cg.typeOf(ty_op.operand).childType(zcu);
		9647	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		9648	const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
		9649	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9650	try ops[0].toOffset(eu_pl_off, cg);
		9651	try ops[0].moveTo(inst, cg);
		9652	},
		9653	.unwrap_errunion_err_ptr => if (use_old) try cg.airUnwrapErrUnionErrPtr(inst) else {
		9654	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9655	const eu_ty = cg.typeOf(ty_op.operand).childType(zcu);
		9656	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		9657	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		9658	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9659	try ops[0].toOffset(eu_err_off, cg);
		9660	var err = try ops[0].load(eu_ty.errorUnionSet(zcu), .{}, cg);
		9661	try ops[0].die(cg);
		9662	try err.moveTo(inst, cg);
		9663	},
		9664	.errunion_payload_ptr_set => if (use_old) try cg.airErrUnionPayloadPtrSet(inst) else {
		9665	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9666	const eu_ty = cg.typeOf(ty_op.operand).childType(zcu);
		9667	const eu_err_ty = eu_ty.errorUnionSet(zcu);
		9668	const eu_pl_ty = eu_ty.errorUnionPayload(zcu);
		9669	const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
		9670	const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu));
		9671	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9672	try ops[0].toOffset(eu_err_off, cg);
		9673	var no_err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 });
		9674	try ops[0].store(&no_err, .{}, cg);
		9675	try no_err.die(cg);
		9676	try ops[0].toOffset(eu_pl_off - eu_err_off, cg);
		9677	try ops[0].moveTo(inst, cg);
		9678	},
		9679	.struct_field_ptr => if (use_old) try cg.airStructFieldPtr(inst) else {
		9680	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		9681	const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data;
		9682	var ops = try cg.tempsFromOperands(inst, .{extra.struct_operand});
		9683	try ops[0].toOffset(cg.fieldOffset(
		9684	cg.typeOf(extra.struct_operand),
		9685	ty_pl.ty.toType(),
		9686	extra.field_index,
		9687	), cg);
		9688	try ops[0].moveTo(inst, cg);
		9689	},
		9690	.struct_field_ptr_index_0,
		9691	.struct_field_ptr_index_1,
		9692	.struct_field_ptr_index_2,
		9693	.struct_field_ptr_index_3,
		9694	=> \|air_tag\| if (use_old) try cg.airStructFieldPtrIndex(inst, switch (air_tag) {
		9695	else => unreachable,
		9696	.struct_field_ptr_index_0 => 0,
		9697	.struct_field_ptr_index_1 => 1,
		9698	.struct_field_ptr_index_2 => 2,
		9699	.struct_field_ptr_index_3 => 3,
		9700	}) else {
		9701	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9702	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9703	try ops[0].toOffset(cg.fieldOffset(
		9704	cg.typeOf(ty_op.operand),
		9705	ty_op.ty.toType(),
		9706	switch (air_tag) {
		9707	else => unreachable,
		9708	.struct_field_ptr_index_0 => 0,
		9709	.struct_field_ptr_index_1 => 1,
		9710	.struct_field_ptr_index_2 => 2,
		9711	.struct_field_ptr_index_3 => 3,
		9712	},
		9713	), cg);
		9714	try ops[0].moveTo(inst, cg);
		9715	},
		9716	.struct_field_val => if (use_old) try cg.airStructFieldVal(inst) else fallback: {
		9717	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		9718	const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data;
		9719	const agg_ty = cg.typeOf(extra.struct_operand);
		9720	const field_ty = ty_pl.ty.toType();
		9721	const field_off: u31 = switch (agg_ty.containerLayout(zcu)) {
		9722	.auto, .@"extern" => @intCast(agg_ty.structFieldOffset(extra.field_index, zcu)),
		9723	.@"packed" => break :fallback try cg.airStructFieldVal(inst),
		9724	};
		9725	var ops = try cg.tempsFromOperands(inst, .{extra.struct_operand});
		9726	// hack around Sema OPV bugs
		9727	var res = if (field_ty.hasRuntimeBitsIgnoreComptime(zcu))
		9728	try ops[0].read(field_ty, .{ .disp = field_off }, cg)
		9729	else
		9730	try cg.tempInit(field_ty, .none);
		9731	for (ops) \|op\| if (op.index != res.index) try op.die(cg);
		9732	try res.moveTo(inst, cg);
		9733	},
		9734	.set_union_tag => if (use_old) try cg.airSetUnionTag(inst) else {
		9735	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
		9736	const union_ty = cg.typeOf(bin_op.lhs).childType(zcu);
		9737	const union_layout = union_ty.unionGetLayout(zcu);
		9738	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		9739	// hack around Sema OPV bugs
		9740	if (union_layout.tag_size > 0) try ops[0].store(&ops[1], .{
		9741	.disp = @intCast(union_layout.tagOffset()),
		9742	}, cg);
		9743	for (ops) \|op\| try op.die(cg);
		9744	},
		9745	.get_union_tag => if (use_old) try cg.airGetUnionTag(inst) else {
		9746	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9747	const union_ty = cg.typeOf(ty_op.operand);
		9748	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9749	const union_layout = union_ty.unionGetLayout(zcu);
		9750	assert(union_layout.tag_size > 0);
		9751	var res = try ops[0].read(ty_op.ty.toType(), .{
		9752	.disp = @intCast(union_layout.tagOffset()),
		9753	}, cg);
		9754	for (ops) \|op\| if (op.index != res.index) try op.die(cg);
		9755	try res.moveTo(inst, cg);
		9756	},
		9757	.slice => if (use_old) try cg.airSlice(inst) else {
		9758	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		9759	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
		9760	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		9761	try ops[0].toPair(&ops[1], cg);
		9762	try ops[0].moveTo(inst, cg);
		9763	},
		9764	.slice_len => if (use_old) try cg.airSliceLen(inst) else {
		9765	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9766	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9767	try ops[0].toSliceLen(cg);
		9768	try ops[0].moveTo(inst, cg);
		9769	},
		9770	.slice_ptr => if (use_old) try cg.airSlicePtr(inst) else {
		9771	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9772	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9773	try ops[0].toSlicePtr(cg);
		9774	try ops[0].moveTo(inst, cg);
		9775	},
		9776	.ptr_slice_len_ptr => if (use_old) try cg.airPtrSliceLenPtr(inst) else {
		9777	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9778	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9779	try ops[0].toOffset(8, cg);
		9780	try ops[0].moveTo(inst, cg);
		9781	},
		9782	.ptr_slice_ptr_ptr => if (use_old) try cg.airPtrSlicePtrPtr(inst) else {
		9783	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9784	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9785	try ops[0].toOffset(0, cg);
		9786	try ops[0].moveTo(inst, cg);
		9787	},
		9788	.slice_elem_val, .ptr_elem_val => \|air_tag\| if (use_old) switch (air_tag) {
		9789	else => unreachable,
		9790	.slice_elem_val => try cg.airSliceElemVal(inst),
		9791	.ptr_elem_val => try cg.airPtrElemVal(inst),
		9792	} else {
		9793	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
		9794	const res_ty = cg.typeOf(bin_op.lhs).elemType2(zcu);
		9795	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		9796	try ops[0].toSlicePtr(cg);
		9797	var res: [1]Temp = undefined;
		9798	if (res_ty.hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{
		9799	.dst_constraints = .{.{ .int = .byte }},
		9800	.patterns = &.{
		9801	.{ .src = .{ .to_gpr, .simm32 } },
		9802	},
		9803	.dst_temps = .{.{ .rc = .general_purpose }},
		9804	.each = .{ .once = &.{
		9805	.{ ._, ._, .movzx, .dst0d, .leaa(.byte, .src0, .add_src0_elem_size_times_src1), ._, ._ },
		9806	} },
		9807	}, .{
		9808	.dst_constraints = .{.{ .int = .byte }},
		9809	.patterns = &.{
		9810	.{ .src = .{ .to_gpr, .to_gpr } },
		9811	},
		9812	.dst_temps = .{.{ .rc = .general_purpose }},
		9813	.each = .{ .once = &.{
		9814	.{ ._, ._, .movzx, .dst0d, .leai(.byte, .src0, .src1), ._, ._ },
		9815	} },
		9816	}, .{
		9817	.dst_constraints = .{.{ .int = .word }},
		9818	.patterns = &.{
		9819	.{ .src = .{ .to_gpr, .simm32 } },
		9820	},
		9821	.dst_temps = .{.{ .rc = .general_purpose }},
		9822	.each = .{ .once = &.{
		9823	.{ ._, ._, .movzx, .dst0d, .leaa(.word, .src0, .add_src0_elem_size_times_src1), ._, ._ },
		9824	} },
		9825	}, .{
		9826	.dst_constraints = .{.{ .int = .word }},
		9827	.patterns = &.{
		9828	.{ .src = .{ .to_gpr, .to_gpr } },
		9829	},
		9830	.dst_temps = .{.{ .rc = .general_purpose }},
		9831	.each = .{ .once = &.{
		9832	.{ ._, ._, .movzx, .dst0d, .leasi(.word, .src0, .@"2", .src1), ._, ._ },
		9833	} },
		9834	}, .{
		9835	.dst_constraints = .{.{ .int = .dword }},
		9836	.patterns = &.{
		9837	.{ .src = .{ .to_gpr, .simm32 } },
		9838	},
		9839	.dst_temps = .{.{ .rc = .general_purpose }},
		9840	.each = .{ .once = &.{
		9841	.{ ._, ._, .mov, .dst0d, .leaa(.dword, .src0, .add_src0_elem_size_times_src1), ._, ._ },
		9842	} },
		9843	}, .{
		9844	.dst_constraints = .{.{ .int = .dword }},
		9845	.patterns = &.{
		9846	.{ .src = .{ .to_gpr, .to_gpr } },
		9847	},
		9848	.dst_temps = .{.{ .rc = .general_purpose }},
		9849	.each = .{ .once = &.{
		9850	.{ ._, ._, .mov, .dst0d, .leasi(.dword, .src0, .@"4", .src1), ._, ._ },
		9851	} },
		9852	}, .{
		9853	.dst_constraints = .{.{ .int = .qword }},
		9854	.patterns = &.{
		9855	.{ .src = .{ .to_gpr, .simm32 } },
		9856	},
		9857	.dst_temps = .{.{ .rc = .general_purpose }},
		9858	.each = .{ .once = &.{
		9859	.{ ._, ._, .mov, .dst0q, .leaa(.qword, .src0, .add_src0_elem_size_times_src1), ._, ._ },
		9860	} },
		9861	}, .{
		9862	.required_features = .{ .@"64bit", null, null, null },
		9863	.dst_constraints = .{.{ .int = .qword }},
		9864	.patterns = &.{
		9865	.{ .src = .{ .to_gpr, .to_gpr } },
		9866	},
		9867	.dst_temps = .{.{ .rc = .general_purpose }},
		9868	.each = .{ .once = &.{
		9869	.{ ._, ._, .mov, .dst0q, .leasi(.qword, .src0, .@"8", .src1), ._, ._ },
		9870	} },
		9871	} }) catch \|err\| switch (err) {
		9872	error.SelectFailed => {
		9873	const elem_size = res_ty.abiSize(zcu);
		9874	while (true) for (&ops) \|*op\| {
		9875	if (try op.toRegClass(true, .general_purpose, cg)) break;
		9876	} else break;
		9877	const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64();
		9878	const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64();
		9879	if (!std.math.isPowerOfTwo(elem_size)) {
		9880	try cg.spillEflagsIfOccupied();
		9881	try cg.asmRegisterRegisterImmediate(
		9882	.{ .i_, .mul },
		9883	rhs_reg,
		9884	rhs_reg,
		9885	.u(elem_size),
		9886	);
		9887	try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		9888	.base = .{ .reg = lhs_reg },
		9889	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
		9890	});
		9891	} else if (elem_size > 8) {
		9892	try cg.spillEflagsIfOccupied();
		9893	try cg.asmRegisterImmediate(
		9894	.{ ._l, .sh },
		9895	rhs_reg,
		9896	.u(std.math.log2_int(u64, elem_size)),
		9897	);
		9898	try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		9899	.base = .{ .reg = lhs_reg },
		9900	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
		9901	});
		9902	} else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		9903	.base = .{ .reg = lhs_reg },
		9904	.mod = .{ .rm = .{
		9905	.size = .qword,
		9906	.index = rhs_reg,
		9907	.scale = .fromFactor(@intCast(elem_size)),
		9908	} },
		9909	});
		9910	res[0] = try ops[0].load(res_ty, .{}, cg);
		9911	},
		9912	else => \|e\| return e,
		9913	} else {
		9914	// hack around Sema OPV bugs
		9915	res[0] = try cg.tempInit(res_ty, .none);
		9916	}
		9917	for (ops) \|op\| for (res) \|r\| {
		9918	if (op.index == r.index) break;
		9919	} else try op.die(cg);
		9920	try res[0].moveTo(inst, cg);
		9921	},
		9922	.slice_elem_ptr, .ptr_elem_ptr => \|air_tag\| if (use_old) switch (air_tag) {
		9923	else => unreachable,
		9924	.slice_elem_ptr => try cg.airSliceElemPtr(inst),
		9925	.ptr_elem_ptr => try cg.airPtrElemPtr(inst),
		9926	} else {
		9927	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		9928	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
		9929	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
		9930	try ops[0].toSlicePtr(cg);
		9931	const dst_ty = ty_pl.ty.toType();
		9932	if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {
		9933	const elem_size = dst_ty.childType(zcu).abiSize(zcu);
		9934	// hack around Sema OPV bugs
		9935	if (elem_size == 0) break :zero_offset;
		9936	while (true) for (&ops) \|*op\| {
		9937	if (try op.toRegClass(true, .general_purpose, cg)) break;
		9938	} else break;
		9939	const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64();
		9940	const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64();
		9941	if (!std.math.isPowerOfTwo(elem_size)) {
		9942	try cg.spillEflagsIfOccupied();
		9943	try cg.asmRegisterRegisterImmediate(
		9944	.{ .i_, .mul },
		9945	rhs_reg,
		9946	rhs_reg,
		9947	.u(elem_size),
		9948	);
		9949	try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		9950	.base = .{ .reg = lhs_reg },
		9951	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
		9952	});
		9953	} else if (elem_size > 8) {
		9954	try cg.spillEflagsIfOccupied();
		9955	try cg.asmRegisterImmediate(
		9956	.{ ._l, .sh },
		9957	rhs_reg,
		9958	.u(std.math.log2_int(u64, elem_size)),
		9959	);
		9960	try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		9961	.base = .{ .reg = lhs_reg },
		9962	.mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
		9963	});
		9964	} else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
		9965	.base = .{ .reg = lhs_reg },
		9966	.mod = .{ .rm = .{
		9967	.size = .qword,
		9968	.index = rhs_reg,
		9969	.scale = .fromFactor(@intCast(elem_size)),
		9970	} },
		9971	});
		9972	}
		9973	try ops[1].die(cg);
		9974	try ops[0].moveTo(inst, cg);
		9975	},
		9976	.array_to_slice => if (use_old) try cg.airArrayToSlice(inst) else {
		9977	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		9978	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		9979	var len = try cg.tempInit(.usize, .{
		9980	.immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu),
		9981	});
		9982	try ops[0].toPair(&len, cg);
		9983	try ops[0].moveTo(inst, cg);
		9984	},
		9985	.error_set_has_value => return cg.fail("TODO implement error_set_has_value", .{}),
		9986	.union_init => if (use_old) try cg.airUnionInit(inst) else {
		9987	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		9988	const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
		9989	const union_ty = ty_pl.ty.toType();
		9990	var ops = try cg.tempsFromOperands(inst, .{extra.init});
		9991	var res = try cg.tempAllocMem(union_ty);
		9992	const union_layout = union_ty.unionGetLayout(zcu);
		9993	if (union_layout.tag_size > 0) {
		9994	var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex(
		9995	union_ty.unionTagTypeSafety(zcu).?,
		9996	extra.field_index,
		9997	));
		9998	try res.write(&tag_temp, .{
		9999	.disp = @intCast(union_layout.tagOffset()),
		10000	}, cg);
		10001	try tag_temp.die(cg);
		10002	}
		10003	try res.write(&ops[0], .{
		10004	.disp = @intCast(union_layout.payloadOffset()),
		10005	}, cg);
		10006	try ops[0].die(cg);
		10007	try res.moveTo(inst, cg);
		10008	},
		10009	.field_parent_ptr => if (use_old) try cg.airFieldParentPtr(inst) else {
		10010	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
		10011	const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
		10012	var ops = try cg.tempsFromOperands(inst, .{extra.field_ptr});
		10013	try ops[0].toOffset(-cg.fieldOffset(
		10014	ty_pl.ty.toType(),
		10015	cg.typeOf(extra.field_ptr),
		10016	extra.field_index,
		10017	), cg);
		10018	try ops[0].moveTo(inst, cg);
		10019	},
		10020
		10021	.is_named_enum_value => return cg.fail("TODO implement is_named_enum_value", .{}),
		10022
		10023	.wasm_memory_size => unreachable,
		10024	.wasm_memory_grow => unreachable,
		10025
		10026	.addrspace_cast => {
		10027	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
		10028	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
		10029	try ops[0].moveTo(inst, cg);
		10030	},
		10031
		10032	.vector_store_elem => return cg.fail("TODO implement vector_store_elem", .{}),
		10033
		10034	.c_va_arg => try cg.airVaArg(inst),
		10035	.c_va_copy => try cg.airVaCopy(inst),
		10036	.c_va_end => try cg.airVaEnd(inst),
		10037	.c_va_start => try cg.airVaStart(inst),
		10038
		10039	.work_item_id => unreachable,
		10040	.work_group_size => unreachable,
		10041	.work_group_id => unreachable,
		10042	}
		10043	cg.resetTemps();
		10044	cg.checkInvariantsAfterAirInst();
		10045	}
		10046	verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
2573	}	10047	}
2574		10048
2575	/// Asserts there is already capacity to insert into top branch inst_table.	10049	fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
2576	fn processDeath(self: *Self, inst: Air.Inst.Index) !void {	10050	const pt = self.pt;
2577	try self.inst_tracking.getPtr(inst).?.die(self, inst);	10051	const zcu = pt.zcu;
		10052	const ip = &zcu.intern_pool;
		10053	switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {
		10054	.@"enum" => {
		10055	const enum_ty: Type = .fromInterned(lazy_sym.ty);
		10056	wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});
		10057
		10058	const param_regs = abi.getCAbiIntParamRegs(.auto);
		10059	const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
		10060	defer for (param_locks) \|lock\| self.register_manager.unlockReg(lock);
		10061
		10062	const ret_reg = param_regs[0];
		10063	const enum_mcv = MCValue{ .register = param_regs[1] };
		10064
		10065	const epilogue_relocs = try self.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu));
		10066	defer self.gpa.free(epilogue_relocs);
		10067
		10068	const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
		10069	const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
		10070	defer self.register_manager.unlockReg(data_lock);
		10071	try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = enum_ty.toIntern() });
		10072
		10073	var data_off: i32 = 0;
		10074	const tag_names = enum_ty.enumFields(zcu);
		10075	for (epilogue_relocs, 0..) \|*epilogue_reloc, tag_index\| {
		10076	const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
		10077	const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index));
		10078	const tag_mcv = try self.genTypedValue(tag_val);
		10079	try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
		10080	const skip_reloc = try self.asmJccReloc(.ne, undefined);
		10081
		10082	try self.genSetMem(
		10083	.{ .reg = ret_reg },
		10084	0,
		10085	.usize,
		10086	.{ .register_offset = .{ .reg = data_reg, .off = data_off } },
		10087	.{},
		10088	);
		10089	try self.genSetMem(.{ .reg = ret_reg }, 8, .usize, .{ .immediate = tag_name_len }, .{});
		10090
		10091	epilogue_reloc.* = try self.asmJmpReloc(undefined);
		10092	self.performReloc(skip_reloc);
		10093
		10094	data_off += @intCast(tag_name_len + 1);
		10095	}
		10096
		10097	try self.asmOpOnly(.{ ._, .ud2 });
		10098
		10099	for (epilogue_relocs) \|reloc\| self.performReloc(reloc);
		10100	try self.asmOpOnly(.{ ._, .ret });
		10101	},
		10102	else => return self.fail(
		10103	"TODO implement {s} for {}",
		10104	.{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },
		10105	),
		10106	}
2578	}	10107	}
2579		10108
2580	/// Called when there are no operands, and the instruction is always unreferenced.	10109	fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void {
2581	fn finishAirBookkeeping(self: *Self) void {	10110	for (value.getRegs()) \|reg\| try self.register_manager.getReg(reg, inst);
2582	if (std.debug.runtime_safety) {	10111	switch (value) {
2583	self.air_bookkeeping += 1;	10112	else => {},
		10113	.eflags, .register_overflow => self.eflags_inst = inst,
		10114	}
		10115	}
		10116
		10117	fn getValueIfFree(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) void {
		10118	for (value.getRegs()) \|reg\| if (self.register_manager.isRegFree(reg))
		10119	self.register_manager.getRegAssumeFree(reg, inst);
		10120	}
		10121
		10122	fn freeReg(self: *CodeGen, reg: Register) !void {
		10123	self.register_manager.freeReg(reg);
		10124	if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
		10125	}
		10126
		10127	fn freeValue(self: *CodeGen, value: MCValue) !void {
		10128	switch (value) {
		10129	.register => \|reg\| try self.freeReg(reg),
		10130	inline .register_pair,
		10131	.register_triple,
		10132	.register_quadruple,
		10133	=> \|regs\| for (regs) \|reg\| try self.freeReg(reg),
		10134	.register_offset, .indirect => \|reg_off\| try self.freeReg(reg_off.reg),
		10135	.register_overflow => \|reg_ov\| {
		10136	try self.freeReg(reg_ov.reg);
		10137	self.eflags_inst = null;
		10138	},
		10139	.register_mask => \|reg_mask\| try self.freeReg(reg_mask.reg),
		10140	.eflags => self.eflags_inst = null,
		10141	else => {}, // TODO process stack allocation death
2584	}	10142	}
2585	}	10143	}
2586		10144
2587	fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {	10145	fn feed(self: CodeGen, bt: Liveness.BigTomb, operand: Air.Inst.Ref) !void {
		10146	if (bt.feed()) if (operand.toIndex()) \|inst\| try self.processDeath(inst);
		10147	}
		10148
		10149	/// Asserts there is already capacity to insert into top branch inst_table.
		10150	fn processDeath(self: *CodeGen, inst: Air.Inst.Index) !void {
		10151	try self.inst_tracking.getPtr(inst).?.die(self, inst);
		10152	}
		10153
		10154	fn finishAirResult(self: *CodeGen, inst: Air.Inst.Index, result: MCValue) void {
2588	if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {	10155	if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {
2589	.none, .dead, .unreach => {},	10156	.none, .dead, .unreach => {},
2590	else => unreachable, // Why didn't the result die?	10157	else => unreachable, // Why didn't the result die?
2591	} else {	10158	} else {
2592	tracking_log.debug("%{d} => {} (birth)", .{ inst, result });	10159	tracking_log.debug("{} => {} (birth)", .{ inst, result });
2593	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));	10160	self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
2594	// In some cases, an operand may be reused as the result.	10161	// In some cases, an operand may be reused as the result.
2595	// If that operand died and was a register, it was freed by	10162	// If that operand died and was a register, it was freed by
2596	// processDeath, so we have to "re-allocate" the register.	10163	// processDeath, so we have to "re-allocate" the register.
2597	self.getValueIfFree(result, inst);	10164	self.getValueIfFree(result, inst);
2598	}	10165	}
2599	self.finishAirBookkeeping();
2600	}	10166	}
2601		10167
2602	fn finishAir(	10168	fn finishAir(
2603	self: *Self,	10169	self: *CodeGen,
2604	inst: Air.Inst.Index,	10170	inst: Air.Inst.Index,
2605	result: MCValue,	10171	result: MCValue,
2606	operands: [Liveness.bpi - 1]Air.Inst.Ref,	10172	operands: [Liveness.bpi - 1]Air.Inst.Ref,
2607	) !void {	10173	) !void {
2608	var tomb_bits = self.liveness.getTombBits(inst);	10174	const tomb_bits = self.liveness.getTombBits(inst);
2609	for (operands) \|op\| {	10175	for (0.., operands) \|op_index, op\| {
2610	const dies = @as(u1, @truncate(tomb_bits)) != 0;	10176	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
2611	tomb_bits >>= 1;	10177	if (self.reused_operands.isSet(op_index)) continue;
2612	if (!dies) continue;
2613	try self.processDeath(op.toIndexAllowNone() orelse continue);	10178	try self.processDeath(op.toIndexAllowNone() orelse continue);
2614	}	10179	}
2615	self.finishAirResult(inst, result);	10180	self.finishAirResult(inst, result);
...	@@ -2622,7 +10187,7 @@ const FrameLayout = struct {	...	@@ -2622,7 +10187,7 @@ const FrameLayout = struct {
2622	};	10187	};
2623		10188
2624	fn setFrameLoc(	10189	fn setFrameLoc(
2625	self: *Self,	10190	self: *CodeGen,
2626	frame_index: FrameIndex,	10191	frame_index: FrameIndex,
2627	base: Register,	10192	base: Register,
2628	offset: *i32,	10193	offset: *i32,
...	@@ -2637,7 +10202,7 @@ fn setFrameLoc(	...	@@ -2637,7 +10202,7 @@ fn setFrameLoc(
2637	offset.* += self.frame_allocs.items(.abi_size)[frame_i];	10202	offset.* += self.frame_allocs.items(.abi_size)[frame_i];
2638	}	10203	}
2639		10204
2640	fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayout {	10205	fn computeFrameLayout(self: *CodeGen, cc: std.builtin.CallingConvention.Tag) !FrameLayout {
2641	const frame_allocs_len = self.frame_allocs.len;	10206	const frame_allocs_len = self.frame_allocs.len;
2642	try self.frame_locs.resize(self.gpa, frame_allocs_len);	10207	try self.frame_locs.resize(self.gpa, frame_allocs_len);
2643	const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);	10208	const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
...	@@ -2657,7 +10222,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo	...	@@ -2657,7 +10222,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2657	}	10222	}
2658	};	10223	};
2659	const sort_context = SortContext{ .frame_align = frame_align };	10224	const sort_context = SortContext{ .frame_align = frame_align };
2660	mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);	10225	std.mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
2661	}	10226	}
2662		10227
2663	const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)];	10228	const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)];
...	@@ -2668,9 +10233,8 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo	...	@@ -2668,9 +10233,8 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2668		10233
2669	// Create list of registers to save in the prologue.	10234	// Create list of registers to save in the prologue.
2670	// TODO handle register classes	10235	// TODO handle register classes
2671	var save_reg_list = Mir.RegisterList{};	10236	var save_reg_list: Mir.RegisterList = .empty;
2672	const callee_preserved_regs =	10237	const callee_preserved_regs = abi.getCalleePreservedRegs(cc);
2673	abi.getCalleePreservedRegs(abi.resolveCallingConvention(cc, self.target.*));
2674	for (callee_preserved_regs) \|reg\| {	10238	for (callee_preserved_regs) \|reg\| {
2675	if (self.register_manager.isRegAllocated(reg)) {	10239	if (self.register_manager.isRegAllocated(reg)) {
2676	save_reg_list.push(callee_preserved_regs, reg);	10240	save_reg_list.push(callee_preserved_regs, reg);
...	@@ -2684,7 +10248,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo	...	@@ -2684,7 +10248,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2684	const stack_frame_align_offset = if (need_align_stack)	10248	const stack_frame_align_offset = if (need_align_stack)
2685	0	10249	0
2686	else	10250	else
2687	save_reg_list.size() + frame_offset[@intFromEnum(FrameIndex.args_frame)];	10251	save_reg_list.size(self.target) + frame_offset[@intFromEnum(FrameIndex.args_frame)];
2688		10252
2689	var rsp_offset: i32 = 0;	10253	var rsp_offset: i32 = 0;
2690	self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true);	10254	self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true);
...	@@ -2697,22 +10261,22 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo	...	@@ -2697,22 +10261,22 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
2697	@intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]);	10261	@intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]);
2698		10262
2699	return .{	10263	return .{
2700	.stack_mask = @as(u32, math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),	10264	.stack_mask = @as(u32, std.math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0),
2701	.stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]),	10265	.stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]),
2702	.save_reg_list = save_reg_list,	10266	.save_reg_list = save_reg_list,
2703	};	10267	};
2704	}	10268	}
2705		10269
2706	fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) Alignment {	10270	fn getFrameAddrAlignment(self: *CodeGen, frame_addr: bits.FrameAddr) InternPool.Alignment {
2707	const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;	10271	const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;
2708	return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));	10272	return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));
2709	}	10273	}
2710		10274
2711	fn getFrameAddrSize(self: *Self, frame_addr: bits.FrameAddr) u32 {	10275	fn getFrameAddrSize(self: *CodeGen, frame_addr: bits.FrameAddr) u32 {
2712	return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off));	10276	return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off));
2713	}	10277	}
2714		10278
2715	fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {	10279	fn allocFrameIndex(self: *CodeGen, alloc: FrameAlloc) !FrameIndex {
2716	const frame_allocs_slice = self.frame_allocs.slice();	10280	const frame_allocs_slice = self.frame_allocs.slice();
2717	const frame_size = frame_allocs_slice.items(.abi_size);	10281	const frame_size = frame_allocs_slice.items(.abi_size);
2718	const frame_align = frame_allocs_slice.items(.abi_align);	10282	const frame_align = frame_allocs_slice.items(.abi_align);
...	@@ -2735,36 +10299,36 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {	...	@@ -2735,36 +10299,36 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
2735	}	10299	}
2736		10300
2737	/// Use a pointer instruction as the basis for allocating stack memory.	10301	/// Use a pointer instruction as the basis for allocating stack memory.
2738	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {	10302	fn allocMemPtr(self: *CodeGen, inst: Air.Inst.Index) !FrameIndex {
2739	const pt = self.pt;	10303	const pt = self.pt;
2740	const zcu = pt.zcu;	10304	const zcu = pt.zcu;
2741	const ptr_ty = self.typeOfIndex(inst);	10305	const ptr_ty = self.typeOfIndex(inst);
2742	const val_ty = ptr_ty.childType(zcu);	10306	const val_ty = ptr_ty.childType(zcu);
2743	return self.allocFrameIndex(FrameAlloc.init(.{	10307	return self.allocFrameIndex(.init(.{
2744	.size = math.cast(u32, val_ty.abiSize(zcu)) orelse {	10308	.size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse {
2745	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});	10309	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)});
2746	},	10310	},
2747	.alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),	10311	.alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),
2748	}));	10312	}));
2749	}	10313	}
2750		10314
2751	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {	10315	fn allocRegOrMem(self: *CodeGen, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
2752	return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok);	10316	return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok);
2753	}	10317	}
2754		10318
2755	fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {	10319	fn allocTempRegOrMem(self: *CodeGen, elem_ty: Type, reg_ok: bool) !MCValue {
2756	return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);	10320	return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);
2757	}	10321	}
2758		10322
2759	fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {	10323	fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
2760	const pt = self.pt;	10324	const pt = self.pt;
2761	const zcu = pt.zcu;	10325	const zcu = pt.zcu;
2762	const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse {	10326	const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
2763	return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(pt)});	10327	return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(pt)});
2764	};	10328	};
2765		10329
2766	if (reg_ok) need_mem: {	10330	if (reg_ok) need_mem: {
2767	if (abi_size <= @as(u32, switch (ty.zigTypeTag(zcu)) {	10331	if (std.math.isPowerOfTwo(abi_size) and abi_size <= @as(u32, switch (ty.zigTypeTag(zcu)) {
2768	.float => switch (ty.floatBits(self.target.*)) {	10332	.float => switch (ty.floatBits(self.target.*)) {
2769	16, 32, 64, 128 => 16,	10333	16, 32, 64, 128 => 16,
2770	80 => break :need_mem,	10334	80 => break :need_mem,
...	@@ -2772,43 +10336,76 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b	...	@@ -2772,43 +10336,76 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2772	},	10336	},
2773	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {	10337	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
2774	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {	10338	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {
2775	16, 32, 64, 128 => if (self.hasFeature(.avx)) 32 else 16,	10339	16, 32, 64, 128 => self.vectorSize(.float),
2776	80 => break :need_mem,	10340	80 => break :need_mem,
2777	else => unreachable,	10341	else => unreachable,
2778	},	10342	},
2779	else => if (self.hasFeature(.avx)) 32 else 16,	10343	else => self.vectorSize(.int),
2780	},	10344	},
2781	else => 8,	10345	else => 8,
2782	})) {	10346	})) {
2783	if (self.register_manager.tryAllocReg(inst, self.regClassForType(ty))) \|reg\| {	10347	if (self.register_manager.tryAllocReg(inst, self.regSetForType(ty))) \|reg\| {
2784	return MCValue{ .register = registerAlias(reg, abi_size) };	10348	return MCValue{ .register = registerAlias(reg, abi_size) };
2785	}	10349	}
2786	}	10350	}
2787	}	10351	}
2788		10352
2789	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, zcu));	10353	const frame_index = try self.allocFrameIndex(.initSpill(ty, zcu));
2790	return .{ .load_frame = .{ .index = frame_index } };	10354	return .{ .load_frame = .{ .index = frame_index } };
2791	}	10355	}
2792		10356
2793	fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {	10357	fn regClassForType(self: *CodeGen, ty: Type) Register.Class {
2794	const pt = self.pt;	10358	const pt = self.pt;
2795	const zcu = pt.zcu;	10359	const zcu = pt.zcu;
2796	return switch (ty.zigTypeTag(zcu)) {	10360	return switch (ty.zigTypeTag(zcu)) {
2797	.float => switch (ty.floatBits(self.target.*)) {	10361	.float => switch (ty.floatBits(self.target.*)) {
2798	80 => abi.RegisterClass.x87,	10362	80 => .x87,
2799	else => abi.RegisterClass.sse,	10363	else => .sse,
2800	},	10364	},
2801	.vector => switch (ty.childType(zcu).toIntern()) {	10365	.vector => switch (ty.childType(zcu).toIntern()) {
2802	.bool_type, .u1_type => abi.RegisterClass.gp,	10366	.bool_type => .general_purpose,
2803	else => if (ty.isAbiInt(zcu) and ty.intInfo(zcu).bits == 1)	10367	else => .sse,
2804	abi.RegisterClass.gp
2805	else
2806	abi.RegisterClass.sse,
2807	},	10368	},
2808	else => abi.RegisterClass.gp,	10369	else => .general_purpose,
2809	};	10370	};
2810	}	10371	}
2811		10372
		10373	fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {
		10374	return switch (rc) {
		10375	.general_purpose => abi.RegisterClass.gp,
		10376	.segment, .ip => unreachable,
		10377	.x87 => abi.RegisterClass.x87,
		10378	.mmx => @panic("TODO"),
		10379	.sse => abi.RegisterClass.sse,
		10380	};
		10381	}
		10382
		10383	fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet {
		10384	return regSetForRegClass(self.regClassForType(ty));
		10385	}
		10386
		10387	fn vectorSize(cg: *CodeGen, kind: enum { int, float }) u6 {
		10388	return if (cg.hasFeature(switch (kind) {
		10389	.int => .avx2,
		10390	.float => .avx,
		10391	})) 32 else if (cg.hasFeature(.sse)) 16 else 8;
		10392	}
		10393
		10394	fn limbType(cg: *CodeGen, ty: Type) Type {
		10395	const pt = cg.pt;
		10396	const zcu = pt.zcu;
		10397	const vector_size = cg.vectorSize(if (ty.isRuntimeFloat()) .float else .int);
		10398	const scalar_ty, const scalar_size = scalar: {
		10399	const scalar_ty = ty.scalarType(zcu);
		10400	const scalar_size = scalar_ty.abiSize(zcu);
		10401	if (scalar_size <= vector_size) break :scalar .{ scalar_ty, scalar_size };
		10402	};
		10403	pt.vectorType(.{
		10404	.len = @divExact(vector_size, scalar_size),
		10405	.child = scalar_ty.toIntern(),
		10406	});
		10407	}
		10408
2812	const State = struct {	10409	const State = struct {
2813	registers: RegisterManager.TrackedRegisters,	10410	registers: RegisterManager.TrackedRegisters,
2814	reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,	10411	reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
...	@@ -2817,14 +10414,14 @@ const State = struct {	...	@@ -2817,14 +10414,14 @@ const State = struct {
2817	scope_generation: u32,	10414	scope_generation: u32,
2818	};	10415	};
2819		10416
2820	fn initRetroactiveState(self: *Self) State {	10417	fn initRetroactiveState(self: *CodeGen) State {
2821	var state: State = undefined;	10418	var state: State = undefined;
2822	state.inst_tracking_len = @intCast(self.inst_tracking.count());	10419	state.inst_tracking_len = @intCast(self.inst_tracking.count());
2823	state.scope_generation = self.scope_generation;	10420	state.scope_generation = self.scope_generation;
2824	return state;	10421	return state;
2825	}	10422	}
2826		10423
2827	fn saveRetroactiveState(self: Self, state: State) !void {	10424	fn saveRetroactiveState(self: CodeGen, state: State) !void {
2828	try self.spillEflagsIfOccupied();	10425	try self.spillEflagsIfOccupied();
2829	const free_registers = self.register_manager.free_registers;	10426	const free_registers = self.register_manager.free_registers;
2830	var it = free_registers.iterator(.{ .kind = .unset });	10427	var it = free_registers.iterator(.{ .kind = .unset });
...	@@ -2836,13 +10433,13 @@ fn saveRetroactiveState(self: Self, state: State) !void {	...	@@ -2836,13 +10433,13 @@ fn saveRetroactiveState(self: Self, state: State) !void {
2836	state.free_registers = free_registers;	10433	state.free_registers = free_registers;
2837	}	10434	}
2838		10435
2839	fn saveState(self: *Self) !State {	10436	fn saveState(self: *CodeGen) !State {
2840	var state = self.initRetroactiveState();	10437	var state = self.initRetroactiveState();
2841	try self.saveRetroactiveState(&state);	10438	try self.saveRetroactiveState(&state);
2842	return state;	10439	return state;
2843	}	10440	}
2844		10441
2845	fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {	10442	fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
2846	emit_instructions: bool,	10443	emit_instructions: bool,
2847	update_tracking: bool,	10444	update_tracking: bool,
2848	resurrect: bool,	10445	resurrect: bool,
...	@@ -2857,9 +10454,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt	...	@@ -2857,9 +10454,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2857	}	10454	}
2858		10455
2859	if (opts.resurrect) for (	10456	if (opts.resurrect) for (
2860	self.inst_tracking.keys()[0..state.inst_tracking_len],	10457	self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len],
2861	self.inst_tracking.values()[0..state.inst_tracking_len],	10458	self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len],
2862	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);	10459	) \|inst, *tracking\| try tracking.resurrect(self, inst, state.scope_generation);
2863	for (deaths) \|death\| try self.processDeath(death);	10460	for (deaths) \|death\| try self.processDeath(death);
2864		10461
2865	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock;	10462	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock;
...	@@ -2876,43 +10473,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt	...	@@ -2876,43 +10473,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2876	reg_locks.deinit();	10473	reg_locks.deinit();
2877	};	10474	};
2878		10475
2879	for (0..state.registers.len) \|index\| {	10476	for (
2880	const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))	10477	0..,
2881	null	10478	self.register_manager.registers,
2882	else	10479	state.registers,
2883	self.register_manager.registers[index];	10480	state.reg_tracking,
2884	const target_maybe_inst = if (state.free_registers.isSet(index))	10481	) \|reg_i, current_slot, target_slot, reg_tracking\| {
2885	null	10482	const reg_index: RegisterManager.TrackedIndex = @intCast(reg_i);
2886	else	10483	const current_maybe_inst = if (self.register_manager.isRegIndexFree(reg_index)) null else current_slot;
2887	state.registers[index];	10484	const target_maybe_inst = if (state.free_registers.isSet(reg_index)) null else target_slot;
2888	if (std.debug.runtime_safety) if (target_maybe_inst) \|target_inst\|	10485	if (std.debug.runtime_safety) if (target_maybe_inst) \|target_inst\|
2889	assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);	10486	assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
2890	if (opts.emit_instructions) {	10487	if (opts.emit_instructions) {
2891	if (current_maybe_inst) \|current_inst\| {	10488	if (current_maybe_inst) \|current_inst\|
2892	try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);	10489	try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
2893	}	10490	if (target_maybe_inst) \|target_inst\|
2894	if (target_maybe_inst) \|target_inst\| {	10491	try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg_tracking);
2895	const target_tracking = self.inst_tracking.getPtr(target_inst).?;
2896	try target_tracking.materialize(self, target_inst, state.reg_tracking[index]);
2897	}
2898	}	10492	}
2899	if (opts.update_tracking) {	10493	if (opts.update_tracking) {
2900	if (current_maybe_inst) \|current_inst\| {	10494	if (current_maybe_inst) \|current_inst\| {
2901	try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);	10495	try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
2902	}	10496	self.register_manager.freeRegIndex(reg_index);
2903	{
2904	const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
2905	self.register_manager.freeReg(reg);
2906	self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
2907	}	10497	}
2908	if (target_maybe_inst) \|target_inst\| {	10498	if (target_maybe_inst) \|target_inst\| {
2909	self.inst_tracking.getPtr(target_inst).?.trackMaterialize(	10499	self.register_manager.getRegIndexAssumeFree(reg_index, target_maybe_inst);
2910	target_inst,	10500	self.inst_tracking.getPtr(target_inst).?.trackMaterialize(target_inst, reg_tracking);
2911	state.reg_tracking[index],
2912	);
2913	}	10501	}
2914	} else if (target_maybe_inst) \|_\|	10502	} else if (target_maybe_inst) \|_\|
2915	try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(@intCast(index)));	10503	try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(reg_index));
2916	}	10504	}
2917	if (opts.emit_instructions) if (self.eflags_inst) \|inst\|	10505	if (opts.emit_instructions) if (self.eflags_inst) \|inst\|
2918	try self.inst_tracking.getPtr(inst).?.spill(self, inst);	10506	try self.inst_tracking.getPtr(inst).?.spill(self, inst);
...	@@ -2930,7 +10518,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt	...	@@ -2930,7 +10518,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2930	}	10518	}
2931	}	10519	}
2932		10520
2933	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {	10521	pub fn spillInstruction(self: *CodeGen, reg: Register, inst: Air.Inst.Index) !void {
2934	const tracking = self.inst_tracking.getPtr(inst) orelse return;	10522	const tracking = self.inst_tracking.getPtr(inst) orelse return;
2935	for (tracking.getRegs()) \|tracked_reg\| {	10523	for (tracking.getRegs()) \|tracked_reg\| {
2936	if (tracked_reg.id() == reg.id()) break;	10524	if (tracked_reg.id() == reg.id()) break;
...	@@ -2939,7 +10527,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void	...	@@ -2939,7 +10527,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
2939	try tracking.trackSpill(self, inst);	10527	try tracking.trackSpill(self, inst);
2940	}	10528	}
2941		10529
2942	pub fn spillEflagsIfOccupied(self: *Self) !void {	10530	pub fn spillEflagsIfOccupied(self: *CodeGen) !void {
2943	if (self.eflags_inst) \|inst\| {	10531	if (self.eflags_inst) \|inst\| {
2944	self.eflags_inst = null;	10532	self.eflags_inst = null;
2945	const tracking = self.inst_tracking.getPtr(inst).?;	10533	const tracking = self.inst_tracking.getPtr(inst).?;
...	@@ -2949,23 +10537,22 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {	...	@@ -2949,23 +10537,22 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
2949	}	10537	}
2950	}	10538	}
2951		10539
2952	pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention) !void {	10540	pub fn spillCallerPreservedRegs(self: *CodeGen, cc: std.builtin.CallingConvention.Tag) !void {
2953	switch (cc) {	10541	switch (cc) {
2954	.x86_64_sysv => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_sysv = .{} })),	10542	inline .auto, .x86_64_sysv, .x86_64_win => \|tag\| try self.spillRegisters(abi.getCallerPreservedRegs(tag)),
2955	.x86_64_win => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_win = .{} })),
2956	else => unreachable,	10543	else => unreachable,
2957	}	10544	}
2958	}	10545	}
2959		10546
2960	pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {	10547	pub fn spillRegisters(self: *CodeGen, comptime registers: []const Register) !void {
2961	inline for (registers) \|reg\| try self.register_manager.getKnownReg(reg, null);	10548	inline for (registers) \|reg\| try self.register_manager.getKnownReg(reg, null);
2962	}	10549	}
2963		10550
2964	/// Copies a value to a register without tracking the register. The register is not considered	10551	/// Copies a value to a register without tracking the register. The register is not considered
2965	/// allocated. A second call to `copyToTmpRegister` may return the same register.	10552	/// allocated. A second call to `copyToTmpRegister` may return the same register.
2966	/// This can have a side effect of spilling instructions to the stack to free up a register.	10553	/// This can have a side effect of spilling instructions to the stack to free up a register.
2967	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {	10554	fn copyToTmpRegister(self: *CodeGen, ty: Type, mcv: MCValue) !Register {
2968	const reg = try self.register_manager.allocReg(null, self.regClassForType(ty));	10555	const reg = try self.register_manager.allocReg(null, self.regSetForType(ty));
2969	try self.genSetReg(reg, ty, mcv, .{});	10556	try self.genSetReg(reg, ty, mcv, .{});
2970	return reg;	10557	return reg;
2971	}	10558	}
...	@@ -2975,22 +10562,22 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {	...	@@ -2975,22 +10562,22 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
2975	/// This can have a side effect of spilling instructions to the stack to free up a register.	10562	/// This can have a side effect of spilling instructions to the stack to free up a register.
2976	/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!	10563	/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
2977	fn copyToRegisterWithInstTracking(	10564	fn copyToRegisterWithInstTracking(
2978	self: *Self,	10565	self: *CodeGen,
2979	reg_owner: Air.Inst.Index,	10566	reg_owner: Air.Inst.Index,
2980	ty: Type,	10567	ty: Type,
2981	mcv: MCValue,	10568	mcv: MCValue,
2982	) !MCValue {	10569	) !MCValue {
2983	const reg: Register = try self.register_manager.allocReg(reg_owner, self.regClassForType(ty));	10570	const reg: Register = try self.register_manager.allocReg(reg_owner, self.regSetForType(ty));
2984	try self.genSetReg(reg, ty, mcv, .{});	10571	try self.genSetReg(reg, ty, mcv, .{});
2985	return MCValue{ .register = reg };	10572	return MCValue{ .register = reg };
2986	}	10573	}
2987		10574
2988	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {	10575	fn airAlloc(self: *CodeGen, inst: Air.Inst.Index) !void {
2989	const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };	10576	const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
2990	return self.finishAir(inst, result, .{ .none, .none, .none });	10577	return self.finishAir(inst, result, .{ .none, .none, .none });
2991	}	10578	}
2992		10579
2993	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {	10580	fn airRetPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
2994	const result: MCValue = switch (self.ret_mcv.long) {	10581	const result: MCValue = switch (self.ret_mcv.long) {
2995	else => unreachable,	10582	else => unreachable,
2996	.none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },	10583	.none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
...	@@ -3006,7 +10593,7 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3006,7 +10593,7 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
3006	return self.finishAir(inst, result, .{ .none, .none, .none });	10593	return self.finishAir(inst, result, .{ .none, .none, .none });
3007	}	10594	}
3008		10595
3009	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	10596	fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
3010	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	10597	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3011	const dst_ty = self.typeOfIndex(inst);	10598	const dst_ty = self.typeOfIndex(inst);
3012	const dst_bits = dst_ty.floatBits(self.target.*);	10599	const dst_bits = dst_ty.floatBits(self.target.*);
...	@@ -3043,7 +10630,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3043,7 +10630,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3043	floatCompilerRtAbiName(src_bits),	10630	floatCompilerRtAbiName(src_bits),
3044	floatCompilerRtAbiName(dst_bits),	10631	floatCompilerRtAbiName(dst_bits),
3045	}) catch unreachable,	10632	}) catch unreachable,
3046	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});	10633	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
3047	}	10634	}
3048		10635
3049	const src_mcv = try self.resolveInst(ty_op.operand);	10636	const src_mcv = try self.resolveInst(ty_op.operand);
...	@@ -3067,18 +10654,18 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3067,18 +10654,18 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3067	.{ .v_, .cvtps2ph },	10654	.{ .v_, .cvtps2ph },
3068	dst_reg,	10655	dst_reg,
3069	mat_src_reg.to128(),	10656	mat_src_reg.to128(),
3070	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	10657	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
3071	);	10658	);
3072	},	10659	},
3073	else => unreachable,	10660	else => unreachable,
3074	}	10661	}
3075	} else {	10662	} else {
3076	assert(src_bits == 64 and dst_bits == 32);	10663	assert(src_bits == 64 and dst_bits == 32);
3077	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(	10664	if (self.hasFeature(.avx)) if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
3078	.{ .v_ss, .cvtsd2 },	10665	.{ .v_ss, .cvtsd2 },
3079	dst_reg,	10666	dst_reg,
3080	dst_reg,	10667	dst_reg,
3081	try src_mcv.mem(self, .qword),	10668	try src_mcv.mem(self, .{ .size = .qword }),
3082	) else try self.asmRegisterRegisterRegister(	10669	) else try self.asmRegisterRegisterRegister(
3083	.{ .v_ss, .cvtsd2 },	10670	.{ .v_ss, .cvtsd2 },
3084	dst_reg,	10671	dst_reg,
...	@@ -3087,10 +10674,10 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3087,10 +10674,10 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3087	src_mcv.getReg().?	10674	src_mcv.getReg().?
3088	else	10675	else
3089	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),	10676	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
3090	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(	10677	) else if (src_mcv.isBase()) try self.asmRegisterMemory(
3091	.{ ._ss, .cvtsd2 },	10678	.{ ._ss, .cvtsd2 },
3092	dst_reg,	10679	dst_reg,
3093	try src_mcv.mem(self, .qword),	10680	try src_mcv.mem(self, .{ .size = .qword }),
3094	) else try self.asmRegisterRegister(	10681	) else try self.asmRegisterRegister(
3095	.{ ._ss, .cvtsd2 },	10682	.{ ._ss, .cvtsd2 },
3096	dst_reg,	10683	dst_reg,
...	@@ -3105,7 +10692,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3105,7 +10692,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3105	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	10692	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3106	}	10693	}
3107		10694
3108	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	10695	fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void {
3109	const pt = self.pt;	10696	const pt = self.pt;
3110	const zcu = pt.zcu;	10697	const zcu = pt.zcu;
3111	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	10698	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -3147,7 +10734,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3147,7 +10734,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3147	floatCompilerRtAbiName(src_bits),	10734	floatCompilerRtAbiName(src_bits),
3148	floatCompilerRtAbiName(dst_bits),	10735	floatCompilerRtAbiName(dst_bits),
3149	}) catch unreachable,	10736	}) catch unreachable,
3150	} }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }});	10737	} }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
3151	}	10738	}
3152		10739
3153	const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));	10740	const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
...	@@ -3186,11 +10773,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3186,11 +10773,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3186	} else {	10773	} else {
3187	assert(src_bits == 32 and dst_bits == 64);	10774	assert(src_bits == 32 and dst_bits == 64);
3188	if (self.hasFeature(.avx)) switch (vec_len) {	10775	if (self.hasFeature(.avx)) switch (vec_len) {
3189	1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(	10776	1 => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
3190	.{ .v_sd, .cvtss2 },	10777	.{ .v_sd, .cvtss2 },
3191	dst_alias,	10778	dst_alias,
3192	dst_alias,	10779	dst_alias,
3193	try src_mcv.mem(self, self.memSize(src_ty)),	10780	try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3194	) else try self.asmRegisterRegisterRegister(	10781	) else try self.asmRegisterRegisterRegister(
3195	.{ .v_sd, .cvtss2 },	10782	.{ .v_sd, .cvtss2 },
3196	dst_alias,	10783	dst_alias,
...	@@ -3200,10 +10787,10 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3200,10 +10787,10 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3200	else	10787	else
3201	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),	10788	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
3202	),	10789	),
3203	2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(	10790	2...4 => if (src_mcv.isBase()) try self.asmRegisterMemory(
3204	.{ .v_pd, .cvtps2 },	10791	.{ .v_pd, .cvtps2 },
3205	dst_alias,	10792	dst_alias,
3206	try src_mcv.mem(self, self.memSize(src_ty)),	10793	try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3207	) else try self.asmRegisterRegister(	10794	) else try self.asmRegisterRegister(
3208	.{ .v_pd, .cvtps2 },	10795	.{ .v_pd, .cvtps2 },
3209	dst_alias,	10796	dst_alias,
...	@@ -3213,14 +10800,14 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3213,14 +10800,14 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3213	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),	10800	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
3214	),	10801	),
3215	else => break :result null,	10802	else => break :result null,
3216	} else if (src_mcv.isMemory()) try self.asmRegisterMemory(	10803	} else if (src_mcv.isBase()) try self.asmRegisterMemory(
3217	switch (vec_len) {	10804	switch (vec_len) {
3218	1 => .{ ._sd, .cvtss2 },	10805	1 => .{ ._sd, .cvtss2 },
3219	2 => .{ ._pd, .cvtps2 },	10806	2 => .{ ._pd, .cvtps2 },
3220	else => break :result null,	10807	else => break :result null,
3221	},	10808	},
3222	dst_alias,	10809	dst_alias,
3223	try src_mcv.mem(self, self.memSize(src_ty)),	10810	try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3224	) else try self.asmRegisterRegister(	10811	) else try self.asmRegisterRegister(
3225	switch (vec_len) {	10812	switch (vec_len) {
3226	1 => .{ ._sd, .cvtss2 },	10813	1 => .{ ._sd, .cvtss2 },
...	@@ -3241,7 +10828,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3241,7 +10828,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3241	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	10828	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3242	}	10829	}
3243		10830
3244	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	10831	fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
3245	const pt = self.pt;	10832	const pt = self.pt;
3246	const zcu = pt.zcu;	10833	const zcu = pt.zcu;
3247	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	10834	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -3249,7 +10836,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3249,7 +10836,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3249	const dst_ty = self.typeOfIndex(inst);	10836	const dst_ty = self.typeOfIndex(inst);
3250		10837
3251	const result = @as(?MCValue, result: {	10838	const result = @as(?MCValue, result: {
3252	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));	10839	const src_abi_size: u31 = @intCast(src_ty.abiSize(zcu));
		10840	const dst_abi_size: u31 = @intCast(dst_ty.abiSize(zcu));
3253		10841
3254	const src_int_info = src_ty.intInfo(zcu);	10842	const src_int_info = src_ty.intInfo(zcu);
3255	const dst_int_info = dst_ty.intInfo(zcu);	10843	const dst_int_info = dst_ty.intInfo(zcu);
...	@@ -3260,14 +10848,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3260,14 +10848,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3260		10848
3261	const src_mcv = try self.resolveInst(ty_op.operand);	10849	const src_mcv = try self.resolveInst(ty_op.operand);
3262	if (dst_ty.isVector(zcu)) {	10850	if (dst_ty.isVector(zcu)) {
3263	const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
3264	const max_abi_size = @max(dst_abi_size, src_abi_size);	10851	const max_abi_size = @max(dst_abi_size, src_abi_size);
3265	if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null;	10852	if (max_abi_size > self.vectorSize(.int)) break :result null;
3266	const has_avx = self.hasFeature(.avx);	10853	const has_avx = self.hasFeature(.avx);
3267		10854
3268	const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu);	10855	const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu);
3269	const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu);	10856	const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu);
3270	switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {	10857	switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) {
3271	.lt => {	10858	.lt => {
3272	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {	10859	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
3273	else => break :result null,	10860	else => break :result null,
...	@@ -3354,10 +10941,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3354,10 +10941,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3354	const dst_reg = dst_mcv.getReg().?;	10941	const dst_reg = dst_mcv.getReg().?;
3355	const dst_alias = registerAlias(dst_reg, dst_abi_size);	10942	const dst_alias = registerAlias(dst_reg, dst_abi_size);
3356		10943
3357	if (src_mcv.isMemory()) try self.asmRegisterMemory(	10944	if (src_mcv.isBase()) try self.asmRegisterMemory(
3358	mir_tag,	10945	mir_tag,
3359	dst_alias,	10946	dst_alias,
3360	try src_mcv.mem(self, self.memSize(src_ty)),	10947	try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3361	) else try self.asmRegisterRegister(	10948	) else try self.asmRegisterRegister(
3362	mir_tag,	10949	mir_tag,
3363	dst_alias,	10950	dst_alias,
...	@@ -3430,9 +11017,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3430,9 +11017,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3430	else => src_int_info.bits,	11017	else => src_int_info.bits,
3431	};	11018	};
3432		11019
3433	const dst_mcv = if (dst_int_info.bits <= src_storage_bits and	11020	const dst_mcv = if ((if (src_mcv.getReg()) \|src_reg\| src_reg.class() == .general_purpose else src_abi_size > 8) and
3434	math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==	11021	dst_int_info.bits <= src_storage_bits and
3435	math.divCeil(u32, src_storage_bits, 64) catch unreachable and	11022	std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
		11023	std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and
3436	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {	11024	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
3437	const dst_mcv = try self.allocRegOrMem(inst, true);	11025	const dst_mcv = try self.allocRegOrMem(inst, true);
3438	try self.genCopy(min_ty, dst_mcv, src_mcv, .{});	11026	try self.genCopy(min_ty, dst_mcv, src_mcv, .{});
...	@@ -3449,10 +11037,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3449,10 +11037,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3449	break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };	11037	break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
3450	}	11038	}
3451		11039
3452	const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;	11040	const src_limbs_len = std.math.divCeil(u31, src_abi_size, 8) catch unreachable;
3453	const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;	11041	const dst_limbs_len = @divExact(dst_abi_size, 8);
3454		11042
3455	const high_mcv: MCValue = if (dst_mcv.isMemory())	11043	const high_mcv: MCValue = if (dst_mcv.isBase())
3456	dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()	11044	dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
3457	else	11045	else
3458	.{ .register = dst_mcv.register_pair[1] };	11046	.{ .register = dst_mcv.register_pair[1] };
...	@@ -3460,8 +11048,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3460,8 +11048,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3460	high_mcv.getReg().?	11048	high_mcv.getReg().?
3461	else	11049	else
3462	try self.copyToTmpRegister(switch (src_int_info.signedness) {	11050	try self.copyToTmpRegister(switch (src_int_info.signedness) {
3463	.signed => Type.isize,	11051	.signed => .isize,
3464	.unsigned => Type.usize,	11052	.unsigned => .usize,
3465	}, high_mcv);	11053	}, high_mcv);
3466	const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);	11054	const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
3467	defer self.register_manager.unlockReg(high_lock);	11055	defer self.register_manager.unlockReg(high_lock);
...	@@ -3469,7 +11057,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3469,7 +11057,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3469	const high_bits = src_int_info.bits % 64;	11057	const high_bits = src_int_info.bits % 64;
3470	if (high_bits > 0) {	11058	if (high_bits > 0) {
3471	try self.truncateRegister(src_ty, high_reg);	11059	try self.truncateRegister(src_ty, high_reg);
3472	const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;	11060	const high_ty: Type = if (dst_int_info.bits >= 64) .usize else dst_ty;
3473	try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{});	11061	try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{});
3474	}	11062	}
3475		11063
...	@@ -3478,13 +11066,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3478,13 +11066,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3478	switch (extend) {	11066	switch (extend) {
3479	.signed => extend: {	11067	.signed => extend: {
3480	const extend_mcv = MCValue{ .register = high_reg };	11068	const extend_mcv = MCValue{ .register = high_reg };
3481	try self.genShiftBinOpMir(	11069	try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, extend_mcv, .u8, .{ .immediate = 63 });
3482	.{ ._r, .sa },
3483	Type.isize,
3484	extend_mcv,
3485	Type.u8,
3486	.{ .immediate = 63 },
3487	);
3488	break :extend extend_mcv;	11070	break :extend extend_mcv;
3489	},	11071	},
3490	.unsigned => .{ .immediate = 0 },	11072	.unsigned => .{ .immediate = 0 },
...	@@ -3500,7 +11082,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3500,7 +11082,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3500	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	11082	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3501	}	11083	}
3502		11084
3503	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	11085	fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
3504	const pt = self.pt;	11086	const pt = self.pt;
3505	const zcu = pt.zcu;	11087	const zcu = pt.zcu;
3506	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	11088	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -3516,7 +11098,8 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3516,7 +11098,8 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3516	if (src_mcv.getReg()) \|reg\| self.register_manager.lockRegAssumeUnused(reg) else null;	11098	if (src_mcv.getReg()) \|reg\| self.register_manager.lockRegAssumeUnused(reg) else null;
3517	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);	11099	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
3518		11100
3519	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))	11101	const dst_mcv = if (src_mcv.isRegister() and src_mcv.getReg().?.class() == self.regClassForType(dst_ty) and
		11102	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3520	src_mcv	11103	src_mcv
3521	else if (dst_abi_size <= 8)	11104	else if (dst_abi_size <= 8)
3522	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)	11105	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)
...	@@ -3570,7 +11153,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3570,7 +11153,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3570	const dst_info = dst_elem_ty.intInfo(zcu);	11153	const dst_info = dst_elem_ty.intInfo(zcu);
3571	const src_info = src_elem_ty.intInfo(zcu);	11154	const src_info = src_elem_ty.intInfo(zcu);
3572		11155
3573	const mask_val = try pt.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));	11156	const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
3574		11157
3575	const splat_ty = try pt.vectorType(.{	11158	const splat_ty = try pt.vectorType(.{
3576	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),	11159	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
...	@@ -3583,10 +11166,10 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3583,10 +11166,10 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3583	.storage = .{ .repeated_elem = mask_val.ip_index },	11166	.storage = .{ .repeated_elem = mask_val.ip_index },
3584	} });	11167	} });
3585		11168
3586	const splat_mcv = try self.genTypedValue(Value.fromInterned(splat_val));	11169	const splat_mcv = try self.genTypedValue(.fromInterned(splat_val));
3587	const splat_addr_mcv: MCValue = switch (splat_mcv) {	11170	const splat_addr_mcv: MCValue = switch (splat_mcv) {
3588	.memory, .indirect, .load_frame => splat_mcv.address(),	11171	.memory, .indirect, .load_frame => splat_mcv.address(),
3589	else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },	11172	else => .{ .register = try self.copyToTmpRegister(.usize, splat_mcv.address()) },
3590	};	11173	};
3591		11174
3592	const dst_reg = dst_mcv.getReg().?;	11175	const dst_reg = dst_mcv.getReg().?;
...	@@ -3596,7 +11179,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3596,7 +11179,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3596	.{ .vp_, .@"and" },	11179	.{ .vp_, .@"and" },
3597	dst_alias,	11180	dst_alias,
3598	dst_alias,	11181	dst_alias,
3599	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),	11182	try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
3600	);	11183	);
3601	if (src_abi_size > 16) {	11184	if (src_abi_size > 16) {
3602	const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);	11185	const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
...	@@ -3607,7 +11190,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3607,7 +11190,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3607	.{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },	11190	.{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
3608	registerAlias(temp_reg, dst_abi_size),	11191	registerAlias(temp_reg, dst_abi_size),
3609	dst_alias,	11192	dst_alias,
3610	Immediate.u(1),	11193	.u(1),
3611	);	11194	);
3612	try self.asmRegisterRegisterRegister(	11195	try self.asmRegisterRegisterRegister(
3613	mir_tag,	11196	mir_tag,
...	@@ -3620,7 +11203,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3620,7 +11203,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3620	try self.asmRegisterMemory(	11203	try self.asmRegisterMemory(
3621	.{ .p_, .@"and" },	11204	.{ .p_, .@"and" },
3622	dst_alias,	11205	dst_alias,
3623	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),	11206	try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
3624	);	11207	);
3625	try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);	11208	try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
3626	}	11209	}
...	@@ -3646,7 +11229,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3646,7 +11229,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3646	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	11229	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3647	}	11230	}
3648		11231
3649	fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {	11232	fn airIntFromBool(self: *CodeGen, inst: Air.Inst.Index) !void {
3650	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	11233	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3651	const ty = self.typeOfIndex(inst);	11234	const ty = self.typeOfIndex(inst);
3652		11235
...	@@ -3659,13 +11242,13 @@ fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3659,13 +11242,13 @@ fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {
3659	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });	11242	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
3660	}	11243	}
3661		11244
3662	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {	11245	fn airSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
3663	const zcu = self.pt.zcu;	11246	const zcu = self.pt.zcu;
3664	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	11247	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3665	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;	11248	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3666		11249
3667	const slice_ty = self.typeOfIndex(inst);	11250	const slice_ty = self.typeOfIndex(inst);
3668	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));	11251	const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
3669		11252
3670	const ptr_ty = self.typeOf(bin_op.lhs);	11253	const ptr_ty = self.typeOf(bin_op.lhs);
3671	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});	11254	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});
...	@@ -3683,13 +11266,13 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3683,13 +11266,13 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
3683	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	11266	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3684	}	11267	}
3685		11268
3686	fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	11269	fn airUnOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3687	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	11270	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3688	const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand);	11271	const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand);
3689	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	11272	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
3690	}	11273	}
3691		11274
3692	fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	11275	fn airBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3693	const pt = self.pt;	11276	const pt = self.pt;
3694	const zcu = pt.zcu;	11277	const zcu = pt.zcu;
3695	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	11278	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -3724,14 +11307,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	...	@@ -3724,14 +11307,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3724	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });	11307	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
3725	}	11308	}
3726		11309
3727	fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	11310	fn airPtrArithmetic(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3728	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	11311	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3729	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;	11312	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3730	const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);	11313	const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
3731	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });	11314	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
3732	}	11315	}
3733		11316
3734	fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {	11317	fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 {
3735	const pt = self.pt;	11318	const pt = self.pt;
3736	const zcu = pt.zcu;	11319	const zcu = pt.zcu;
3737	const air_tag = self.air.instructions.items(.tag);	11320	const air_tag = self.air.instructions.items(.tag);
...	@@ -3766,7 +11349,7 @@ fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {	...	@@ -3766,7 +11349,7 @@ fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {
3766	return dst_info.bits;	11349	return dst_info.bits;
3767	}	11350	}
3768		11351
3769	fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	11352	fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
3770	const pt = self.pt;	11353	const pt = self.pt;
3771	const zcu = pt.zcu;	11354	const zcu = pt.zcu;
3772	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	11355	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -3802,11 +11385,11 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3802,11 +11385,11 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3802	state: State,	11385	state: State,
3803	reloc: Mir.Inst.Index,	11386	reloc: Mir.Inst.Index,
3804	} = if (signed and tag == .div_floor) state: {	11387	} = if (signed and tag == .div_floor) state: {
3805	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(Type.usize, zcu));	11388	const frame_index = try self.allocFrameIndex(.initType(.usize, zcu));
3806	try self.asmMemoryImmediate(	11389	try self.asmMemoryImmediate(
3807	.{ ._, .mov },	11390	.{ ._, .mov },
3808	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },	11391	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
3809	Immediate.u(0),	11392	.u(0),
3810	);	11393	);
3811		11394
3812	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	11395	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	@@ -3817,7 +11400,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3817,7 +11400,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3817	const mat_lhs_mcv = switch (lhs_mcv) {	11400	const mat_lhs_mcv = switch (lhs_mcv) {
3818	.load_symbol => mat_lhs_mcv: {	11401	.load_symbol => mat_lhs_mcv: {
3819	// TODO clean this up!	11402	// TODO clean this up!
3820	const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());	11403	const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
3821	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	11404	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
3822	},	11405	},
3823	else => lhs_mcv,	11406	else => lhs_mcv,
...	@@ -3827,10 +11410,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3827,10 +11410,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3827	else => null,	11410	else => null,
3828	};	11411	};
3829	defer if (mat_lhs_lock) \|lock\| self.register_manager.unlockReg(lock);	11412	defer if (mat_lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3830	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(	11413	if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
3831	.{ ._, .mov },	11414	.{ ._, .mov },
3832	tmp_reg,	11415	tmp_reg,
3833	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),	11416	try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
3834	) else try self.asmRegisterRegister(	11417	) else try self.asmRegisterRegister(
3835	.{ ._, .mov },	11418	.{ ._, .mov },
3836	tmp_reg,	11419	tmp_reg,
...	@@ -3841,7 +11424,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3841,7 +11424,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3841	const mat_rhs_mcv = switch (rhs_mcv) {	11424	const mat_rhs_mcv = switch (rhs_mcv) {
3842	.load_symbol => mat_rhs_mcv: {	11425	.load_symbol => mat_rhs_mcv: {
3843	// TODO clean this up!	11426	// TODO clean this up!
3844	const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());	11427	const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
3845	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	11428	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
3846	},	11429	},
3847	else => rhs_mcv,	11430	else => rhs_mcv,
...	@@ -3851,10 +11434,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3851,10 +11434,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3851	else => null,	11434	else => null,
3852	};	11435	};
3853	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	11436	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3854	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(	11437	if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
3855	.{ ._, .xor },	11438	.{ ._, .xor },
3856	tmp_reg,	11439	tmp_reg,
3857	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),	11440	try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
3858	) else try self.asmRegisterRegister(	11441	) else try self.asmRegisterRegister(
3859	.{ ._, .xor },	11442	.{ ._, .xor },
3860	tmp_reg,	11443	tmp_reg,
...	@@ -3882,6 +11465,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3882,6 +11465,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3882	} },	11465	} },
3883	&.{ src_ty, src_ty },	11466	&.{ src_ty, src_ty },
3884	&.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },	11467	&.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },
		11468	.{},
3885	);	11469	);
3886	break :result if (signed) switch (tag) {	11470	break :result if (signed) switch (tag) {
3887	.div_floor => {	11471	.div_floor => {
...	@@ -3911,6 +11495,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3911,6 +11495,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3911	} },	11495	} },
3912	&.{ src_ty, src_ty },	11496	&.{ src_ty, src_ty },
3913	&.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },	11497	&.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },
		11498	.{},
3914	);	11499	);
3915	try self.asmRegisterMemory(	11500	try self.asmRegisterMemory(
3916	.{ ._, .sub },	11501	.{ ._, .sub },
...	@@ -3920,11 +11505,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3920,11 +11505,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3920	.mod = .{ .rm = .{ .size = .qword } },	11505	.mod = .{ .rm = .{ .size = .qword } },
3921	},	11506	},
3922	);	11507	);
3923	try self.asmRegisterImmediate(	11508	try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0));
3924	.{ ._, .sbb },
3925	dst_mcv.register_pair[1],
3926	Immediate.u(0),
3927	);
3928	try self.freeValue(	11509	try self.freeValue(
3929	.{ .load_frame = .{ .index = signed_div_floor_state.frame_index } },	11510	.{ .load_frame = .{ .index = signed_div_floor_state.frame_index } },
3930	);	11511	);
...	@@ -3936,7 +11517,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3936,7 +11517,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3936	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);	11517	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);
3937		11518
3938	const tmp_regs =	11519	const tmp_regs =
3939	try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);	11520	try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
3940	const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);	11521	const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
3941	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);	11522	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);
3942		11523
...	@@ -3944,7 +11525,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3944,7 +11525,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3944	const mat_rhs_mcv = switch (rhs_mcv) {	11525	const mat_rhs_mcv = switch (rhs_mcv) {
3945	.load_symbol => mat_rhs_mcv: {	11526	.load_symbol => mat_rhs_mcv: {
3946	// TODO clean this up!	11527	// TODO clean this up!
3947	const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());	11528	const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
3948	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	11529	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
3949	},	11530	},
3950	else => rhs_mcv,	11531	else => rhs_mcv,
...	@@ -3957,16 +11538,16 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3957,16 +11538,16 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3957		11538
3958	for (tmp_regs, dst_regs) \|tmp_reg, dst_reg\|	11539	for (tmp_regs, dst_regs) \|tmp_reg, dst_reg\|
3959	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg);	11540	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg);
3960	if (mat_rhs_mcv.isMemory()) {	11541	if (mat_rhs_mcv.isBase()) {
3961	try self.asmRegisterMemory(	11542	try self.asmRegisterMemory(
3962	.{ ._, .add },	11543	.{ ._, .add },
3963	tmp_regs[0],	11544	tmp_regs[0],
3964	try mat_rhs_mcv.mem(self, .qword),	11545	try mat_rhs_mcv.mem(self, .{ .size = .qword }),
3965	);	11546	);
3966	try self.asmRegisterMemory(	11547	try self.asmRegisterMemory(
3967	.{ ._, .adc },	11548	.{ ._, .adc },
3968	tmp_regs[1],	11549	tmp_regs[1],
3969	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),	11550	try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
3970	);	11551	);
3971	} else for (	11552	} else for (
3972	[_]Mir.Inst.Tag{ .add, .adc },	11553	[_]Mir.Inst.Tag{ .add, .adc },
...	@@ -3996,7 +11577,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3996,7 +11577,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3996	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	11577	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3997	}	11578	}
3998		11579
3999	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	11580	fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
4000	const pt = self.pt;	11581	const pt = self.pt;
4001	const zcu = pt.zcu;	11582	const zcu = pt.zcu;
4002	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	11583	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -4031,22 +11612,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4031,22 +11612,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
4031	const reg_extra_bits = self.regExtraBits(ty);	11612	const reg_extra_bits = self.regExtraBits(ty);
4032	const cc: Condition = if (ty.isSignedInt(zcu)) cc: {	11613	const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
4033	if (reg_extra_bits > 0) {	11614	if (reg_extra_bits > 0) {
4034	try self.genShiftBinOpMir(	11615	try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
4035	.{ ._l, .sa },
4036	ty,
4037	dst_mcv,
4038	Type.u8,
4039	.{ .immediate = reg_extra_bits },
4040	);
4041	}	11616	}
4042	try self.genSetReg(limit_reg, ty, dst_mcv, .{});	11617	try self.genSetReg(limit_reg, ty, dst_mcv, .{});
4043	try self.genShiftBinOpMir(	11618	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
4044	.{ ._r, .sa },
4045	ty,
4046	limit_mcv,
4047	Type.u8,
4048	.{ .immediate = reg_bits - 1 },
4049	);
4050	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{	11619	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
4051	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,	11620	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
4052	});	11621	});
...	@@ -4056,19 +11625,13 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4056,19 +11625,13 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
4056	const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);	11625	const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
4057	defer self.register_manager.unlockReg(shifted_rhs_lock);	11626	defer self.register_manager.unlockReg(shifted_rhs_lock);
4058		11627
4059	try self.genShiftBinOpMir(	11628	try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
4060	.{ ._l, .sa },
4061	ty,
4062	shifted_rhs_mcv,
4063	Type.u8,
4064	.{ .immediate = reg_extra_bits },
4065	);
4066	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv);	11629	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv);
4067	} else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);	11630	} else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
4068	break :cc .o;	11631	break :cc .o;
4069	} else cc: {	11632	} else cc: {
4070	try self.genSetReg(limit_reg, ty, .{	11633	try self.genSetReg(limit_reg, ty, .{
4071	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),	11634	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),
4072	}, .{});	11635	}, .{});
4073		11636
4074	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);	11637	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
...	@@ -4086,18 +11649,13 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4086,18 +11649,13 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
4086	registerAlias(limit_reg, cmov_abi_size),	11649	registerAlias(limit_reg, cmov_abi_size),
4087	);	11650	);
4088		11651
4089	if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) try self.genShiftBinOpMir(	11652	if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
4090	.{ ._r, .sa },	11653	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
4091	ty,
4092	dst_mcv,
4093	Type.u8,
4094	.{ .immediate = reg_extra_bits },
4095	);
4096		11654
4097	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });	11655	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
4098	}	11656	}
4099		11657
4100	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {	11658	fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
4101	const pt = self.pt;	11659	const pt = self.pt;
4102	const zcu = pt.zcu;	11660	const zcu = pt.zcu;
4103	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	11661	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -4132,22 +11690,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4132,22 +11690,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
4132	const reg_extra_bits = self.regExtraBits(ty);	11690	const reg_extra_bits = self.regExtraBits(ty);
4133	const cc: Condition = if (ty.isSignedInt(zcu)) cc: {	11691	const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
4134	if (reg_extra_bits > 0) {	11692	if (reg_extra_bits > 0) {
4135	try self.genShiftBinOpMir(	11693	try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
4136	.{ ._l, .sa },
4137	ty,
4138	dst_mcv,
4139	Type.u8,
4140	.{ .immediate = reg_extra_bits },
4141	);
4142	}	11694	}
4143	try self.genSetReg(limit_reg, ty, dst_mcv, .{});	11695	try self.genSetReg(limit_reg, ty, dst_mcv, .{});
4144	try self.genShiftBinOpMir(	11696	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
4145	.{ ._r, .sa },
4146	ty,
4147	limit_mcv,
4148	Type.u8,
4149	.{ .immediate = reg_bits - 1 },
4150	);
4151	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{	11697	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
4152	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,	11698	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
4153	});	11699	});
...	@@ -4157,13 +11703,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4157,13 +11703,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
4157	const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);	11703	const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
4158	defer self.register_manager.unlockReg(shifted_rhs_lock);	11704	defer self.register_manager.unlockReg(shifted_rhs_lock);
4159		11705
4160	try self.genShiftBinOpMir(	11706	try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
4161	.{ ._l, .sa },
4162	ty,
4163	shifted_rhs_mcv,
4164	Type.u8,
4165	.{ .immediate = reg_extra_bits },
4166	);
4167	try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv);	11707	try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv);
4168	} else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);	11708	} else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
4169	break :cc .o;	11709	break :cc .o;
...	@@ -4180,18 +11720,13 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4180,18 +11720,13 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
4180	registerAlias(limit_reg, cmov_abi_size),	11720	registerAlias(limit_reg, cmov_abi_size),
4181	);	11721	);
4182		11722
4183	if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) try self.genShiftBinOpMir(	11723	if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
4184	.{ ._r, .sa },	11724	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
4185	ty,
4186	dst_mcv,
4187	Type.u8,
4188	.{ .immediate = reg_extra_bits },
4189	);
4190		11725
4191	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });	11726	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
4192	}	11727	}
4193		11728
4194	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	11729	fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
4195	const pt = self.pt;	11730	const pt = self.pt;
4196	const zcu = pt.zcu;	11731	const zcu = pt.zcu;
4197	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	11732	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -4199,18 +11734,18 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4199,18 +11734,18 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4199		11734
4200	const result = result: {	11735	const result = result: {
4201	if (ty.toIntern() == .i128_type) {	11736	if (ty.toIntern() == .i128_type) {
4202	const ptr_c_int = try pt.singleMutPtrType(Type.c_int);	11737	const ptr_c_int = try pt.singleMutPtrType(.c_int);
4203	const overflow = try self.allocTempRegOrMem(Type.c_int, false);	11738	const overflow = try self.allocTempRegOrMem(.c_int, false);
4204		11739
4205	const dst_mcv = try self.genCall(.{ .lib = .{	11740	const dst_mcv = try self.genCall(.{ .lib = .{
4206	.return_type = .i128_type,	11741	.return_type = .i128_type,
4207	.param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },	11742	.param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
4208	.callee = "__muloti4",	11743	.callee = "__muloti4",
4209	} }, &.{ Type.i128, Type.i128, ptr_c_int }, &.{	11744	} }, &.{ .i128, .i128, ptr_c_int }, &.{
4210	.{ .air_ref = bin_op.lhs },	11745	.{ .air_ref = bin_op.lhs },
4211	.{ .air_ref = bin_op.rhs },	11746	.{ .air_ref = bin_op.rhs },
4212	overflow.address(),	11747	overflow.address(),
4213	});	11748	}, .{});
4214	const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_mcv.register_pair);	11749	const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_mcv.register_pair);
4215	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);	11750	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);
4216		11751
...	@@ -4222,7 +11757,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4222,7 +11757,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4222	const mat_lhs_mcv = switch (lhs_mcv) {	11757	const mat_lhs_mcv = switch (lhs_mcv) {
4223	.load_symbol => mat_lhs_mcv: {	11758	.load_symbol => mat_lhs_mcv: {
4224	// TODO clean this up!	11759	// TODO clean this up!
4225	const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());	11760	const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
4226	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	11761	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
4227	},	11762	},
4228	else => lhs_mcv,	11763	else => lhs_mcv,
...	@@ -4232,10 +11767,10 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4232,10 +11767,10 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4232	else => null,	11767	else => null,
4233	};	11768	};
4234	defer if (mat_lhs_lock) \|lock\| self.register_manager.unlockReg(lock);	11769	defer if (mat_lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
4235	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(	11770	if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
4236	.{ ._, .mov },	11771	.{ ._, .mov },
4237	tmp_reg,	11772	tmp_reg,
4238	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),	11773	try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4239	) else try self.asmRegisterRegister(	11774	) else try self.asmRegisterRegister(
4240	.{ ._, .mov },	11775	.{ ._, .mov },
4241	tmp_reg,	11776	tmp_reg,
...	@@ -4246,7 +11781,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4246,7 +11781,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4246	const mat_rhs_mcv = switch (rhs_mcv) {	11781	const mat_rhs_mcv = switch (rhs_mcv) {
4247	.load_symbol => mat_rhs_mcv: {	11782	.load_symbol => mat_rhs_mcv: {
4248	// TODO clean this up!	11783	// TODO clean this up!
4249	const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());	11784	const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
4250	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	11785	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
4251	},	11786	},
4252	else => rhs_mcv,	11787	else => rhs_mcv,
...	@@ -4256,22 +11791,22 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4256,22 +11791,22 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4256	else => null,	11791	else => null,
4257	};	11792	};
4258	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	11793	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
4259	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(	11794	if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
4260	.{ ._, .xor },	11795	.{ ._, .xor },
4261	tmp_reg,	11796	tmp_reg,
4262	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),	11797	try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4263	) else try self.asmRegisterRegister(	11798	) else try self.asmRegisterRegister(
4264	.{ ._, .xor },	11799	.{ ._, .xor },
4265	tmp_reg,	11800	tmp_reg,
4266	mat_rhs_mcv.register_pair[1],	11801	mat_rhs_mcv.register_pair[1],
4267	);	11802	);
4268		11803
4269	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));	11804	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
4270	try self.asmRegister(.{ ._, .not }, tmp_reg);	11805	try self.asmRegister(.{ ._, .not }, tmp_reg);
4271	try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));	11806	try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .{ .size = .dword }), .s(0));
4272	try self.freeValue(overflow);	11807	try self.freeValue(overflow);
4273	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);	11808	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
4274	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));	11809	try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));
4275	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);	11810	try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);
4276	break :result dst_mcv;	11811	break :result dst_mcv;
4277	}	11812	}
...	@@ -4308,20 +11843,14 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4308,20 +11843,14 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4308	const cc: Condition = if (ty.isSignedInt(zcu)) cc: {	11843	const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
4309	try self.genSetReg(limit_reg, ty, lhs_mcv, .{});	11844	try self.genSetReg(limit_reg, ty, lhs_mcv, .{});
4310	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);	11845	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
4311	try self.genShiftBinOpMir(	11846	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
4312	.{ ._r, .sa },
4313	ty,
4314	limit_mcv,
4315	Type.u8,
4316	.{ .immediate = reg_bits - 1 },
4317	);
4318	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{	11847	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
4319	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,	11848	.immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
4320	});	11849	});
4321	break :cc .o;	11850	break :cc .o;
4322	} else cc: {	11851	} else cc: {
4323	try self.genSetReg(limit_reg, ty, .{	11852	try self.genSetReg(limit_reg, ty, .{
4324	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),	11853	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits),
4325	}, .{});	11854	}, .{});
4326	break :cc .c;	11855	break :cc .c;
4327	};	11856	};
...	@@ -4338,7 +11867,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4338,7 +11867,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4338	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	11867	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4339	}	11868	}
4340		11869
4341	fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	11870	fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
4342	const pt = self.pt;	11871	const pt = self.pt;
4343	const zcu = pt.zcu;	11872	const zcu = pt.zcu;
4344	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	11873	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -4366,7 +11895,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4366,7 +11895,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4366	};	11895	};
4367		11896
4368	const tuple_ty = self.typeOfIndex(inst);	11897	const tuple_ty = self.typeOfIndex(inst);
4369	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {	11898	if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
4370	switch (partial_mcv) {	11899	switch (partial_mcv) {
4371	.register => \|reg\| {	11900	.register => \|reg\| {
4372	self.eflags_inst = inst;	11901	self.eflags_inst = inst;
...	@@ -4375,12 +11904,11 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4375,12 +11904,11 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4375	else => {},	11904	else => {},
4376	}	11905	}
4377		11906
4378	const frame_index =	11907	const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4379	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4380	try self.genSetMem(	11908	try self.genSetMem(
4381	.{ .frame = frame_index },	11909	.{ .frame = frame_index },
4382	@intCast(tuple_ty.structFieldOffset(1, zcu)),	11910	@intCast(tuple_ty.structFieldOffset(1, zcu)),
4383	Type.u1,	11911	.u1,
4384	.{ .eflags = cc },	11912	.{ .eflags = cc },
4385	.{},	11913	.{},
4386	);	11914	);
...	@@ -4394,8 +11922,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4394,8 +11922,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4394	break :result .{ .load_frame = .{ .index = frame_index } };	11922	break :result .{ .load_frame = .{ .index = frame_index } };
4395	}	11923	}
4396		11924
4397	const frame_index =	11925	const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4398	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4399	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);	11926	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4400	break :result .{ .load_frame = .{ .index = frame_index } };	11927	break :result .{ .load_frame = .{ .index = frame_index } };
4401	},	11928	},
...	@@ -4405,7 +11932,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4405,7 +11932,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4405	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	11932	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4406	}	11933	}
4407		11934
4408	fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	11935	fn airShlWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
4409	const pt = self.pt;	11936	const pt = self.pt;
4410	const zcu = pt.zcu;	11937	const zcu = pt.zcu;
4411	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	11938	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -4444,7 +11971,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4444,7 +11971,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4444	const cc = Condition.ne;	11971	const cc = Condition.ne;
4445		11972
4446	const tuple_ty = self.typeOfIndex(inst);	11973	const tuple_ty = self.typeOfIndex(inst);
4447	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {	11974	if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
4448	switch (partial_mcv) {	11975	switch (partial_mcv) {
4449	.register => \|reg\| {	11976	.register => \|reg\| {
4450	self.eflags_inst = inst;	11977	self.eflags_inst = inst;
...	@@ -4453,8 +11980,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4453,8 +11980,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4453	else => {},	11980	else => {},
4454	}	11981	}
4455		11982
4456	const frame_index =	11983	const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4457	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));
4458	try self.genSetMem(	11984	try self.genSetMem(
4459	.{ .frame = frame_index },	11985	.{ .frame = frame_index },
4460	@intCast(tuple_ty.structFieldOffset(1, zcu)),	11986	@intCast(tuple_ty.structFieldOffset(1, zcu)),
...	@@ -4473,7 +11999,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4473,7 +11999,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4473	}	11999	}
4474		12000
4475	const frame_index =	12001	const frame_index =
4476	try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));	12002	try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4477	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);	12003	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4478	break :result .{ .load_frame = .{ .index = frame_index } };	12004	break :result .{ .load_frame = .{ .index = frame_index } };
4479	},	12005	},
...	@@ -4484,7 +12010,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4484,7 +12010,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4484	}	12010	}
4485		12011
4486	fn genSetFrameTruncatedOverflowCompare(	12012	fn genSetFrameTruncatedOverflowCompare(
4487	self: *Self,	12013	self: *CodeGen,
4488	tuple_ty: Type,	12014	tuple_ty: Type,
4489	frame_index: FrameIndex,	12015	frame_index: FrameIndex,
4490	src_mcv: MCValue,	12016	src_mcv: MCValue,
...	@@ -4499,6 +12025,7 @@ fn genSetFrameTruncatedOverflowCompare(	...	@@ -4499,6 +12025,7 @@ fn genSetFrameTruncatedOverflowCompare(
4499	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);	12025	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
4500		12026
4501	const ty = tuple_ty.fieldType(0, zcu);	12027	const ty = tuple_ty.fieldType(0, zcu);
		12028	const ty_size = ty.abiSize(zcu);
4502	const int_info = ty.intInfo(zcu);	12029	const int_info = ty.intInfo(zcu);
4503		12030
4504	const hi_bits = (int_info.bits - 1) % 64 + 1;	12031	const hi_bits = (int_info.bits - 1) % 64 + 1;
...	@@ -4510,7 +12037,7 @@ fn genSetFrameTruncatedOverflowCompare(	...	@@ -4510,7 +12037,7 @@ fn genSetFrameTruncatedOverflowCompare(
4510	const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits);	12037	const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits);
4511		12038
4512	const temp_regs =	12039	const temp_regs =
4513	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);	12040	try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
4514	const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);	12041	const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
4515	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);	12042	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
4516		12043
...	@@ -4530,13 +12057,15 @@ fn genSetFrameTruncatedOverflowCompare(	...	@@ -4530,13 +12057,15 @@ fn genSetFrameTruncatedOverflowCompare(
4530	const eq_reg = temp_regs[2];	12057	const eq_reg = temp_regs[2];
4531	if (overflow_cc) \|_\| {	12058	if (overflow_cc) \|_\| {
4532	try self.asmSetccRegister(.ne, eq_reg.to8());	12059	try self.asmSetccRegister(.ne, eq_reg.to8());
4533	try self.genBinOpMir(	12060	try self.genBinOpMir(.{ ._, .@"or" }, .u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
4534	.{ ._, .@"or" },
4535	Type.u8,
4536	.{ .register = overflow_reg },
4537	.{ .register = eq_reg },
4538	);
4539	}	12061	}
		12062	try self.genSetMem(
		12063	.{ .frame = frame_index },
		12064	@intCast(tuple_ty.structFieldOffset(1, zcu)),
		12065	tuple_ty.fieldType(1, zcu),
		12066	if (overflow_cc) \|_\| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
		12067	.{},
		12068	);
4540		12069
4541	const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));	12070	const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));
4542	if (hi_limb_off > 0) try self.genSetMem(	12071	if (hi_limb_off > 0) try self.genSetMem(
...	@@ -4546,23 +12075,30 @@ fn genSetFrameTruncatedOverflowCompare(	...	@@ -4546,23 +12075,30 @@ fn genSetFrameTruncatedOverflowCompare(
4546	src_mcv,	12075	src_mcv,
4547	.{},	12076	.{},
4548	);	12077	);
4549	try self.genSetMem(	12078	try self.genSetMem(
4550	.{ .frame = frame_index },	12079	.{ .frame = frame_index },
4551	payload_off + hi_limb_off,	12080	payload_off + hi_limb_off,
4552	limb_ty,	12081	limb_ty,
4553	.{ .register = scratch_reg },	12082	.{ .register = scratch_reg },
4554	.{},
4555	);
4556	try self.genSetMem(
4557	.{ .frame = frame_index },
4558	@intCast(tuple_ty.structFieldOffset(1, zcu)),
4559	tuple_ty.fieldType(1, zcu),
4560	if (overflow_cc) \|_\| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
4561	.{},	12083	.{},
4562	);	12084	);
		12085	var ext_off: i32 = hi_limb_off + 8;
		12086	if (ext_off < ty_size) {
		12087	switch (int_info.signedness) {
		12088	.signed => try self.asmRegisterImmediate(.{ ._r, .sa }, scratch_reg.to64(), .s(63)),
		12089	.unsigned => try self.asmRegisterRegister(.{ ._, .xor }, scratch_reg.to32(), scratch_reg.to32()),
		12090	}
		12091	while (ext_off < ty_size) : (ext_off += 8) try self.genSetMem(
		12092	.{ .frame = frame_index },
		12093	payload_off + ext_off,
		12094	limb_ty,
		12095	.{ .register = scratch_reg },
		12096	.{},
		12097	);
		12098	}
4563	}	12099	}
4564		12100
4565	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	12101	fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
4566	const pt = self.pt;	12102	const pt = self.pt;
4567	const zcu = pt.zcu;	12103	const zcu = pt.zcu;
4568	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	12104	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -4576,7 +12112,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4576,7 +12112,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4576	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {	12112	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
4577	const slow_inc = self.hasFeature(.slow_incdec);	12113	const slow_inc = self.hasFeature(.slow_incdec);
4578	const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));	12114	const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
4579	const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;	12115	const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
4580		12116
4581	try self.spillRegisters(&.{ .rax, .rcx, .rdx });	12117	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
4582	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });	12118	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
...	@@ -4593,7 +12129,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4593,7 +12129,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4593	const rhs_mcv = try self.resolveInst(bin_op.rhs);	12129	const rhs_mcv = try self.resolveInst(bin_op.rhs);
4594		12130
4595	const temp_regs =	12131	const temp_regs =
4596	try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);	12132	try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
4597	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);	12133	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
4598	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);	12134	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
4599		12135
...	@@ -4618,7 +12154,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4618,7 +12154,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4618	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);	12154	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
4619		12155
4620	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	12156	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4621	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));	12157	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
4622	try self.asmMemoryRegister(.{ ._, .adc }, .{	12158	try self.asmMemoryRegister(.{ ._, .adc }, .{
4623	.base = .{ .frame = dst_mcv.load_frame.index },	12159	.base = .{ .frame = dst_mcv.load_frame.index },
4624	.mod = .{ .rm = .{	12160	.mod = .{ .rm = .{
...	@@ -4642,7 +12178,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4642,7 +12178,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4642	});	12178	});
4643	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());	12179	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
4644		12180
4645	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));	12181	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
4646	try self.asmMemoryRegister(.{ ._, .adc }, .{	12182	try self.asmMemoryRegister(.{ ._, .adc }, .{
4647	.base = .{ .frame = dst_mcv.load_frame.index },	12183	.base = .{ .frame = dst_mcv.load_frame.index },
4648	.mod = .{ .rm = .{	12184	.mod = .{ .rm = .{
...	@@ -4656,30 +12192,22 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4656,30 +12192,22 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4656	try self.asmSetccRegister(.c, .ch);	12192	try self.asmSetccRegister(.c, .ch);
4657		12193
4658	if (slow_inc) {	12194	if (slow_inc) {
4659	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));	12195	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
4660	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));	12196	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
4661	} else {	12197	} else {
4662	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());	12198	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
4663	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());	12199	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
4664	}	12200	}
4665	try self.asmRegisterImmediate(	12201	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
4666	.{ ._, .cmp },
4667	temp_regs[3].to32(),
4668	Immediate.u(limb_len),
4669	);
4670	_ = try self.asmJccReloc(.b, inner_loop);	12202	_ = try self.asmJccReloc(.b, inner_loop);
4671		12203
4672	try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx);	12204	try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx);
4673	const overflow = try self.asmJccReloc(.nz, undefined);	12205	const overflow = try self.asmJccReloc(.nz, undefined);
4674	const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	12206	const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4675	try self.asmRegisterImmediate(	12207	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len));
4676	.{ ._, .cmp },
4677	temp_regs[2].to32(),
4678	Immediate.u(limb_len),
4679	);
4680	const no_overflow = try self.asmJccReloc(.nb, undefined);	12208	const no_overflow = try self.asmJccReloc(.nb, undefined);
4681	if (slow_inc) {	12209	if (slow_inc) {
4682	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));	12210	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
4683	} else {	12211	} else {
4684	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());	12212	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
4685	}	12213	}
...	@@ -4691,7 +12219,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4691,7 +12219,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4691	.scale = .@"8",	12219	.scale = .@"8",
4692	.disp = lhs_mcv.load_frame.off - 8,	12220	.disp = lhs_mcv.load_frame.off - 8,
4693	} },	12221	} },
4694	}, Immediate.u(0));	12222	}, .u(0));
4695	_ = try self.asmJccReloc(.z, overflow_loop);	12223	_ = try self.asmJccReloc(.z, overflow_loop);
4696	self.performReloc(overflow);	12224	self.performReloc(overflow);
4697	try self.asmMemoryImmediate(.{ ._, .mov }, .{	12225	try self.asmMemoryImmediate(.{ ._, .mov }, .{
...	@@ -4701,20 +12229,16 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4701,20 +12229,16 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4701	.disp = dst_mcv.load_frame.off +	12229	.disp = dst_mcv.load_frame.off +
4702	@as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),	12230	@as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
4703	} },	12231	} },
4704	}, Immediate.u(1));	12232	}, .u(1));
4705	self.performReloc(no_overflow);	12233	self.performReloc(no_overflow);
4706		12234
4707	self.performReloc(skip_inner);	12235	self.performReloc(skip_inner);
4708	if (slow_inc) {	12236	if (slow_inc) {
4709	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	12237	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
4710	} else {	12238	} else {
4711	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	12239	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
4712	}	12240	}
4713	try self.asmRegisterImmediate(	12241	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
4714	.{ ._, .cmp },
4715	temp_regs[0].to32(),
4716	Immediate.u(limb_len),
4717	);
4718	_ = try self.asmJccReloc(.b, outer_loop);	12242	_ = try self.asmJccReloc(.b, outer_loop);
4719		12243
4720	break :result dst_mcv;	12244	break :result dst_mcv;
...	@@ -4727,17 +12251,17 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4727,17 +12251,17 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4727	if (src_bits > 64 and src_bits <= 128 and	12251	if (src_bits > 64 and src_bits <= 128 and
4728	dst_info.bits > 64 and dst_info.bits <= 128) switch (dst_info.signedness) {	12252	dst_info.bits > 64 and dst_info.bits <= 128) switch (dst_info.signedness) {
4729	.signed => {	12253	.signed => {
4730	const ptr_c_int = try pt.singleMutPtrType(Type.c_int);	12254	const ptr_c_int = try pt.singleMutPtrType(.c_int);
4731	const overflow = try self.allocTempRegOrMem(Type.c_int, false);	12255	const overflow = try self.allocTempRegOrMem(.c_int, false);
4732	const result = try self.genCall(.{ .lib = .{	12256	const result = try self.genCall(.{ .lib = .{
4733	.return_type = .i128_type,	12257	.return_type = .i128_type,
4734	.param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },	12258	.param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
4735	.callee = "__muloti4",	12259	.callee = "__muloti4",
4736	} }, &.{ Type.i128, Type.i128, ptr_c_int }, &.{	12260	} }, &.{ .i128, .i128, ptr_c_int }, &.{
4737	.{ .air_ref = bin_op.lhs },	12261	.{ .air_ref = bin_op.lhs },
4738	.{ .air_ref = bin_op.rhs },	12262	.{ .air_ref = bin_op.rhs },
4739	overflow.address(),	12263	overflow.address(),
4740	});	12264	}, .{});
4741		12265
4742	const dst_mcv = try self.allocRegOrMem(inst, false);	12266	const dst_mcv = try self.allocRegOrMem(inst, false);
4743	try self.genSetMem(	12267	try self.genSetMem(
...	@@ -4749,8 +12273,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4749,8 +12273,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4749	);	12273	);
4750	try self.asmMemoryImmediate(	12274	try self.asmMemoryImmediate(
4751	.{ ._, .cmp },	12275	.{ ._, .cmp },
4752	try overflow.mem(self, self.memSize(Type.c_int)),	12276	try overflow.mem(self, .{ .size = self.memSize(.c_int) }),
4753	Immediate.s(0),	12277	.s(0),
4754	);	12278	);
4755	try self.genSetMem(	12279	try self.genSetMem(
4756	.{ .frame = dst_mcv.load_frame.index },	12280	.{ .frame = dst_mcv.load_frame.index },
...	@@ -4769,52 +12293,74 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4769,52 +12293,74 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4769	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);	12293	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
4770		12294
4771	const tmp_regs =	12295	const tmp_regs =
4772	try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);	12296	try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
4773	const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs);	12297	const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs);
4774	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);	12298	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);
4775		12299
4776	const lhs_mcv = try self.resolveInst(bin_op.lhs);	12300	const lhs_mcv = try self.resolveInst(bin_op.lhs);
4777	const rhs_mcv = try self.resolveInst(bin_op.rhs);	12301	const rhs_mcv = try self.resolveInst(bin_op.rhs);
4778	const mat_lhs_mcv = switch (lhs_mcv) {	12302	const mat_lhs_mcv = mat_lhs_mcv: switch (lhs_mcv) {
4779	.load_symbol => mat_lhs_mcv: {	12303	.register => \|lhs_reg\| switch (lhs_reg.class()) {
		12304	else => lhs_mcv,
		12305	.sse => {
		12306	const mat_lhs_mcv: MCValue = .{
		12307	.register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
		12308	};
		12309	try self.genCopy(dst_ty, mat_lhs_mcv, lhs_mcv, .{});
		12310	break :mat_lhs_mcv mat_lhs_mcv;
		12311	},
		12312	},
		12313	.load_symbol => {
4780	// TODO clean this up!	12314	// TODO clean this up!
4781	const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());	12315	const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
4782	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	12316	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
4783	},	12317	},
4784	else => lhs_mcv,	12318	else => lhs_mcv,
4785	};	12319	};
4786	const mat_lhs_lock = switch (mat_lhs_mcv) {	12320	const mat_lhs_locks: [2]?RegisterLock = switch (mat_lhs_mcv) {
4787	.indirect => \|reg_off\| self.register_manager.lockReg(reg_off.reg),	12321	.register_pair => \|mat_lhs_regs\| self.register_manager.lockRegs(2, mat_lhs_regs),
4788	else => null,	12322	.indirect => \|reg_off\| .{ self.register_manager.lockReg(reg_off.reg), null },
		12323	else => @splat(null),
4789	};	12324	};
4790	defer if (mat_lhs_lock) \|lock\| self.register_manager.unlockReg(lock);	12325	defer for (mat_lhs_locks) \|mat_lhs_lock\| if (mat_lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
4791	const mat_rhs_mcv = switch (rhs_mcv) {	12326	const mat_rhs_mcv = mat_rhs_mcv: switch (rhs_mcv) {
4792	.load_symbol => mat_rhs_mcv: {	12327	.register => \|rhs_reg\| switch (rhs_reg.class()) {
		12328	else => rhs_mcv,
		12329	.sse => {
		12330	const mat_rhs_mcv: MCValue = .{
		12331	.register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
		12332	};
		12333	try self.genCopy(dst_ty, mat_rhs_mcv, rhs_mcv, .{});
		12334	break :mat_rhs_mcv mat_rhs_mcv;
		12335	},
		12336	},
		12337	.load_symbol => {
4793	// TODO clean this up!	12338	// TODO clean this up!
4794	const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());	12339	const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
4795	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	12340	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
4796	},	12341	},
4797	else => rhs_mcv,	12342	else => rhs_mcv,
4798	};	12343	};
4799	const mat_rhs_lock = switch (mat_rhs_mcv) {	12344	const mat_rhs_locks: [2]?RegisterLock = switch (mat_rhs_mcv) {
4800	.indirect => \|reg_off\| self.register_manager.lockReg(reg_off.reg),	12345	.register_pair => \|mat_rhs_regs\| self.register_manager.lockRegs(2, mat_rhs_regs),
4801	else => null,	12346	.indirect => \|reg_off\| .{ self.register_manager.lockReg(reg_off.reg), null },
		12347	else => @splat(null),
4802	};	12348	};
4803	defer if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	12349	defer for (mat_rhs_locks) \|mat_rhs_lock\| if (mat_rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
4804		12350
4805	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(	12351	if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
4806	.{ ._, .mov },	12352	.{ ._, .mov },
4807	.rax,	12353	.rax,
4808	try mat_lhs_mcv.mem(self, .qword),	12354	try mat_lhs_mcv.mem(self, .{ .size = .qword }),
4809	) else try self.asmRegisterRegister(	12355	) else try self.asmRegisterRegister(
4810	.{ ._, .mov },	12356	.{ ._, .mov },
4811	.rax,	12357	.rax,
4812	mat_lhs_mcv.register_pair[0],	12358	mat_lhs_mcv.register_pair[0],
4813	);	12359	);
4814	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(	12360	if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
4815	.{ ._, .mov },	12361	.{ ._, .mov },
4816	tmp_regs[0],	12362	tmp_regs[0],
4817	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),	12363	try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4818	) else try self.asmRegisterRegister(	12364	) else try self.asmRegisterRegister(
4819	.{ ._, .mov },	12365	.{ ._, .mov },
4820	tmp_regs[0],	12366	tmp_regs[0],
...	@@ -4824,17 +12370,17 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4824,17 +12370,17 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4824	try self.asmSetccRegister(.nz, tmp_regs[1].to8());	12370	try self.asmSetccRegister(.nz, tmp_regs[1].to8());
4825	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);	12371	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
4826	try self.asmSetccRegister(.o, tmp_regs[2].to8());	12372	try self.asmSetccRegister(.o, tmp_regs[2].to8());
4827	if (mat_rhs_mcv.isMemory())	12373	if (mat_rhs_mcv.isBase())
4828	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))	12374	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
4829	else	12375	else
4830	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);	12376	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
4831	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);	12377	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
4832	try self.asmSetccRegister(.c, tmp_regs[3].to8());	12378	try self.asmSetccRegister(.c, tmp_regs[3].to8());
4833	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8());	12379	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8());
4834	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(	12380	if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
4835	.{ ._, .mov },	12381	.{ ._, .mov },
4836	tmp_regs[0],	12382	tmp_regs[0],
4837	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),	12383	try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4838	) else try self.asmRegisterRegister(	12384	) else try self.asmRegisterRegister(
4839	.{ ._, .mov },	12385	.{ ._, .mov },
4840	tmp_regs[0],	12386	tmp_regs[0],
...	@@ -4848,10 +12394,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4848,10 +12394,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4848	tmp_regs[3].to8(),	12394	tmp_regs[3].to8(),
4849	);	12395	);
4850	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());	12396	try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
4851	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(	12397	if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
4852	.{ .i_, .mul },	12398	.{ .i_, .mul },
4853	tmp_regs[0],	12399	tmp_regs[0],
4854	try mat_rhs_mcv.mem(self, .qword),	12400	try mat_rhs_mcv.mem(self, .{ .size = .qword }),
4855	) else try self.asmRegisterRegister(	12401	) else try self.asmRegisterRegister(
4856	.{ .i_, .mul },	12402	.{ .i_, .mul },
4857	tmp_regs[0],	12403	tmp_regs[0],
...	@@ -4906,7 +12452,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4906,7 +12452,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4906	self.eflags_inst = inst;	12452	self.eflags_inst = inst;
4907	break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };	12453	break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
4908	} else {	12454	} else {
4909	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));	12455	const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4910	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);	12456	try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4911	break :result .{ .load_frame = .{ .index = frame_index } };	12457	break :result .{ .load_frame = .{ .index = frame_index } };
4912	},	12458	},
...	@@ -4917,7 +12463,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4917,7 +12463,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4917	src_ty.fmt(pt), dst_ty.fmt(pt),	12463	src_ty.fmt(pt), dst_ty.fmt(pt),
4918	});	12464	});
4919		12465
4920	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, zcu));	12466	const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
4921	if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {	12467	if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
4922	try self.genSetMem(	12468	try self.genSetMem(
4923	.{ .frame = frame_index },	12469	.{ .frame = frame_index },
...	@@ -4951,7 +12497,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4951,7 +12497,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4951	/// Generates signed or unsigned integer multiplication/division.	12497	/// Generates signed or unsigned integer multiplication/division.
4952	/// Clobbers .rax and .rdx registers.	12498	/// Clobbers .rax and .rdx registers.
4953	/// Quotient is saved in .rax and remainder in .rdx.	12499	/// Quotient is saved in .rax and remainder in .rdx.
4954	fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {	12500	fn genIntMulDivOpMir(self: *CodeGen, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
4955	const pt = self.pt;	12501	const pt = self.pt;
4956	const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));	12502	const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
4957	const bit_size: u32 = @intCast(self.regBitSize(ty));	12503	const bit_size: u32 = @intCast(self.regBitSize(ty));
...	@@ -4992,7 +12538,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue	...	@@ -4992,7 +12538,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
4992	.register => \|reg\| try self.asmRegister(tag, registerAlias(reg, abi_size)),	12538	.register => \|reg\| try self.asmRegister(tag, registerAlias(reg, abi_size)),
4993	.memory, .indirect, .load_frame => try self.asmMemory(	12539	.memory, .indirect, .load_frame => try self.asmMemory(
4994	tag,	12540	tag,
4995	try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),	12541	try mat_rhs.mem(self, .{ .size = .fromSize(abi_size) }),
4996	),	12542	),
4997	else => unreachable,	12543	else => unreachable,
4998	}	12544	}
...	@@ -5001,7 +12547,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue	...	@@ -5001,7 +12547,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
5001		12547
5002	/// Always returns a register.	12548	/// Always returns a register.
5003	/// Clobbers .rax and .rdx registers.	12549	/// Clobbers .rax and .rdx registers.
5004	fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {	12550	fn genInlineIntDivFloor(self: *CodeGen, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
5005	const pt = self.pt;	12551	const pt = self.pt;
5006	const zcu = pt.zcu;	12552	const zcu = pt.zcu;
5007	const abi_size: u32 = @intCast(ty.abiSize(zcu));	12553	const abi_size: u32 = @intCast(ty.abiSize(zcu));
...	@@ -5038,7 +12584,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -5038,7 +12584,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
5038	try self.asmRegisterImmediate(	12584	try self.asmRegisterImmediate(
5039	.{ ._r, .sa },	12585	.{ ._r, .sa },
5040	registerAlias(divisor, abi_size),	12586	registerAlias(divisor, abi_size),
5041	Immediate.u(int_info.bits - 1),	12587	.u(int_info.bits - 1),
5042	);	12588	);
5043	try self.asmRegisterRegister(	12589	try self.asmRegisterRegister(
5044	.{ ._, .@"test" },	12590	.{ ._, .@"test" },
...	@@ -5054,7 +12600,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -5054,7 +12600,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
5054	return MCValue{ .register = divisor };	12600	return MCValue{ .register = divisor };
5055	}	12601	}
5056		12602
5057	fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	12603	fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
5058	const pt = self.pt;	12604	const pt = self.pt;
5059	const zcu = pt.zcu;	12605	const zcu = pt.zcu;
5060	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	12606	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -5084,7 +12630,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5084,7 +12630,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5084	defer self.register_manager.unlockReg(tmp_lock);	12630	defer self.register_manager.unlockReg(tmp_lock);
5085		12631
5086	const lhs_bits: u31 = @intCast(lhs_ty.bitSize(zcu));	12632	const lhs_bits: u31 = @intCast(lhs_ty.bitSize(zcu));
5087	const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;	12633	const tmp_ty: Type = if (lhs_bits > 64) .usize else lhs_ty;
5088	const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;	12634	const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
5089	try self.genSetReg(	12635	try self.genSetReg(
5090	tmp_reg,	12636	tmp_reg,
...	@@ -5217,8 +12763,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5217,8 +12763,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5217	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|	12763	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\|
5218	self.register_manager.unlockReg(lock);	12764	self.register_manager.unlockReg(lock);
5219		12765
5220	const shift_imm =	12766	const shift_imm: Immediate =
5221	Immediate.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));	12767	.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));
5222	if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(	12768	if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
5223	mir_tag,	12769	mir_tag,
5224	registerAlias(dst_reg, abi_size),	12770	registerAlias(dst_reg, abi_size),
...	@@ -5264,16 +12810,15 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5264,16 +12810,15 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5264	defer self.register_manager.unlockReg(shift_lock);	12810	defer self.register_manager.unlockReg(shift_lock);
5265		12811
5266	const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type });	12812	const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type });
5267	const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{	12813	const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
5268	.ty = mask_ty.toIntern(),	12814	.ty = mask_ty.toIntern(),
5269	.storage = .{ .elems = &([1]InternPool.Index{	12815	.storage = .{ .elems = &([1]InternPool.Index{
5270	(try rhs_ty.childType(zcu).maxIntScalar(pt, Type.u8)).toIntern(),	12816	(try rhs_ty.childType(zcu).maxIntScalar(pt, .u8)).toIntern(),
5271	} ++ [1]InternPool.Index{	12817	} ++ [1]InternPool.Index{
5272	(try pt.intValue(Type.u8, 0)).toIntern(),	12818	(try pt.intValue(.u8, 0)).toIntern(),
5273	} ** 15) },	12819	} ** 15) },
5274	} })));	12820	} })));
5275	const mask_addr_reg =	12821	const mask_addr_reg = try self.copyToTmpRegister(.usize, mask_mcv.address());
5276	try self.copyToTmpRegister(Type.usize, mask_mcv.address());
5277	const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);	12822	const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
5278	defer self.register_manager.unlockReg(mask_addr_lock);	12823	defer self.register_manager.unlockReg(mask_addr_lock);
5279		12824
...	@@ -5322,14 +12867,14 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5322,14 +12867,14 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5322	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	12867	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
5323	}	12868	}
5324		12869
5325	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {	12870	fn airShlSat(self: *CodeGen, inst: Air.Inst.Index) !void {
5326	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	12871	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5327	_ = bin_op;	12872	_ = bin_op;
5328	return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});	12873	return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
5329	//return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	12874	//return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
5330	}	12875	}
5331		12876
5332	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {	12877	fn airOptionalPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
5333	const zcu = self.pt.zcu;	12878	const zcu = self.pt.zcu;
5334	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	12879	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5335	const result: MCValue = result: {	12880	const result: MCValue = result: {
...	@@ -5362,7 +12907,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5362,7 +12907,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
5362	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	12907	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5363	}	12908	}
5364		12909
5365	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {	12910	fn airOptionalPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5366	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	12911	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5367		12912
5368	const dst_ty = self.typeOfIndex(inst);	12913	const dst_ty = self.typeOfIndex(inst);
...	@@ -5375,7 +12920,7 @@ fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5375,7 +12920,7 @@ fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
5375	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	12920	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
5376	}	12921	}
5377		12922
5378	fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	12923	fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
5379	const pt = self.pt;	12924	const pt = self.pt;
5380	const zcu = pt.zcu;	12925	const zcu = pt.zcu;
5381	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	12926	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -5407,7 +12952,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5407,7 +12952,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5407	try self.genSetMem(	12952	try self.genSetMem(
5408	.{ .reg = dst_mcv.getReg().? },	12953	.{ .reg = dst_mcv.getReg().? },
5409	pl_abi_size,	12954	pl_abi_size,
5410	Type.bool,	12955	.bool,
5411	.{ .immediate = 1 },	12956	.{ .immediate = 1 },
5412	.{},	12957	.{},
5413	);	12958	);
...	@@ -5416,7 +12961,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5416,7 +12961,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5416	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	12961	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5417	}	12962	}
5418		12963
5419	fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	12964	fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
5420	const pt = self.pt;	12965	const pt = self.pt;
5421	const zcu = pt.zcu;	12966	const zcu = pt.zcu;
5422	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	12967	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -5434,7 +12979,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5434,7 +12979,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5434	break :result operand;	12979	break :result operand;
5435	}	12980	}
5436		12981
5437	const err_off = errUnionErrorOffset(payload_ty, zcu);	12982	const err_off = codegen.errUnionErrorOffset(payload_ty, zcu);
5438	switch (operand) {	12983	switch (operand) {
5439	.register => \|reg\| {	12984	.register => \|reg\| {
5440	// TODO reuse operand	12985	// TODO reuse operand
...	@@ -5446,9 +12991,9 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5446,9 +12991,9 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5446	.{ ._r, .sh },	12991	.{ ._r, .sh },
5447	err_union_ty,	12992	err_union_ty,
5448	result,	12993	result,
5449	Type.u8,	12994	.u8,
5450	.{ .immediate = @as(u6, @intCast(err_off * 8)) },	12995	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
5451	) else try self.truncateRegister(Type.anyerror, result.register);	12996	) else try self.truncateRegister(.anyerror, result.register);
5452	break :result result;	12997	break :result result;
5453	},	12998	},
5454	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{	12999	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
...	@@ -5461,7 +13006,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5461,7 +13006,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5461	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	13006	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5462	}	13007	}
5463		13008
5464	fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	13009	fn airUnwrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
5465	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13010	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5466	const operand_ty = self.typeOf(ty_op.operand);	13011	const operand_ty = self.typeOf(ty_op.operand);
5467	const operand = try self.resolveInst(ty_op.operand);	13012	const operand = try self.resolveInst(ty_op.operand);
...	@@ -5470,7 +13015,7 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5470,7 +13015,7 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
5470	}	13015	}
5471		13016
5472	// *(E!T) -> E	13017	// *(E!T) -> E
5473	fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {	13018	fn airUnwrapErrUnionErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5474	const pt = self.pt;	13019	const pt = self.pt;
5475	const zcu = pt.zcu;	13020	const zcu = pt.zcu;
5476	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13021	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -5492,7 +13037,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5492,7 +13037,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
5492	const eu_ty = src_ty.childType(zcu);	13037	const eu_ty = src_ty.childType(zcu);
5493	const pl_ty = eu_ty.errorUnionPayload(zcu);	13038	const pl_ty = eu_ty.errorUnionPayload(zcu);
5494	const err_ty = eu_ty.errorUnionSet(zcu);	13039	const err_ty = eu_ty.errorUnionSet(zcu);
5495	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	13040	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5496	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));	13041	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
5497	try self.asmRegisterMemory(	13042	try self.asmRegisterMemory(
5498	.{ ._, .mov },	13043	.{ ._, .mov },
...	@@ -5500,7 +13045,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5500,7 +13045,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
5500	.{	13045	.{
5501	.base = .{ .reg = src_reg },	13046	.base = .{ .reg = src_reg },
5502	.mod = .{ .rm = .{	13047	.mod = .{ .rm = .{
5503	.size = Memory.Size.fromSize(err_abi_size),	13048	.size = .fromSize(err_abi_size),
5504	.disp = err_off,	13049	.disp = err_off,
5505	} },	13050	} },
5506	},	13051	},
...	@@ -5510,7 +13055,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5510,7 +13055,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
5510	}	13055	}
5511		13056
5512	// (E!T) -> T	13057	// (E!T) -> T
5513	fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {	13058	fn airUnwrapErrUnionPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5514	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13059	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5515	const operand_ty = self.typeOf(ty_op.operand);	13060	const operand_ty = self.typeOf(ty_op.operand);
5516	const operand = try self.resolveInst(ty_op.operand);	13061	const operand = try self.resolveInst(ty_op.operand);
...	@@ -5518,7 +13063,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5518,7 +13063,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
5518	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	13063	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5519	}	13064	}
5520		13065
5521	fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	13066	fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
5522	const pt = self.pt;	13067	const pt = self.pt;
5523	const zcu = pt.zcu;	13068	const zcu = pt.zcu;
5524	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13069	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -5535,18 +13080,18 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5535,18 +13080,18 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5535	const eu_ty = src_ty.childType(zcu);	13080	const eu_ty = src_ty.childType(zcu);
5536	const pl_ty = eu_ty.errorUnionPayload(zcu);	13081	const pl_ty = eu_ty.errorUnionPayload(zcu);
5537	const err_ty = eu_ty.errorUnionSet(zcu);	13082	const err_ty = eu_ty.errorUnionSet(zcu);
5538	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	13083	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5539	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));	13084	const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
5540	try self.asmMemoryImmediate(	13085	try self.asmMemoryImmediate(
5541	.{ ._, .mov },	13086	.{ ._, .mov },
5542	.{	13087	.{
5543	.base = .{ .reg = src_reg },	13088	.base = .{ .reg = src_reg },
5544	.mod = .{ .rm = .{	13089	.mod = .{ .rm = .{
5545	.size = Memory.Size.fromSize(err_abi_size),	13090	.size = .fromSize(err_abi_size),
5546	.disp = err_off,	13091	.disp = err_off,
5547	} },	13092	} },
5548	},	13093	},
5549	Immediate.u(0),	13094	.u(0),
5550	);	13095	);
5551		13096
5552	if (self.liveness.isUnused(inst)) break :result .unreach;	13097	if (self.liveness.isUnused(inst)) break :result .unreach;
...	@@ -5559,7 +13104,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5559,7 +13104,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5559	const dst_lock = self.register_manager.lockReg(dst_reg);	13104	const dst_lock = self.register_manager.lockReg(dst_reg);
5560	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	13105	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5561		13106
5562	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));	13107	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
5563	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));	13108	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
5564	try self.asmRegisterMemory(	13109	try self.asmRegisterMemory(
5565	.{ ._, .lea },	13110	.{ ._, .lea },
...	@@ -5575,7 +13120,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5575,7 +13120,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5575	}	13120	}
5576		13121
5577	fn genUnwrapErrUnionPayloadMir(	13122	fn genUnwrapErrUnionPayloadMir(
5578	self: *Self,	13123	self: *CodeGen,
5579	maybe_inst: ?Air.Inst.Index,	13124	maybe_inst: ?Air.Inst.Index,
5580	err_union_ty: Type,	13125	err_union_ty: Type,
5581	err_union: MCValue,	13126	err_union: MCValue,
...	@@ -5587,7 +13132,7 @@ fn genUnwrapErrUnionPayloadMir(	...	@@ -5587,7 +13132,7 @@ fn genUnwrapErrUnionPayloadMir(
5587	const result: MCValue = result: {	13132	const result: MCValue = result: {
5588	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;	13133	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
5589		13134
5590	const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu));	13135	const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu));
5591	switch (err_union) {	13136	switch (err_union) {
5592	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{	13137	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
5593	.index = frame_addr.index,	13138	.index = frame_addr.index,
...	@@ -5598,7 +13143,7 @@ fn genUnwrapErrUnionPayloadMir(	...	@@ -5598,7 +13143,7 @@ fn genUnwrapErrUnionPayloadMir(
5598	const eu_lock = self.register_manager.lockReg(reg);	13143	const eu_lock = self.register_manager.lockReg(reg);
5599	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);	13144	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
5600		13145
5601	const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);	13146	const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp);
5602	const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)	13147	const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
5603	try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)	13148	try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
5604	else	13149	else
...	@@ -5607,7 +13152,7 @@ fn genUnwrapErrUnionPayloadMir(	...	@@ -5607,7 +13152,7 @@ fn genUnwrapErrUnionPayloadMir(
5607	.{ ._r, .sh },	13152	.{ ._r, .sh },
5608	err_union_ty,	13153	err_union_ty,
5609	result_mcv,	13154	result_mcv,
5610	Type.u8,	13155	.u8,
5611	.{ .immediate = @as(u6, @intCast(payload_off * 8)) },	13156	.{ .immediate = @as(u6, @intCast(payload_off * 8)) },
5612	) else try self.truncateRegister(payload_ty, result_mcv.register);	13157	) else try self.truncateRegister(payload_ty, result_mcv.register);
5613	break :result if (payload_in_gp)	13158	break :result if (payload_in_gp)
...	@@ -5625,7 +13170,7 @@ fn genUnwrapErrUnionPayloadMir(	...	@@ -5625,7 +13170,7 @@ fn genUnwrapErrUnionPayloadMir(
5625	}	13170	}
5626		13171
5627	fn genUnwrapErrUnionPayloadPtrMir(	13172	fn genUnwrapErrUnionPayloadPtrMir(
5628	self: *Self,	13173	self: *CodeGen,
5629	maybe_inst: ?Air.Inst.Index,	13174	maybe_inst: ?Air.Inst.Index,
5630	ptr_ty: Type,	13175	ptr_ty: Type,
5631	ptr_mcv: MCValue,	13176	ptr_mcv: MCValue,
...	@@ -5636,7 +13181,7 @@ fn genUnwrapErrUnionPayloadPtrMir(	...	@@ -5636,7 +13181,7 @@ fn genUnwrapErrUnionPayloadPtrMir(
5636	const payload_ty = err_union_ty.errorUnionPayload(zcu);	13181	const payload_ty = err_union_ty.errorUnionPayload(zcu);
5637		13182
5638	const result: MCValue = result: {	13183	const result: MCValue = result: {
5639	const payload_off = errUnionPayloadOffset(payload_ty, zcu);	13184	const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu);
5640	const result_mcv: MCValue = if (maybe_inst) \|inst\|	13185	const result_mcv: MCValue = if (maybe_inst) \|inst\|
5641	try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)	13186	try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)
5642	else	13187	else
...	@@ -5648,23 +13193,23 @@ fn genUnwrapErrUnionPayloadPtrMir(	...	@@ -5648,23 +13193,23 @@ fn genUnwrapErrUnionPayloadPtrMir(
5648	return result;	13193	return result;
5649	}	13194	}
5650		13195
5651	fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {	13196	fn airErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
5652	_ = inst;	13197	_ = inst;
5653	return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});	13198	return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
5654	//return self.finishAir(inst, result, .{ .none, .none, .none });	13199	//return self.finishAir(inst, result, .{ .none, .none, .none });
5655	}	13200	}
5656		13201
5657	fn airSetErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {	13202	fn airSetErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
5658	_ = inst;	13203	_ = inst;
5659	return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});	13204	return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});
5660	}	13205	}
5661		13206
5662	fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {	13207	fn airSaveErrReturnTraceIndex(self: *CodeGen, inst: Air.Inst.Index) !void {
5663	_ = inst;	13208	_ = inst;
5664	return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});	13209	return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});
5665	}	13210	}
5666		13211
5667	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	13212	fn airWrapOptional(self: *CodeGen, inst: Air.Inst.Index) !void {
5668	const pt = self.pt;	13213	const pt = self.pt;
5669	const zcu = pt.zcu;	13214	const zcu = pt.zcu;
5670	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13215	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -5696,7 +13241,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5696,7 +13241,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
5696	try self.asmRegisterImmediate(	13241	try self.asmRegisterImmediate(
5697	.{ ._s, .bt },	13242	.{ ._s, .bt },
5698	opt_reg,	13243	opt_reg,
5699	Immediate.u(@as(u6, @intCast(pl_abi_size * 8))),	13244	.u(@as(u6, @intCast(pl_abi_size * 8))),
5700	);	13245	);
5701	},	13246	},
5702		13247
...	@@ -5709,7 +13254,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5709,7 +13254,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
5709	.disp = frame_addr.off + pl_abi_size,	13254	.disp = frame_addr.off + pl_abi_size,
5710	} },	13255	} },
5711	},	13256	},
5712	Immediate.u(1),	13257	.u(1),
5713	),	13258	),
5714	}	13259	}
5715	}	13260	}
...	@@ -5719,7 +13264,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5719,7 +13264,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
5719	}	13264	}
5720		13265
5721	/// T to E!T	13266	/// T to E!T
5722	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	13267	fn airWrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
5723	const pt = self.pt;	13268	const pt = self.pt;
5724	const zcu = pt.zcu;	13269	const zcu = pt.zcu;
5725	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13270	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -5732,9 +13277,9 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5732,9 +13277,9 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
5732	const result: MCValue = result: {	13277	const result: MCValue = result: {
5733	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };	13278	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
5734		13279
5735	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));	13280	const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
5736	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));	13281	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
5737	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	13282	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5738	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});	13283	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
5739	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});	13284	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});
5740	break :result .{ .load_frame = .{ .index = frame_index } };	13285	break :result .{ .load_frame = .{ .index = frame_index } };
...	@@ -5743,7 +13288,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5743,7 +13288,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
5743	}	13288	}
5744		13289
5745	/// E to E!T	13290	/// E to E!T
5746	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	13291	fn airWrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
5747	const pt = self.pt;	13292	const pt = self.pt;
5748	const zcu = pt.zcu;	13293	const zcu = pt.zcu;
5749	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13294	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -5755,9 +13300,9 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5755,9 +13300,9 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5755	const result: MCValue = result: {	13300	const result: MCValue = result: {
5756	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);	13301	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
5757		13302
5758	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));	13303	const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
5759	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));	13304	const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
5760	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));	13305	const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
5761	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});	13306	try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
5762	const operand = try self.resolveInst(ty_op.operand);	13307	const operand = try self.resolveInst(ty_op.operand);
5763	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});	13308	try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});
...	@@ -5766,44 +13311,58 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5766,44 +13311,58 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5766	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	13311	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5767	}	13312	}
5768		13313
5769	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {	13314	fn airSlicePtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5770	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13315	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5771	const result = result: {	13316	const result = result: {
5772	const src_mcv = try self.resolveInst(ty_op.operand);	13317	const src_mcv = try self.resolveInst(ty_op.operand);
5773	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;	13318	const ptr_mcv: MCValue = switch (src_mcv) {
		13319	.register_pair => \|regs\| .{ .register = regs[0] },
		13320	else => src_mcv,
		13321	};
		13322	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
		13323	switch (src_mcv) {
		13324	.register_pair => \|regs\| try self.freeValue(.{ .register = regs[1] }),
		13325	else => {},
		13326	}
		13327	break :result ptr_mcv;
		13328	}
5774		13329
5775	const dst_mcv = try self.allocRegOrMem(inst, true);	13330	const dst_mcv = try self.allocRegOrMem(inst, true);
5776	const dst_ty = self.typeOfIndex(inst);	13331	try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{});
5777	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
5778	break :result dst_mcv;	13332	break :result dst_mcv;
5779	};	13333	};
5780	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	13334	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5781	}	13335	}
5782		13336
5783	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {	13337	fn airSliceLen(self: *CodeGen, inst: Air.Inst.Index) !void {
5784	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13338	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5785		13339	const result = result: {
5786	const result: MCValue = result: {
5787	const src_mcv = try self.resolveInst(ty_op.operand);	13340	const src_mcv = try self.resolveInst(ty_op.operand);
5788	switch (src_mcv) {	13341	const len_mcv: MCValue = switch (src_mcv) {
5789	.load_frame => \|frame_addr\| {	13342	.register_pair => \|regs\| .{ .register = regs[1] },
5790	const len_mcv: MCValue = .{ .load_frame = .{	13343	.load_frame => \|frame_addr\| .{ .load_frame = .{
5791	.index = frame_addr.index,	13344	.index = frame_addr.index,
5792	.off = frame_addr.off + 8,	13345	.off = frame_addr.off + 8,
5793	} };	13346	} },
5794	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
5795
5796	const dst_mcv = try self.allocRegOrMem(inst, true);
5797	try self.genCopy(Type.usize, dst_mcv, len_mcv, .{});
5798	break :result dst_mcv;
5799	},
5800	else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),	13347	else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),
		13348	};
		13349	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
		13350	switch (src_mcv) {
		13351	.register_pair => \|regs\| try self.freeValue(.{ .register = regs[0] }),
		13352	.load_frame => {},
		13353	else => unreachable,
		13354	}
		13355	break :result len_mcv;
5801	}	13356	}
		13357
		13358	const dst_mcv = try self.allocRegOrMem(inst, true);
		13359	try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{});
		13360	break :result dst_mcv;
5802	};	13361	};
5803	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	13362	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5804	}	13363	}
5805		13364
5806	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {	13365	fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5807	const pt = self.pt;	13366	const pt = self.pt;
5808	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13367	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5809		13368
...	@@ -5838,7 +13397,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5838,7 +13397,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
5838	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	13397	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
5839	}	13398	}
5840		13399
5841	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {	13400	fn airPtrSlicePtrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5842	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13401	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5843		13402
5844	const dst_ty = self.typeOfIndex(inst);	13403	const dst_ty = self.typeOfIndex(inst);
...	@@ -5851,7 +13410,7 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5851,7 +13410,7 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
5851	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	13410	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
5852	}	13411	}
5853		13412
5854	fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {	13413	fn elemOffset(self: *CodeGen, index_ty: Type, index: MCValue, elem_size: u64) !Register {
5855	const reg: Register = blk: {	13414	const reg: Register = blk: {
5856	switch (index) {	13415	switch (index) {
5857	.immediate => \|imm\| {	13416	.immediate => \|imm\| {
...	@@ -5871,7 +13430,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi	...	@@ -5871,7 +13430,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
5871	return reg;	13430	return reg;
5872	}	13431	}
5873		13432
5874	fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	13433	fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
5875	const pt = self.pt;	13434	const pt = self.pt;
5876	const zcu = pt.zcu;	13435	const zcu = pt.zcu;
5877	const slice_ty = self.typeOf(lhs);	13436	const slice_ty = self.typeOf(lhs);
...	@@ -5899,7 +13458,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	...	@@ -5899,7 +13458,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
5899	defer self.register_manager.unlockReg(offset_reg_lock);	13458	defer self.register_manager.unlockReg(offset_reg_lock);
5900		13459
5901	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	13460	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5902	try self.genSetReg(addr_reg, Type.usize, slice_mcv, .{});	13461	try self.genSetReg(addr_reg, .usize, slice_mcv, .{});
5903	// TODO we could allocate register here, but need to expect addr register and potentially	13462	// TODO we could allocate register here, but need to expect addr register and potentially
5904	// offset register.	13463	// offset register.
5905	try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{	13464	try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
...	@@ -5908,7 +13467,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	...	@@ -5908,7 +13467,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
5908	return MCValue{ .register = addr_reg.to64() };	13467	return MCValue{ .register = addr_reg.to64() };
5909	}	13468	}
5910		13469
5911	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	13470	fn airSliceElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
5912	const pt = self.pt;	13471	const pt = self.pt;
5913	const zcu = pt.zcu;	13472	const zcu = pt.zcu;
5914	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	13473	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -5927,14 +13486,14 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5927,14 +13486,14 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
5927	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	13486	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
5928	}	13487	}
5929		13488
5930	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {	13489	fn airSliceElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5931	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	13490	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5932	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;	13491	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
5933	const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs);	13492	const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs);
5934	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });	13493	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
5935	}	13494	}
5936		13495
5937	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	13496	fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
5938	const pt = self.pt;	13497	const pt = self.pt;
5939	const zcu = pt.zcu;	13498	const zcu = pt.zcu;
5940	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	13499	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -5952,7 +13511,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5952,7 +13511,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
5952		13511
5953	const index_ty = self.typeOf(bin_op.rhs);	13512	const index_ty = self.typeOf(bin_op.rhs);
5954	const index_mcv = try self.resolveInst(bin_op.rhs);	13513	const index_mcv = try self.resolveInst(bin_op.rhs);
5955	const index_lock: ?RegisterLock = switch (index_mcv) {	13514	const index_lock = switch (index_mcv) {
5956	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	13515	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
5957	else => null,	13516	else => null,
5958	};	13517	};
...	@@ -5960,48 +13519,111 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5960,48 +13519,111 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
5960		13519
5961	try self.spillEflagsIfOccupied();	13520	try self.spillEflagsIfOccupied();
5962	if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) {	13521	if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) {
5963	const index_reg = switch (index_mcv) {	13522	const array_mat_mcv: MCValue = switch (array_mcv) {
5964	.register => \|reg\| reg,	13523	else => array_mcv,
5965	else => try self.copyToTmpRegister(index_ty, index_mcv),	13524	.register_mask => .{ .register = try self.copyToTmpRegister(array_ty, array_mcv) },
		13525	};
		13526	const array_mat_lock = switch (array_mat_mcv) {
		13527	.register => \|reg\| self.register_manager.lockReg(reg),
		13528	else => null,
5966	};	13529	};
5967	switch (array_mcv) {	13530	defer if (array_mat_lock) \|lock\| self.register_manager.unlockReg(lock);
		13531
		13532	switch (array_mat_mcv) {
5968	.register => \|array_reg\| switch (array_reg.class()) {	13533	.register => \|array_reg\| switch (array_reg.class()) {
5969	.general_purpose => try self.asmRegisterRegister(	13534	.general_purpose => switch (index_mcv) {
5970	.{ ._, .bt },	13535	.immediate => \|index_imm\| try self.asmRegisterImmediate(
5971	array_reg.to64(),
5972	index_reg.to64(),
5973	),
5974	.sse => {
5975	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
5976	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
5977	try self.asmMemoryRegister(
5978	.{ ._, .bt },	13536	.{ ._, .bt },
5979	.{	13537	array_reg.to64(),
5980	.base = .{ .frame = frame_index },	13538	.u(index_imm),
5981	.mod = .{ .rm = .{ .size = .qword } },	13539	),
5982	},	13540	else => try self.asmRegisterRegister(
5983	index_reg.to64(),	13541	.{ ._, .bt },
5984	);	13542	array_reg.to64(),
		13543	switch (index_mcv) {
		13544	.register => \|index_reg\| index_reg,
		13545	else => try self.copyToTmpRegister(index_ty, index_mcv),
		13546	}.to64(),
		13547	),
		13548	},
		13549	.sse => {
		13550	const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
		13551	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mat_mcv, .{});
		13552	switch (index_mcv) {
		13553	.immediate => \|index_imm\| try self.asmMemoryImmediate(
		13554	.{ ._, .bt },
		13555	.{
		13556	.base = .{ .frame = frame_index },
		13557	.mod = .{ .rm = .{
		13558	.size = .qword,
		13559	.disp = @intCast(index_imm / 64 * 8),
		13560	} },
		13561	},
		13562	.u(index_imm % 64),
		13563	),
		13564	else => try self.asmMemoryRegister(
		13565	.{ ._, .bt },
		13566	.{
		13567	.base = .{ .frame = frame_index },
		13568	.mod = .{ .rm = .{ .size = .qword } },
		13569	},
		13570	switch (index_mcv) {
		13571	.register => \|index_reg\| index_reg,
		13572	else => try self.copyToTmpRegister(index_ty, index_mcv),
		13573	}.to64(),
		13574	),
		13575	}
5985	},	13576	},
5986	else => unreachable,	13577	else => unreachable,
5987	},	13578	},
5988	.load_frame => try self.asmMemoryRegister(	13579	.load_frame => switch (index_mcv) {
5989	.{ ._, .bt },	13580	.immediate => \|index_imm\| try self.asmMemoryImmediate(
5990	try array_mcv.mem(self, .qword),	13581	.{ ._, .bt },
5991	index_reg.to64(),	13582	try array_mat_mcv.mem(self, .{
5992	),	13583	.size = .qword,
5993	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister(	13584	.disp = @intCast(index_imm / 64 * 8),
5994	.{ ._, .bt },	13585	}),
5995	.{	13586	.u(index_imm % 64),
5996	.base = .{	13587	),
5997	.reg = try self.copyToTmpRegister(Type.usize, array_mcv.address()),	13588	else => try self.asmMemoryRegister(
		13589	.{ ._, .bt },
		13590	try array_mat_mcv.mem(self, .{ .size = .qword }),
		13591	switch (index_mcv) {
		13592	.register => \|index_reg\| index_reg,
		13593	else => try self.copyToTmpRegister(index_ty, index_mcv),
		13594	}.to64(),
		13595	),
		13596	},
		13597	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => switch (index_mcv) {
		13598	.immediate => \|index_imm\| try self.asmMemoryImmediate(
		13599	.{ ._, .bt },
		13600	.{
		13601	.base = .{
		13602	.reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
		13603	},
		13604	.mod = .{ .rm = .{
		13605	.size = .qword,
		13606	.disp = @intCast(index_imm / 64 * 8),
		13607	} },
5998	},	13608	},
5999	.mod = .{ .rm = .{ .size = .qword } },	13609	.u(index_imm % 64),
6000	},	13610	),
6001	index_reg.to64(),	13611	else => try self.asmMemoryRegister(
6002	),	13612	.{ ._, .bt },
		13613	.{
		13614	.base = .{
		13615	.reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
		13616	},
		13617	.mod = .{ .rm = .{ .size = .qword } },
		13618	},
		13619	switch (index_mcv) {
		13620	.register => \|index_reg\| index_reg,
		13621	else => try self.copyToTmpRegister(index_ty, index_mcv),
		13622	}.to64(),
		13623	),
		13624	},
6003	else => return self.fail("TODO airArrayElemVal for {s} of {}", .{	13625	else => return self.fail("TODO airArrayElemVal for {s} of {}", .{
6004	@tagName(array_mcv), array_ty.fmt(pt),	13626	@tagName(array_mat_mcv), array_ty.fmt(pt),
6005	}),	13627	}),
6006	}	13628	}
6007		13629
...	@@ -6017,7 +13639,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6017,7 +13639,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
6017		13639
6018	switch (array_mcv) {	13640	switch (array_mcv) {
6019	.register => {	13641	.register => {
6020	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));	13642	const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
6021	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});	13643	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
6022	try self.asmRegisterMemory(	13644	try self.asmRegisterMemory(
6023	.{ ._, .lea },	13645	.{ ._, .lea },
...	@@ -6038,7 +13660,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6038,7 +13660,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
6038	.load_direct,	13660	.load_direct,
6039	.load_got,	13661	.load_got,
6040	.load_tlv,	13662	.load_tlv,
6041	=> try self.genSetReg(addr_reg, Type.usize, array_mcv.address(), .{}),	13663	=> try self.genSetReg(addr_reg, .usize, array_mcv.address(), .{}),
6042	.lea_symbol, .lea_direct, .lea_tlv => unreachable,	13664	.lea_symbol, .lea_direct, .lea_tlv => unreachable,
6043	else => return self.fail("TODO airArrayElemVal_val for {s} of {}", .{	13665	else => return self.fail("TODO airArrayElemVal_val for {s} of {}", .{
6044	@tagName(array_mcv), array_ty.fmt(pt),	13666	@tagName(array_mcv), array_ty.fmt(pt),
...	@@ -6052,19 +13674,14 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6052,19 +13674,14 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
6052	// TODO we could allocate register here, but need to expect addr register and potentially	13674	// TODO we could allocate register here, but need to expect addr register and potentially
6053	// offset register.	13675	// offset register.
6054	const dst_mcv = try self.allocRegOrMem(inst, false);	13676	const dst_mcv = try self.allocRegOrMem(inst, false);
6055	try self.genBinOpMir(	13677	try self.genBinOpMir(.{ ._, .add }, .usize, .{ .register = addr_reg }, .{ .register = offset_reg });
6056	.{ ._, .add },
6057	Type.usize,
6058	.{ .register = addr_reg },
6059	.{ .register = offset_reg },
6060	);
6061	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{});	13678	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{});
6062	break :result dst_mcv;	13679	break :result dst_mcv;
6063	};	13680	};
6064	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	13681	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6065	}	13682	}
6066		13683
6067	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	13684	fn airPtrElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
6068	const pt = self.pt;	13685	const pt = self.pt;
6069	const zcu = pt.zcu;	13686	const zcu = pt.zcu;
6070	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	13687	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -6115,7 +13732,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6115,7 +13732,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
6115	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	13732	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6116	}	13733	}
6117		13734
6118	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {	13735	fn airPtrElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
6119	const pt = self.pt;	13736	const pt = self.pt;
6120	const zcu = pt.zcu;	13737	const zcu = pt.zcu;
6121	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	13738	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -6161,7 +13778,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6161,7 +13778,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
6161	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });	13778	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
6162	}	13779	}
6163		13780
6164	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	13781	fn airSetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
6165	const pt = self.pt;	13782	const pt = self.pt;
6166	const zcu = pt.zcu;	13783	const zcu = pt.zcu;
6167	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	13784	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -6206,7 +13823,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6206,7 +13823,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
6206	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });	13823	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
6207	}	13824	}
6208		13825
6209	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	13826	fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
6210	const zcu = self.pt.zcu;	13827	const zcu = self.pt.zcu;
6211	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13828	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6212		13829
...	@@ -6245,13 +13862,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6245,13 +13862,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
6245	.register => {	13862	.register => {
6246	const shift: u6 = @intCast(layout.tagOffset() * 8);	13863	const shift: u6 = @intCast(layout.tagOffset() * 8);
6247	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);	13864	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
6248	try self.genShiftBinOpMir(	13865	try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, result, .u8, .{ .immediate = shift });
6249	.{ ._r, .sh },
6250	Type.usize,
6251	result,
6252	Type.u8,
6253	.{ .immediate = shift },
6254	);
6255	break :blk MCValue{	13866	break :blk MCValue{
6256	.register = registerAlias(result.register, @intCast(layout.tag_size)),	13867	.register = registerAlias(result.register, @intCast(layout.tag_size)),
6257	};	13868	};
...	@@ -6263,7 +13874,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6263,7 +13874,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
6263	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	13874	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
6264	}	13875	}
6265		13876
6266	fn airClz(self: *Self, inst: Air.Inst.Index) !void {	13877	fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
6267	const pt = self.pt;	13878	const pt = self.pt;
6268	const zcu = pt.zcu;	13879	const zcu = pt.zcu;
6269	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	13880	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -6296,53 +13907,62 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6296,53 +13907,62 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6296	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));	13907	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
6297	const has_lzcnt = self.hasFeature(.lzcnt);	13908	const has_lzcnt = self.hasFeature(.lzcnt);
6298	if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {	13909	if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {
6299	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	13910	const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) {
		13911	.load_frame => \|src_frame_addr\| src_frame_addr,
		13912	else => {
		13913	const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu));
		13914	try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{});
		13915	break :src_frame_addr .{ .index = src_frame_addr };
		13916	},
		13917	};
		13918
		13919	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6300	const extra_bits = abi_size * 8 - src_bits;	13920	const extra_bits = abi_size * 8 - src_bits;
6301		13921
6302	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	13922	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6303	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);	13923	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
6304	defer self.register_manager.unlockReg(index_lock);	13924	defer self.register_manager.unlockReg(index_lock);
6305		13925
6306	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.u(limbs_len));	13926	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len));
6307	switch (extra_bits) {	13927	switch (extra_bits) {
6308	1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),	13928	1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
6309	else => try self.asmRegisterImmediate(	13929	else => try self.asmRegisterImmediate(
6310	.{ ._, .mov },	13930	.{ ._, .mov },
6311	dst_reg.to32(),	13931	dst_reg.to32(),
6312	Immediate.s(@as(i32, extra_bits) - 1),	13932	.s(@as(i32, extra_bits) - 1),
6313	),	13933	),
6314	}	13934	}
6315	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	13935	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6316	try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());	13936	try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());
6317	const zero = try self.asmJccReloc(.z, undefined);	13937	const zero = try self.asmJccReloc(.z, undefined);
6318	if (self.hasFeature(.slow_incdec)) {	13938	if (self.hasFeature(.slow_incdec)) {
6319	try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), Immediate.u(1));	13939	try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1));
6320	} else {	13940	} else {
6321	try self.asmRegister(.{ ._, .dec }, index_reg.to32());	13941	try self.asmRegister(.{ ._, .dec }, index_reg.to32());
6322	}	13942	}
6323	try self.asmMemoryImmediate(.{ ._, .cmp }, .{	13943	try self.asmMemoryImmediate(.{ ._, .cmp }, .{
6324	.base = .{ .frame = src_mcv.load_frame.index },	13944	.base = .{ .frame = src_frame_addr.index },
6325	.mod = .{ .rm = .{	13945	.mod = .{ .rm = .{
6326	.size = .qword,	13946	.size = .qword,
6327	.index = index_reg.to64(),	13947	.index = index_reg.to64(),
6328	.scale = .@"8",	13948	.scale = .@"8",
6329	.disp = src_mcv.load_frame.off,	13949	.disp = src_frame_addr.off,
6330	} },	13950	} },
6331	}, Immediate.u(0));	13951	}, .u(0));
6332	_ = try self.asmJccReloc(.e, loop);	13952	_ = try self.asmJccReloc(.e, loop);
6333	try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{	13953	try self.asmRegisterMemory(.{ ._r, .bs }, dst_reg.to64(), .{
6334	.base = .{ .frame = src_mcv.load_frame.index },	13954	.base = .{ .frame = src_frame_addr.index },
6335	.mod = .{ .rm = .{	13955	.mod = .{ .rm = .{
6336	.size = .qword,	13956	.size = .qword,
6337	.index = index_reg.to64(),	13957	.index = index_reg.to64(),
6338	.scale = .@"8",	13958	.scale = .@"8",
6339	.disp = src_mcv.load_frame.off,	13959	.disp = src_frame_addr.off,
6340	} },	13960	} },
6341	});	13961	});
6342	self.performReloc(zero);	13962	self.performReloc(zero);
6343	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));	13963	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
6344	try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());	13964	try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());
6345	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(src_bits - 1));	13965	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1));
6346	try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());	13966	try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());
6347	break :result dst_mcv;	13967	break :result dst_mcv;
6348	}	13968	}
...	@@ -6351,7 +13971,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6351,7 +13971,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6351	if (src_bits <= 8) {	13971	if (src_bits <= 8) {
6352	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);	13972	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
6353	try self.truncateRegister(src_ty, wide_reg);	13973	try self.truncateRegister(src_ty, wide_reg);
6354	try self.genBinOpMir(.{ ._, .lzcnt }, Type.u32, dst_mcv, .{ .register = wide_reg });	13974	try self.genBinOpMir(.{ ._, .lzcnt }, .u32, dst_mcv, .{ .register = wide_reg });
6355	try self.genBinOpMir(	13975	try self.genBinOpMir(
6356	.{ ._, .sub },	13976	.{ ._, .sub },
6357	dst_ty,	13977	dst_ty,
...	@@ -6371,25 +13991,15 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6371,25 +13991,15 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6371	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	13991	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6372	defer self.register_manager.unlockReg(tmp_lock);	13992	defer self.register_manager.unlockReg(tmp_lock);
6373		13993
6374	try self.genBinOpMir(	13994	try self.genBinOpMir(.{ ._, .lzcnt }, .u64, dst_mcv, if (mat_src_mcv.isBase())
6375	.{ ._, .lzcnt },	13995	mat_src_mcv
6376	Type.u64,	13996	else
6377	dst_mcv,	13997	.{ .register = mat_src_mcv.register_pair[0] });
6378	if (mat_src_mcv.isMemory())
6379	mat_src_mcv
6380	else
6381	.{ .register = mat_src_mcv.register_pair[0] },
6382	);
6383	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });	13998	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
6384	try self.genBinOpMir(	13999	try self.genBinOpMir(.{ ._, .lzcnt }, .u64, tmp_mcv, if (mat_src_mcv.isBase())
6385	.{ ._, .lzcnt },	14000	mat_src_mcv.address().offset(8).deref()
6386	Type.u64,	14001	else
6387	tmp_mcv,	14002	.{ .register = mat_src_mcv.register_pair[1] });
6388	if (mat_src_mcv.isMemory())
6389	mat_src_mcv.address().offset(8).deref()
6390	else
6391	.{ .register = mat_src_mcv.register_pair[1] },
6392	);
6393	try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());	14003	try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
6394		14004
6395	if (src_bits < 128) try self.genBinOpMir(	14005	if (src_bits < 128) try self.genBinOpMir(
...	@@ -6404,7 +14014,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6404,7 +14014,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6404		14014
6405	assert(src_bits <= 64);	14015	assert(src_bits <= 64);
6406	const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);	14016	const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
6407	if (math.isPowerOfTwo(src_bits)) {	14017	if (std.math.isPowerOfTwo(src_bits)) {
6408	const imm_reg = try self.copyToTmpRegister(dst_ty, .{	14018	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
6409	.immediate = src_bits ^ (src_bits - 1),	14019	.immediate = src_bits ^ (src_bits - 1),
6410	});	14020	});
...	@@ -6417,8 +14027,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6417,8 +14027,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6417	defer self.register_manager.unlockReg(wide_lock);	14027	defer self.register_manager.unlockReg(wide_lock);
6418		14028
6419	try self.truncateRegister(src_ty, wide_reg);	14029	try self.truncateRegister(src_ty, wide_reg);
6420	try self.genBinOpMir(.{ ._, .bsr }, Type.u16, dst_mcv, .{ .register = wide_reg });	14030	try self.genBinOpMir(.{ ._r, .bs }, .u16, dst_mcv, .{ .register = wide_reg });
6421	} else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);	14031	} else try self.genBinOpMir(.{ ._r, .bs }, src_ty, dst_mcv, mat_src_mcv);
6422		14032
6423	try self.asmCmovccRegisterRegister(	14033	try self.asmCmovccRegisterRegister(
6424	.z,	14034	.z,
...	@@ -6429,7 +14039,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6429,7 +14039,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6429	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });	14039	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
6430	} else {	14040	} else {
6431	const imm_reg = try self.copyToTmpRegister(dst_ty, .{	14041	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
6432	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),	14042	.immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
6433	});	14043	});
6434	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);	14044	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
6435	defer self.register_manager.unlockReg(imm_lock);	14045	defer self.register_manager.unlockReg(imm_lock);
...	@@ -6440,8 +14050,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6440,8 +14050,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6440		14050
6441	try self.truncateRegister(src_ty, wide_reg);	14051	try self.truncateRegister(src_ty, wide_reg);
6442	try self.genBinOpMir(	14052	try self.genBinOpMir(
6443	.{ ._, .bsr },	14053	.{ ._r, .bs },
6444	if (src_bits <= 8) Type.u16 else src_ty,	14054	if (src_bits <= 8) .u16 else src_ty,
6445	dst_mcv,	14055	dst_mcv,
6446	.{ .register = wide_reg },	14056	.{ .register = wide_reg },
6447	);	14057	);
...	@@ -6460,7 +14070,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6460,7 +14070,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6460	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	14070	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6461	}	14071	}
6462		14072
6463	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	14073	fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
6464	const pt = self.pt;	14074	const pt = self.pt;
6465	const zcu = pt.zcu;	14075	const zcu = pt.zcu;
6466	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	14076	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -6493,57 +14103,66 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6493,57 +14103,66 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6493	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));	14103	const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
6494	const has_bmi = self.hasFeature(.bmi);	14104	const has_bmi = self.hasFeature(.bmi);
6495	if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {	14105	if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {
6496	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	14106	const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) {
		14107	.load_frame => \|src_frame_addr\| src_frame_addr,
		14108	else => {
		14109	const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu));
		14110	try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{});
		14111	break :src_frame_addr .{ .index = src_frame_addr };
		14112	},
		14113	};
		14114
		14115	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6497	const extra_bits = abi_size * 8 - src_bits;	14116	const extra_bits = abi_size * 8 - src_bits;
6498		14117
6499	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	14118	const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6500	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);	14119	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
6501	defer self.register_manager.unlockReg(index_lock);	14120	defer self.register_manager.unlockReg(index_lock);
6502		14121
6503	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.s(-1));	14122	try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1));
6504	switch (extra_bits) {	14123	switch (extra_bits) {
6505	0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),	14124	0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
6506	1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),	14125	1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),
6507	else => try self.asmRegisterImmediate(	14126	else => try self.asmRegisterImmediate(
6508	.{ ._, .mov },	14127	.{ ._, .mov },
6509	dst_reg.to32(),	14128	dst_reg.to32(),
6510	Immediate.s(-@as(i32, extra_bits)),	14129	.s(-@as(i32, extra_bits)),
6511	),	14130	),
6512	}	14131	}
6513	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	14132	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6514	if (self.hasFeature(.slow_incdec)) {	14133	if (self.hasFeature(.slow_incdec)) {
6515	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));	14134	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
6516	} else {	14135	} else {
6517	try self.asmRegister(.{ ._, .inc }, index_reg.to32());	14136	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
6518	}	14137	}
6519	try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), Immediate.u(limbs_len));	14138	try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len));
6520	const zero = try self.asmJccReloc(.nb, undefined);	14139	const zero = try self.asmJccReloc(.nb, undefined);
6521	try self.asmMemoryImmediate(.{ ._, .cmp }, .{	14140	try self.asmMemoryImmediate(.{ ._, .cmp }, .{
6522	.base = .{ .frame = src_mcv.load_frame.index },	14141	.base = .{ .frame = src_frame_addr.index },
6523	.mod = .{ .rm = .{	14142	.mod = .{ .rm = .{
6524	.size = .qword,	14143	.size = .qword,
6525	.index = index_reg.to64(),	14144	.index = index_reg.to64(),
6526	.scale = .@"8",	14145	.scale = .@"8",
6527	.disp = src_mcv.load_frame.off,	14146	.disp = src_frame_addr.off,
6528	} },	14147	} },
6529	}, Immediate.u(0));	14148	}, .u(0));
6530	_ = try self.asmJccReloc(.e, loop);	14149	_ = try self.asmJccReloc(.e, loop);
6531	try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{	14150	try self.asmRegisterMemory(.{ ._f, .bs }, dst_reg.to64(), .{
6532	.base = .{ .frame = src_mcv.load_frame.index },	14151	.base = .{ .frame = src_frame_addr.index },
6533	.mod = .{ .rm = .{	14152	.mod = .{ .rm = .{
6534	.size = .qword,	14153	.size = .qword,
6535	.index = index_reg.to64(),	14154	.index = index_reg.to64(),
6536	.scale = .@"8",	14155	.scale = .@"8",
6537	.disp = src_mcv.load_frame.off,	14156	.disp = src_frame_addr.off,
6538	} },	14157	} },
6539	});	14158	});
6540	self.performReloc(zero);	14159	self.performReloc(zero);
6541	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));	14160	try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
6542	try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());	14161	try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());
6543	break :result dst_mcv;	14162	break :result dst_mcv;
6544	}	14163	}
6545		14164
6546	const wide_ty = if (src_bits <= 8) Type.u16 else src_ty;	14165	const wide_ty: Type = if (src_bits <= 8) .u16 else src_ty;
6547	if (has_bmi) {	14166	if (has_bmi) {
6548	if (src_bits <= 64) {	14167	if (src_bits <= 64) {
6549	const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);	14168	const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);
...	@@ -6558,7 +14177,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6558,7 +14177,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6558	.{ ._, .@"or" },	14177	.{ ._, .@"or" },
6559	wide_ty,	14178	wide_ty,
6560	tmp_mcv,	14179	tmp_mcv,
6561	.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<	14180	.{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
6562	@intCast(src_bits) },	14181	@intCast(src_bits) },
6563	);	14182	);
6564	break :masked tmp_mcv;	14183	break :masked tmp_mcv;
...	@@ -6571,27 +14190,27 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6571,27 +14190,27 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6571	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	14190	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6572	defer self.register_manager.unlockReg(tmp_lock);	14191	defer self.register_manager.unlockReg(tmp_lock);
6573		14192
6574	const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory())	14193	const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isBase())
6575	mat_src_mcv	14194	mat_src_mcv
6576	else	14195	else
6577	.{ .register = mat_src_mcv.register_pair[0] };	14196	.{ .register = mat_src_mcv.register_pair[0] };
6578	const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory())	14197	const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isBase())
6579	mat_src_mcv.address().offset(8).deref()	14198	mat_src_mcv.address().offset(8).deref()
6580	else	14199	else
6581	.{ .register = mat_src_mcv.register_pair[1] };	14200	.{ .register = mat_src_mcv.register_pair[1] };
6582	const masked_mcv = if (src_bits < 128) masked: {	14201	const masked_mcv = if (src_bits < 128) masked: {
6583	try self.genCopy(Type.u64, dst_mcv, hi_mat_src_mcv, .{});	14202	try self.genCopy(.u64, dst_mcv, hi_mat_src_mcv, .{});
6584	try self.genBinOpMir(	14203	try self.genBinOpMir(
6585	.{ ._, .@"or" },	14204	.{ ._, .@"or" },
6586	Type.u64,	14205	.u64,
6587	dst_mcv,	14206	dst_mcv,
6588	.{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(src_bits - 64) },	14207	.{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) },
6589	);	14208	);
6590	break :masked dst_mcv;	14209	break :masked dst_mcv;
6591	} else hi_mat_src_mcv;	14210	} else hi_mat_src_mcv;
6592	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);	14211	try self.genBinOpMir(.{ ._, .tzcnt }, .u64, dst_mcv, masked_mcv);
6593	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });	14212	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
6594	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, lo_mat_src_mcv);	14213	try self.genBinOpMir(.{ ._, .tzcnt }, .u64, tmp_mcv, lo_mat_src_mcv);
6595	try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());	14214	try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
6596	}	14215	}
6597	break :result dst_mcv;	14216	break :result dst_mcv;
...	@@ -6602,14 +14221,14 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6602,14 +14221,14 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6602	const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);	14221	const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
6603	defer self.register_manager.unlockReg(width_lock);	14222	defer self.register_manager.unlockReg(width_lock);
6604		14223
6605	if (src_bits <= 8 or !math.isPowerOfTwo(src_bits)) {	14224	if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) {
6606	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);	14225	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
6607	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);	14226	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
6608	defer self.register_manager.unlockReg(wide_lock);	14227	defer self.register_manager.unlockReg(wide_lock);
6609		14228
6610	try self.truncateRegister(src_ty, wide_reg);	14229	try self.truncateRegister(src_ty, wide_reg);
6611	try self.genBinOpMir(.{ ._, .bsf }, wide_ty, dst_mcv, .{ .register = wide_reg });	14230	try self.genBinOpMir(.{ ._f, .bs }, wide_ty, dst_mcv, .{ .register = wide_reg });
6612	} else try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);	14231	} else try self.genBinOpMir(.{ ._f, .bs }, src_ty, dst_mcv, mat_src_mcv);
6613		14232
6614	const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);	14233	const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
6615	try self.asmCmovccRegisterRegister(	14234	try self.asmCmovccRegisterRegister(
...	@@ -6622,7 +14241,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6622,7 +14241,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
6622	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	14241	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6623	}	14242	}
6624		14243
6625	fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {	14244	fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void {
6626	const pt = self.pt;	14245	const pt = self.pt;
6627	const zcu = pt.zcu;	14246	const zcu = pt.zcu;
6628	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	14247	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -6664,13 +14283,13 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6664,13 +14283,13 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
6664	const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);	14283	const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
6665	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);	14284	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);
6666		14285
6667	try self.genPopCount(tmp_regs[0], Type.usize, if (mat_src_mcv.isMemory())	14286	try self.genPopCount(tmp_regs[0], .usize, if (mat_src_mcv.isBase())
6668	mat_src_mcv	14287	mat_src_mcv
6669	else	14288	else
6670	.{ .register = mat_src_mcv.register_pair[0] }, false);	14289	.{ .register = mat_src_mcv.register_pair[0] }, false);
6671	const src_info = src_ty.intInfo(zcu);	14290	const src_info = src_ty.intInfo(zcu);
6672	const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);	14291	const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);
6673	try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory())	14292	try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isBase())
6674	mat_src_mcv.address().offset(8).deref()	14293	mat_src_mcv.address().offset(8).deref()
6675	else	14294	else
6676	.{ .register = mat_src_mcv.register_pair[1] }, false);	14295	.{ .register = mat_src_mcv.register_pair[1] }, false);
...	@@ -6681,7 +14300,7 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6681,7 +14300,7 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
6681	}	14300	}
6682		14301
6683	fn genPopCount(	14302	fn genPopCount(
6684	self: *Self,	14303	self: *CodeGen,
6685	dst_reg: Register,	14304	dst_reg: Register,
6686	src_ty: Type,	14305	src_ty: Type,
6687	src_mcv: MCValue,	14306	src_mcv: MCValue,
...	@@ -6692,7 +14311,7 @@ fn genPopCount(	...	@@ -6692,7 +14311,7 @@ fn genPopCount(
6692	const src_abi_size: u32 = @intCast(src_ty.abiSize(pt.zcu));	14311	const src_abi_size: u32 = @intCast(src_ty.abiSize(pt.zcu));
6693	if (self.hasFeature(.popcnt)) return self.genBinOpMir(	14312	if (self.hasFeature(.popcnt)) return self.genBinOpMir(
6694	.{ ._, .popcnt },	14313	.{ ._, .popcnt },
6695	if (src_abi_size > 1) src_ty else Type.u32,	14314	if (src_abi_size > 1) src_ty else .u32,
6696	.{ .register = dst_reg },	14315	.{ .register = dst_reg },
6697	if (src_abi_size > 1) src_mcv else src: {	14316	if (src_abi_size > 1) src_mcv else src: {
6698	if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{});	14317	if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{});
...	@@ -6701,11 +14320,11 @@ fn genPopCount(	...	@@ -6701,11 +14320,11 @@ fn genPopCount(
6701	},	14320	},
6702	);	14321	);
6703		14322
6704	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);	14323	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
6705	const imm_0_1 = Immediate.u(mask / 0b1_1);	14324	const imm_0_1: Immediate = .u(mask / 0b1_1);
6706	const imm_00_11 = Immediate.u(mask / 0b01_01);	14325	const imm_00_11: Immediate = .u(mask / 0b01_01);
6707	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);	14326	const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
6708	const imm_0000_0001 = Immediate.u(mask / 0b1111_1111);	14327	const imm_0000_0001: Immediate = .u(mask / 0b1111_1111);
6709		14328
6710	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	14329	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6711	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	14330	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...	@@ -6722,7 +14341,7 @@ fn genPopCount(	...	@@ -6722,7 +14341,7 @@ fn genPopCount(
6722	// dst = operand	14341	// dst = operand
6723	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	14342	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
6724	// tmp = operand	14343	// tmp = operand
6725	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));	14344	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
6726	// tmp = operand >> 1	14345	// tmp = operand >> 1
6727	if (src_abi_size > 4) {	14346	if (src_abi_size > 4) {
6728	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);	14347	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
...	@@ -6733,7 +14352,7 @@ fn genPopCount(	...	@@ -6733,7 +14352,7 @@ fn genPopCount(
6733	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)	14352	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)
6734	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	14353	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
6735	// tmp = temp1	14354	// tmp = temp1
6736	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));	14355	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
6737	// dst = temp1 >> 2	14356	// dst = temp1 >> 2
6738	if (src_abi_size > 4) {	14357	if (src_abi_size > 4) {
6739	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);	14358	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
...	@@ -6749,7 +14368,7 @@ fn genPopCount(	...	@@ -6749,7 +14368,7 @@ fn genPopCount(
6749	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)	14368	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
6750	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);	14369	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
6751	// dst = temp2	14370	// dst = temp2
6752	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));	14371	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4));
6753	// tmp = temp2 >> 4	14372	// tmp = temp2 >> 4
6754	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);	14373	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
6755	// dst = temp2 + (temp2 >> 4)	14374	// dst = temp2 + (temp2 >> 4)
...	@@ -6767,13 +14386,13 @@ fn genPopCount(	...	@@ -6767,13 +14386,13 @@ fn genPopCount(
6767	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f	14386	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
6768	// dst = temp3 * 0x01...01	14387	// dst = temp3 * 0x01...01
6769	if (src_abi_size > 1) {	14388	if (src_abi_size > 1) {
6770	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));	14389	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8));
6771	}	14390	}
6772	// dst = (temp3 * 0x01...01) >> (bits - 8)	14391	// dst = (temp3 * 0x01...01) >> (bits - 8)
6773	}	14392	}
6774		14393
6775	fn genByteSwap(	14394	fn genByteSwap(
6776	self: *Self,	14395	self: *CodeGen,
6777	inst: Air.Inst.Index,	14396	inst: Air.Inst.Index,
6778	src_ty: Type,	14397	src_ty: Type,
6779	src_mcv: MCValue,	14398	src_mcv: MCValue,
...	@@ -6814,13 +14433,18 @@ fn genByteSwap(	...	@@ -6814,13 +14433,18 @@ fn genByteSwap(
6814	return src_mcv;	14433	return src_mcv;
6815	},	14434	},
6816	9...16 => {	14435	9...16 => {
6817	switch (src_mcv) {	14436	const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) {
		14437	.register => {
		14438	const frame_index = try self.allocFrameIndex(.initSpill(src_ty, zcu));
		14439	try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{});
		14440	break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } };
		14441	},
6818	.register_pair => \|src_regs\| if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {	14442	.register_pair => \|src_regs\| if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
6819	for (src_regs) \|src_reg\| try self.asmRegister(.{ ._, .bswap }, src_reg.to64());	14443	for (src_regs) \|src_reg\| try self.asmRegister(.{ ._, .bswap }, src_reg.to64());
6820	return .{ .register_pair = .{ src_regs[1], src_regs[0] } };	14444	return .{ .register_pair = .{ src_regs[1], src_regs[0] } };
6821	},	14445	} else src_mcv,
6822	else => {},	14446	else => src_mcv,
6823	}	14447	};
6824		14448
6825	const dst_regs =	14449	const dst_regs =
6826	try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);	14450	try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
...	@@ -6828,18 +14452,18 @@ fn genByteSwap(	...	@@ -6828,18 +14452,18 @@ fn genByteSwap(
6828	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);	14452	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);
6829		14453
6830	for (dst_regs, 0..) \|dst_reg, limb_index\| {	14454	for (dst_regs, 0..) \|dst_reg, limb_index\| {
6831	if (src_mcv.isMemory()) {	14455	if (mat_src_mcv.isBase()) {
6832	try self.asmRegisterMemory(	14456	try self.asmRegisterMemory(
6833	.{ ._, if (has_movbe) .movbe else .mov },	14457	.{ ._, if (has_movbe) .movbe else .mov },
6834	dst_reg.to64(),	14458	dst_reg.to64(),
6835	try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .qword),	14459	try mat_src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .{ .size = .qword }),
6836	);	14460	);
6837	if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());	14461	if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
6838	} else {	14462	} else {
6839	try self.asmRegisterRegister(	14463	try self.asmRegisterRegister(
6840	.{ ._, .mov },	14464	.{ ._, .mov },
6841	dst_reg.to64(),	14465	dst_reg.to64(),
6842	src_mcv.register_pair[limb_index].to64(),	14466	mat_src_mcv.register_pair[limb_index].to64(),
6843	);	14467	);
6844	try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());	14468	try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
6845	}	14469	}
...	@@ -6847,20 +14471,16 @@ fn genByteSwap(	...	@@ -6847,20 +14471,16 @@ fn genByteSwap(
6847	return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } };	14471	return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } };
6848	},	14472	},
6849	else => {	14473	else => {
6850	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	14474	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
6851		14475
6852	const temp_regs =	14476	const temp_regs =
6853	try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);	14477	try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
6854	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);	14478	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
6855	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);	14479	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
6856		14480
6857	const dst_mcv = try self.allocRegOrMem(inst, false);	14481	const dst_mcv = try self.allocRegOrMem(inst, false);
6858	try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());	14482	try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
6859	try self.asmRegisterImmediate(	14483	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
6860	.{ ._, .mov },
6861	temp_regs[1].to32(),
6862	Immediate.u(limbs_len - 1),
6863	);
6864		14484
6865	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	14485	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6866	try self.asmRegisterMemory(	14486	try self.asmRegisterMemory(
...	@@ -6912,8 +14532,8 @@ fn genByteSwap(	...	@@ -6912,8 +14532,8 @@ fn genByteSwap(
6912	} },	14532	} },
6913	}, temp_regs[2].to64());	14533	}, temp_regs[2].to64());
6914	if (self.hasFeature(.slow_incdec)) {	14534	if (self.hasFeature(.slow_incdec)) {
6915	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	14535	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
6916	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));	14536	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
6917	} else {	14537	} else {
6918	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	14538	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
6919	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());	14539	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
...	@@ -6942,7 +14562,7 @@ fn genByteSwap(	...	@@ -6942,7 +14562,7 @@ fn genByteSwap(
6942	return dst_mcv;	14562	return dst_mcv;
6943	}	14563	}
6944		14564
6945	fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {	14565	fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void {
6946	const pt = self.pt;	14566	const pt = self.pt;
6947	const zcu = pt.zcu;	14567	const zcu = pt.zcu;
6948	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	14568	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -6959,13 +14579,13 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6959,13 +14579,13 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
6959	} },	14579	} },
6960	src_ty,	14580	src_ty,
6961	dst_mcv,	14581	dst_mcv,
6962	if (src_bits > 256) Type.u16 else Type.u8,	14582	if (src_bits > 256) .u16 else .u8,
6963	.{ .immediate = src_ty.abiSize(zcu) * 8 - src_bits },	14583	.{ .immediate = src_ty.abiSize(zcu) * 8 - src_bits },
6964	);	14584	);
6965	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	14585	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
6966	}	14586	}
6967		14587
6968	fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	14588	fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void {
6969	const pt = self.pt;	14589	const pt = self.pt;
6970	const zcu = pt.zcu;	14590	const zcu = pt.zcu;
6971	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	14591	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -6994,10 +14614,10 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6994,10 +14614,10 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
6994	else	14614	else
6995	undefined;	14615	undefined;
6996		14616
6997	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);	14617	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);
6998	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);	14618	const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
6999	const imm_00_11 = Immediate.u(mask / 0b01_01);	14619	const imm_00_11: Immediate = .u(mask / 0b01_01);
7000	const imm_0_1 = Immediate.u(mask / 0b1_1);	14620	const imm_0_1: Immediate = .u(mask / 0b1_1);
7001		14621
7002	for (dst_mcv.getRegs()) \|dst_reg\| {	14622	for (dst_mcv.getRegs()) \|dst_reg\| {
7003	const dst = registerAlias(dst_reg, limb_abi_size);	14623	const dst = registerAlias(dst_reg, limb_abi_size);
...	@@ -7005,7 +14625,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7005,7 +14625,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7005	// dst = temp1 = bswap(operand)	14625	// dst = temp1 = bswap(operand)
7006	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	14626	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
7007	// tmp = temp1	14627	// tmp = temp1
7008	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));	14628	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4));
7009	// dst = temp1 >> 4	14629	// dst = temp1 >> 4
7010	if (limb_abi_size > 4) {	14630	if (limb_abi_size > 4) {
7011	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);	14631	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
...	@@ -7017,13 +14637,13 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7017,13 +14637,13 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7017	}	14637	}
7018	// tmp = temp1 & 0x0F...0F	14638	// tmp = temp1 & 0x0F...0F
7019	// dst = (temp1 >> 4) & 0x0F...0F	14639	// dst = (temp1 >> 4) & 0x0F...0F
7020	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));	14640	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4));
7021	// tmp = (temp1 & 0x0F...0F) << 4	14641	// tmp = (temp1 & 0x0F...0F) << 4
7022	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);	14642	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
7023	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)	14643	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)
7024	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);	14644	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
7025	// tmp = temp2	14645	// tmp = temp2
7026	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));	14646	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
7027	// dst = temp2 >> 2	14647	// dst = temp2 >> 2
7028	if (limb_abi_size > 4) {	14648	if (limb_abi_size > 4) {
7029	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);	14649	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
...	@@ -7050,7 +14670,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7050,7 +14670,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7050	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)	14670	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
7051	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);	14671	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
7052	// dst = temp3	14672	// dst = temp3
7053	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));	14673	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
7054	// tmp = temp3 >> 1	14674	// tmp = temp3 >> 1
7055	if (limb_abi_size > 4) {	14675	if (limb_abi_size > 4) {
7056	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);	14676	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
...	@@ -7083,12 +14703,12 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7083,12 +14703,12 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7083	if (extra_bits > 0) try self.genShiftBinOpMir(switch (signedness) {	14703	if (extra_bits > 0) try self.genShiftBinOpMir(switch (signedness) {
7084	.signed => .{ ._r, .sa },	14704	.signed => .{ ._r, .sa },
7085	.unsigned => .{ ._r, .sh },	14705	.unsigned => .{ ._r, .sh },
7086	}, src_ty, dst_mcv, Type.u8, .{ .immediate = extra_bits });	14706	}, src_ty, dst_mcv, .u8, .{ .immediate = extra_bits });
7087		14707
7088	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	14708	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
7089	}	14709	}
7090		14710
7091	fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {	14711	fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {
7092	const pt = self.pt;	14712	const pt = self.pt;
7093	const zcu = pt.zcu;	14713	const zcu = pt.zcu;
7094	const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];	14714	const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
...	@@ -7150,12 +14770,12 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)	...	@@ -7150,12 +14770,12 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
7150	.abs => try vec_ty.maxInt(pt, vec_ty),	14770	.abs => try vec_ty.maxInt(pt, vec_ty),
7151	else => unreachable,	14771	else => unreachable,
7152	});	14772	});
7153	const sign_mem: Memory = if (sign_mcv.isMemory())	14773	const sign_mem: Memory = if (sign_mcv.isBase())
7154	try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))	14774	try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) })
7155	else	14775	else
7156	.{	14776	.{
7157	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },	14777	.base = .{ .reg = try self.copyToTmpRegister(.usize, sign_mcv.address()) },
7158	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } },	14778	.mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
7159	};	14779	};
7160		14780
7161	if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(	14781	if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
...	@@ -7216,7 +14836,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)	...	@@ -7216,7 +14836,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
7216	return self.finishAir(inst, result, .{ operand, .none, .none });	14836	return self.finishAir(inst, result, .{ operand, .none, .none });
7217	}	14837	}
7218		14838
7219	fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {	14839	fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index) !void {
7220	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	14840	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7221	const ty = self.typeOf(un_op);	14841	const ty = self.typeOf(un_op);
7222	return self.floatSign(inst, un_op, ty);	14842	return self.floatSign(inst, un_op, ty);
...	@@ -7241,7 +14861,7 @@ const RoundMode = packed struct(u5) {	...	@@ -7241,7 +14861,7 @@ const RoundMode = packed struct(u5) {
7241	} = .normal,	14861	} = .normal,
7242	};	14862	};
7243		14863
7244	fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {	14864	fn airRound(self: *CodeGen, inst: Air.Inst.Index, mode: RoundMode) !void {
7245	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	14865	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7246	const ty = self.typeOf(un_op);	14866	const ty = self.typeOf(un_op);
7247		14867
...	@@ -7265,7 +14885,7 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {	...	@@ -7265,7 +14885,7 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {
7265	return self.finishAir(inst, result, .{ un_op, .none, .none });	14885	return self.finishAir(inst, result, .{ un_op, .none, .none });
7266	}	14886	}
7267		14887
7268	fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {	14888	fn getRoundTag(self: *CodeGen, ty: Type) ?Mir.Inst.FixedTag {
7269	const pt = self.pt;	14889	const pt = self.pt;
7270	const zcu = pt.zcu;	14890	const zcu = pt.zcu;
7271	return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) {	14891	return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) {
...	@@ -7298,7 +14918,7 @@ fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {	...	@@ -7298,7 +14918,7 @@ fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {
7298	} else null;	14918	} else null;
7299	}	14919	}
7300		14920
7301	fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {	14921	fn genRoundLibcall(self: *CodeGen, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {
7302	const pt = self.pt;	14922	const pt = self.pt;
7303	const zcu = pt.zcu;	14923	const zcu = pt.zcu;
7304	if (self.getRoundTag(ty)) \|_\| return .none;	14924	if (self.getRoundTag(ty)) \|_\| return .none;
...	@@ -7320,10 +14940,10 @@ fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MC	...	@@ -7320,10 +14940,10 @@ fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MC
7320	},	14940	},
7321	floatLibcAbiSuffix(ty),	14941	floatLibcAbiSuffix(ty),
7322	}) catch unreachable,	14942	}) catch unreachable,
7323	} }, &.{ty}, &.{src_mcv});	14943	} }, &.{ty}, &.{src_mcv}, .{});
7324	}	14944	}
7325		14945
7326	fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {	14946	fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
7327	const pt = self.pt;	14947	const pt = self.pt;
7328	const mir_tag = self.getRoundTag(ty) orelse {	14948	const mir_tag = self.getRoundTag(ty) orelse {
7329	const result = try self.genRoundLibcall(ty, src_mcv, mode);	14949	const result = try self.genRoundLibcall(ty, src_mcv, mode);
...	@@ -7332,12 +14952,12 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro	...	@@ -7332,12 +14952,12 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7332	const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));	14952	const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
7333	const dst_alias = registerAlias(dst_reg, abi_size);	14953	const dst_alias = registerAlias(dst_reg, abi_size);
7334	switch (mir_tag[0]) {	14954	switch (mir_tag[0]) {
7335	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	14955	.v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
7336	mir_tag,	14956	mir_tag,
7337	dst_alias,	14957	dst_alias,
7338	dst_alias,	14958	dst_alias,
7339	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),	14959	try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
7340	Immediate.u(@as(u5, @bitCast(mode))),	14960	.u(@as(u5, @bitCast(mode))),
7341	) else try self.asmRegisterRegisterRegisterImmediate(	14961	) else try self.asmRegisterRegisterRegisterImmediate(
7342	mir_tag,	14962	mir_tag,
7343	dst_alias,	14963	dst_alias,
...	@@ -7346,13 +14966,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro	...	@@ -7346,13 +14966,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7346	src_mcv.getReg().?	14966	src_mcv.getReg().?
7347	else	14967	else
7348	try self.copyToTmpRegister(ty, src_mcv), abi_size),	14968	try self.copyToTmpRegister(ty, src_mcv), abi_size),
7349	Immediate.u(@as(u5, @bitCast(mode))),	14969	.u(@as(u5, @bitCast(mode))),
7350	),	14970	),
7351	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	14971	else => if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
7352	mir_tag,	14972	mir_tag,
7353	dst_alias,	14973	dst_alias,
7354	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),	14974	try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
7355	Immediate.u(@as(u5, @bitCast(mode))),	14975	.u(@as(u5, @bitCast(mode))),
7356	) else try self.asmRegisterRegisterImmediate(	14976	) else try self.asmRegisterRegisterImmediate(
7357	mir_tag,	14977	mir_tag,
7358	dst_alias,	14978	dst_alias,
...	@@ -7360,12 +14980,12 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro	...	@@ -7360,12 +14980,12 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7360	src_mcv.getReg().?	14980	src_mcv.getReg().?
7361	else	14981	else
7362	try self.copyToTmpRegister(ty, src_mcv), abi_size),	14982	try self.copyToTmpRegister(ty, src_mcv), abi_size),
7363	Immediate.u(@as(u5, @bitCast(mode))),	14983	.u(@as(u5, @bitCast(mode))),
7364	),	14984	),
7365	}	14985	}
7366	}	14986	}
7367		14987
7368	fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	14988	fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void {
7369	const pt = self.pt;	14989	const pt = self.pt;
7370	const zcu = pt.zcu;	14990	const zcu = pt.zcu;
7371	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	14991	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -7393,7 +15013,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7393,7 +15013,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7393	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(	15013	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
7394	.l,	15014	.l,
7395	registerAlias(dst_mcv.register, cmov_abi_size),	15015	registerAlias(dst_mcv.register, cmov_abi_size),
7396	try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),	15016	try src_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
7397	),	15017	),
7398	else => {	15018	else => {
7399	const val_reg = try self.copyToTmpRegister(ty, src_mcv);	15019	const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...	@@ -7433,7 +15053,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7433,7 +15053,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7433	defer self.register_manager.unlockReg(tmp_lock);	15053	defer self.register_manager.unlockReg(tmp_lock);
7434		15054
7435	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);	15055	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
7436	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));	15056	try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
7437	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg);	15057	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg);
7438	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg);	15058	try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg);
7439	try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg);	15059	try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg);
...	@@ -7443,10 +15063,10 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7443,10 +15063,10 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7443	},	15063	},
7444	else => {	15064	else => {
7445	const abi_size: u31 = @intCast(ty.abiSize(zcu));	15065	const abi_size: u31 = @intCast(ty.abiSize(zcu));
7446	const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;	15066	const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
7447		15067
7448	const tmp_regs =	15068	const tmp_regs =
7449	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);	15069	try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
7450	const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs);	15070	const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs);
7451	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);	15071	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);
7452		15072
...	@@ -7459,8 +15079,8 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7459,8 +15079,8 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7459		15079
7460	try self.asmMemoryImmediate(	15080	try self.asmMemoryImmediate(
7461	.{ ._, .cmp },	15081	.{ ._, .cmp },
7462	try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword),	15082	try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .{ .size = .qword }),
7463	Immediate.u(0),	15083	.u(0),
7464	);	15084	);
7465	const positive = try self.asmJccReloc(.ns, undefined);	15085	const positive = try self.asmJccReloc(.ns, undefined);
7466		15086
...	@@ -7469,7 +15089,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7469,7 +15089,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7469		15089
7470	const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	15090	const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
7471	try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32());	15091	try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32());
7472	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), Immediate.u(1));	15092	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1));
7473	try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{	15093	try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{
7474	.base = .{ .frame = dst_mcv.load_frame.index },	15094	.base = .{ .frame = dst_mcv.load_frame.index },
7475	.mod = .{ .rm = .{	15095	.mod = .{ .rm = .{
...	@@ -7491,11 +15111,11 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7491,11 +15111,11 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7491	}, tmp_regs[2].to64());	15111	}, tmp_regs[2].to64());
7492		15112
7493	if (self.hasFeature(.slow_incdec)) {	15113	if (self.hasFeature(.slow_incdec)) {
7494	try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), Immediate.u(1));	15114	try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1));
7495	} else {	15115	} else {
7496	try self.asmRegister(.{ ._, .inc }, tmp_regs[0].to32());	15116	try self.asmRegister(.{ ._, .inc }, tmp_regs[0].to32());
7497	}	15117	}
7498	try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), Immediate.u(limb_len));	15118	try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len));
7499	_ = try self.asmJccReloc(.b, neg_loop);	15119	_ = try self.asmJccReloc(.b, neg_loop);
7500		15120
7501	self.performReloc(positive);	15121	self.performReloc(positive);
...	@@ -7547,12 +15167,12 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7547,12 +15167,12 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7547	const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))	15167	const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
7548	src_mcv.getReg().?	15168	src_mcv.getReg().?
7549	else	15169	else
7550	try self.register_manager.allocReg(inst, self.regClassForType(ty));	15170	try self.register_manager.allocReg(inst, self.regSetForType(ty));
7551	const dst_alias = registerAlias(dst_reg, abi_size);	15171	const dst_alias = registerAlias(dst_reg, abi_size);
7552	if (src_mcv.isMemory()) try self.asmRegisterMemory(	15172	if (src_mcv.isBase()) try self.asmRegisterMemory(
7553	mir_tag,	15173	mir_tag,
7554	dst_alias,	15174	dst_alias,
7555	try src_mcv.mem(self, self.memSize(ty)),	15175	try src_mcv.mem(self, .{ .size = self.memSize(ty) }),
7556	) else try self.asmRegisterRegister(	15176	) else try self.asmRegisterRegister(
7557	mir_tag,	15177	mir_tag,
7558	dst_alias,	15178	dst_alias,
...	@@ -7566,7 +15186,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7566,7 +15186,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7566	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	15186	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7567	}	15187	}
7568		15188
7569	fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	15189	fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void {
7570	const pt = self.pt;	15190	const pt = self.pt;
7571	const zcu = pt.zcu;	15191	const zcu = pt.zcu;
7572	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	15192	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
...	@@ -7591,7 +15211,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7591,7 +15211,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7591	floatLibcAbiPrefix(ty),	15211	floatLibcAbiPrefix(ty),
7592	floatLibcAbiSuffix(ty),	15212	floatLibcAbiSuffix(ty),
7593	}) catch unreachable,	15213	}) catch unreachable,
7594	} }, &.{ty}, &.{.{ .air_ref = un_op }});	15214	} }, &.{ty}, &.{.{ .air_ref = un_op }}, .{});
7595	}	15215	}
7596	},	15216	},
7597	else => {},	15217	else => {},
...	@@ -7620,7 +15240,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7620,7 +15240,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7620	.{ .v_, .cvtps2ph },	15240	.{ .v_, .cvtps2ph },
7621	dst_reg,	15241	dst_reg,
7622	dst_reg,	15242	dst_reg,
7623	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	15243	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7624	);	15244	);
7625	break :result dst_mcv;	15245	break :result dst_mcv;
7626	},	15246	},
...	@@ -7650,18 +15270,18 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7650,18 +15270,18 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7650	.{ .v_, .cvtps2ph },	15270	.{ .v_, .cvtps2ph },
7651	dst_reg,	15271	dst_reg,
7652	dst_reg,	15272	dst_reg,
7653	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	15273	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7654	);	15274	);
7655	break :result dst_mcv;	15275	break :result dst_mcv;
7656	},	15276	},
7657	2...8 => {	15277	2...8 => {
7658	const wide_reg = registerAlias(dst_reg, abi_size * 2);	15278	const wide_reg = registerAlias(dst_reg, abi_size * 2);
7659	if (src_mcv.isMemory()) try self.asmRegisterMemory(	15279	if (src_mcv.isBase()) try self.asmRegisterMemory(
7660	.{ .v_ps, .cvtph2 },	15280	.{ .v_ps, .cvtph2 },
7661	wide_reg,	15281	wide_reg,
7662	try src_mcv.mem(self, Memory.Size.fromSize(	15282	try src_mcv.mem(self, .{ .size = .fromSize(
7663	@intCast(@divExact(wide_reg.bitSize(), 16)),	15283	@intCast(@divExact(wide_reg.bitSize(), 16)),
7664	)),	15284	) }),
7665	) else try self.asmRegisterRegister(	15285	) else try self.asmRegisterRegister(
7666	.{ .v_ps, .cvtph2 },	15286	.{ .v_ps, .cvtph2 },
7667	wide_reg,	15287	wide_reg,
...	@@ -7675,7 +15295,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7675,7 +15295,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7675	.{ .v_, .cvtps2ph },	15295	.{ .v_, .cvtps2ph },
7676	dst_reg,	15296	dst_reg,
7677	wide_reg,	15297	wide_reg,
7678	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	15298	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
7679	);	15299	);
7680	break :result dst_mcv;	15300	break :result dst_mcv;
7681	},	15301	},
...	@@ -7699,15 +15319,13 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7699,15 +15319,13 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7699	else => unreachable,	15319	else => unreachable,
7700	},	15320	},
7701	else => unreachable,	15321	else => unreachable,
7702	}) orelse return self.fail("TODO implement airSqrt for {}", .{	15322	}) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)});
7703	ty.fmt(pt),
7704	});
7705	switch (mir_tag[0]) {	15323	switch (mir_tag[0]) {
7706	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(	15324	.v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
7707	mir_tag,	15325	mir_tag,
7708	dst_reg,	15326	dst_reg,
7709	dst_reg,	15327	dst_reg,
7710	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),	15328	try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
7711	) else try self.asmRegisterRegisterRegister(	15329	) else try self.asmRegisterRegisterRegister(
7712	mir_tag,	15330	mir_tag,
7713	dst_reg,	15331	dst_reg,
...	@@ -7717,10 +15335,10 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7717,10 +15335,10 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7717	else	15335	else
7718	try self.copyToTmpRegister(ty, src_mcv), abi_size),	15336	try self.copyToTmpRegister(ty, src_mcv), abi_size),
7719	),	15337	),
7720	else => if (src_mcv.isMemory()) try self.asmRegisterMemory(	15338	else => if (src_mcv.isBase()) try self.asmRegisterMemory(
7721	mir_tag,	15339	mir_tag,
7722	dst_reg,	15340	dst_reg,
7723	try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),	15341	try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
7724	) else try self.asmRegisterRegister(	15342	) else try self.asmRegisterRegister(
7725	mir_tag,	15343	mir_tag,
7726	dst_reg,	15344	dst_reg,
...	@@ -7735,7 +15353,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7735,7 +15353,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
7735	return self.finishAir(inst, result, .{ un_op, .none, .none });	15353	return self.finishAir(inst, result, .{ un_op, .none, .none });
7736	}	15354	}
7737		15355
7738	fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	15356	fn airUnaryMath(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
7739	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	15357	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7740	const ty = self.typeOf(un_op);	15358	const ty = self.typeOf(un_op);
7741	var callee_buf: ["__round?".len]u8 = undefined;	15359	var callee_buf: ["__round?".len]u8 = undefined;
...	@@ -7759,12 +15377,12 @@ fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	...	@@ -7759,12 +15377,12 @@ fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
7759	},	15377	},
7760	floatLibcAbiSuffix(ty),	15378	floatLibcAbiSuffix(ty),
7761	}) catch unreachable,	15379	}) catch unreachable,
7762	} }, &.{ty}, &.{.{ .air_ref = un_op }});	15380	} }, &.{ty}, &.{.{ .air_ref = un_op }}, .{});
7763	return self.finishAir(inst, result, .{ un_op, .none, .none });	15381	return self.finishAir(inst, result, .{ un_op, .none, .none });
7764	}	15382	}
7765		15383
7766	fn reuseOperand(	15384	fn reuseOperand(
7767	self: *Self,	15385	self: *CodeGen,
7768	inst: Air.Inst.Index,	15386	inst: Air.Inst.Index,
7769	operand: Air.Inst.Ref,	15387	operand: Air.Inst.Ref,
7770	op_index: Liveness.OperandInt,	15388	op_index: Liveness.OperandInt,
...	@@ -7774,7 +15392,7 @@ fn reuseOperand(	...	@@ -7774,7 +15392,7 @@ fn reuseOperand(
7774	}	15392	}
7775		15393
7776	fn reuseOperandAdvanced(	15394	fn reuseOperandAdvanced(
7777	self: *Self,	15395	self: *CodeGen,
7778	inst: Air.Inst.Index,	15396	inst: Air.Inst.Index,
7779	operand: Air.Inst.Ref,	15397	operand: Air.Inst.Ref,
7780	op_index: Liveness.OperandInt,	15398	op_index: Liveness.OperandInt,
...	@@ -7785,7 +15403,7 @@ fn reuseOperandAdvanced(	...	@@ -7785,7 +15403,7 @@ fn reuseOperandAdvanced(
7785	return false;	15403	return false;
7786		15404
7787	switch (mcv) {	15405	switch (mcv) {
7788	.register, .register_pair, .register_overflow => for (mcv.getRegs()) \|reg\| {	15406	.register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) \|reg\| {
7789	// If it's in the registers table, need to associate the register(s) with the	15407	// If it's in the registers table, need to associate the register(s) with the
7790	// new instruction.	15408	// new instruction.
7791	if (maybe_tracked_inst) \|tracked_inst\| {	15409	if (maybe_tracked_inst) \|tracked_inst\| {
...	@@ -7805,19 +15423,19 @@ fn reuseOperandAdvanced(	...	@@ -7805,19 +15423,19 @@ fn reuseOperandAdvanced(
7805	}	15423	}
7806		15424
7807	// Prevent the operand deaths processing code from deallocating it.	15425	// Prevent the operand deaths processing code from deallocating it.
7808	self.liveness.clearOperandDeath(inst, op_index);	15426	self.reused_operands.set(op_index);
7809	const op_inst = operand.toIndex().?;	15427	const op_inst = operand.toIndex().?;
7810	self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst);	15428	self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst);
7811		15429
7812	return true;	15430	return true;
7813	}	15431	}
7814		15432
7815	fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {	15433	fn packedLoad(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
7816	const pt = self.pt;	15434	const pt = self.pt;
7817	const zcu = pt.zcu;	15435	const zcu = pt.zcu;
7818		15436
7819	const ptr_info = ptr_ty.ptrInfo(zcu);	15437	const ptr_info = ptr_ty.ptrInfo(zcu);
7820	const val_ty = Type.fromInterned(ptr_info.child);	15438	const val_ty: Type = .fromInterned(ptr_info.child);
7821	if (!val_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;	15439	if (!val_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
7822	const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));	15440	const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
7823		15441
...	@@ -7851,9 +15469,9 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -7851,9 +15469,9 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7851	defer self.register_manager.unlockReg(tmp_lock);	15469	defer self.register_manager.unlockReg(tmp_lock);
7852		15470
7853	const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();	15471	const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
7854	try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});	15472	try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
7855	try self.truncateRegister(val_ty, tmp_reg);	15473	try self.truncateRegister(val_ty, tmp_reg);
7856	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});	15474	try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
7857	}	15475	}
7858	}	15476	}
7859	return;	15477	return;
...	@@ -7885,12 +15503,12 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -7885,12 +15503,12 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7885	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{	15503	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
7886	.base = .{ .reg = ptr_reg },	15504	.base = .{ .reg = ptr_reg },
7887	.mod = .{ .rm = .{	15505	.mod = .{ .rm = .{
7888	.size = Memory.Size.fromSize(load_abi_size),	15506	.size = .fromSize(load_abi_size),
7889	.disp = val_byte_off,	15507	.disp = val_byte_off,
7890	} },	15508	} },
7891	});	15509	});
7892	try self.spillEflagsIfOccupied();	15510	try self.spillEflagsIfOccupied();
7893	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));	15511	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(val_bit_off));
7894	} else {	15512	} else {
7895	const tmp_reg =	15513	const tmp_reg =
7896	registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size);	15514	registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size);
...	@@ -7901,31 +15519,26 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -7901,31 +15519,26 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7901	try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{	15519	try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
7902	.base = .{ .reg = ptr_reg },	15520	.base = .{ .reg = ptr_reg },
7903	.mod = .{ .rm = .{	15521	.mod = .{ .rm = .{
7904	.size = Memory.Size.fromSize(val_abi_size),	15522	.size = .fromSize(val_abi_size),
7905	.disp = val_byte_off,	15523	.disp = val_byte_off,
7906	} },	15524	} },
7907	});	15525	});
7908	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{	15526	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
7909	.base = .{ .reg = ptr_reg },	15527	.base = .{ .reg = ptr_reg },
7910	.mod = .{ .rm = .{	15528	.mod = .{ .rm = .{
7911	.size = Memory.Size.fromSize(val_abi_size),	15529	.size = .fromSize(val_abi_size),
7912	.disp = val_byte_off + limb_abi_size,	15530	.disp = val_byte_off + limb_abi_size,
7913	} },	15531	} },
7914	});	15532	});
7915	try self.spillEflagsIfOccupied();	15533	try self.spillEflagsIfOccupied();
7916	try self.asmRegisterRegisterImmediate(	15534	try self.asmRegisterRegisterImmediate(.{ ._rd, .sh }, dst_alias, tmp_reg, .u(val_bit_off));
7917	.{ ._rd, .sh },
7918	dst_alias,
7919	tmp_reg,
7920	Immediate.u(val_bit_off),
7921	);
7922	}	15535	}
7923		15536
7924	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);	15537	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
7925	try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{});	15538	try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{});
7926	}	15539	}
7927		15540
7928	fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {	15541	fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
7929	const pt = self.pt;	15542	const pt = self.pt;
7930	const zcu = pt.zcu;	15543	const zcu = pt.zcu;
7931	const dst_ty = ptr_ty.childType(zcu);	15544	const dst_ty = ptr_ty.childType(zcu);
...	@@ -7937,7 +15550,10 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro	...	@@ -7937,7 +15550,10 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
7937	.undef,	15550	.undef,
7938	.eflags,	15551	.eflags,
7939	.register_pair,	15552	.register_pair,
		15553	.register_triple,
		15554	.register_quadruple,
7940	.register_overflow,	15555	.register_overflow,
		15556	.register_mask,
7941	.elementwise_regs_then_frame,	15557	.elementwise_regs_then_frame,
7942	.reserved_frame,	15558	.reserved_frame,
7943	=> unreachable, // not a valid pointer	15559	=> unreachable, // not a valid pointer
...	@@ -7968,7 +15584,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro	...	@@ -7968,7 +15584,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
7968	}	15584	}
7969	}	15585	}
7970		15586
7971	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {	15587	fn airLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
7972	const pt = self.pt;	15588	const pt = self.pt;
7973	const zcu = pt.zcu;	15589	const zcu = pt.zcu;
7974	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	15590	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -7983,12 +15599,12 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7983,12 +15599,12 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
7983	const ptr_ty = self.typeOf(ty_op.operand);	15599	const ptr_ty = self.typeOf(ty_op.operand);
7984	const elem_size = elem_ty.abiSize(zcu);	15600	const elem_size = elem_ty.abiSize(zcu);
7985		15601
7986	const elem_rc = self.regClassForType(elem_ty);	15602	const elem_rs = self.regSetForType(elem_ty);
7987	const ptr_rc = self.regClassForType(ptr_ty);	15603	const ptr_rs = self.regSetForType(ptr_ty);
7988		15604
7989	const ptr_mcv = try self.resolveInst(ty_op.operand);	15605	const ptr_mcv = try self.resolveInst(ty_op.operand);
7990	const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and	15606	const dst_mcv = if (elem_size <= 8 and std.math.isPowerOfTwo(elem_size) and
7991	self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))	15607	elem_rs.supersetOf(ptr_rs) and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
7992	// The MCValue that holds the pointer can be re-used as the value.	15608	// The MCValue that holds the pointer can be re-used as the value.
7993	ptr_mcv	15609	ptr_mcv
7994	else	15610	else
...	@@ -8010,13 +15626,13 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8010,13 +15626,13 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
8010	const high_reg = if (high_mcv.isRegister())	15626	const high_reg = if (high_mcv.isRegister())
8011	high_mcv.getReg().?	15627	high_mcv.getReg().?
8012	else	15628	else
8013	try self.copyToTmpRegister(Type.usize, high_mcv);	15629	try self.copyToTmpRegister(.usize, high_mcv);
8014	const high_lock = self.register_manager.lockReg(high_reg);	15630	const high_lock = self.register_manager.lockReg(high_reg);
8015	defer if (high_lock) \|lock\| self.register_manager.unlockReg(lock);	15631	defer if (high_lock) \|lock\| self.register_manager.unlockReg(lock);
8016		15632
8017	try self.truncateRegister(elem_ty, high_reg);	15633	try self.truncateRegister(elem_ty, high_reg);
8018	if (!high_mcv.isRegister()) try self.genCopy(	15634	if (!high_mcv.isRegister()) try self.genCopy(
8019	if (elem_size <= 8) elem_ty else Type.usize,	15635	if (elem_size <= 8) elem_ty else .usize,
8020	high_mcv,	15636	high_mcv,
8021	.{ .register = high_reg },	15637	.{ .register = high_reg },
8022	.{},	15638	.{},
...	@@ -8027,11 +15643,11 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8027,11 +15643,11 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
8027	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	15643	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8028	}	15644	}
8029		15645
8030	fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {	15646	fn packedStore(self: *CodeGen, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
8031	const pt = self.pt;	15647	const pt = self.pt;
8032	const zcu = pt.zcu;	15648	const zcu = pt.zcu;
8033	const ptr_info = ptr_ty.ptrInfo(zcu);	15649	const ptr_info = ptr_ty.ptrInfo(zcu);
8034	const src_ty = Type.fromInterned(ptr_info.child);	15650	const src_ty: Type = .fromInterned(ptr_info.child);
8035	if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;	15651	if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return;
8036		15652
8037	const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8);	15653	const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8);
...	@@ -8051,6 +15667,15 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -8051,6 +15667,15 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8051	const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);	15667	const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
8052	defer self.register_manager.unlockReg(ptr_lock);	15668	defer self.register_manager.unlockReg(ptr_lock);
8053		15669
		15670	const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) {
		15671	.register => if (src_bit_size > 64) {
		15672	const frame_index = try self.allocFrameIndex(.initSpill(src_ty, self.pt.zcu));
		15673	try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{});
		15674	break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } };
		15675	} else src_mcv,
		15676	else => src_mcv,
		15677	};
		15678
8054	var limb_i: u16 = 0;	15679	var limb_i: u16 = 0;
8055	while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) {	15680	while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) {
8056	const part_bit_off = if (limb_i == 0) src_bit_off else 0;	15681	const part_bit_off = if (limb_i == 0) src_bit_off else 0;
...	@@ -8059,21 +15684,21 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -8059,21 +15684,21 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8059	const limb_mem: Memory = .{	15684	const limb_mem: Memory = .{
8060	.base = .{ .reg = ptr_reg },	15685	.base = .{ .reg = ptr_reg },
8061	.mod = .{ .rm = .{	15686	.mod = .{ .rm = .{
8062	.size = Memory.Size.fromSize(limb_abi_size),	15687	.size = .fromSize(limb_abi_size),
8063	.disp = src_byte_off + limb_i * limb_abi_size,	15688	.disp = src_byte_off + limb_i * limb_abi_size,
8064	} },	15689	} },
8065	};	15690	};
8066		15691
8067	const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<	15692	const part_mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
8068	@intCast(part_bit_off);	15693	@intCast(part_bit_off);
8069	const part_mask_not = part_mask ^ (@as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));	15694	const part_mask_not = part_mask ^ (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_bits));
8070	if (limb_abi_size <= 4) {	15695	if (limb_abi_size <= 4) {
8071	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));	15696	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .u(part_mask_not));
8072	} else if (math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {	15697	} else if (std.math.cast(i32, @as(i64, @bitCast(part_mask_not)))) \|small\| {
8073	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));	15698	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .s(small));
8074	} else {	15699	} else {
8075	const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	15700	const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
8076	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));	15701	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, .u(part_mask_not));
8077	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);	15702	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
8078	}	15703	}
8079		15704
...	@@ -8083,20 +15708,20 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -8083,20 +15708,20 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8083	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	15708	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
8084	defer self.register_manager.unlockReg(tmp_lock);	15709	defer self.register_manager.unlockReg(tmp_lock);
8085		15710
8086	try self.genSetReg(tmp_reg, limb_ty, src_mcv, .{});	15711	try self.genSetReg(tmp_reg, limb_ty, mat_src_mcv, .{});
8087	switch (limb_i) {	15712	switch (limb_i) {
8088	0 => try self.genShiftBinOpMir(	15713	0 => try self.genShiftBinOpMir(
8089	.{ ._l, .sh },	15714	.{ ._l, .sh },
8090	limb_ty,	15715	limb_ty,
8091	tmp_mcv,	15716	tmp_mcv,
8092	Type.u8,	15717	.u8,
8093	.{ .immediate = src_bit_off },	15718	.{ .immediate = src_bit_off },
8094	),	15719	),
8095	1 => try self.genShiftBinOpMir(	15720	1 => try self.genShiftBinOpMir(
8096	.{ ._r, .sh },	15721	.{ ._r, .sh },
8097	limb_ty,	15722	limb_ty,
8098	tmp_mcv,	15723	tmp_mcv,
8099	Type.u8,	15724	.u8,
8100	.{ .immediate = limb_abi_bits - src_bit_off },	15725	.{ .immediate = limb_abi_bits - src_bit_off },
8101	),	15726	),
8102	else => unreachable,	15727	else => unreachable,
...	@@ -8114,8 +15739,8 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -8114,8 +15739,8 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8114	defer self.register_manager.unlockReg(tmp_lock);	15739	defer self.register_manager.unlockReg(tmp_lock);
8115		15740
8116	try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {	15741	try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {
8117	0 => src_mcv,	15742	0 => mat_src_mcv,
8118	else => src_mcv.address().offset(limb_i * limb_abi_size).deref(),	15743	else => mat_src_mcv.address().offset(limb_i * limb_abi_size).deref(),
8119	}, .{});	15744	}, .{});
8120	try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });	15745	try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });
8121	try self.asmMemoryRegister(	15746	try self.asmMemoryRegister(
...	@@ -8128,7 +15753,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -8128,7 +15753,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8128	}	15753	}
8129		15754
8130	fn store(	15755	fn store(
8131	self: *Self,	15756	self: *CodeGen,
8132	ptr_ty: Type,	15757	ptr_ty: Type,
8133	ptr_mcv: MCValue,	15758	ptr_mcv: MCValue,
8134	src_mcv: MCValue,	15759	src_mcv: MCValue,
...	@@ -8145,7 +15770,10 @@ fn store(	...	@@ -8145,7 +15770,10 @@ fn store(
8145	.undef,	15770	.undef,
8146	.eflags,	15771	.eflags,
8147	.register_pair,	15772	.register_pair,
		15773	.register_triple,
		15774	.register_quadruple,
8148	.register_overflow,	15775	.register_overflow,
		15776	.register_mask,
8149	.elementwise_regs_then_frame,	15777	.elementwise_regs_then_frame,
8150	.reserved_frame,	15778	.reserved_frame,
8151	=> unreachable, // not a valid pointer	15779	=> unreachable, // not a valid pointer
...	@@ -8176,7 +15804,7 @@ fn store(	...	@@ -8176,7 +15804,7 @@ fn store(
8176	}	15804	}
8177	}	15805	}
8178		15806
8179	fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	15807	fn airStore(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
8180	const pt = self.pt;	15808	const pt = self.pt;
8181	const zcu = pt.zcu;	15809	const zcu = pt.zcu;
8182	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	15810	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -8202,32 +15830,21 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -8202,32 +15830,21 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8202	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });	15830	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
8203	}	15831	}
8204		15832
8205	fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {	15833	fn airStructFieldPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
8206	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	15834	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8207	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;	15835	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
8208	const result = try self.fieldPtr(inst, extra.struct_operand, extra.field_index);	15836	const result = try self.fieldPtr(inst, extra.struct_operand, extra.field_index);
8209	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });	15837	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
8210	}	15838	}
8211		15839
8212	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {	15840	fn airStructFieldPtrIndex(self: *CodeGen, inst: Air.Inst.Index, field_index: u8) !void {
8213	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	15841	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8214	const result = try self.fieldPtr(inst, ty_op.operand, index);	15842	const result = try self.fieldPtr(inst, ty_op.operand, field_index);
8215	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	15843	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8216	}	15844	}
8217		15845
8218	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {	15846	fn fieldPtr(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {
8219	const pt = self.pt;
8220	const zcu = pt.zcu;
8221	const ptr_field_ty = self.typeOfIndex(inst);	15847	const ptr_field_ty = self.typeOfIndex(inst);
8222	const ptr_container_ty = self.typeOf(operand);
8223	const container_ty = ptr_container_ty.childType(zcu);
8224
8225	const field_off: i32 = switch (container_ty.containerLayout(zcu)) {
8226	.auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)),
8227	.@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8228	(if (zcu.typeToStruct(container_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, index) else 0) -
8229	ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
8230	};
8231		15848
8232	const src_mcv = try self.resolveInst(operand);	15849	const src_mcv = try self.resolveInst(operand);
8233	const dst_mcv = if (switch (src_mcv) {	15850	const dst_mcv = if (switch (src_mcv) {
...	@@ -8235,10 +15852,22 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32	...	@@ -8235,10 +15852,22 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
8235	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),	15852	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),
8236	else => false,	15853	else => false,
8237	}) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv);	15854	}) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv);
8238	return dst_mcv.offset(field_off);	15855	return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index));
		15856	}
		15857
		15858	fn fieldOffset(self: *CodeGen, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {
		15859	const pt = self.pt;
		15860	const zcu = pt.zcu;
		15861	const agg_ty = ptr_agg_ty.childType(zcu);
		15862	return switch (agg_ty.containerLayout(zcu)) {
		15863	.auto, .@"extern" => @intCast(agg_ty.structFieldOffset(field_index, zcu)),
		15864	.@"packed" => @divExact(@as(i32, ptr_agg_ty.ptrInfo(zcu).packed_offset.bit_offset) +
		15865	(if (zcu.typeToStruct(agg_ty)) \|loaded_struct\| pt.structPackedFieldBitOffset(loaded_struct, field_index) else 0) -
		15866	ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
		15867	};
8239	}	15868	}
8240		15869
8241	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	15870	fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
8242	const pt = self.pt;	15871	const pt = self.pt;
8243	const zcu = pt.zcu;	15872	const zcu = pt.zcu;
8244	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	15873	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -8248,16 +15877,19 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8248,16 +15877,19 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8248	const index = extra.field_index;	15877	const index = extra.field_index;
8249		15878
8250	const container_ty = self.typeOf(operand);	15879	const container_ty = self.typeOf(operand);
8251	const container_rc = self.regClassForType(container_ty);	15880	const container_rc = self.regSetForType(container_ty);
8252	const field_ty = container_ty.fieldType(index, zcu);	15881	const field_ty = container_ty.fieldType(index, zcu);
8253	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;	15882	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
8254	const field_rc = self.regClassForType(field_ty);	15883	const field_rc = self.regSetForType(field_ty);
8255	const field_is_gp = field_rc.supersetOf(abi.RegisterClass.gp);	15884	const field_is_gp = field_rc.supersetOf(abi.RegisterClass.gp);
8256		15885
8257	const src_mcv = try self.resolveInst(operand);	15886	const src_mcv = try self.resolveInst(operand);
8258	const field_off: u32 = switch (container_ty.containerLayout(zcu)) {	15887	const field_off: u32 = switch (container_ty.containerLayout(zcu)) {
8259	.auto, .@"extern" => @intCast(container_ty.structFieldOffset(extra.field_index, zcu) * 8),	15888	.auto, .@"extern" => @intCast(container_ty.structFieldOffset(extra.field_index, zcu) * 8),
8260	.@"packed" => if (zcu.typeToStruct(container_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0,	15889	.@"packed" => if (zcu.typeToStruct(container_ty)) \|loaded_struct\|
		15890	pt.structPackedFieldBitOffset(loaded_struct, extra.field_index)
		15891	else
		15892	0,
8261	};	15893	};
8262		15894
8263	switch (src_mcv) {	15895	switch (src_mcv) {
...	@@ -8272,20 +15904,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8272,20 +15904,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8272	else if (field_off == 0)	15904	else if (field_off == 0)
8273	(try self.copyToRegisterWithInstTracking(inst, field_ty, src_mcv)).register	15905	(try self.copyToRegisterWithInstTracking(inst, field_ty, src_mcv)).register
8274	else	15906	else
8275	try self.copyToTmpRegister(Type.usize, .{ .register = src_reg });	15907	try self.copyToTmpRegister(.usize, .{ .register = src_reg });
8276	const dst_mcv: MCValue = .{ .register = dst_reg };	15908	const dst_mcv: MCValue = .{ .register = dst_reg };
8277	const dst_lock = self.register_manager.lockReg(dst_reg);	15909	const dst_lock = self.register_manager.lockReg(dst_reg);
8278	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	15910	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
8279		15911
8280	if (field_off > 0) {	15912	if (field_off > 0) {
8281	try self.spillEflagsIfOccupied();	15913	try self.spillEflagsIfOccupied();
8282	try self.genShiftBinOpMir(	15914	try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, dst_mcv, .u8, .{ .immediate = field_off });
8283	.{ ._r, .sh },
8284	Type.usize,
8285	dst_mcv,
8286	Type.u8,
8287	.{ .immediate = field_off },
8288	);
8289	}	15915	}
8290	if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and	15916	if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and
8291	container_ty.abiSize(zcu) * 8 > field_ty.bitSize(zcu))	15917	container_ty.abiSize(zcu) * 8 > field_ty.bitSize(zcu))
...	@@ -8309,7 +15935,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8309,7 +15935,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8309	const dst_regs: [2]Register = if (field_rc.supersetOf(container_rc) and	15935	const dst_regs: [2]Register = if (field_rc.supersetOf(container_rc) and
8310	self.reuseOperand(inst, operand, 0, src_mcv)) src_regs else dst: {	15936	self.reuseOperand(inst, operand, 0, src_mcv)) src_regs else dst: {
8311	const dst_regs =	15937	const dst_regs =
8312	try self.register_manager.allocRegs(2, .{null} ** 2, field_rc);	15938	try self.register_manager.allocRegs(2, @splat(null), field_rc);
8313	const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);	15939	const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
8314	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);	15940	defer for (dst_locks) \|lock\| self.register_manager.unlockReg(lock);
8315		15941
...	@@ -8323,13 +15949,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8323,13 +15949,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8323		15949
8324	if (field_off > 0) {	15950	if (field_off > 0) {
8325	try self.spillEflagsIfOccupied();	15951	try self.spillEflagsIfOccupied();
8326	try self.genShiftBinOpMir(	15952	try self.genShiftBinOpMir(.{ ._r, .sh }, .u128, dst_mcv, .u8, .{ .immediate = field_off });
8327	.{ ._r, .sh },
8328	Type.u128,
8329	dst_mcv,
8330	Type.u8,
8331	.{ .immediate = field_off },
8332	);
8333	}	15953	}
8334		15954
8335	if (field_bit_size <= 64) {	15955	if (field_bit_size <= 64) {
...	@@ -8353,20 +15973,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8353,20 +15973,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8353	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);	15973	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
8354	};	15974	};
8355		15975
8356	const dst_reg = try self.copyToTmpRegister(Type.usize, .{ .register = src_reg });	15976	const dst_reg = try self.copyToTmpRegister(.usize, .{ .register = src_reg });
8357	const dst_mcv = MCValue{ .register = dst_reg };	15977	const dst_mcv = MCValue{ .register = dst_reg };
8358	const dst_lock = self.register_manager.lockReg(dst_reg);	15978	const dst_lock = self.register_manager.lockReg(dst_reg);
8359	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	15979	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
8360		15980
8361	if (field_off % 64 > 0) {	15981	if (field_off % 64 > 0) {
8362	try self.spillEflagsIfOccupied();	15982	try self.spillEflagsIfOccupied();
8363	try self.genShiftBinOpMir(	15983	try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, dst_mcv, .u8, .{ .immediate = field_off % 64 });
8364	.{ ._r, .sh },
8365	Type.usize,
8366	dst_mcv,
8367	Type.u8,
8368	.{ .immediate = field_off % 64 },
8369	);
8370	}	15984	}
8371	if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);	15985	if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
8372		15986
...	@@ -8381,11 +15995,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8381,11 +15995,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8381	0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {	15995	0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {
8382	self.eflags_inst = null; // actually stop tracking the overflow part	15996	self.eflags_inst = null; // actually stop tracking the overflow part
8383	break :result .{ .register = ro.reg };	15997	break :result .{ .register = ro.reg };
8384	} else break :result try self.copyToRegisterWithInstTracking(	15998	} else break :result try self.copyToRegisterWithInstTracking(inst, .usize, .{ .register = ro.reg }),
8385	inst,
8386	Type.usize,
8387	.{ .register = ro.reg },
8388	),
8389	// Get overflow bit.	15999	// Get overflow bit.
8390	1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {	16000	1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {
8391	self.eflags_inst = inst; // actually keep tracking the overflow part	16001	self.eflags_inst = inst; // actually keep tracking the overflow part
...	@@ -8436,16 +16046,16 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8436,16 +16046,16 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8436	try self.genCopy(field_ty, dst_mcv, off_mcv, .{});	16046	try self.genCopy(field_ty, dst_mcv, off_mcv, .{});
8437	break :dst dst_mcv;	16047	break :dst dst_mcv;
8438	};	16048	};
8439	if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {	16049	if (field_abi_size * 8 > field_bit_size and dst_mcv.isBase()) {
8440	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	16050	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
8441	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	16051	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
8442	defer self.register_manager.unlockReg(tmp_lock);	16052	defer self.register_manager.unlockReg(tmp_lock);
8443		16053
8444	const hi_mcv =	16054	const hi_mcv =
8445	dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();	16055	dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
8446	try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});	16056	try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
8447	try self.truncateRegister(field_ty, tmp_reg);	16057	try self.truncateRegister(field_ty, tmp_reg);
8448	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});	16058	try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
8449	}	16059	}
8450	break :result dst_mcv;	16060	break :result dst_mcv;
8451	}	16061	}
...	@@ -8471,12 +16081,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8471,12 +16081,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8471	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{	16081	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
8472	.base = .{ .frame = frame_addr.index },	16082	.base = .{ .frame = frame_addr.index },
8473	.mod = .{ .rm = .{	16083	.mod = .{ .rm = .{
8474	.size = Memory.Size.fromSize(load_abi_size),	16084	.size = .fromSize(load_abi_size),
8475	.disp = frame_addr.off + field_byte_off,	16085	.disp = frame_addr.off + field_byte_off,
8476	} },	16086	} },
8477	});	16087	});
8478	try self.spillEflagsIfOccupied();	16088	try self.spillEflagsIfOccupied();
8479	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));	16089	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(field_bit_off));
8480	} else {	16090	} else {
8481	const tmp_reg = registerAlias(	16091	const tmp_reg = registerAlias(
8482	try self.register_manager.allocReg(null, abi.RegisterClass.gp),	16092	try self.register_manager.allocReg(null, abi.RegisterClass.gp),
...	@@ -8492,7 +16102,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8492,7 +16102,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8492	.{	16102	.{
8493	.base = .{ .frame = frame_addr.index },	16103	.base = .{ .frame = frame_addr.index },
8494	.mod = .{ .rm = .{	16104	.mod = .{ .rm = .{
8495	.size = Memory.Size.fromSize(field_abi_size),	16105	.size = .fromSize(field_abi_size),
8496	.disp = frame_addr.off + field_byte_off,	16106	.disp = frame_addr.off + field_byte_off,
8497	} },	16107	} },
8498	},	16108	},
...	@@ -8500,7 +16110,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8500,7 +16110,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8500	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{	16110	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
8501	.base = .{ .frame = frame_addr.index },	16111	.base = .{ .frame = frame_addr.index },
8502	.mod = .{ .rm = .{	16112	.mod = .{ .rm = .{
8503	.size = Memory.Size.fromSize(field_abi_size),	16113	.size = .fromSize(field_abi_size),
8504	.disp = frame_addr.off + field_byte_off + limb_abi_size,	16114	.disp = frame_addr.off + field_byte_off + limb_abi_size,
8505	} },	16115	} },
8506	});	16116	});
...	@@ -8509,7 +16119,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8509,7 +16119,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8509	.{ ._rd, .sh },	16119	.{ ._rd, .sh },
8510	dst_alias,	16120	dst_alias,
8511	tmp_reg,	16121	tmp_reg,
8512	Immediate.u(field_bit_off),	16122	.u(field_bit_off),
8513	);	16123	);
8514	}	16124	}
8515		16125
...	@@ -8527,32 +16137,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8527,32 +16137,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8527	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });	16137	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
8528	}	16138	}
8529		16139
8530	fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {	16140	fn airFieldParentPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
8531	const pt = self.pt;
8532	const zcu = pt.zcu;
8533	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	16141	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8534	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;	16142	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
8535		16143
8536	const inst_ty = self.typeOfIndex(inst);	16144	const ptr_agg_ty = self.typeOfIndex(inst);
8537	const parent_ty = inst_ty.childType(zcu);
8538	const field_off: i32 = switch (parent_ty.containerLayout(zcu)) {
8539	.auto, .@"extern" => @intCast(parent_ty.structFieldOffset(extra.field_index, zcu)),
8540	.@"packed" => @divExact(@as(i32, inst_ty.ptrInfo(zcu).packed_offset.bit_offset) +
8541	(if (zcu.typeToStruct(parent_ty)) \|struct_obj\| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0) -
8542	self.typeOf(extra.field_ptr).ptrInfo(zcu).packed_offset.bit_offset, 8),
8543	};
8544
8545	const src_mcv = try self.resolveInst(extra.field_ptr);	16145	const src_mcv = try self.resolveInst(extra.field_ptr);
8546	const dst_mcv = if (src_mcv.isRegisterOffset() and	16146	const dst_mcv = if (src_mcv.isRegisterOffset() and
8547	self.reuseOperand(inst, extra.field_ptr, 0, src_mcv))	16147	self.reuseOperand(inst, extra.field_ptr, 0, src_mcv))
8548	src_mcv	16148	src_mcv
8549	else	16149	else
8550	try self.copyToRegisterWithInstTracking(inst, inst_ty, src_mcv);	16150	try self.copyToRegisterWithInstTracking(inst, ptr_agg_ty, src_mcv);
8551	const result = dst_mcv.offset(-field_off);	16151	const result = dst_mcv.offset(-self.fieldOffset(ptr_agg_ty, self.typeOf(extra.field_ptr), extra.field_index));
8552	return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });	16152	return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
8553	}	16153	}
8554		16154
8555	fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {	16155	fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
8556	const pt = self.pt;	16156	const pt = self.pt;
8557	const zcu = pt.zcu;	16157	const zcu = pt.zcu;
8558	const src_ty = self.typeOf(src_air);	16158	const src_ty = self.typeOf(src_air);
...	@@ -8596,8 +16196,8 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:	...	@@ -8596,8 +16196,8 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
8596	switch (tag) {	16196	switch (tag) {
8597	.not => {	16197	.not => {
8598	const limb_abi_size: u16 = @min(abi_size, 8);	16198	const limb_abi_size: u16 = @min(abi_size, 8);
8599	const int_info = if (src_ty.ip_index == .bool_type)	16199	const int_info: InternPool.Key.IntType = if (src_ty.ip_index == .bool_type)
8600	std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 }	16200	.{ .signedness = .unsigned, .bits = 1 }
8601	else	16201	else
8602	src_ty.intInfo(zcu);	16202	src_ty.intInfo(zcu);
8603	var byte_off: i32 = 0;	16203	var byte_off: i32 = 0;
...	@@ -8613,7 +16213,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:	...	@@ -8613,7 +16213,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
8613	};	16213	};
8614		16214
8615	if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {	16215	if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {
8616	const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_bits);	16216	const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits);
8617	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });	16217	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
8618	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);	16218	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
8619	}	16219	}
...	@@ -8630,9 +16230,9 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:	...	@@ -8630,9 +16230,9 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
8630	defer self.register_manager.unlockReg(tmp_lock);	16230	defer self.register_manager.unlockReg(tmp_lock);
8631		16231
8632	const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();	16232	const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
8633	try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});	16233	try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
8634	try self.truncateRegister(src_ty, tmp_reg);	16234	try self.truncateRegister(src_ty, tmp_reg);
8635	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});	16235	try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
8636	}	16236	}
8637	}	16237	}
8638	},	16238	},
...	@@ -8641,7 +16241,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:	...	@@ -8641,7 +16241,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
8641	return dst_mcv;	16241	return dst_mcv;
8642	}	16242	}
8643		16243
8644	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {	16244	fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
8645	const pt = self.pt;	16245	const pt = self.pt;
8646	const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));	16246	const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
8647	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{ mir_tag, dst_ty.fmt(pt) });	16247	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{ mir_tag, dst_ty.fmt(pt) });
...	@@ -8654,6 +16254,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC	...	@@ -8654,6 +16254,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
8654	.register_offset,	16254	.register_offset,
8655	.eflags,	16255	.eflags,
8656	.register_overflow,	16256	.register_overflow,
		16257	.register_mask,
8657	.lea_symbol,	16258	.lea_symbol,
8658	.lea_direct,	16259	.lea_direct,
8659	.lea_got,	16260	.lea_got,
...	@@ -8664,27 +16265,27 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC	...	@@ -8664,27 +16265,27 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
8664	.air_ref,	16265	.air_ref,
8665	=> unreachable, // unmodifiable destination	16266	=> unreachable, // unmodifiable destination
8666	.register => \|dst_reg\| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),	16267	.register => \|dst_reg\| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
8667	.register_pair => unreachable, // unimplemented	16268	.register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented
8668	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => {	16269	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => {
8669	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	16270	const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
8670	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	16271	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
8671	defer self.register_manager.unlockReg(addr_reg_lock);	16272	defer self.register_manager.unlockReg(addr_reg_lock);
8672		16273
8673	try self.genSetReg(addr_reg, Type.usize, dst_mcv.address(), .{});	16274	try self.genSetReg(addr_reg, .usize, dst_mcv.address(), .{});
8674	try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{	16275	try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
8675	.size = Memory.Size.fromSize(abi_size),	16276	.size = .fromSize(abi_size),
8676	} } });	16277	} } });
8677	},	16278	},
8678	.indirect, .load_frame => try self.asmMemory(	16279	.indirect, .load_frame => try self.asmMemory(
8679	mir_tag,	16280	mir_tag,
8680	try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),	16281	try dst_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
8681	),	16282	),
8682	}	16283	}
8683	}	16284	}
8684		16285
8685	/// Clobbers .rcx for non-immediate shift value.	16286	/// Clobbers .rcx for non-immediate shift value.
8686	fn genShiftBinOpMir(	16287	fn genShiftBinOpMir(
8687	self: *Self,	16288	self: *CodeGen,
8688	tag: Mir.Inst.FixedTag,	16289	tag: Mir.Inst.FixedTag,
8689	lhs_ty: Type,	16290	lhs_ty: Type,
8690	lhs_mcv: MCValue,	16291	lhs_mcv: MCValue,
...	@@ -8698,7 +16299,7 @@ fn genShiftBinOpMir(	...	@@ -8698,7 +16299,7 @@ fn genShiftBinOpMir(
8698	try self.spillEflagsIfOccupied();	16299	try self.spillEflagsIfOccupied();
8699		16300
8700	if (abi_size > 16) {	16301	if (abi_size > 16) {
8701	const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;	16302	const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
8702	assert(shift_abi_size >= 1 and shift_abi_size <= 2);	16303	assert(shift_abi_size >= 1 and shift_abi_size <= 2);
8703		16304
8704	const rcx_lock: ?RegisterLock = switch (rhs_mcv) {	16305	const rcx_lock: ?RegisterLock = switch (rhs_mcv) {
...	@@ -8719,18 +16320,18 @@ fn genShiftBinOpMir(	...	@@ -8719,18 +16320,18 @@ fn genShiftBinOpMir(
8719	};	16320	};
8720	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);	16321	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
8721		16322
8722	const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);	16323	const temp_regs = try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
8723	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);	16324	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
8724	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);	16325	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
8725		16326
8726	switch (tag[0]) {	16327	switch (tag[0]) {
8727	._l => {	16328	._l => {
8728	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), Immediate.u(limbs_len - 1));	16329	try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
8729	switch (rhs_mcv) {	16330	switch (rhs_mcv) {
8730	.immediate => \|shift_imm\| try self.asmRegisterImmediate(	16331	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8731	.{ ._, .mov },	16332	.{ ._, .mov },
8732	temp_regs[0].to32(),	16333	temp_regs[0].to32(),
8733	Immediate.u(limbs_len - (shift_imm >> 6) - 1),	16334	.u(limbs_len - (shift_imm >> 6) - 1),
8734	),	16335	),
8735	else => {	16336	else => {
8736	try self.asmRegisterRegister(	16337	try self.asmRegisterRegister(
...	@@ -8738,16 +16339,8 @@ fn genShiftBinOpMir(	...	@@ -8738,16 +16339,8 @@ fn genShiftBinOpMir(
8738	temp_regs[2].to32(),	16339	temp_regs[2].to32(),
8739	registerAlias(.rcx, shift_abi_size),	16340	registerAlias(.rcx, shift_abi_size),
8740	);	16341	);
8741	try self.asmRegisterImmediate(	16342	try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
8742	.{ ._, .@"and" },	16343	try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[2].to32(), .u(6));
8743	.cl,
8744	Immediate.u(math.maxInt(u6)),
8745	);
8746	try self.asmRegisterImmediate(
8747	.{ ._r, .sh },
8748	temp_regs[2].to32(),
8749	Immediate.u(6),
8750	);
8751	try self.asmRegisterRegister(	16344	try self.asmRegisterRegister(
8752	.{ ._, .mov },	16345	.{ ._, .mov },
8753	temp_regs[0].to32(),	16346	temp_regs[0].to32(),
...	@@ -8767,7 +16360,7 @@ fn genShiftBinOpMir(	...	@@ -8767,7 +16360,7 @@ fn genShiftBinOpMir(
8767	.immediate => \|shift_imm\| try self.asmRegisterImmediate(	16360	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8768	.{ ._, .mov },	16361	.{ ._, .mov },
8769	temp_regs[0].to32(),	16362	temp_regs[0].to32(),
8770	Immediate.u(shift_imm >> 6),	16363	.u(shift_imm >> 6),
8771	),	16364	),
8772	else => {	16365	else => {
8773	try self.asmRegisterRegister(	16366	try self.asmRegisterRegister(
...	@@ -8775,16 +16368,8 @@ fn genShiftBinOpMir(	...	@@ -8775,16 +16368,8 @@ fn genShiftBinOpMir(
8775	temp_regs[0].to32(),	16368	temp_regs[0].to32(),
8776	registerAlias(.rcx, shift_abi_size),	16369	registerAlias(.rcx, shift_abi_size),
8777	);	16370	);
8778	try self.asmRegisterImmediate(	16371	try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6)));
8779	.{ ._, .@"and" },	16372	try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[0].to32(), .u(6));
8780	.cl,
8781	Immediate.u(math.maxInt(u6)),
8782	);
8783	try self.asmRegisterImmediate(
8784	.{ ._r, .sh },
8785	temp_regs[0].to32(),
8786	Immediate.u(6),
8787	);
8788	},	16373	},
8789	}	16374	}
8790	},	16375	},
...	@@ -8813,7 +16398,7 @@ fn genShiftBinOpMir(	...	@@ -8813,7 +16398,7 @@ fn genShiftBinOpMir(
8813	try self.asmRegisterImmediate(	16398	try self.asmRegisterImmediate(
8814	.{ ._, .cmp },	16399	.{ ._, .cmp },
8815	temp_regs[0].to32(),	16400	temp_regs[0].to32(),
8816	Immediate.u(limbs_len - 1),	16401	.u(limbs_len - 1),
8817	);	16402	);
8818	break :skip try self.asmJccReloc(.nb, undefined);	16403	break :skip try self.asmJccReloc(.nb, undefined);
8819	},	16404	},
...	@@ -8843,7 +16428,7 @@ fn genShiftBinOpMir(	...	@@ -8843,7 +16428,7 @@ fn genShiftBinOpMir(
8843	}, .sh },	16428	}, .sh },
8844	temp_regs[2].to64(),	16429	temp_regs[2].to64(),
8845	temp_regs[3].to64(),	16430	temp_regs[3].to64(),
8846	Immediate.u(shift_imm & math.maxInt(u6)),	16431	.u(shift_imm & std.math.maxInt(u6)),
8847	),	16432	),
8848	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {	16433	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
8849	._l => ._ld,	16434	._l => ._ld,
...	@@ -8864,8 +16449,8 @@ fn genShiftBinOpMir(	...	@@ -8864,8 +16449,8 @@ fn genShiftBinOpMir(
8864	switch (tag[0]) {	16449	switch (tag[0]) {
8865	._l => {	16450	._l => {
8866	if (slow_inc_dec) {	16451	if (slow_inc_dec) {
8867	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));	16452	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
8868	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), Immediate.u(1));	16453	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), .u(1));
8869	} else {	16454	} else {
8870	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());	16455	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
8871	try self.asmRegister(.{ ._, .dec }, temp_regs[0].to32());	16456	try self.asmRegister(.{ ._, .dec }, temp_regs[0].to32());
...	@@ -8874,8 +16459,8 @@ fn genShiftBinOpMir(	...	@@ -8874,8 +16459,8 @@ fn genShiftBinOpMir(
8874	},	16459	},
8875	._r => {	16460	._r => {
8876	if (slow_inc_dec) {	16461	if (slow_inc_dec) {
8877	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));	16462	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
8878	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	16463	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
8879	} else {	16464	} else {
8880	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());	16465	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
8881	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	16466	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
...	@@ -8883,7 +16468,7 @@ fn genShiftBinOpMir(	...	@@ -8883,7 +16468,7 @@ fn genShiftBinOpMir(
8883	try self.asmRegisterImmediate(	16468	try self.asmRegisterImmediate(
8884	.{ ._, .cmp },	16469	.{ ._, .cmp },
8885	temp_regs[0].to32(),	16470	temp_regs[0].to32(),
8886	Immediate.u(limbs_len - 1),	16471	.u(limbs_len - 1),
8887	);	16472	);
8888	_ = try self.asmJccReloc(.b, loop);	16473	_ = try self.asmJccReloc(.b, loop);
8889	},	16474	},
...	@@ -8898,7 +16483,7 @@ fn genShiftBinOpMir(	...	@@ -8898,7 +16483,7 @@ fn genShiftBinOpMir(
8898	.immediate => \|shift_imm\| try self.asmRegisterImmediate(	16483	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8899	tag,	16484	tag,
8900	temp_regs[2].to64(),	16485	temp_regs[2].to64(),
8901	Immediate.u(shift_imm & math.maxInt(u6)),	16486	.u(shift_imm & std.math.maxInt(u6)),
8902	),	16487	),
8903	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),	16488	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
8904	}	16489	}
...	@@ -8914,7 +16499,7 @@ fn genShiftBinOpMir(	...	@@ -8914,7 +16499,7 @@ fn genShiftBinOpMir(
8914	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(	16499	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
8915	tag,	16500	tag,
8916	temp_regs[2].to64(),	16501	temp_regs[2].to64(),
8917	Immediate.u(63),	16502	.u(63),
8918	);	16503	);
8919	if (switch (rhs_mcv) {	16504	if (switch (rhs_mcv) {
8920	.immediate => \|shift_imm\| shift_imm >> 6 > 0,	16505	.immediate => \|shift_imm\| shift_imm >> 6 > 0,
...	@@ -8935,7 +16520,7 @@ fn genShiftBinOpMir(	...	@@ -8935,7 +16520,7 @@ fn genShiftBinOpMir(
8935	try self.asmRegisterImmediate(	16520	try self.asmRegisterImmediate(
8936	.{ ._, .cmp },	16521	.{ ._, .cmp },
8937	temp_regs[1].to32(),	16522	temp_regs[1].to32(),
8938	Immediate.u(limbs_len - 1),	16523	.u(limbs_len - 1),
8939	);	16524	);
8940	break :skip try self.asmJccReloc(.nb, undefined);	16525	break :skip try self.asmJccReloc(.nb, undefined);
8941	},	16526	},
...	@@ -8945,12 +16530,12 @@ fn genShiftBinOpMir(	...	@@ -8945,12 +16530,12 @@ fn genShiftBinOpMir(
8945	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	16530	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
8946	switch (tag[0]) {	16531	switch (tag[0]) {
8947	._l => if (slow_inc_dec) {	16532	._l => if (slow_inc_dec) {
8948	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1));	16533	try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
8949	} else {	16534	} else {
8950	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());	16535	try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32());
8951	},	16536	},
8952	._r => if (slow_inc_dec) {	16537	._r => if (slow_inc_dec) {
8953	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1));	16538	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1));
8954	} else {	16539	} else {
8955	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());	16540	try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32());
8956	},	16541	},
...	@@ -8972,14 +16557,14 @@ fn genShiftBinOpMir(	...	@@ -8972,14 +16557,14 @@ fn genShiftBinOpMir(
8972	.scale = .@"8",	16557	.scale = .@"8",
8973	.disp = lhs_mcv.load_frame.off,	16558	.disp = lhs_mcv.load_frame.off,
8974	} },	16559	} },
8975	}, Immediate.u(0));	16560	}, .u(0));
8976	switch (tag[0]) {	16561	switch (tag[0]) {
8977	._l => _ = try self.asmJccReloc(.nz, loop),	16562	._l => _ = try self.asmJccReloc(.nz, loop),
8978	._r => {	16563	._r => {
8979	try self.asmRegisterImmediate(	16564	try self.asmRegisterImmediate(
8980	.{ ._, .cmp },	16565	.{ ._, .cmp },
8981	temp_regs[1].to32(),	16566	temp_regs[1].to32(),
8982	Immediate.u(limbs_len - 1),	16567	.u(limbs_len - 1),
8983	);	16568	);
8984	_ = try self.asmJccReloc(.b, loop);	16569	_ = try self.asmJccReloc(.b, loop);
8985	},	16570	},
...	@@ -9021,12 +16606,12 @@ fn genShiftBinOpMir(	...	@@ -9021,12 +16606,12 @@ fn genShiftBinOpMir(
9021	info.double_tag,	16606	info.double_tag,
9022	lhs_regs[info.indices[1]],	16607	lhs_regs[info.indices[1]],
9023	lhs_regs[info.indices[0]],	16608	lhs_regs[info.indices[0]],
9024	Immediate.u(shift_imm),	16609	.u(shift_imm),
9025	);	16610	);
9026	try self.asmRegisterImmediate(	16611	try self.asmRegisterImmediate(
9027	tag,	16612	tag,
9028	lhs_regs[info.indices[0]],	16613	lhs_regs[info.indices[0]],
9029	Immediate.u(shift_imm),	16614	.u(shift_imm),
9030	);	16615	);
9031	return;	16616	return;
9032	} else {	16617	} else {
...	@@ -9039,7 +16624,7 @@ fn genShiftBinOpMir(	...	@@ -9039,7 +16624,7 @@ fn genShiftBinOpMir(
9039	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(	16624	if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate(
9040	tag,	16625	tag,
9041	lhs_regs[info.indices[0]],	16626	lhs_regs[info.indices[0]],
9042	Immediate.u(63),	16627	.u(63),
9043	) else try self.asmRegisterRegister(	16628	) else try self.asmRegisterRegister(
9044	.{ ._, .xor },	16629	.{ ._, .xor },
9045	lhs_regs[info.indices[0]],	16630	lhs_regs[info.indices[0]],
...	@@ -9048,7 +16633,7 @@ fn genShiftBinOpMir(	...	@@ -9048,7 +16633,7 @@ fn genShiftBinOpMir(
9048	if (shift_imm > 64) try self.asmRegisterImmediate(	16633	if (shift_imm > 64) try self.asmRegisterImmediate(
9049	tag,	16634	tag,
9050	lhs_regs[info.indices[1]],	16635	lhs_regs[info.indices[1]],
9051	Immediate.u(shift_imm - 64),	16636	.u(shift_imm - 64),
9052	);	16637	);
9053	return;	16638	return;
9054	},	16639	},
...	@@ -9059,7 +16644,7 @@ fn genShiftBinOpMir(	...	@@ -9059,7 +16644,7 @@ fn genShiftBinOpMir(
9059		16644
9060	if (tag[0] == ._r and tag[1] == .sa) {	16645	if (tag[0] == ._r and tag[1] == .sa) {
9061	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]);	16646	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]);
9062	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));	16647	try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
9063	} else try self.asmRegisterRegister(	16648	} else try self.asmRegisterRegister(
9064	.{ ._, .xor },	16649	.{ ._, .xor },
9065	tmp_reg.to32(),	16650	tmp_reg.to32(),
...	@@ -9076,11 +16661,7 @@ fn genShiftBinOpMir(	...	@@ -9076,11 +16661,7 @@ fn genShiftBinOpMir(
9076	lhs_regs[info.indices[0]],	16661	lhs_regs[info.indices[0]],
9077	registerAlias(shift_reg, 1),	16662	registerAlias(shift_reg, 1),
9078	);	16663	);
9079	try self.asmRegisterImmediate(	16664	try self.asmRegisterImmediate(.{ ._, .cmp }, registerAlias(shift_reg, 1), .u(64));
9080	.{ ._, .cmp },
9081	registerAlias(shift_reg, 1),
9082	Immediate.u(64),
9083	);
9084	try self.asmCmovccRegisterRegister(	16665	try self.asmCmovccRegisterRegister(
9085	.ae,	16666	.ae,
9086	lhs_regs[info.indices[1]],	16667	lhs_regs[info.indices[1]],
...	@@ -9119,7 +16700,7 @@ fn genShiftBinOpMir(	...	@@ -9119,7 +16700,7 @@ fn genShiftBinOpMir(
9119	} },	16700	} },
9120	},	16701	},
9121	tmp_reg,	16702	tmp_reg,
9122	Immediate.u(shift_imm),	16703	.u(shift_imm),
9123	);	16704	);
9124	try self.asmMemoryImmediate(	16705	try self.asmMemoryImmediate(
9125	tag,	16706	tag,
...	@@ -9130,7 +16711,7 @@ fn genShiftBinOpMir(	...	@@ -9130,7 +16711,7 @@ fn genShiftBinOpMir(
9130	.disp = dst_frame_addr.off + info.indices[0] * 8,	16711	.disp = dst_frame_addr.off + info.indices[0] * 8,
9131	} },	16712	} },
9132	},	16713	},
9133	Immediate.u(shift_imm),	16714	.u(shift_imm),
9134	);	16715	);
9135	return;	16716	return;
9136	} else {	16717	} else {
...	@@ -9149,7 +16730,7 @@ fn genShiftBinOpMir(	...	@@ -9149,7 +16730,7 @@ fn genShiftBinOpMir(
9149	if (shift_imm > 64) try self.asmRegisterImmediate(	16730	if (shift_imm > 64) try self.asmRegisterImmediate(
9150	tag,	16731	tag,
9151	tmp_reg,	16732	tmp_reg,
9152	Immediate.u(shift_imm - 64),	16733	.u(shift_imm - 64),
9153	);	16734	);
9154	try self.asmMemoryRegister(	16735	try self.asmMemoryRegister(
9155	.{ ._, .mov },	16736	.{ ._, .mov },
...	@@ -9171,7 +16752,7 @@ fn genShiftBinOpMir(	...	@@ -9171,7 +16752,7 @@ fn genShiftBinOpMir(
9171	.disp = dst_frame_addr.off + info.indices[0] * 8,	16752	.disp = dst_frame_addr.off + info.indices[0] * 8,
9172	} },	16753	} },
9173	},	16754	},
9174	Immediate.u(63),	16755	.u(63),
9175	) else {	16756	) else {
9176	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());	16757	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
9177	try self.asmMemoryRegister(	16758	try self.asmMemoryRegister(
...	@@ -9223,7 +16804,7 @@ fn genShiftBinOpMir(	...	@@ -9223,7 +16804,7 @@ fn genShiftBinOpMir(
9223	);	16804	);
9224	if (tag[0] == ._r and tag[1] == .sa) {	16805	if (tag[0] == ._r and tag[1] == .sa) {
9225	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);	16806	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
9226	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));	16807	try self.asmRegisterImmediate(tag, tmp_reg, .u(63));
9227	} else try self.asmRegisterRegister(	16808	} else try self.asmRegisterRegister(
9228	.{ ._, .xor },	16809	.{ ._, .xor },
9229	tmp_reg.to32(),	16810	tmp_reg.to32(),
...	@@ -9239,7 +16820,7 @@ fn genShiftBinOpMir(	...	@@ -9239,7 +16820,7 @@ fn genShiftBinOpMir(
9239	try self.asmRegisterImmediate(	16820	try self.asmRegisterImmediate(
9240	.{ ._, .cmp },	16821	.{ ._, .cmp },
9241	registerAlias(shift_reg, 1),	16822	registerAlias(shift_reg, 1),
9242	Immediate.u(64),	16823	.u(64),
9243	);	16824	);
9244	try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg);	16825	try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg);
9245	try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);	16826	try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);
...	@@ -9277,7 +16858,7 @@ fn genShiftBinOpMir(	...	@@ -9277,7 +16858,7 @@ fn genShiftBinOpMir(
9277	.immediate => \|shift_imm\| return self.asmRegisterImmediate(	16858	.immediate => \|shift_imm\| return self.asmRegisterImmediate(
9278	tag,	16859	tag,
9279	registerAlias(lhs_reg, abi_size),	16860	registerAlias(lhs_reg, abi_size),
9280	Immediate.u(shift_imm),	16861	.u(shift_imm),
9281	),	16862	),
9282	.register => \|shift_reg\| return self.asmRegisterRegister(	16863	.register => \|shift_reg\| return self.asmRegisterRegister(
9283	tag,	16864	tag,
...	@@ -9291,8 +16872,8 @@ fn genShiftBinOpMir(	...	@@ -9291,8 +16872,8 @@ fn genShiftBinOpMir(
9291	.memory => \|addr\| .{	16872	.memory => \|addr\| .{
9292	.base = .{ .reg = .ds },	16873	.base = .{ .reg = .ds },
9293	.mod = .{ .rm = .{	16874	.mod = .{ .rm = .{
9294	.size = Memory.Size.fromSize(abi_size),	16875	.size = .fromSize(abi_size),
9295	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse	16876	.disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
9296	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{	16877	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
9297	@tagName(lhs_mcv),	16878	@tagName(lhs_mcv),
9298	@tagName(shift_mcv),	16879	@tagName(shift_mcv),
...	@@ -9302,25 +16883,21 @@ fn genShiftBinOpMir(	...	@@ -9302,25 +16883,21 @@ fn genShiftBinOpMir(
9302	.indirect => \|reg_off\| .{	16883	.indirect => \|reg_off\| .{
9303	.base = .{ .reg = reg_off.reg },	16884	.base = .{ .reg = reg_off.reg },
9304	.mod = .{ .rm = .{	16885	.mod = .{ .rm = .{
9305	.size = Memory.Size.fromSize(abi_size),	16886	.size = .fromSize(abi_size),
9306	.disp = reg_off.off,	16887	.disp = reg_off.off,
9307	} },	16888	} },
9308	},	16889	},
9309	.load_frame => \|frame_addr\| .{	16890	.load_frame => \|frame_addr\| .{
9310	.base = .{ .frame = frame_addr.index },	16891	.base = .{ .frame = frame_addr.index },
9311	.mod = .{ .rm = .{	16892	.mod = .{ .rm = .{
9312	.size = Memory.Size.fromSize(abi_size),	16893	.size = .fromSize(abi_size),
9313	.disp = frame_addr.off,	16894	.disp = frame_addr.off,
9314	} },	16895	} },
9315	},	16896	},
9316	else => unreachable,	16897	else => unreachable,
9317	};	16898	};
9318	switch (shift_mcv) {	16899	switch (shift_mcv) {
9319	.immediate => \|shift_imm\| return self.asmMemoryImmediate(	16900	.immediate => \|shift_imm\| return self.asmMemoryImmediate(tag, lhs_mem, .u(shift_imm)),
9320	tag,
9321	lhs_mem,
9322	Immediate.u(shift_imm),
9323	),
9324	.register => \|shift_reg\| return self.asmMemoryRegister(	16901	.register => \|shift_reg\| return self.asmMemoryRegister(
9325	tag,	16902	tag,
9326	lhs_mem,	16903	lhs_mem,
...	@@ -9341,7 +16918,7 @@ fn genShiftBinOpMir(	...	@@ -9341,7 +16918,7 @@ fn genShiftBinOpMir(
9341	/// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront.	16918	/// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront.
9342	/// Asserts .rcx is free.	16919	/// Asserts .rcx is free.
9343	fn genShiftBinOp(	16920	fn genShiftBinOp(
9344	self: *Self,	16921	self: *CodeGen,
9345	air_tag: Air.Inst.Tag,	16922	air_tag: Air.Inst.Tag,
9346	maybe_inst: ?Air.Inst.Index,	16923	maybe_inst: ?Air.Inst.Index,
9347	lhs_mcv: MCValue,	16924	lhs_mcv: MCValue,
...	@@ -9359,7 +16936,18 @@ fn genShiftBinOp(	...	@@ -9359,7 +16936,18 @@ fn genShiftBinOp(
9359	const rcx_lock = self.register_manager.lockReg(.rcx);	16936	const rcx_lock = self.register_manager.lockReg(.rcx);
9360	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);	16937	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
9361		16938
9362	const lhs_lock = switch (lhs_mcv) {	16939	const mat_lhs_mcv: MCValue, const can_reuse_lhs = switch (lhs_mcv) {
		16940	.register => \|lhs_reg\| switch (lhs_reg.class()) {
		16941	.general_purpose => .{ lhs_mcv, true },
		16942	else => lhs: {
		16943	const mat_lhs_mcv = try self.allocTempRegOrMem(lhs_ty, true);
		16944	try self.genCopy(lhs_ty, mat_lhs_mcv, lhs_mcv, .{});
		16945	break :lhs .{ mat_lhs_mcv, false };
		16946	},
		16947	},
		16948	else => .{ lhs_mcv, true },
		16949	};
		16950	const lhs_lock = switch (mat_lhs_mcv) {
9363	.register => \|reg\| self.register_manager.lockReg(reg),	16951	.register => \|reg\| self.register_manager.lockReg(reg),
9364	else => null,	16952	else => null,
9365	};	16953	};
...	@@ -9372,12 +16960,12 @@ fn genShiftBinOp(	...	@@ -9372,12 +16960,12 @@ fn genShiftBinOp(
9372	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	16960	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
9373		16961
9374	const dst_mcv: MCValue = dst: {	16962	const dst_mcv: MCValue = dst: {
9375	if (maybe_inst) \|inst\| {	16963	if (can_reuse_lhs) if (maybe_inst) \|inst\| {
9376	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	16964	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9377	if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;	16965	if (self.reuseOperand(inst, bin_op.lhs, 0, mat_lhs_mcv)) break :dst mat_lhs_mcv;
9378	}	16966	};
9379	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);	16967	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
9380	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{});	16968	try self.genCopy(lhs_ty, dst_mcv, mat_lhs_mcv, .{});
9381	break :dst dst_mcv;	16969	break :dst dst_mcv;
9382	};	16970	};
9383		16971
...	@@ -9400,7 +16988,7 @@ fn genShiftBinOp(	...	@@ -9400,7 +16988,7 @@ fn genShiftBinOp(
9400	/// Clobbers .rax and .rdx therefore care is needed to spill .rax and .rdx upfront.	16988	/// Clobbers .rax and .rdx therefore care is needed to spill .rax and .rdx upfront.
9401	/// Asserts .rax and .rdx are free.	16989	/// Asserts .rax and .rdx are free.
9402	fn genMulDivBinOp(	16990	fn genMulDivBinOp(
9403	self: *Self,	16991	self: *CodeGen,
9404	tag: Air.Inst.Tag,	16992	tag: Air.Inst.Tag,
9405	maybe_inst: ?Air.Inst.Index,	16993	maybe_inst: ?Air.Inst.Index,
9406	dst_ty: Type,	16994	dst_ty: Type,
...	@@ -9430,7 +17018,7 @@ fn genMulDivBinOp(	...	@@ -9430,7 +17018,7 @@ fn genMulDivBinOp(
9430	const mat_lhs_mcv = switch (lhs_mcv) {	17018	const mat_lhs_mcv = switch (lhs_mcv) {
9431	.load_symbol => mat_lhs_mcv: {	17019	.load_symbol => mat_lhs_mcv: {
9432	// TODO clean this up!	17020	// TODO clean this up!
9433	const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());	17021	const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
9434	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	17022	break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
9435	},	17023	},
9436	else => lhs_mcv,	17024	else => lhs_mcv,
...	@@ -9443,7 +17031,7 @@ fn genMulDivBinOp(	...	@@ -9443,7 +17031,7 @@ fn genMulDivBinOp(
9443	const mat_rhs_mcv = switch (rhs_mcv) {	17031	const mat_rhs_mcv = switch (rhs_mcv) {
9444	.load_symbol => mat_rhs_mcv: {	17032	.load_symbol => mat_rhs_mcv: {
9445	// TODO clean this up!	17033	// TODO clean this up!
9446	const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());	17034	const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
9447	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };	17035	break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
9448	},	17036	},
9449	else => rhs_mcv,	17037	else => rhs_mcv,
...	@@ -9458,28 +17046,28 @@ fn genMulDivBinOp(	...	@@ -9458,28 +17046,28 @@ fn genMulDivBinOp(
9458	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	17046	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
9459	defer self.register_manager.unlockReg(tmp_lock);	17047	defer self.register_manager.unlockReg(tmp_lock);
9460		17048
9461	if (mat_lhs_mcv.isMemory())	17049	if (mat_lhs_mcv.isBase())
9462	try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))	17050	try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .{ .size = .qword }))
9463	else	17051	else
9464	try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);	17052	try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
9465	if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(	17053	if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
9466	.{ ._, .mov },	17054	.{ ._, .mov },
9467	tmp_reg,	17055	tmp_reg,
9468	try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),	17056	try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
9469	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);	17057	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
9470	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);	17058	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
9471	if (mat_rhs_mcv.isMemory())	17059	if (mat_rhs_mcv.isBase())
9472	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))	17060	try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
9473	else	17061	else
9474	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);	17062	try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
9475	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);	17063	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
9476	if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(	17064	if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
9477	.{ ._, .mov },	17065	.{ ._, .mov },
9478	tmp_reg,	17066	tmp_reg,
9479	try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),	17067	try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
9480	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);	17068	) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
9481	if (mat_rhs_mcv.isMemory())	17069	if (mat_rhs_mcv.isBase())
9482	try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))	17070	try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
9483	else	17071	else
9484	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);	17072	try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
9485	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);	17073	try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
...	@@ -9495,7 +17083,7 @@ fn genMulDivBinOp(	...	@@ -9495,7 +17083,7 @@ fn genMulDivBinOp(
9495	switch (tag) {	17083	switch (tag) {
9496	.mul, .mul_wrap => {	17084	.mul, .mul_wrap => {
9497	const slow_inc = self.hasFeature(.slow_incdec);	17085	const slow_inc = self.hasFeature(.slow_incdec);
9498	const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;	17086	const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;
9499		17087
9500	try self.spillRegisters(&.{ .rax, .rcx, .rdx });	17088	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
9501	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });	17089	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
...	@@ -9511,7 +17099,7 @@ fn genMulDivBinOp(	...	@@ -9511,7 +17099,7 @@ fn genMulDivBinOp(
9511	);	17099	);
9512		17100
9513	const temp_regs =	17101	const temp_regs =
9514	try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);	17102	try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
9515	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);	17103	const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
9516	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);	17104	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
9517		17105
...	@@ -9536,7 +17124,7 @@ fn genMulDivBinOp(	...	@@ -9536,7 +17124,7 @@ fn genMulDivBinOp(
9536	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);	17124	try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
9537		17125
9538	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	17126	const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
9539	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1));	17127	try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
9540	try self.asmMemoryRegister(.{ ._, .adc }, .{	17128	try self.asmMemoryRegister(.{ ._, .adc }, .{
9541	.base = .{ .frame = dst_mcv.load_frame.index },	17129	.base = .{ .frame = dst_mcv.load_frame.index },
9542	.mod = .{ .rm = .{	17130	.mod = .{ .rm = .{
...	@@ -9559,7 +17147,7 @@ fn genMulDivBinOp(	...	@@ -9559,7 +17147,7 @@ fn genMulDivBinOp(
9559	});	17147	});
9560	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());	17148	try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
9561		17149
9562	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1));	17150	try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
9563	try self.asmMemoryRegister(.{ ._, .adc }, .{	17151	try self.asmMemoryRegister(.{ ._, .adc }, .{
9564	.base = .{ .frame = dst_mcv.load_frame.index },	17152	.base = .{ .frame = dst_mcv.load_frame.index },
9565	.mod = .{ .rm = .{	17153	.mod = .{ .rm = .{
...	@@ -9572,30 +17160,22 @@ fn genMulDivBinOp(	...	@@ -9572,30 +17160,22 @@ fn genMulDivBinOp(
9572	try self.asmSetccRegister(.c, .ch);	17160	try self.asmSetccRegister(.c, .ch);
9573		17161
9574	if (slow_inc) {	17162	if (slow_inc) {
9575	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1));	17163	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
9576	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1));	17164	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
9577	} else {	17165	} else {
9578	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());	17166	try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32());
9579	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());	17167	try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32());
9580	}	17168	}
9581	try self.asmRegisterImmediate(	17169	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
9582	.{ ._, .cmp },
9583	temp_regs[3].to32(),
9584	Immediate.u(limb_len),
9585	);
9586	_ = try self.asmJccReloc(.b, inner_loop);	17170	_ = try self.asmJccReloc(.b, inner_loop);
9587		17171
9588	self.performReloc(skip_inner);	17172	self.performReloc(skip_inner);
9589	if (slow_inc) {	17173	if (slow_inc) {
9590	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1));	17174	try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
9591	} else {	17175	} else {
9592	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());	17176	try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32());
9593	}	17177	}
9594	try self.asmRegisterImmediate(	17178	try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
9595	.{ ._, .cmp },
9596	temp_regs[0].to32(),
9597	Immediate.u(limb_len),
9598	);
9599	_ = try self.asmJccReloc(.b, outer_loop);	17179	_ = try self.asmJccReloc(.b, outer_loop);
9600		17180
9601	return dst_mcv;	17181	return dst_mcv;
...	@@ -9639,13 +17219,13 @@ fn genMulDivBinOp(	...	@@ -9639,13 +17219,13 @@ fn genMulDivBinOp(
9639	manyptr_u32_ty,	17219	manyptr_u32_ty,
9640	manyptr_const_u32_ty,	17220	manyptr_const_u32_ty,
9641	manyptr_const_u32_ty,	17221	manyptr_const_u32_ty,
9642	Type.usize,	17222	.usize,
9643	}, &.{	17223	}, &.{
9644	dst_mcv.address(),	17224	dst_mcv.address(),
9645	lhs_mcv.address(),	17225	lhs_mcv.address(),
9646	rhs_mcv.address(),	17226	rhs_mcv.address(),
9647	.{ .immediate = src_info.bits },	17227	.{ .immediate = 8 * src_abi_size },
9648	});	17228	}, .{});
9649	return dst_mcv;	17229	return dst_mcv;
9650	},	17230	},
9651	},	17231	},
...	@@ -9662,7 +17242,8 @@ fn genMulDivBinOp(	...	@@ -9662,7 +17242,8 @@ fn genMulDivBinOp(
9662	const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });	17242	const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
9663	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);	17243	defer for (reg_locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
9664		17244
9665	const signedness = ty.intInfo(zcu).signedness;	17245	const int_info = ty.intInfo(zcu);
		17246	const signedness = int_info.signedness;
9666	switch (tag) {	17247	switch (tag) {
9667	.mul,	17248	.mul,
9668	.mul_wrap,	17249	.mul_wrap,
...	@@ -9695,6 +17276,15 @@ fn genMulDivBinOp(	...	@@ -9695,6 +17276,15 @@ fn genMulDivBinOp(
9695	},	17276	},
9696	}, ty, lhs_mcv, rhs_mcv);	17277	}, ty, lhs_mcv, rhs_mcv);
9697		17278
		17279	switch (tag) {
		17280	.mul, .rem, .div_trunc, .div_exact => {},
		17281	.mul_wrap => if (dst_ty.intInfo(zcu).bits < 8 * dst_abi_size) try self.truncateRegister(
		17282	dst_ty,
		17283	if (dst_abi_size <= 8) .rax else .rdx,
		17284	),
		17285	else => unreachable,
		17286	}
		17287
9698	if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {	17288	if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {
9699	.mul, .mul_wrap, .div_trunc, .div_exact => .rax,	17289	.mul, .mul_wrap, .div_trunc, .div_exact => .rax,
9700	.rem => .rdx,	17290	.rem => .rdx,
...	@@ -9814,7 +17404,7 @@ fn genMulDivBinOp(	...	@@ -9814,7 +17404,7 @@ fn genMulDivBinOp(
9814	}	17404	}
9815		17405
9816	fn genBinOp(	17406	fn genBinOp(
9817	self: *Self,	17407	self: *CodeGen,
9818	maybe_inst: ?Air.Inst.Index,	17408	maybe_inst: ?Air.Inst.Index,
9819	air_tag: Air.Inst.Tag,	17409	air_tag: Air.Inst.Tag,
9820	lhs_air: Air.Inst.Ref,	17410	lhs_air: Air.Inst.Ref,
...	@@ -9869,7 +17459,7 @@ fn genBinOp(	...	@@ -9869,7 +17459,7 @@ fn genBinOp(
9869	.return_type = lhs_ty.toIntern(),	17459	.return_type = lhs_ty.toIntern(),
9870	.param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },	17460	.param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },
9871	.callee = callee,	17461	.callee = callee,
9872	} }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } });	17462	} }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } }, .{});
9873	return switch (air_tag) {	17463	return switch (air_tag) {
9874	.mod => result: {	17464	.mod => result: {
9875	const adjusted: MCValue = if (type_needs_libcall) adjusted: {	17465	const adjusted: MCValue = if (type_needs_libcall) adjusted: {
...	@@ -9883,7 +17473,7 @@ fn genBinOp(	...	@@ -9883,7 +17473,7 @@ fn genBinOp(
9883	.callee = std.fmt.bufPrint(&add_callee_buf, "__add{c}f3", .{	17473	.callee = std.fmt.bufPrint(&add_callee_buf, "__add{c}f3", .{
9884	floatCompilerRtAbiName(float_bits),	17474	floatCompilerRtAbiName(float_bits),
9885	}) catch unreachable,	17475	}) catch unreachable,
9886	} }, &.{ lhs_ty, rhs_ty }, &.{ result, .{ .air_ref = rhs_air } });	17476	} }, &.{ lhs_ty, rhs_ty }, &.{ result, .{ .air_ref = rhs_air } }, .{});
9887	} else switch (float_bits) {	17477	} else switch (float_bits) {
9888	16, 32, 64 => adjusted: {	17478	16, 32, 64 => adjusted: {
9889	const dst_reg = switch (result) {	17479	const dst_reg = switch (result) {
...	@@ -9906,12 +17496,12 @@ fn genBinOp(	...	@@ -9906,12 +17496,12 @@ fn genBinOp(
9906	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	17496	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
9907	defer self.register_manager.unlockReg(tmp_lock);	17497	defer self.register_manager.unlockReg(tmp_lock);
9908		17498
9909	if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	17499	if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
9910	.{ .vp_w, .insr },	17500	.{ .vp_w, .insr },
9911	dst_reg,	17501	dst_reg,
9912	dst_reg,	17502	dst_reg,
9913	try rhs_mcv.mem(self, .word),	17503	try rhs_mcv.mem(self, .{ .size = .word }),
9914	Immediate.u(1),	17504	.u(1),
9915	) else try self.asmRegisterRegisterRegister(	17505	) else try self.asmRegisterRegisterRegister(
9916	.{ .vp_, .unpcklwd },	17506	.{ .vp_, .unpcklwd },
9917	dst_reg,	17507	dst_reg,
...	@@ -9931,11 +17521,11 @@ fn genBinOp(	...	@@ -9931,11 +17521,11 @@ fn genBinOp(
9931	64 => .{ .v_sd, .add },	17521	64 => .{ .v_sd, .add },
9932	else => unreachable,	17522	else => unreachable,
9933	};	17523	};
9934	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(	17524	if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
9935	mir_tag,	17525	mir_tag,
9936	dst_reg,	17526	dst_reg,
9937	dst_reg,	17527	dst_reg,
9938	try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),	17528	try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
9939	) else try self.asmRegisterRegisterRegister(	17529	) else try self.asmRegisterRegisterRegister(
9940	mir_tag,	17530	mir_tag,
9941	dst_reg,	17531	dst_reg,
...	@@ -9951,10 +17541,10 @@ fn genBinOp(	...	@@ -9951,10 +17541,10 @@ fn genBinOp(
9951	64 => .{ ._sd, .add },	17541	64 => .{ ._sd, .add },
9952	else => unreachable,	17542	else => unreachable,
9953	};	17543	};
9954	if (src_mcv.isMemory()) try self.asmRegisterMemory(	17544	if (src_mcv.isBase()) try self.asmRegisterMemory(
9955	mir_tag,	17545	mir_tag,
9956	dst_reg,	17546	dst_reg,
9957	try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),	17547	try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
9958	) else try self.asmRegisterRegister(	17548	) else try self.asmRegisterRegister(
9959	mir_tag,	17549	mir_tag,
9960	dst_reg,	17550	dst_reg,
...	@@ -9969,7 +17559,7 @@ fn genBinOp(	...	@@ -9969,7 +17559,7 @@ fn genBinOp(
9969	.{ .v_, .cvtps2ph },	17559	.{ .v_, .cvtps2ph },
9970	dst_reg,	17560	dst_reg,
9971	dst_reg,	17561	dst_reg,
9972	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	17562	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
9973	);	17563	);
9974	break :adjusted .{ .register = dst_reg };	17564	break :adjusted .{ .register = dst_reg };
9975	},	17565	},
...	@@ -9982,7 +17572,7 @@ fn genBinOp(	...	@@ -9982,7 +17572,7 @@ fn genBinOp(
9982	.return_type = lhs_ty.toIntern(),	17572	.return_type = lhs_ty.toIntern(),
9983	.param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },	17573	.param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },
9984	.callee = callee,	17574	.callee = callee,
9985	} }, &.{ lhs_ty, rhs_ty }, &.{ adjusted, .{ .air_ref = rhs_air } });	17575	} }, &.{ lhs_ty, rhs_ty }, &.{ adjusted, .{ .air_ref = rhs_air } }, .{});
9986	},	17576	},
9987	.div_trunc, .div_floor => try self.genRoundLibcall(lhs_ty, result, .{	17577	.div_trunc, .div_floor => try self.genRoundLibcall(lhs_ty, result, .{
9988	.mode = switch (air_tag) {	17578	.mode = switch (air_tag) {
...	@@ -10006,7 +17596,7 @@ fn genBinOp(	...	@@ -10006,7 +17596,7 @@ fn genBinOp(
10006	};	17596	};
10007	if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and	17597	if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and
10008	lhs_ty.scalarType(zcu).floatBits(self.target.*) == 80) or	17598	lhs_ty.scalarType(zcu).floatBits(self.target.*) == 80) or
10009	lhs_ty.abiSize(zcu) > @as(u6, if (self.hasFeature(.avx)) 32 else 16)))	17599	lhs_ty.abiSize(zcu) > self.vectorSize(.float)))
10010	return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(pt) });	17600	return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(pt) });
10011		17601
10012	const maybe_mask_reg = switch (air_tag) {	17602	const maybe_mask_reg = switch (air_tag) {
...	@@ -10085,14 +17675,14 @@ fn genBinOp(	...	@@ -10085,14 +17675,14 @@ fn genBinOp(
10085	const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);	17675	const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
10086	break :locks .{ locks[0], locks[1] };	17676	break :locks .{ locks[0], locks[1] };
10087	},	17677	},
10088	else => .{null} ** 2,	17678	else => @splat(null),
10089	};	17679	};
10090	defer for (lhs_locks) \|lhs_lock\| if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);	17680	defer for (lhs_locks) \|lhs_lock\| if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
10091		17681
10092	const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {	17682	const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
10093	.register => \|rhs_reg\| .{ self.register_manager.lockReg(rhs_reg), null },	17683	.register => \|rhs_reg\| .{ self.register_manager.lockReg(rhs_reg), null },
10094	.register_pair => \|rhs_regs\| self.register_manager.lockRegs(2, rhs_regs),	17684	.register_pair => \|rhs_regs\| self.register_manager.lockRegs(2, rhs_regs),
10095	else => .{null} ** 2,	17685	else => @splat(null),
10096	};	17686	};
10097	defer for (rhs_locks) \|rhs_lock\| if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	17687	defer for (rhs_locks) \|rhs_lock\| if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
10098		17688
...	@@ -10125,7 +17715,7 @@ fn genBinOp(	...	@@ -10125,7 +17715,7 @@ fn genBinOp(
10125	const dst_locks: [2]?RegisterLock = switch (dst_mcv) {	17715	const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
10126	.register => \|dst_reg\| .{ self.register_manager.lockReg(dst_reg), null },	17716	.register => \|dst_reg\| .{ self.register_manager.lockReg(dst_reg), null },
10127	.register_pair => \|dst_regs\| self.register_manager.lockRegs(2, dst_regs),	17717	.register_pair => \|dst_regs\| self.register_manager.lockRegs(2, dst_regs),
10128	else => .{null} ** 2,	17718	else => @splat(null),
10129	};	17719	};
10130	defer for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	17720	defer for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
10131		17721
...	@@ -10141,7 +17731,7 @@ fn genBinOp(	...	@@ -10141,7 +17731,7 @@ fn genBinOp(
10141	const src_locks: [2]?RegisterLock = switch (src_mcv) {	17731	const src_locks: [2]?RegisterLock = switch (src_mcv) {
10142	.register => \|src_reg\| .{ self.register_manager.lockReg(src_reg), null },	17732	.register => \|src_reg\| .{ self.register_manager.lockReg(src_reg), null },
10143	.register_pair => \|src_regs\| self.register_manager.lockRegs(2, src_regs),	17733	.register_pair => \|src_regs\| self.register_manager.lockRegs(2, src_regs),
10144	else => .{null} ** 2,	17734	else => @splat(null),
10145	};	17735	};
10146	defer for (src_locks) \|src_lock\| if (src_lock) \|lock\| self.register_manager.unlockReg(lock);	17736	defer for (src_locks) \|src_lock\| if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
10147		17737
...	@@ -10199,11 +17789,7 @@ fn genBinOp(	...	@@ -10199,11 +17789,7 @@ fn genBinOp(
10199	const dst_regs = switch (dst_mcv) {	17789	const dst_regs = switch (dst_mcv) {
10200	.register_pair => \|dst_regs\| dst_regs,	17790	.register_pair => \|dst_regs\| dst_regs,
10201	else => dst: {	17791	else => dst: {
10202	const dst_regs = try self.register_manager.allocRegs(	17792	const dst_regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
10203	2,
10204	.{null} ** 2,
10205	abi.RegisterClass.gp,
10206	);
10207	const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);	17793	const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
10208	defer for (dst_regs_locks) \|lock\| self.register_manager.unlockReg(lock);	17794	defer for (dst_regs_locks) \|lock\| self.register_manager.unlockReg(lock);
10209		17795
...	@@ -10227,26 +17813,26 @@ fn genBinOp(	...	@@ -10227,26 +17813,26 @@ fn genBinOp(
10227	};	17813	};
10228		17814
10229	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);	17815	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
10230	if (src_mcv.isMemory()) {	17816	if (src_mcv.isBase()) {
10231	try self.asmRegisterMemory(	17817	try self.asmRegisterMemory(
10232	.{ ._, .cmp },	17818	.{ ._, .cmp },
10233	dst_regs[0],	17819	dst_regs[0],
10234	try src_mcv.mem(self, .qword),	17820	try src_mcv.mem(self, .{ .size = .qword }),
10235	);	17821	);
10236	try self.asmRegisterMemory(	17822	try self.asmRegisterMemory(
10237	.{ ._, .sbb },	17823	.{ ._, .sbb },
10238	tmp_reg,	17824	tmp_reg,
10239	try src_mcv.address().offset(8).deref().mem(self, .qword),	17825	try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
10240	);	17826	);
10241	try self.asmCmovccRegisterMemory(	17827	try self.asmCmovccRegisterMemory(
10242	cc,	17828	cc,
10243	dst_regs[0],	17829	dst_regs[0],
10244	try src_mcv.mem(self, .qword),	17830	try src_mcv.mem(self, .{ .size = .qword }),
10245	);	17831	);
10246	try self.asmCmovccRegisterMemory(	17832	try self.asmCmovccRegisterMemory(
10247	cc,	17833	cc,
10248	dst_regs[1],	17834	dst_regs[1],
10249	try src_mcv.address().offset(8).deref().mem(self, .qword),	17835	try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
10250	);	17836	);
10251	} else {	17837	} else {
10252	try self.asmRegisterRegister(	17838	try self.asmRegisterRegister(
...	@@ -10278,7 +17864,7 @@ fn genBinOp(	...	@@ -10278,7 +17864,7 @@ fn genBinOp(
10278	.lea_tlv,	17864	.lea_tlv,
10279	.lea_frame,	17865	.lea_frame,
10280	=> true,	17866	=> true,
10281	.memory => \|addr\| math.cast(i32, @as(i64, @bitCast(addr))) == null,	17867	.memory => \|addr\| std.math.cast(i32, @as(i64, @bitCast(addr))) == null,
10282	else => false,	17868	else => false,
10283	.register_pair,	17869	.register_pair,
10284	.register_overflow,	17870	.register_overflow,
...	@@ -10324,8 +17910,11 @@ fn genBinOp(	...	@@ -10324,8 +17910,11 @@ fn genBinOp(
10324	.immediate,	17910	.immediate,
10325	.eflags,	17911	.eflags,
10326	.register_pair,	17912	.register_pair,
		17913	.register_triple,
		17914	.register_quadruple,
10327	.register_offset,	17915	.register_offset,
10328	.register_overflow,	17916	.register_overflow,
		17917	.register_mask,
10329	.load_symbol,	17918	.load_symbol,
10330	.lea_symbol,	17919	.lea_symbol,
10331	.load_direct,	17920	.load_direct,
...	@@ -10351,21 +17940,21 @@ fn genBinOp(	...	@@ -10351,21 +17940,21 @@ fn genBinOp(
10351	.memory => \|addr\| .{	17940	.memory => \|addr\| .{
10352	.base = .{ .reg = .ds },	17941	.base = .{ .reg = .ds },
10353	.mod = .{ .rm = .{	17942	.mod = .{ .rm = .{
10354	.size = Memory.Size.fromSize(cmov_abi_size),	17943	.size = .fromSize(cmov_abi_size),
10355	.disp = @intCast(@as(i64, @bitCast(addr))),	17944	.disp = @intCast(@as(i64, @bitCast(addr))),
10356	} },	17945	} },
10357	},	17946	},
10358	.indirect => \|reg_off\| .{	17947	.indirect => \|reg_off\| .{
10359	.base = .{ .reg = reg_off.reg },	17948	.base = .{ .reg = reg_off.reg },
10360	.mod = .{ .rm = .{	17949	.mod = .{ .rm = .{
10361	.size = Memory.Size.fromSize(cmov_abi_size),	17950	.size = .fromSize(cmov_abi_size),
10362	.disp = reg_off.off,	17951	.disp = reg_off.off,
10363	} },	17952	} },
10364	},	17953	},
10365	.load_frame => \|frame_addr\| .{	17954	.load_frame => \|frame_addr\| .{
10366	.base = .{ .frame = frame_addr.index },	17955	.base = .{ .frame = frame_addr.index },
10367	.mod = .{ .rm = .{	17956	.mod = .{ .rm = .{
10368	.size = Memory.Size.fromSize(cmov_abi_size),	17957	.size = .fromSize(cmov_abi_size),
10369	.disp = frame_addr.off,	17958	.disp = frame_addr.off,
10370	} },	17959	} },
10371	},	17960	},
...	@@ -10400,21 +17989,22 @@ fn genBinOp(	...	@@ -10400,21 +17989,22 @@ fn genBinOp(
10400	.float => switch (lhs_ty.floatBits(self.target.*)) {	17989	.float => switch (lhs_ty.floatBits(self.target.*)) {
10401	16 => {	17990	16 => {
10402	assert(self.hasFeature(.f16c));	17991	assert(self.hasFeature(.f16c));
10403	const tmp_reg =	17992	const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
10404	(try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();	17993
		17994	const tmp_reg = (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
10405	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	17995	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
10406	defer self.register_manager.unlockReg(tmp_lock);	17996	defer self.register_manager.unlockReg(tmp_lock);
10407		17997
10408	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	17998	if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
10409	.{ .vp_w, .insr },	17999	.{ .vp_w, .insr },
10410	dst_reg,	18000	dst_reg,
10411	dst_reg,	18001	lhs_reg,
10412	try src_mcv.mem(self, .word),	18002	try src_mcv.mem(self, .{ .size = .word }),
10413	Immediate.u(1),	18003	.u(1),
10414	) else try self.asmRegisterRegisterRegister(	18004	) else try self.asmRegisterRegisterRegister(
10415	.{ .vp_, .unpcklwd },	18005	.{ .vp_, .unpcklwd },
10416	dst_reg,	18006	dst_reg,
10417	dst_reg,	18007	lhs_reg,
10418	(if (src_mcv.isRegister())	18008	(if (src_mcv.isRegister())
10419	src_mcv.getReg().?	18009	src_mcv.getReg().?
10420	else	18010	else
...	@@ -10442,7 +18032,7 @@ fn genBinOp(	...	@@ -10442,7 +18032,7 @@ fn genBinOp(
10442	dst_reg,	18032	dst_reg,
10443	dst_reg,	18033	dst_reg,
10444	dst_reg,	18034	dst_reg,
10445	Immediate.u(@as(u5, @bitCast(RoundMode{	18035	.u(@as(u5, @bitCast(RoundMode{
10446	.mode = switch (air_tag) {	18036	.mode = switch (air_tag) {
10447	.div_trunc => .zero,	18037	.div_trunc => .zero,
10448	.div_floor => .down,	18038	.div_floor => .down,
...	@@ -10457,7 +18047,7 @@ fn genBinOp(	...	@@ -10457,7 +18047,7 @@ fn genBinOp(
10457	.{ .v_, .cvtps2ph },	18047	.{ .v_, .cvtps2ph },
10458	dst_reg,	18048	dst_reg,
10459	dst_reg,	18049	dst_reg,
10460	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	18050	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10461	);	18051	);
10462	return dst_mcv;	18052	return dst_mcv;
10463	},	18053	},
...	@@ -10842,25 +18432,24 @@ fn genBinOp(	...	@@ -10842,25 +18432,24 @@ fn genBinOp(
10842	.float => switch (lhs_ty.childType(zcu).floatBits(self.target.*)) {	18432	.float => switch (lhs_ty.childType(zcu).floatBits(self.target.*)) {
10843	16 => tag: {	18433	16 => tag: {
10844	assert(self.hasFeature(.f16c));	18434	assert(self.hasFeature(.f16c));
		18435	const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
10845	switch (lhs_ty.vectorLen(zcu)) {	18436	switch (lhs_ty.vectorLen(zcu)) {
10846	1 => {	18437	1 => {
10847	const tmp_reg = (try self.register_manager.allocReg(	18438	const tmp_reg =
10848	null,	18439	(try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
10849	abi.RegisterClass.sse,
10850	)).to128();
10851	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	18440	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
10852	defer self.register_manager.unlockReg(tmp_lock);	18441	defer self.register_manager.unlockReg(tmp_lock);
10853		18442
10854	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	18443	if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
10855	.{ .vp_w, .insr },	18444	.{ .vp_w, .insr },
10856	dst_reg,	18445	dst_reg,
10857	dst_reg,	18446	lhs_reg,
10858	try src_mcv.mem(self, .word),	18447	try src_mcv.mem(self, .{ .size = .word }),
10859	Immediate.u(1),	18448	.u(1),
10860	) else try self.asmRegisterRegisterRegister(	18449	) else try self.asmRegisterRegisterRegister(
10861	.{ .vp_, .unpcklwd },	18450	.{ .vp_, .unpcklwd },
10862	dst_reg,	18451	dst_reg,
10863	dst_reg,	18452	lhs_reg,
10864	(if (src_mcv.isRegister())	18453	(if (src_mcv.isRegister())
10865	src_mcv.getReg().?	18454	src_mcv.getReg().?
10866	else	18455	else
...	@@ -10886,7 +18475,7 @@ fn genBinOp(	...	@@ -10886,7 +18475,7 @@ fn genBinOp(
10886	.{ .v_, .cvtps2ph },	18475	.{ .v_, .cvtps2ph },
10887	dst_reg,	18476	dst_reg,
10888	dst_reg,	18477	dst_reg,
10889	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	18478	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10890	);	18479	);
10891	return dst_mcv;	18480	return dst_mcv;
10892	},	18481	},
...	@@ -10898,15 +18487,16 @@ fn genBinOp(	...	@@ -10898,15 +18487,16 @@ fn genBinOp(
10898	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	18487	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
10899	defer self.register_manager.unlockReg(tmp_lock);	18488	defer self.register_manager.unlockReg(tmp_lock);
10900		18489
10901	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	18490	if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
10902	.{ .vp_d, .insr },	18491	.{ .vp_d, .insr },
10903	dst_reg,	18492	dst_reg,
10904	try src_mcv.mem(self, .dword),	18493	lhs_reg,
10905	Immediate.u(1),	18494	try src_mcv.mem(self, .{ .size = .dword }),
		18495	.u(1),
10906	) else try self.asmRegisterRegisterRegister(	18496	) else try self.asmRegisterRegisterRegister(
10907	.{ .v_ps, .unpckl },	18497	.{ .v_ps, .unpckl },
10908	dst_reg,	18498	dst_reg,
10909	dst_reg,	18499	lhs_reg,
10910	(if (src_mcv.isRegister())	18500	(if (src_mcv.isRegister())
10911	src_mcv.getReg().?	18501	src_mcv.getReg().?
10912	else	18502	else
...	@@ -10937,7 +18527,7 @@ fn genBinOp(	...	@@ -10937,7 +18527,7 @@ fn genBinOp(
10937	.{ .v_, .cvtps2ph },	18527	.{ .v_, .cvtps2ph },
10938	dst_reg,	18528	dst_reg,
10939	dst_reg,	18529	dst_reg,
10940	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	18530	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10941	);	18531	);
10942	return dst_mcv;	18532	return dst_mcv;
10943	},	18533	},
...	@@ -10949,11 +18539,11 @@ fn genBinOp(	...	@@ -10949,11 +18539,11 @@ fn genBinOp(
10949	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	18539	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
10950	defer self.register_manager.unlockReg(tmp_lock);	18540	defer self.register_manager.unlockReg(tmp_lock);
10951		18541
10952	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);	18542	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, lhs_reg);
10953	if (src_mcv.isMemory()) try self.asmRegisterMemory(	18543	if (src_mcv.isBase()) try self.asmRegisterMemory(
10954	.{ .v_ps, .cvtph2 },	18544	.{ .v_ps, .cvtph2 },
10955	tmp_reg,	18545	tmp_reg,
10956	try src_mcv.mem(self, .qword),	18546	try src_mcv.mem(self, .{ .size = .qword }),
10957	) else try self.asmRegisterRegister(	18547	) else try self.asmRegisterRegister(
10958	.{ .v_ps, .cvtph2 },	18548	.{ .v_ps, .cvtph2 },
10959	tmp_reg,	18549	tmp_reg,
...	@@ -10980,7 +18570,7 @@ fn genBinOp(	...	@@ -10980,7 +18570,7 @@ fn genBinOp(
10980	.{ .v_, .cvtps2ph },	18570	.{ .v_, .cvtps2ph },
10981	dst_reg,	18571	dst_reg,
10982	dst_reg,	18572	dst_reg,
10983	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	18573	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
10984	);	18574	);
10985	return dst_mcv;	18575	return dst_mcv;
10986	},	18576	},
...	@@ -10992,11 +18582,11 @@ fn genBinOp(	...	@@ -10992,11 +18582,11 @@ fn genBinOp(
10992	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	18582	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
10993	defer self.register_manager.unlockReg(tmp_lock);	18583	defer self.register_manager.unlockReg(tmp_lock);
10994		18584
10995	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg);	18585	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), lhs_reg);
10996	if (src_mcv.isMemory()) try self.asmRegisterMemory(	18586	if (src_mcv.isBase()) try self.asmRegisterMemory(
10997	.{ .v_ps, .cvtph2 },	18587	.{ .v_ps, .cvtph2 },
10998	tmp_reg,	18588	tmp_reg,
10999	try src_mcv.mem(self, .xword),	18589	try src_mcv.mem(self, .{ .size = .xword }),
11000	) else try self.asmRegisterRegister(	18590	) else try self.asmRegisterRegister(
11001	.{ .v_ps, .cvtph2 },	18591	.{ .v_ps, .cvtph2 },
11002	tmp_reg,	18592	tmp_reg,
...	@@ -11023,7 +18613,7 @@ fn genBinOp(	...	@@ -11023,7 +18613,7 @@ fn genBinOp(
11023	.{ .v_, .cvtps2ph },	18613	.{ .v_, .cvtps2ph },
11024	dst_reg,	18614	dst_reg,
11025	dst_reg.to256(),	18615	dst_reg.to256(),
11026	Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),	18616	.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))),
11027	);	18617	);
11028	return dst_mcv;	18618	return dst_mcv;
11029	},	18619	},
...	@@ -11153,16 +18743,15 @@ fn genBinOp(	...	@@ -11153,16 +18743,15 @@ fn genBinOp(
11153		18743
11154	switch (mir_tag[1]) {	18744	switch (mir_tag[1]) {
11155	else => if (self.hasFeature(.avx)) {	18745	else => if (self.hasFeature(.avx)) {
11156	const lhs_reg =	18746	const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
11157	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);	18747	if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
11158	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
11159	mir_tag,	18748	mir_tag,
11160	dst_reg,	18749	dst_reg,
11161	lhs_reg,	18750	lhs_reg,
11162	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {	18751	try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11163	else => Memory.Size.fromSize(abi_size),	18752	else => .fromSize(abi_size),
11164	.vector => Memory.Size.fromBitSize(dst_reg.bitSize()),	18753	.vector => .fromBitSize(dst_reg.bitSize()),
11165	}),	18754	} }),
11166	) else try self.asmRegisterRegisterRegister(	18755	) else try self.asmRegisterRegisterRegister(
11167	mir_tag,	18756	mir_tag,
11168	dst_reg,	18757	dst_reg,
...	@@ -11174,13 +18763,13 @@ fn genBinOp(	...	@@ -11174,13 +18763,13 @@ fn genBinOp(
11174	);	18763	);
11175	} else {	18764	} else {
11176	assert(copied_to_dst);	18765	assert(copied_to_dst);
11177	if (src_mcv.isMemory()) try self.asmRegisterMemory(	18766	if (src_mcv.isBase()) try self.asmRegisterMemory(
11178	mir_tag,	18767	mir_tag,
11179	dst_reg,	18768	dst_reg,
11180	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {	18769	try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11181	else => Memory.Size.fromSize(abi_size),	18770	else => .fromSize(abi_size),
11182	.vector => Memory.Size.fromBitSize(dst_reg.bitSize()),	18771	.vector => .fromBitSize(dst_reg.bitSize()),
11183	}),	18772	} }),
11184	) else try self.asmRegisterRegister(	18773	) else try self.asmRegisterRegister(
11185	mir_tag,	18774	mir_tag,
11186	dst_reg,	18775	dst_reg,
...	@@ -11191,7 +18780,7 @@ fn genBinOp(	...	@@ -11191,7 +18780,7 @@ fn genBinOp(
11191	);	18780	);
11192	},	18781	},
11193	.cmp => {	18782	.cmp => {
11194	const imm = Immediate.u(switch (air_tag) {	18783	const imm: Immediate = .u(switch (air_tag) {
11195	.cmp_eq => 0,	18784	.cmp_eq => 0,
11196	.cmp_lt, .cmp_gt => 1,	18785	.cmp_lt, .cmp_gt => 1,
11197	.cmp_lte, .cmp_gte => 2,	18786	.cmp_lte, .cmp_gte => 2,
...	@@ -11201,14 +18790,14 @@ fn genBinOp(	...	@@ -11201,14 +18790,14 @@ fn genBinOp(
11201	if (self.hasFeature(.avx)) {	18790	if (self.hasFeature(.avx)) {
11202	const lhs_reg =	18791	const lhs_reg =
11203	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);	18792	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
11204	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	18793	if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
11205	mir_tag,	18794	mir_tag,
11206	dst_reg,	18795	dst_reg,
11207	lhs_reg,	18796	lhs_reg,
11208	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {	18797	try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11209	else => Memory.Size.fromSize(abi_size),	18798	else => .fromSize(abi_size),
11210	.vector => Memory.Size.fromBitSize(dst_reg.bitSize()),	18799	.vector => .fromBitSize(dst_reg.bitSize()),
11211	}),	18800	} }),
11212	imm,	18801	imm,
11213	) else try self.asmRegisterRegisterRegisterImmediate(	18802	) else try self.asmRegisterRegisterRegisterImmediate(
11214	mir_tag,	18803	mir_tag,
...	@@ -11222,13 +18811,13 @@ fn genBinOp(	...	@@ -11222,13 +18811,13 @@ fn genBinOp(
11222	);	18811	);
11223	} else {	18812	} else {
11224	assert(copied_to_dst);	18813	assert(copied_to_dst);
11225	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	18814	if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
11226	mir_tag,	18815	mir_tag,
11227	dst_reg,	18816	dst_reg,
11228	try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {	18817	try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11229	else => Memory.Size.fromSize(abi_size),	18818	else => .fromSize(abi_size),
11230	.vector => Memory.Size.fromBitSize(dst_reg.bitSize()),	18819	.vector => .fromBitSize(dst_reg.bitSize()),
11231	}),	18820	} }),
11232	imm,	18821	imm,
11233	) else try self.asmRegisterRegisterImmediate(	18822	) else try self.asmRegisterRegisterImmediate(
11234	mir_tag,	18823	mir_tag,
...	@@ -11289,7 +18878,7 @@ fn genBinOp(	...	@@ -11289,7 +18878,7 @@ fn genBinOp(
11289	mask_reg,	18878	mask_reg,
11290	rhs_copy_reg,	18879	rhs_copy_reg,
11291	rhs_copy_reg,	18880	rhs_copy_reg,
11292	Immediate.u(3), // unord	18881	.u(3), // unord
11293	);	18882	);
11294	try self.asmRegisterRegisterRegisterRegister(	18883	try self.asmRegisterRegisterRegisterRegister(
11295	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {	18884	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
...	@@ -11356,7 +18945,7 @@ fn genBinOp(	...	@@ -11356,7 +18945,7 @@ fn genBinOp(
11356	}),	18945	}),
11357	mask_reg,	18946	mask_reg,
11358	mask_reg,	18947	mask_reg,
11359	Immediate.u(if (has_blend) 3 else 7), // unord, ord	18948	.u(if (has_blend) 3 else 7), // unord, ord
11360	);	18949	);
11361	if (has_blend) try self.asmRegisterRegisterRegister(	18950	if (has_blend) try self.asmRegisterRegisterRegister(
11362	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {	18951	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
...	@@ -11433,12 +19022,12 @@ fn genBinOp(	...	@@ -11433,12 +19022,12 @@ fn genBinOp(
11433	=> {	19022	=> {
11434	const unsigned_ty = try lhs_ty.toUnsigned(pt);	19023	const unsigned_ty = try lhs_ty.toUnsigned(pt);
11435	const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));	19024	const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));
11436	const not_mem: Memory = if (not_mcv.isMemory())	19025	const not_mem: Memory = if (not_mcv.isBase())
11437	try not_mcv.mem(self, Memory.Size.fromSize(abi_size))	19026	try not_mcv.mem(self, .{ .size = .fromSize(abi_size) })
11438	else	19027	else
11439	.{ .base = .{	19028	.{ .base = .{
11440	.reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),	19029	.reg = try self.copyToTmpRegister(.usize, not_mcv.address()),
11441	}, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };	19030	}, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } };
11442	switch (mir_tag[0]) {	19031	switch (mir_tag[0]) {
11443	.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(	19032	.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
11444	.{ .vp_, .xor },	19033	.{ .vp_, .xor },
...	@@ -11495,7 +19084,7 @@ fn genBinOp(	...	@@ -11495,7 +19084,7 @@ fn genBinOp(
11495	}	19084	}
11496		19085
11497	fn genBinOpMir(	19086	fn genBinOpMir(
11498	self: *Self,	19087	self: *CodeGen,
11499	mir_tag: Mir.Inst.FixedTag,	19088	mir_tag: Mir.Inst.FixedTag,
11500	ty: Type,	19089	ty: Type,
11501	dst_mcv: MCValue,	19090	dst_mcv: MCValue,
...	@@ -11513,6 +19102,7 @@ fn genBinOpMir(	...	@@ -11513,6 +19102,7 @@ fn genBinOpMir(
11513	.immediate,	19102	.immediate,
11514	.eflags,	19103	.eflags,
11515	.register_overflow,	19104	.register_overflow,
		19105	.register_mask,
11516	.lea_direct,	19106	.lea_direct,
11517	.lea_got,	19107	.lea_got,
11518	.lea_tlv,	19108	.lea_tlv,
...	@@ -11522,9 +19112,9 @@ fn genBinOpMir(	...	@@ -11522,9 +19112,9 @@ fn genBinOpMir(
11522	.reserved_frame,	19112	.reserved_frame,
11523	.air_ref,	19113	.air_ref,
11524	=> unreachable, // unmodifiable destination	19114	=> unreachable, // unmodifiable destination
11525	.register, .register_pair, .register_offset => {	19115	.register, .register_pair, .register_triple, .register_quadruple, .register_offset => {
11526	switch (dst_mcv) {	19116	switch (dst_mcv) {
11527	.register, .register_pair => {},	19117	.register, .register_pair, .register_triple, .register_quadruple => {},
11528	.register_offset => \|ro\| assert(ro.off == 0),	19118	.register_offset => \|ro\| assert(ro.off == 0),
11529	else => unreachable,	19119	else => unreachable,
11530	}	19120	}
...	@@ -11553,10 +19143,15 @@ fn genBinOpMir(	...	@@ -11553,10 +19143,15 @@ fn genBinOpMir(
11553	.dead,	19143	.dead,
11554	.undef,	19144	.undef,
11555	.register_overflow,	19145	.register_overflow,
		19146	.register_mask,
11556	.elementwise_regs_then_frame,	19147	.elementwise_regs_then_frame,
11557	.reserved_frame,	19148	.reserved_frame,
11558	=> unreachable,	19149	=> unreachable,
11559	.register, .register_pair => try self.asmRegisterRegister(	19150	.register,
		19151	.register_pair,
		19152	.register_triple,
		19153	.register_quadruple,
		19154	=> try self.asmRegisterRegister(
11560	mir_limb_tag,	19155	mir_limb_tag,
11561	dst_alias,	19156	dst_alias,
11562	registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size),	19157	registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size),
...	@@ -11567,29 +19162,29 @@ fn genBinOpMir(	...	@@ -11567,29 +19162,29 @@ fn genBinOpMir(
11567	8 => try self.asmRegisterImmediate(	19162	8 => try self.asmRegisterImmediate(
11568	mir_limb_tag,	19163	mir_limb_tag,
11569	dst_alias,	19164	dst_alias,
11570	if (math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|	19165	if (std.math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
11571	Immediate.s(small)	19166	.s(small)
11572	else	19167	else
11573	Immediate.u(@as(u8, @intCast(imm))),	19168	.u(@as(u8, @intCast(imm))),
11574	),	19169	),
11575	16 => try self.asmRegisterImmediate(	19170	16 => try self.asmRegisterImmediate(
11576	mir_limb_tag,	19171	mir_limb_tag,
11577	dst_alias,	19172	dst_alias,
11578	if (math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|	19173	if (std.math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
11579	Immediate.s(small)	19174	.s(small)
11580	else	19175	else
11581	Immediate.u(@as(u16, @intCast(imm))),	19176	.u(@as(u16, @intCast(imm))),
11582	),	19177	),
11583	32 => try self.asmRegisterImmediate(	19178	32 => try self.asmRegisterImmediate(
11584	mir_limb_tag,	19179	mir_limb_tag,
11585	dst_alias,	19180	dst_alias,
11586	if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	19181	if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11587	Immediate.s(small)	19182	.s(small)
11588	else	19183	else
11589	Immediate.u(@as(u32, @intCast(imm))),	19184	.u(@as(u32, @intCast(imm))),
11590	),	19185	),
11591	64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	19186	64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11592	try self.asmRegisterImmediate(mir_limb_tag, dst_alias, Immediate.s(small))	19187	try self.asmRegisterImmediate(mir_limb_tag, dst_alias, .s(small))
11593	else	19188	else
11594	try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias(	19189	try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias(
11595	try self.copyToTmpRegister(ty, src_mcv),	19190	try self.copyToTmpRegister(ty, src_mcv),
...	@@ -11618,21 +19213,21 @@ fn genBinOpMir(	...	@@ -11618,21 +19213,21 @@ fn genBinOpMir(
11618	.memory => \|addr\| .{	19213	.memory => \|addr\| .{
11619	.base = .{ .reg = .ds },	19214	.base = .{ .reg = .ds },
11620	.mod = .{ .rm = .{	19215	.mod = .{ .rm = .{
11621	.size = Memory.Size.fromSize(limb_abi_size),	19216	.size = .fromSize(limb_abi_size),
11622	.disp = math.cast(i32, addr + off) orelse break :direct,	19217	.disp = std.math.cast(i32, addr + off) orelse break :direct,
11623	} },	19218	} },
11624	},	19219	},
11625	.indirect => \|reg_off\| .{	19220	.indirect => \|reg_off\| .{
11626	.base = .{ .reg = reg_off.reg },	19221	.base = .{ .reg = reg_off.reg },
11627	.mod = .{ .rm = .{	19222	.mod = .{ .rm = .{
11628	.size = Memory.Size.fromSize(limb_abi_size),	19223	.size = .fromSize(limb_abi_size),
11629	.disp = reg_off.off + off,	19224	.disp = reg_off.off + off,
11630	} },	19225	} },
11631	},	19226	},
11632	.load_frame => \|frame_addr\| .{	19227	.load_frame => \|frame_addr\| .{
11633	.base = .{ .frame = frame_addr.index },	19228	.base = .{ .frame = frame_addr.index },
11634	.mod = .{ .rm = .{	19229	.mod = .{ .rm = .{
11635	.size = Memory.Size.fromSize(limb_abi_size),	19230	.size = .fromSize(limb_abi_size),
11636	.disp = frame_addr.off + off,	19231	.disp = frame_addr.off + off,
11637	} },	19232	} },
11638	},	19233	},
...	@@ -11695,7 +19290,7 @@ fn genBinOpMir(	...	@@ -11695,7 +19290,7 @@ fn genBinOpMir(
11695	const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);	19290	const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
11696	errdefer self.register_manager.unlockReg(dst_addr_lock);	19291	errdefer self.register_manager.unlockReg(dst_addr_lock);
11697		19292
11698	try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address(), .{});	19293	try self.genSetReg(dst_addr_reg, .usize, dst_mcv.address(), .{});
11699	break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock };	19294	break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock };
11700	},	19295	},
11701	.load_frame => null,	19296	.load_frame => null,
...	@@ -11712,6 +19307,7 @@ fn genBinOpMir(	...	@@ -11712,6 +19307,7 @@ fn genBinOpMir(
11712	.dead,	19307	.dead,
11713	.undef,	19308	.undef,
11714	.register_overflow,	19309	.register_overflow,
		19310	.register_mask,
11715	.elementwise_regs_then_frame,	19311	.elementwise_regs_then_frame,
11716	.reserved_frame,	19312	.reserved_frame,
11717	.air_ref,	19313	.air_ref,
...	@@ -11720,6 +19316,8 @@ fn genBinOpMir(	...	@@ -11720,6 +19316,8 @@ fn genBinOpMir(
11720	.eflags,	19316	.eflags,
11721	.register,	19317	.register,
11722	.register_pair,	19318	.register_pair,
		19319	.register_triple,
		19320	.register_quadruple,
11723	.register_offset,	19321	.register_offset,
11724	.indirect,	19322	.indirect,
11725	.lea_direct,	19323	.lea_direct,
...	@@ -11731,8 +19329,8 @@ fn genBinOpMir(	...	@@ -11731,8 +19329,8 @@ fn genBinOpMir(
11731	=> null,	19329	=> null,
11732	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {	19330	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
11733	switch (resolved_src_mcv) {	19331	switch (resolved_src_mcv) {
11734	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr))) != null and	19332	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr))) != null and
11735	math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)	19333	std.math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null)
11736	break :src null,	19334	break :src null,
11737	.load_symbol, .load_got, .load_direct, .load_tlv => {},	19335	.load_symbol, .load_got, .load_direct, .load_tlv => {},
11738	else => unreachable,	19336	else => unreachable,
...	@@ -11743,7 +19341,7 @@ fn genBinOpMir(	...	@@ -11743,7 +19341,7 @@ fn genBinOpMir(
11743	const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);	19341	const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
11744	errdefer self.register_manager.unlockReg(src_addr_lock);	19342	errdefer self.register_manager.unlockReg(src_addr_lock);
11745		19343
11746	try self.genSetReg(src_addr_reg, Type.usize, resolved_src_mcv.address(), .{});	19344	try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{});
11747	break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };	19345	break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
11748	},	19346	},
11749	};	19347	};
...	@@ -11751,9 +19349,9 @@ fn genBinOpMir(	...	@@ -11751,9 +19349,9 @@ fn genBinOpMir(
11751		19349
11752	const ty_signedness =	19350	const ty_signedness =
11753	if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned;	19351	if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned;
11754	const limb_ty = if (abi_size <= 8) ty else switch (ty_signedness) {	19352	const limb_ty: Type = if (abi_size <= 8) ty else switch (ty_signedness) {
11755	.signed => Type.usize,	19353	.signed => .usize,
11756	.unsigned => Type.isize,	19354	.unsigned => .isize,
11757	};	19355	};
11758	var limb_i: usize = 0;	19356	var limb_i: usize = 0;
11759	var off: i32 = 0;	19357	var off: i32 = 0;
...	@@ -11781,21 +19379,21 @@ fn genBinOpMir(	...	@@ -11781,21 +19379,21 @@ fn genBinOpMir(
11781	=> .{	19379	=> .{
11782	.base = .{ .reg = dst_info.?.addr_reg },	19380	.base = .{ .reg = dst_info.?.addr_reg },
11783	.mod = .{ .rm = .{	19381	.mod = .{ .rm = .{
11784	.size = Memory.Size.fromSize(limb_abi_size),	19382	.size = .fromSize(limb_abi_size),
11785	.disp = off,	19383	.disp = off,
11786	} },	19384	} },
11787	},	19385	},
11788	.indirect => \|reg_off\| .{	19386	.indirect => \|reg_off\| .{
11789	.base = .{ .reg = reg_off.reg },	19387	.base = .{ .reg = reg_off.reg },
11790	.mod = .{ .rm = .{	19388	.mod = .{ .rm = .{
11791	.size = Memory.Size.fromSize(limb_abi_size),	19389	.size = .fromSize(limb_abi_size),
11792	.disp = reg_off.off + off,	19390	.disp = reg_off.off + off,
11793	} },	19391	} },
11794	},	19392	},
11795	.load_frame => \|frame_addr\| .{	19393	.load_frame => \|frame_addr\| .{
11796	.base = .{ .frame = frame_addr.index },	19394	.base = .{ .frame = frame_addr.index },
11797	.mod = .{ .rm = .{	19395	.mod = .{ .rm = .{
11798	.size = Memory.Size.fromSize(limb_abi_size),	19396	.size = .fromSize(limb_abi_size),
11799	.disp = frame_addr.off + off,	19397	.disp = frame_addr.off + off,
11800	} },	19398	} },
11801	},	19399	},
...	@@ -11807,6 +19405,7 @@ fn genBinOpMir(	...	@@ -11807,6 +19405,7 @@ fn genBinOpMir(
11807	.dead,	19405	.dead,
11808	.undef,	19406	.undef,
11809	.register_overflow,	19407	.register_overflow,
		19408	.register_mask,
11810	.elementwise_regs_then_frame,	19409	.elementwise_regs_then_frame,
11811	.reserved_frame,	19410	.reserved_frame,
11812	.air_ref,	19411	.air_ref,
...	@@ -11823,33 +19422,29 @@ fn genBinOpMir(	...	@@ -11823,33 +19422,29 @@ fn genBinOpMir(
11823	8 => try self.asmMemoryImmediate(	19422	8 => try self.asmMemoryImmediate(
11824	mir_limb_tag,	19423	mir_limb_tag,
11825	dst_limb_mem,	19424	dst_limb_mem,
11826	if (math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|	19425	if (std.math.cast(i8, @as(i64, @bitCast(imm)))) \|small\|
11827	Immediate.s(small)	19426	.s(small)
11828	else	19427	else
11829	Immediate.u(@as(u8, @intCast(imm))),	19428	.u(@as(u8, @intCast(imm))),
11830	),	19429	),
11831	16 => try self.asmMemoryImmediate(	19430	16 => try self.asmMemoryImmediate(
11832	mir_limb_tag,	19431	mir_limb_tag,
11833	dst_limb_mem,	19432	dst_limb_mem,
11834	if (math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|	19433	if (std.math.cast(i16, @as(i64, @bitCast(imm)))) \|small\|
11835	Immediate.s(small)	19434	.s(small)
11836	else	19435	else
11837	Immediate.u(@as(u16, @intCast(imm))),	19436	.u(@as(u16, @intCast(imm))),
11838	),	19437	),
11839	32 => try self.asmMemoryImmediate(	19438	32 => try self.asmMemoryImmediate(
11840	mir_limb_tag,	19439	mir_limb_tag,
11841	dst_limb_mem,	19440	dst_limb_mem,
11842	if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	19441	if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11843	Immediate.s(small)	19442	.s(small)
11844	else	19443	else
11845	Immediate.u(@as(u32, @intCast(imm))),	19444	.u(@as(u32, @intCast(imm))),
11846	),	19445	),
11847	64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|	19446	64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\|
11848	try self.asmMemoryImmediate(	19447	try self.asmMemoryImmediate(mir_limb_tag, dst_limb_mem, .s(small))
11849	mir_limb_tag,
11850	dst_limb_mem,
11851	Immediate.s(small),
11852	)
11853	else	19448	else
11854	try self.asmMemoryRegister(	19449	try self.asmMemoryRegister(
11855	mir_limb_tag,	19450	mir_limb_tag,
...	@@ -11864,6 +19459,8 @@ fn genBinOpMir(	...	@@ -11864,6 +19459,8 @@ fn genBinOpMir(
11864	},	19459	},
11865	.register,	19460	.register,
11866	.register_pair,	19461	.register_pair,
		19462	.register_triple,
		19463	.register_quadruple,
11867	.register_offset,	19464	.register_offset,
11868	.eflags,	19465	.eflags,
11869	.memory,	19466	.memory,
...	@@ -11882,7 +19479,7 @@ fn genBinOpMir(	...	@@ -11882,7 +19479,7 @@ fn genBinOpMir(
11882	const src_limb_mcv: MCValue = if (src_info) \|info\| .{	19479	const src_limb_mcv: MCValue = if (src_info) \|info\| .{
11883	.indirect = .{ .reg = info.addr_reg, .off = off },	19480	.indirect = .{ .reg = info.addr_reg, .off = off },
11884	} else switch (resolved_src_mcv) {	19481	} else switch (resolved_src_mcv) {
11885	.register, .register_pair => .{	19482	.register, .register_pair, .register_triple, .register_quadruple => .{
11886	.register = resolved_src_mcv.getRegs()[limb_i],	19483	.register = resolved_src_mcv.getRegs()[limb_i],
11887	},	19484	},
11888	.eflags,	19485	.eflags,
...	@@ -11925,7 +19522,7 @@ fn genBinOpMir(	...	@@ -11925,7 +19522,7 @@ fn genBinOpMir(
11925		19522
11926	/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.	19523	/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
11927	/// Does not support byte-size operands.	19524	/// Does not support byte-size operands.
11928	fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {	19525	fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
11929	const pt = self.pt;	19526	const pt = self.pt;
11930	const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));	19527	const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
11931	try self.spillEflagsIfOccupied();	19528	try self.spillEflagsIfOccupied();
...	@@ -11938,6 +19535,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -11938,6 +19535,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
11938	.eflags,	19535	.eflags,
11939	.register_offset,	19536	.register_offset,
11940	.register_overflow,	19537	.register_overflow,
		19538	.register_mask,
11941	.lea_symbol,	19539	.lea_symbol,
11942	.lea_direct,	19540	.lea_direct,
11943	.lea_got,	19541	.lea_got,
...	@@ -11948,10 +19546,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -11948,10 +19546,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
11948	.air_ref,	19546	.air_ref,
11949	=> unreachable, // unmodifiable destination	19547	=> unreachable, // unmodifiable destination
11950	.register => \|dst_reg\| {	19548	.register => \|dst_reg\| {
11951	const dst_alias = registerAlias(dst_reg, abi_size);	19549	const alias_size = switch (abi_size) {
		19550	1 => 4,
		19551	else => abi_size,
		19552	};
		19553	const dst_alias = registerAlias(dst_reg, alias_size);
11952	const dst_lock = self.register_manager.lockReg(dst_reg);	19554	const dst_lock = self.register_manager.lockReg(dst_reg);
11953	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	19555	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
11954		19556
		19557	switch (abi_size) {
		19558	1 => try self.asmRegisterRegister(.{ ._, .movzx }, dst_reg.to32(), dst_reg.to8()),
		19559	else => {},
		19560	}
		19561
11955	const resolved_src_mcv = switch (src_mcv) {	19562	const resolved_src_mcv = switch (src_mcv) {
11956	else => src_mcv,	19563	else => src_mcv,
11957	.air_ref => \|src_ref\| try self.resolveInst(src_ref),	19564	.air_ref => \|src_ref\| try self.resolveInst(src_ref),
...	@@ -11962,24 +19569,28 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -11962,24 +19569,28 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
11962	.dead,	19569	.dead,
11963	.undef,	19570	.undef,
11964	.register_pair,	19571	.register_pair,
		19572	.register_triple,
		19573	.register_quadruple,
11965	.register_overflow,	19574	.register_overflow,
		19575	.register_mask,
11966	.elementwise_regs_then_frame,	19576	.elementwise_regs_then_frame,
11967	.reserved_frame,	19577	.reserved_frame,
11968	.air_ref,	19578	.air_ref,
11969	=> unreachable,	19579	=> unreachable,
11970	.register => \|src_reg\| try self.asmRegisterRegister(	19580	.register => \|src_reg\| {
11971	.{ .i_, .mul },	19581	switch (abi_size) {
11972	dst_alias,	19582	1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()),
11973	registerAlias(src_reg, abi_size),	19583	else => {},
11974	),	19584	}
		19585	try self.asmRegisterRegister(
		19586	.{ .i_, .mul },
		19587	dst_alias,
		19588	registerAlias(src_reg, alias_size),
		19589	);
		19590	},
11975	.immediate => \|imm\| {	19591	.immediate => \|imm\| {
11976	if (math.cast(i32, imm)) \|small\| {	19592	if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {
11977	try self.asmRegisterRegisterImmediate(	19593	try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst_alias, dst_alias, .s(small));
11978	.{ .i_, .mul },
11979	dst_alias,
11980	dst_alias,
11981	Immediate.s(small),
11982	);
11983	} else {	19594	} else {
11984	const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);	19595	const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
11985	return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });	19596	return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
...	@@ -11996,50 +19607,60 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -11996,50 +19607,60 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
11996	.load_tlv,	19607	.load_tlv,
11997	.lea_tlv,	19608	.lea_tlv,
11998	.lea_frame,	19609	.lea_frame,
11999	=> try self.asmRegisterRegister(	19610	=> {
12000	.{ .i_, .mul },	19611	const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
12001	dst_alias,	19612	switch (abi_size) {
12002	registerAlias(try self.copyToTmpRegister(dst_ty, resolved_src_mcv), abi_size),	19613	1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()),
12003	),	19614	else => {},
12004	.memory, .indirect, .load_frame => try self.asmRegisterMemory(	19615	}
12005	.{ .i_, .mul },	19616	try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size));
12006	dst_alias,	19617	},
12007	switch (resolved_src_mcv) {	19618	.memory, .indirect, .load_frame => switch (abi_size) {
12008	.memory => \|addr\| .{	19619	1 => {
12009	.base = .{ .reg = .ds },	19620	const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
12010	.mod = .{ .rm = .{	19621	try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8());
12011	.size = Memory.Size.fromSize(abi_size),	19622	try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size));
12012	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse	19623	},
12013	return self.asmRegisterRegister(	19624	else => try self.asmRegisterMemory(
12014	.{ .i_, .mul },	19625	.{ .i_, .mul },
12015	dst_alias,	19626	dst_alias,
12016	registerAlias(	19627	switch (resolved_src_mcv) {
12017	try self.copyToTmpRegister(dst_ty, resolved_src_mcv),	19628	.memory => \|addr\| .{
12018	abi_size,	19629	.base = .{ .reg = .ds },
		19630	.mod = .{ .rm = .{
		19631	.size = .fromSize(abi_size),
		19632	.disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
		19633	return self.asmRegisterRegister(
		19634	.{ .i_, .mul },
		19635	dst_alias,
		19636	registerAlias(
		19637	try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
		19638	abi_size,
		19639	),
12019	),	19640	),
12020	),	19641	} },
12021	} },	19642	},
12022	},	19643	.indirect => \|reg_off\| .{
12023	.indirect => \|reg_off\| .{	19644	.base = .{ .reg = reg_off.reg },
12024	.base = .{ .reg = reg_off.reg },	19645	.mod = .{ .rm = .{
12025	.mod = .{ .rm = .{	19646	.size = .fromSize(abi_size),
12026	.size = Memory.Size.fromSize(abi_size),	19647	.disp = reg_off.off,
12027	.disp = reg_off.off,	19648	} },
12028	} },	19649	},
12029	},	19650	.load_frame => \|frame_addr\| .{
12030	.load_frame => \|frame_addr\| .{	19651	.base = .{ .frame = frame_addr.index },
12031	.base = .{ .frame = frame_addr.index },	19652	.mod = .{ .rm = .{
12032	.mod = .{ .rm = .{	19653	.size = .fromSize(abi_size),
12033	.size = Memory.Size.fromSize(abi_size),	19654	.disp = frame_addr.off,
12034	.disp = frame_addr.off,	19655	} },
12035	} },	19656	},
		19657	else => unreachable,
12036	},	19658	},
12037	else => unreachable,	19659	),
12038	},	19660	},
12039	),
12040	}	19661	}
12041	},	19662	},
12042	.register_pair => unreachable, // unimplemented	19663	.register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented
12043	.memory, .indirect, .load_symbol, .load_direct, .load_got, .load_tlv, .load_frame => {	19664	.memory, .indirect, .load_symbol, .load_direct, .load_got, .load_tlv, .load_frame => {
12044	const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);	19665	const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
12045	const tmp_mcv = MCValue{ .register = tmp_reg };	19666	const tmp_mcv = MCValue{ .register = tmp_reg };
...	@@ -12052,7 +19673,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -12052,7 +19673,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
12052	}	19673	}
12053	}	19674	}
12054		19675
12055	fn airArg(self: *Self, inst: Air.Inst.Index) !void {	19676	fn airArg(self: *CodeGen, inst: Air.Inst.Index) !void {
12056	const pt = self.pt;	19677	const pt = self.pt;
12057	const zcu = pt.zcu;	19678	const zcu = pt.zcu;
12058	// skip zero-bit arguments as they don't have a corresponding arg instruction	19679	// skip zero-bit arguments as they don't have a corresponding arg instruction
...	@@ -12078,8 +19699,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12078,8 +19699,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12078	try self.spillEflagsIfOccupied();	19699	try self.spillEflagsIfOccupied();
12079		19700
12080	const fn_info = zcu.typeToFunc(self.fn_type).?;	19701	const fn_info = zcu.typeToFunc(self.fn_type).?;
12081	const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);	19702	const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc);
12082	const param_int_regs = abi.getCAbiIntParamRegs(cc);
12083	var prev_reg: Register = undefined;	19703	var prev_reg: Register = undefined;
12084	for (	19704	for (
12085	param_int_regs[param_int_regs.len - regs_frame_addr.regs ..],	19705	param_int_regs[param_int_regs.len - regs_frame_addr.regs ..],
...	@@ -12087,11 +19707,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12087,11 +19707,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12087	) \|dst_reg, elem_index\| {	19707	) \|dst_reg, elem_index\| {
12088	assert(self.register_manager.isRegFree(dst_reg));	19708	assert(self.register_manager.isRegFree(dst_reg));
12089	if (elem_index > 0) {	19709	if (elem_index > 0) {
12090	try self.asmRegisterImmediate(	19710	try self.asmRegisterImmediate(.{ ._l, .sh }, dst_reg.to8(), .u(elem_index));
12091	.{ ._l, .sh },
12092	dst_reg.to8(),
12093	Immediate.u(elem_index),
12094	);
12095	try self.asmRegisterRegister(	19711	try self.asmRegisterRegister(
12096	.{ ._, .@"or" },	19712	.{ ._, .@"or" },
12097	dst_reg.to8(),	19713	dst_reg.to8(),
...	@@ -12110,7 +19726,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12110,7 +19726,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12110	const dst_mcv = try self.allocRegOrMem(inst, false);	19726	const dst_mcv = try self.allocRegOrMem(inst, false);
12111	if (regs_frame_addr.regs > 0) try self.asmMemoryRegister(	19727	if (regs_frame_addr.regs > 0) try self.asmMemoryRegister(
12112	.{ ._, .mov },	19728	.{ ._, .mov },
12113	try dst_mcv.mem(self, .byte),	19729	try dst_mcv.mem(self, .{ .size = .byte }),
12114	prev_reg.to8(),	19730	prev_reg.to8(),
12115	);	19731	);
12116	try self.genInlineMemset(	19732	try self.genInlineMemset(
...	@@ -12127,7 +19743,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12127,7 +19743,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12127	try self.asmRegisterImmediate(	19743	try self.asmRegisterImmediate(
12128	.{ ._, .mov },	19744	.{ ._, .mov },
12129	index_reg.to32(),	19745	index_reg.to32(),
12130	Immediate.u(regs_frame_addr.regs),	19746	.u(regs_frame_addr.regs),
12131	);	19747	);
12132	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	19748	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12133	try self.asmMemoryImmediate(.{ ._, .cmp }, .{	19749	try self.asmMemoryImmediate(.{ ._, .cmp }, .{
...	@@ -12142,19 +19758,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12142,19 +19758,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12142	const unset = try self.asmJccReloc(.e, undefined);	19758	const unset = try self.asmJccReloc(.e, undefined);
12143	try self.asmMemoryRegister(	19759	try self.asmMemoryRegister(
12144	.{ ._s, .bt },	19760	.{ ._s, .bt },
12145	try dst_mcv.mem(self, .dword),	19761	try dst_mcv.mem(self, .{ .size = .dword }),
12146	index_reg.to32(),	19762	index_reg.to32(),
12147	);	19763	);
12148	self.performReloc(unset);	19764	self.performReloc(unset);
12149	if (self.hasFeature(.slow_incdec)) {	19765	if (self.hasFeature(.slow_incdec)) {
12150	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));	19766	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
12151	} else {	19767	} else {
12152	try self.asmRegister(.{ ._, .inc }, index_reg.to32());	19768	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
12153	}	19769	}
12154	try self.asmRegisterImmediate(	19770	try self.asmRegisterImmediate(
12155	.{ ._, .cmp },	19771	.{ ._, .cmp },
12156	index_reg.to32(),	19772	index_reg.to32(),
12157	Immediate.u(arg_ty.vectorLen(zcu)),	19773	.u(arg_ty.vectorLen(zcu)),
12158	);	19774	);
12159	_ = try self.asmJccReloc(.b, loop);	19775	_ = try self.asmJccReloc(.b, loop);
12160		19776
...	@@ -12166,7 +19782,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12166,7 +19782,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12166	return self.finishAir(inst, result, .{ .none, .none, .none });	19782	return self.finishAir(inst, result, .{ .none, .none, .none });
12167	}	19783	}
12168		19784
12169	fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {	19785	fn airDbgArg(self: *CodeGen, inst: Air.Inst.Index) !void {
12170	// skip zero-bit arguments as they don't have a corresponding arg instruction	19786	// skip zero-bit arguments as they don't have a corresponding arg instruction
12171	var arg_index = self.arg_index;	19787	var arg_index = self.arg_index;
12172	while (self.args[arg_index] == .none) arg_index += 1;	19788	while (self.args[arg_index] == .none) arg_index += 1;
...	@@ -12180,16 +19796,14 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12180,16 +19796,14 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
12180	for (self.args[self.arg_index..]) \|arg\| {	19796	for (self.args[self.arg_index..]) \|arg\| {
12181	if (arg != .none) break;	19797	if (arg != .none) break;
12182	} else try self.airDbgVarArgs();	19798	} else try self.airDbgVarArgs();
12183	self.finishAirBookkeeping();
12184	}	19799	}
12185		19800
12186	fn airDbgVarArgs(self: *Self) !void {	19801	fn airDbgVarArgs(self: *CodeGen) !void {
12187	if (self.pt.zcu.typeToFunc(self.fn_type).?.is_var_args)	19802	if (self.pt.zcu.typeToFunc(self.fn_type).?.is_var_args) try self.asmPseudo(.pseudo_dbg_var_args_none);
12188	try self.asmPseudo(.pseudo_dbg_var_args_none);
12189	}	19803	}
12190		19804
12191	fn genLocalDebugInfo(	19805	fn genLocalDebugInfo(
12192	self: *Self,	19806	self: *CodeGen,
12193	inst: Air.Inst.Index,	19807	inst: Air.Inst.Index,
12194	mcv: MCValue,	19808	mcv: MCValue,
12195	) !void {	19809	) !void {
...	@@ -12200,9 +19814,9 @@ fn genLocalDebugInfo(	...	@@ -12200,9 +19814,9 @@ fn genLocalDebugInfo(
12200	switch (mcv) {	19814	switch (mcv) {
12201	.none => try self.asmAir(.dbg_local, inst),	19815	.none => try self.asmAir(.dbg_local, inst),
12202	.unreach, .dead, .elementwise_regs_then_frame, .reserved_frame, .air_ref => unreachable,	19816	.unreach, .dead, .elementwise_regs_then_frame, .reserved_frame, .air_ref => unreachable,
12203	.immediate => \|imm\| try self.asmAirImmediate(.dbg_local, inst, Immediate.u(imm)),	19817	.immediate => \|imm\| try self.asmAirImmediate(.dbg_local, inst, .u(imm)),
12204	.lea_frame => \|frame_addr\| try self.asmAirFrameAddress(.dbg_local, inst, frame_addr),	19818	.lea_frame => \|frame_addr\| try self.asmAirFrameAddress(.dbg_local, inst, frame_addr),
12205	.lea_symbol => \|sym_off\| try self.asmAirImmediate(.dbg_local, inst, Immediate.rel(sym_off)),	19819	.lea_symbol => \|sym_off\| try self.asmAirImmediate(.dbg_local, inst, .rel(sym_off)),
12206	else => {	19820	else => {
12207	const ty = switch (tag) {	19821	const ty = switch (tag) {
12208	else => unreachable,	19822	else => unreachable,
...	@@ -12211,7 +19825,7 @@ fn genLocalDebugInfo(	...	@@ -12211,7 +19825,7 @@ fn genLocalDebugInfo(
12211	self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op.operand,	19825	self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op.operand,
12212	),	19826	),
12213	};	19827	};
12214	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, self.pt.zcu));	19828	const frame_index = try self.allocFrameIndex(.initSpill(ty, self.pt.zcu));
12215	try self.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{});	19829	try self.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{});
12216	try self.asmAirMemory(.dbg_local, inst, .{	19830	try self.asmAirMemory(.dbg_local, inst, .{
12217	.base = .{ .frame = frame_index },	19831	.base = .{ .frame = frame_index },
...	@@ -12245,29 +19859,19 @@ fn genLocalDebugInfo(	...	@@ -12245,29 +19859,19 @@ fn genLocalDebugInfo(
12245	}	19859	}
12246	}	19860	}
12247		19861
12248	fn airTrap(self: *Self) !void {	19862	fn airRetAddr(self: *CodeGen, inst: Air.Inst.Index) !void {
12249	try self.asmOpOnly(.{ ._, .ud2 });
12250	self.finishAirBookkeeping();
12251	}
12252
12253	fn airBreakpoint(self: *Self) !void {
12254	try self.asmOpOnly(.{ ._, .int3 });
12255	self.finishAirBookkeeping();
12256	}
12257
12258	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
12259	const dst_mcv = try self.allocRegOrMem(inst, true);	19863	const dst_mcv = try self.allocRegOrMem(inst, true);
12260	try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});	19864	try self.genCopy(.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
12261	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });	19865	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
12262	}	19866	}
12263		19867
12264	fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {	19868	fn airFrameAddress(self: *CodeGen, inst: Air.Inst.Index) !void {
12265	const dst_mcv = try self.allocRegOrMem(inst, true);	19869	const dst_mcv = try self.allocRegOrMem(inst, true);
12266	try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});	19870	try self.genCopy(.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
12267	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });	19871	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
12268	}	19872	}
12269		19873
12270	fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {	19874	fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier, opts: CopyOptions) !void {
12271	if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{});	19875	if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{});
12272		19876
12273	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	19877	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
...	@@ -12291,7 +19895,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -12291,7 +19895,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
12291	defer allocator.free(arg_vals);	19895	defer allocator.free(arg_vals);
12292	for (arg_vals, arg_refs) \|arg_val, arg_ref\| arg_val. = .{ .air_ref = arg_ref };	19896	for (arg_vals, arg_refs) \|arg_val, arg_ref\| arg_val. = .{ .air_ref = arg_ref };
12293		19897
12294	const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals);	19898	const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals, opts);
12295		19899
12296	var bt = self.liveness.iterateBigTomb(inst);	19900	var bt = self.liveness.iterateBigTomb(inst);
12297	try self.feed(&bt, pl_op.operand);	19901	try self.feed(&bt, pl_op.operand);
...	@@ -12301,7 +19905,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -12301,7 +19905,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
12301	return self.finishAirResult(inst, result);	19905	return self.finishAirResult(inst, result);
12302	}	19906	}
12303		19907
12304	fn genCall(self: *Self, info: union(enum) {	19908	fn genCall(self: *CodeGen, info: union(enum) {
12305	air: Air.Inst.Ref,	19909	air: Air.Inst.Ref,
12306	lib: struct {	19910	lib: struct {
12307	return_type: InternPool.Index,	19911	return_type: InternPool.Index,
...	@@ -12309,7 +19913,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12309,7 +19913,7 @@ fn genCall(self: *Self, info: union(enum) {
12309	lib: ?[]const u8 = null,	19913	lib: ?[]const u8 = null,
12310	callee: []const u8,	19914	callee: []const u8,
12311	},	19915	},
12312	}, arg_types: []const Type, args: []const MCValue) !MCValue {	19916	}, arg_types: []const Type, args: []const MCValue, opts: CopyOptions) !MCValue {
12313	const pt = self.pt;	19917	const pt = self.pt;
12314	const zcu = pt.zcu;	19918	const zcu = pt.zcu;
12315	const ip = &zcu.intern_pool;	19919	const ip = &zcu.intern_pool;
...	@@ -12330,7 +19934,6 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12330,7 +19934,6 @@ fn genCall(self: *Self, info: union(enum) {
12330	}),	19934	}),
12331	};	19935	};
12332	const fn_info = zcu.typeToFunc(fn_ty).?;	19936	const fn_info = zcu.typeToFunc(fn_ty).?;
12333	const resolved_cc = abi.resolveCallingConvention(fn_info.cc, self.target.*);
12334		19937
12335	const ExpectedContents = extern struct {	19938	const ExpectedContents = extern struct {
12336	var_args: [16][@sizeOf(Type)]u8 align(@alignOf(Type)),	19939	var_args: [16][@sizeOf(Type)]u8 align(@alignOf(Type)),
...	@@ -12348,7 +19951,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12348,7 +19951,7 @@ fn genCall(self: *Self, info: union(enum) {
12348	const frame_indices = try allocator.alloc(FrameIndex, args.len);	19951	const frame_indices = try allocator.alloc(FrameIndex, args.len);
12349	defer allocator.free(frame_indices);	19952	defer allocator.free(frame_indices);
12350		19953
12351	var reg_locks = std.ArrayList(?RegisterLock).init(allocator);	19954	var reg_locks: std.ArrayList(?RegisterLock) = .init(allocator);
12352	defer reg_locks.deinit();	19955	defer reg_locks.deinit();
12353	try reg_locks.ensureTotalCapacity(16);	19956	try reg_locks.ensureTotalCapacity(16);
12354	defer for (reg_locks.items) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);	19957	defer for (reg_locks.items) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
...	@@ -12358,7 +19961,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12358,7 +19961,7 @@ fn genCall(self: *Self, info: union(enum) {
12358		19961
12359	// We need a properly aligned and sized call frame to be able to call this function.	19962	// We need a properly aligned and sized call frame to be able to call this function.
12360	{	19963	{
12361	const needed_call_frame = FrameAlloc.init(.{	19964	const needed_call_frame: FrameAlloc = .init(.{
12362	.size = call_info.stack_byte_count,	19965	.size = call_info.stack_byte_count,
12363	.alignment = call_info.stack_align,	19966	.alignment = call_info.stack_align,
12364	});	19967	});
...	@@ -12372,7 +19975,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12372,7 +19975,7 @@ fn genCall(self: *Self, info: union(enum) {
12372	}	19975	}
12373		19976
12374	try self.spillEflagsIfOccupied();	19977	try self.spillEflagsIfOccupied();
12375	try self.spillCallerPreservedRegs(resolved_cc);	19978	try self.spillCallerPreservedRegs(fn_info.cc);
12376		19979
12377	// set stack arguments first because this can clobber registers	19980	// set stack arguments first because this can clobber registers
12378	// also clobber spill arguments as we go	19981	// also clobber spill arguments as we go
...	@@ -12393,13 +19996,13 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12393,13 +19996,13 @@ fn genCall(self: *Self, info: union(enum) {
12393	try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));	19996	try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
12394	},	19997	},
12395	.indirect => \|reg_off\| {	19998	.indirect => \|reg_off\| {
12396	frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu));	19999	frame_index.* = try self.allocFrameIndex(.initType(arg_ty, zcu));
12397	try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, .{});	20000	try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts);
12398	try self.register_manager.getReg(reg_off.reg, null);	20001	try self.register_manager.getReg(reg_off.reg, null);
12399	try reg_locks.append(self.register_manager.lockReg(reg_off.reg));	20002	try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
12400	},	20003	},
12401	.load_frame => {	20004	.load_frame => {
12402	try self.genCopy(arg_ty, dst_arg, src_arg, .{});	20005	try self.genCopy(arg_ty, dst_arg, src_arg, opts);
12403	try self.freeValue(src_arg);	20006	try self.freeValue(src_arg);
12404	},	20007	},
12405	.elementwise_regs_then_frame => \|regs_frame_addr\| {	20008	.elementwise_regs_then_frame => \|regs_frame_addr\| {
...	@@ -12407,14 +20010,11 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12407,14 +20010,11 @@ fn genCall(self: *Self, info: union(enum) {
12407	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);	20010	const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
12408	defer self.register_manager.unlockReg(index_lock);	20011	defer self.register_manager.unlockReg(index_lock);
12409		20012
12410	const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{	20013	const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{
12411	.base = .{ .reg = try self.copyToTmpRegister(	20014	.base = .{ .reg = try self.copyToTmpRegister(.usize, switch (src_arg) {
12412	Type.usize,	20015	else => src_arg,
12413	switch (src_arg) {	20016	.air_ref => \|src_ref\| try self.resolveInst(src_ref),
12414	else => src_arg,	20017	}.address()) },
12415	.air_ref => \|src_ref\| try self.resolveInst(src_ref),
12416	}.address(),
12417	) },
12418	.mod = .{ .rm = .{ .size = .dword } },	20018	.mod = .{ .rm = .{ .size = .dword } },
12419	};	20019	};
12420	const src_lock = switch (src_mem.base) {	20020	const src_lock = switch (src_mem.base) {
...	@@ -12426,7 +20026,7 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12426,7 +20026,7 @@ fn genCall(self: *Self, info: union(enum) {
12426	try self.asmRegisterImmediate(	20026	try self.asmRegisterImmediate(
12427	.{ ._, .mov },	20027	.{ ._, .mov },
12428	index_reg.to32(),	20028	index_reg.to32(),
12429	Immediate.u(regs_frame_addr.regs),	20029	.u(regs_frame_addr.regs),
12430	);	20030	);
12431	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);	20031	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
12432	try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32());	20032	try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32());
...	@@ -12440,18 +20040,18 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12440,18 +20040,18 @@ fn genCall(self: *Self, info: union(enum) {
12440	} },	20040	} },
12441	});	20041	});
12442	if (self.hasFeature(.slow_incdec)) {	20042	if (self.hasFeature(.slow_incdec)) {
12443	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));	20043	try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
12444	} else {	20044	} else {
12445	try self.asmRegister(.{ ._, .inc }, index_reg.to32());	20045	try self.asmRegister(.{ ._, .inc }, index_reg.to32());
12446	}	20046	}
12447	try self.asmRegisterImmediate(	20047	try self.asmRegisterImmediate(
12448	.{ ._, .cmp },	20048	.{ ._, .cmp },
12449	index_reg.to32(),	20049	index_reg.to32(),
12450	Immediate.u(arg_ty.vectorLen(zcu)),	20050	.u(arg_ty.vectorLen(zcu)),
12451	);	20051	);
12452	_ = try self.asmJccReloc(.b, loop);	20052	_ = try self.asmJccReloc(.b, loop);
12453		20053
12454	const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc);	20054	const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc);
12455	for (param_int_regs[param_int_regs.len - regs_frame_addr.regs ..]) \|dst_reg\| {	20055	for (param_int_regs[param_int_regs.len - regs_frame_addr.regs ..]) \|dst_reg\| {
12456	try self.register_manager.getReg(dst_reg, null);	20056	try self.register_manager.getReg(dst_reg, null);
12457	try reg_locks.append(self.register_manager.lockReg(dst_reg));	20057	try reg_locks.append(self.register_manager.lockReg(dst_reg));
...	@@ -12464,9 +20064,9 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12464,9 +20064,9 @@ fn genCall(self: *Self, info: union(enum) {
12464	switch (call_info.return_value.long) {	20064	switch (call_info.return_value.long) {
12465	.none, .unreach => {},	20065	.none, .unreach => {},
12466	.indirect => \|reg_off\| {	20066	.indirect => \|reg_off\| {
12467	const ret_ty = Type.fromInterned(fn_info.return_type);	20067	const ret_ty: Type = .fromInterned(fn_info.return_type);
12468	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu));	20068	const frame_index = try self.allocFrameIndex(.initSpill(ret_ty, zcu));
12469	try self.genSetReg(reg_off.reg, Type.usize, .{	20069	try self.genSetReg(reg_off.reg, .usize, .{
12470	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },	20070	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
12471	}, .{});	20071	}, .{});
12472	call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } };	20072	call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
...	@@ -12479,29 +20079,27 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12479,29 +20079,27 @@ fn genCall(self: *Self, info: union(enum) {
12479	switch (dst_arg) {	20079	switch (dst_arg) {
12480	.none, .load_frame => {},	20080	.none, .load_frame => {},
12481	.register => \|dst_reg\| switch (fn_info.cc) {	20081	.register => \|dst_reg\| switch (fn_info.cc) {
12482	else => try self.genSetReg(	20082	else => try self.genSetReg(registerAlias(
12483	registerAlias(dst_reg, @intCast(arg_ty.abiSize(zcu))),	20083	dst_reg,
12484	arg_ty,	20084	@intCast(arg_ty.abiSize(zcu)),
12485	src_arg,	20085	), arg_ty, src_arg, opts),
12486	.{},
12487	),
12488	.x86_64_sysv, .x86_64_win => {	20086	.x86_64_sysv, .x86_64_win => {
12489	const promoted_ty = self.promoteInt(arg_ty);	20087	const promoted_ty = self.promoteInt(arg_ty);
12490	const promoted_abi_size: u32 = @intCast(promoted_ty.abiSize(zcu));	20088	const promoted_abi_size: u32 = @intCast(promoted_ty.abiSize(zcu));
12491	const dst_alias = registerAlias(dst_reg, promoted_abi_size);	20089	const dst_alias = registerAlias(dst_reg, promoted_abi_size);
12492	try self.genSetReg(dst_alias, promoted_ty, src_arg, .{});	20090	try self.genSetReg(dst_alias, promoted_ty, src_arg, opts);
12493	if (promoted_ty.toIntern() != arg_ty.toIntern())	20091	if (promoted_ty.toIntern() != arg_ty.toIntern())
12494	try self.truncateRegister(arg_ty, dst_alias);	20092	try self.truncateRegister(arg_ty, dst_alias);
12495	},	20093	},
12496	},	20094	},
12497	.register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, .{}),	20095	.register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, opts),
12498	.indirect => \|reg_off\| try self.genSetReg(reg_off.reg, Type.usize, .{	20096	.indirect => \|reg_off\| try self.genSetReg(reg_off.reg, .usize, .{
12499	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },	20097	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
12500	}, .{}),	20098	}, .{}),
12501	.elementwise_regs_then_frame => \|regs_frame_addr\| {	20099	.elementwise_regs_then_frame => \|regs_frame_addr\| {
12502	const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{	20100	const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{
12503	.base = .{ .reg = try self.copyToTmpRegister(	20101	.base = .{ .reg = try self.copyToTmpRegister(
12504	Type.usize,	20102	.usize,
12505	switch (src_arg) {	20103	switch (src_arg) {
12506	else => src_arg,	20104	else => src_arg,
12507	.air_ref => \|src_ref\| try self.resolveInst(src_ref),	20105	.air_ref => \|src_ref\| try self.resolveInst(src_ref),
...	@@ -12515,25 +20113,20 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12515,25 +20113,20 @@ fn genCall(self: *Self, info: union(enum) {
12515	};	20113	};
12516	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);	20114	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
12517		20115
12518	const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc);	20116	const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc);
12519	for (	20117	for (
12520	param_int_regs[param_int_regs.len - regs_frame_addr.regs ..],	20118	param_int_regs[param_int_regs.len - regs_frame_addr.regs ..],
12521	0..,	20119	0..,
12522	) \|dst_reg, elem_index\| {	20120	) \|dst_reg, elem_index\| {
12523	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());	20121	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12524	try self.asmMemoryImmediate(	20122	try self.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index));
12525	.{ ._, .bt },
12526	src_mem,
12527	Immediate.u(elem_index),
12528	);
12529	try self.asmSetccRegister(.c, dst_reg.to8());	20123	try self.asmSetccRegister(.c, dst_reg.to8());
12530	}	20124	}
12531	},	20125	},
12532	else => unreachable,	20126	else => unreachable,
12533	};	20127	};
12534		20128
12535	if (fn_info.is_var_args)	20129	if (fn_info.is_var_args) try self.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count));
12536	try self.asmRegisterImmediate(.{ ._, .mov }, .al, Immediate.u(call_info.fp_count));
12537		20130
12538	// Due to incremental compilation, how function calls are generated depends	20131	// Due to incremental compilation, how function calls are generated depends
12539	// on linking.	20132	// on linking.
...	@@ -12551,17 +20144,18 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12551,17 +20144,18 @@ fn genCall(self: *Self, info: union(enum) {
12551	if (self.bin_file.cast(.elf)) \|elf_file\| {	20144	if (self.bin_file.cast(.elf)) \|elf_file\| {
12552	const zo = elf_file.zigObjectPtr().?;	20145	const zo = elf_file.zigObjectPtr().?;
12553	const sym_index = try zo.getOrCreateMetadataForNav(zcu, func.owner_nav);	20146	const sym_index = try zo.getOrCreateMetadataForNav(zcu, func.owner_nav);
12554	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index }));	20147	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index }));
12555	} else if (self.bin_file.cast(.coff)) \|coff_file\| {	20148	} else if (self.bin_file.cast(.coff)) \|coff_file\| {
12556	const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);	20149	const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);
12557	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;	20150	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
12558	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index }, .{});	20151	const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc);
12559	try self.asmRegister(.{ ._, .call }, .rax);	20152	try self.genSetReg(scratch_reg, .usize, .{ .lea_got = sym_index }, .{});
		20153	try self.asmRegister(.{ ._, .call }, scratch_reg);
12560	} else if (self.bin_file.cast(.macho)) \|macho_file\| {	20154	} else if (self.bin_file.cast(.macho)) \|macho_file\| {
12561	const zo = macho_file.getZigObject().?;	20155	const zo = macho_file.getZigObject().?;
12562	const sym_index = try zo.getOrCreateMetadataForNav(macho_file, func.owner_nav);	20156	const sym_index = try zo.getOrCreateMetadataForNav(macho_file, func.owner_nav);
12563	const sym = zo.symbols.items[sym_index];	20157	const sym = zo.symbols.items[sym_index];
12564	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym.nlist_idx }));	20158	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym.nlist_idx }));
12565	} else if (self.bin_file.cast(.plan9)) \|p9\| {	20159	} else if (self.bin_file.cast(.plan9)) \|p9\| {
12566	const atom_index = try p9.seeNav(pt, func.owner_nav);	20160	const atom_index = try p9.seeNav(pt, func.owner_nav);
12567	const atom = p9.getAtom(atom_index);	20161	const atom = p9.getAtom(atom_index);
...	@@ -12579,13 +20173,13 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12579,13 +20173,13 @@ fn genCall(self: *Self, info: union(enum) {
12579	@"extern".name.toSlice(ip),	20173	@"extern".name.toSlice(ip),
12580	@"extern".lib_name.toSlice(ip),	20174	@"extern".lib_name.toSlice(ip),
12581	);	20175	);
12582	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	20176	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12583	} else if (self.bin_file.cast(.macho)) \|macho_file\| {	20177	} else if (self.bin_file.cast(.macho)) \|macho_file\| {
12584	const target_sym_index = try macho_file.getGlobalSymbol(	20178	const target_sym_index = try macho_file.getGlobalSymbol(
12585	@"extern".name.toSlice(ip),	20179	@"extern".name.toSlice(ip),
12586	@"extern".lib_name.toSlice(ip),	20180	@"extern".lib_name.toSlice(ip),
12587	);	20181	);
12588	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	20182	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12589	} else try self.genExternSymbolRef(	20183	} else try self.genExternSymbolRef(
12590	.call,	20184	.call,
12591	@"extern".lib_name.toSlice(ip),	20185	@"extern".lib_name.toSlice(ip),
...	@@ -12595,21 +20189,22 @@ fn genCall(self: *Self, info: union(enum) {	...	@@ -12595,21 +20189,22 @@ fn genCall(self: *Self, info: union(enum) {
12595	}	20189	}
12596	} else {	20190	} else {
12597	assert(self.typeOf(callee).zigTypeTag(zcu) == .pointer);	20191	assert(self.typeOf(callee).zigTypeTag(zcu) == .pointer);
12598	try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee }, .{});	20192	const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc);
12599	try self.asmRegister(.{ ._, .call }, .rax);	20193	try self.genSetReg(scratch_reg, .usize, .{ .air_ref = callee }, .{});
		20194	try self.asmRegister(.{ ._, .call }, scratch_reg);
12600	},	20195	},
12601	.lib => \|lib\| if (self.bin_file.cast(.elf)) \|elf_file\| {	20196	.lib => \|lib\| if (self.bin_file.cast(.elf)) \|elf_file\| {
12602	const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib);	20197	const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib);
12603	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	20198	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12604	} else if (self.bin_file.cast(.macho)) \|macho_file\| {	20199	} else if (self.bin_file.cast(.macho)) \|macho_file\| {
12605	const target_sym_index = try macho_file.getGlobalSymbol(lib.callee, lib.lib);	20200	const target_sym_index = try macho_file.getGlobalSymbol(lib.callee, lib.lib);
12606	try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index }));	20201	try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index }));
12607	} else try self.genExternSymbolRef(.call, lib.lib, lib.callee),	20202	} else try self.genExternSymbolRef(.call, lib.lib, lib.callee),
12608	}	20203	}
12609	return call_info.return_value.short;	20204	return call_info.return_value.short;
12610	}	20205	}
12611		20206
12612	fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	20207	fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
12613	const pt = self.pt;	20208	const pt = self.pt;
12614	const zcu = pt.zcu;	20209	const zcu = pt.zcu;
12615	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	20210	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
...	@@ -12619,13 +20214,15 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -12619,13 +20214,15 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
12619	.none => {},	20214	.none => {},
12620	.register,	20215	.register,
12621	.register_pair,	20216	.register_pair,
		20217	.register_triple,
		20218	.register_quadruple,
12622	=> try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }),	20219	=> try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }),
12623	.indirect => \|reg_off\| {	20220	.indirect => \|reg_off\| {
12624	try self.register_manager.getReg(reg_off.reg, null);	20221	try self.register_manager.getReg(reg_off.reg, null);
12625	const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);	20222	const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
12626	defer self.register_manager.unlockReg(lock);	20223	defer self.register_manager.unlockReg(lock);
12627		20224
12628	try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long, .{});	20225	try self.genSetReg(reg_off.reg, .usize, self.ret_mcv.long, .{});
12629	try self.genSetMem(	20226	try self.genSetMem(
12630	.{ .reg = reg_off.reg },	20227	.{ .reg = reg_off.reg },
12631	reg_off.off,	20228	reg_off.off,
...	@@ -12642,10 +20239,10 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -12642,10 +20239,10 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
12642	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction	20239	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
12643	// which is available if the jump is 127 bytes or less forward.	20240	// which is available if the jump is 127 bytes or less forward.
12644	const jmp_reloc = try self.asmJmpReloc(undefined);	20241	const jmp_reloc = try self.asmJmpReloc(undefined);
12645	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);	20242	try self.epilogue_relocs.append(self.gpa, jmp_reloc);
12646	}	20243	}
12647		20244
12648	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	20245	fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
12649	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	20246	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
12650	const ptr = try self.resolveInst(un_op);	20247	const ptr = try self.resolveInst(un_op);
12651		20248
...	@@ -12662,10 +20259,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12662,10 +20259,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
12662	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction	20259	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
12663	// which is available if the jump is 127 bytes or less forward.	20260	// which is available if the jump is 127 bytes or less forward.
12664	const jmp_reloc = try self.asmJmpReloc(undefined);	20261	const jmp_reloc = try self.asmJmpReloc(undefined);
12665	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);	20262	try self.epilogue_relocs.append(self.gpa, jmp_reloc);
12666	}	20263	}
12667		20264
12668	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	20265	fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
12669	const pt = self.pt;	20266	const pt = self.pt;
12670	const zcu = pt.zcu;	20267	const zcu = pt.zcu;
12671	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	20268	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -12686,7 +20283,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12686,7 +20283,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12686	self.register_manager.lockRegAssumeUnused(lhs_ro.reg),	20283	self.register_manager.lockRegAssumeUnused(lhs_ro.reg),
12687	null,	20284	null,
12688	},	20285	},
12689	else => .{null} ** 2,	20286	else => @splat(null),
12690	};	20287	};
12691	defer for (lhs_locks) \|lhs_lock\| if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);	20288	defer for (lhs_locks) \|lhs_lock\| if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
12692		20289
...	@@ -12695,7 +20292,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12695,7 +20292,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12695	.register => \|rhs_reg\| .{ self.register_manager.lockReg(rhs_reg), null },	20292	.register => \|rhs_reg\| .{ self.register_manager.lockReg(rhs_reg), null },
12696	.register_pair => \|rhs_regs\| self.register_manager.lockRegs(2, rhs_regs),	20293	.register_pair => \|rhs_regs\| self.register_manager.lockRegs(2, rhs_regs),
12697	.register_offset => \|rhs_ro\| .{ self.register_manager.lockReg(rhs_ro.reg), null },	20294	.register_offset => \|rhs_ro\| .{ self.register_manager.lockReg(rhs_ro.reg), null },
12698	else => .{null} ** 2,	20295	else => @splat(null),
12699	};	20296	};
12700	defer for (rhs_locks) \|rhs_lock\| if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	20297	defer for (rhs_locks) \|rhs_lock\| if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
12701		20298
...	@@ -12723,8 +20320,8 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12723,8 +20320,8 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12723	},	20320	},
12724	floatCompilerRtAbiName(float_bits),	20321	floatCompilerRtAbiName(float_bits),
12725	}) catch unreachable,	20322	}) catch unreachable,
12726	} }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });	20323	} }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, .{});
12727	try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);	20324	try self.genBinOpMir(.{ ._, .@"test" }, .i32, ret, ret);
12728	break :result switch (op) {	20325	break :result switch (op) {
12729	.eq => .e,	20326	.eq => .e,
12730	.neq => .ne,	20327	.neq => .ne,
...	@@ -12745,23 +20342,23 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12745,23 +20342,23 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12745	const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);	20342	const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);
12746	defer self.register_manager.unlockReg(temp_lhs_lock);	20343	defer self.register_manager.unlockReg(temp_lhs_lock);
12747		20344
12748	if (lhs_mcv.isMemory()) try self.asmRegisterMemory(	20345	if (lhs_mcv.isBase()) try self.asmRegisterMemory(
12749	.{ ._, .mov },	20346	.{ ._, .mov },
12750	temp_lhs_reg.to8(),	20347	temp_lhs_reg.to8(),
12751	try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),	20348	try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }),
12752	) else {	20349	) else {
12753	try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});	20350	try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
12754	try self.asmRegisterImmediate(	20351	try self.asmRegisterImmediate(
12755	.{ ._r, .sh },	20352	.{ ._r, .sh },
12756	registerAlias(temp_lhs_reg, opt_abi_size),	20353	registerAlias(temp_lhs_reg, opt_abi_size),
12757	Immediate.u(payload_abi_size * 8),	20354	.u(payload_abi_size * 8),
12758	);	20355	);
12759	}	20356	}
12760		20357
12761	const payload_compare = payload_compare: {	20358	const payload_compare = payload_compare: {
12762	if (rhs_mcv.isMemory()) {	20359	if (rhs_mcv.isBase()) {
12763	const rhs_mem =	20360	const rhs_mem =
12764	try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);	20361	try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte });
12765	try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());	20362	try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
12766	const payload_compare = try self.asmJccReloc(.nz, undefined);	20363	const payload_compare = try self.asmJccReloc(.nz, undefined);
12767	try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);	20364	try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
...	@@ -12775,7 +20372,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12775,7 +20372,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12775	try self.asmRegisterImmediate(	20372	try self.asmRegisterImmediate(
12776	.{ ._r, .sh },	20373	.{ ._r, .sh },
12777	registerAlias(temp_rhs_reg, opt_abi_size),	20374	registerAlias(temp_rhs_reg, opt_abi_size),
12778	Immediate.u(payload_abi_size * 8),	20375	.u(payload_abi_size * 8),
12779	);	20376	);
12780	try self.asmRegisterRegister(	20377	try self.asmRegisterRegister(
12781	.{ ._, .@"test" },	20378	.{ ._, .@"test" },
...	@@ -12810,13 +20407,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12810,13 +20407,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12810	} else .may_flip;	20407	} else .may_flip;
12811		20408
12812	const flipped = switch (may_flip) {	20409	const flipped = switch (may_flip) {
12813	.may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isMemory(),	20410	.may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isBase(),
12814	.must_flip => true,	20411	.must_flip => true,
12815	.must_not_flip => false,	20412	.must_not_flip => false,
12816	};	20413	};
12817	const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv;	20414	const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
12818	const dst_mcv = if (unmat_dst_mcv.isRegister() or	20415	const dst_mcv = if (unmat_dst_mcv.isRegister() or
12819	(abi_size <= 8 and unmat_dst_mcv.isMemory())) unmat_dst_mcv else dst: {	20416	(abi_size <= 8 and unmat_dst_mcv.isBase())) unmat_dst_mcv else dst: {
12820	const dst_mcv = try self.allocTempRegOrMem(ty, true);	20417	const dst_mcv = try self.allocTempRegOrMem(ty, true);
12821	try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{});	20418	try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{});
12822	break :dst dst_mcv;	20419	break :dst dst_mcv;
...	@@ -12830,7 +20427,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12830,7 +20427,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12830	if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;	20427	if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
12831	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);	20428	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
12832		20429
12833	break :result Condition.fromCompareOperator(	20430	break :result .fromCompareOperator(
12834	if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned,	20431	if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned,
12835	result_op: {	20432	result_op: {
12836	const flipped_op = if (flipped) op.reverse() else op;	20433	const flipped_op = if (flipped) op.reverse() else op;
...	@@ -12851,9 +20448,9 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12851,9 +20448,9 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12851	.undef,	20448	.undef,
12852	.immediate,	20449	.immediate,
12853	.eflags,	20450	.eflags,
12854	.register,
12855	.register_offset,	20451	.register_offset,
12856	.register_overflow,	20452	.register_overflow,
		20453	.register_mask,
12857	.indirect,	20454	.indirect,
12858	.lea_direct,	20455	.lea_direct,
12859	.lea_got,	20456	.lea_got,
...	@@ -12864,13 +20461,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12864,13 +20461,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12864	.reserved_frame,	20461	.reserved_frame,
12865	.air_ref,	20462	.air_ref,
12866	=> unreachable,	20463	=> unreachable,
12867	.register_pair, .load_frame => null,	20464	.register, .register_pair, .register_triple, .register_quadruple, .load_frame => null,
12868	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: {	20465	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: {
12869	switch (resolved_dst_mcv) {	20466	switch (resolved_dst_mcv) {
12870	.memory => \|addr\| if (math.cast(	20467	.memory => \|addr\| if (std.math.cast(
12871	i32,	20468	i32,
12872	@as(i64, @bitCast(addr)),	20469	@as(i64, @bitCast(addr)),
12873	) != null and math.cast(	20470	) != null and std.math.cast(
12874	i32,	20471	i32,
12875	@as(i64, @bitCast(addr)) + abi_size - 8,	20472	@as(i64, @bitCast(addr)) + abi_size - 8,
12876	) != null) break :dst null,	20473	) != null) break :dst null,
...	@@ -12886,20 +20483,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12886,20 +20483,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12886	self.register_manager.lockRegAssumeUnused(dst_addr_reg);	20483	self.register_manager.lockRegAssumeUnused(dst_addr_reg);
12887	errdefer self.register_manager.unlockReg(dst_addr_lock);	20484	errdefer self.register_manager.unlockReg(dst_addr_lock);
12888		20485
12889	try self.genSetReg(	20486	try self.genSetReg(dst_addr_reg, .usize, resolved_dst_mcv.address(), .{});
12890	dst_addr_reg,
12891	Type.usize,
12892	resolved_dst_mcv.address(),
12893	.{},
12894	);
12895	break :dst .{	20487	break :dst .{
12896	.addr_reg = dst_addr_reg,	20488	.addr_reg = dst_addr_reg,
12897	.addr_lock = dst_addr_lock,	20489	.addr_lock = dst_addr_lock,
12898	};	20490	};
12899	},	20491	},
12900	};	20492	};
12901	defer if (dst_info) \|info\|	20493	defer if (dst_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
12902	self.register_manager.unlockReg(info.addr_lock);
12903		20494
12904	const resolved_src_mcv = switch (src_mcv) {	20495	const resolved_src_mcv = switch (src_mcv) {
12905	else => src_mcv,	20496	else => src_mcv,
...	@@ -12915,6 +20506,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12915,6 +20506,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12915	.register,	20506	.register,
12916	.register_offset,	20507	.register_offset,
12917	.register_overflow,	20508	.register_overflow,
		20509	.register_mask,
12918	.indirect,	20510	.indirect,
12919	.lea_symbol,	20511	.lea_symbol,
12920	.lea_direct,	20512	.lea_direct,
...	@@ -12925,13 +20517,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12925,13 +20517,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12925	.reserved_frame,	20517	.reserved_frame,
12926	.air_ref,	20518	.air_ref,
12927	=> unreachable,	20519	=> unreachable,
12928	.register_pair, .load_frame => null,	20520	.register_pair, .register_triple, .register_quadruple, .load_frame => null,
12929	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {	20521	.memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: {
12930	switch (resolved_src_mcv) {	20522	switch (resolved_src_mcv) {
12931	.memory => \|addr\| if (math.cast(	20523	.memory => \|addr\| if (std.math.cast(
12932	i32,	20524	i32,
12933	@as(i64, @bitCast(addr)),	20525	@as(i64, @bitCast(addr)),
12934	) != null and math.cast(	20526	) != null and std.math.cast(
12935	i32,	20527	i32,
12936	@as(i64, @bitCast(addr)) + abi_size - 8,	20528	@as(i64, @bitCast(addr)) + abi_size - 8,
12937	) != null) break :src null,	20529	) != null) break :src null,
...	@@ -12947,12 +20539,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12947,12 +20539,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12947	self.register_manager.lockRegAssumeUnused(src_addr_reg);	20539	self.register_manager.lockRegAssumeUnused(src_addr_reg);
12948	errdefer self.register_manager.unlockReg(src_addr_lock);	20540	errdefer self.register_manager.unlockReg(src_addr_lock);
12949		20541
12950	try self.genSetReg(	20542	try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{});
12951	src_addr_reg,
12952	Type.usize,
12953	resolved_src_mcv.address(),
12954	.{},
12955	);
12956	break :src .{	20543	break :src .{
12957	.addr_reg = src_addr_reg,	20544	.addr_reg = src_addr_reg,
12958	.addr_lock = src_addr_lock,	20545	.addr_lock = src_addr_lock,
...	@@ -12962,25 +20549,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12962,25 +20549,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12962	defer if (src_info) \|info\|	20549	defer if (src_info) \|info\|
12963	self.register_manager.unlockReg(info.addr_lock);	20550	self.register_manager.unlockReg(info.addr_lock);
12964		20551
12965	const regs = try self.register_manager.allocRegs(	20552	const regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
12966	2,
12967	.{null} ** 2,
12968	abi.RegisterClass.gp,
12969	);
12970	const acc_reg = regs[0].to64();	20553	const acc_reg = regs[0].to64();
12971	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);	20554	const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
12972	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);	20555	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);
12973		20556
12974	const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;	20557	const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
12975	var limb_i: u16 = 0;	20558	var limb_i: u16 = 0;
12976	while (limb_i < limbs_len) : (limb_i += 1) {	20559	while (limb_i < limbs_len) : (limb_i += 1) {
12977	const off = limb_i * 8;	20560	const off = limb_i * 8;
12978	const tmp_reg = regs[@min(limb_i, 1)].to64();	20561	const tmp_reg = regs[@min(limb_i, 1)].to64();
12979		20562
12980	try self.genSetReg(tmp_reg, Type.usize, if (dst_info) \|info\| .{	20563	try self.genSetReg(tmp_reg, .usize, if (dst_info) \|info\| .{
12981	.indirect = .{ .reg = info.addr_reg, .off = off },	20564	.indirect = .{ .reg = info.addr_reg, .off = off },
12982	} else switch (resolved_dst_mcv) {	20565	} else switch (resolved_dst_mcv) {
12983	.register_pair => \|dst_regs\| .{ .register = dst_regs[limb_i] },	20566	inline .register_pair,
		20567	.register_triple,
		20568	.register_quadruple,
		20569	=> \|dst_regs\| .{ .register = dst_regs[limb_i] },
12984	.memory => \|dst_addr\| .{	20570	.memory => \|dst_addr\| .{
12985	.memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off),	20571	.memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off),
12986	},	20572	},
...	@@ -12997,14 +20583,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12997,14 +20583,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12997		20583
12998	try self.genBinOpMir(	20584	try self.genBinOpMir(
12999	.{ ._, .xor },	20585	.{ ._, .xor },
13000	Type.usize,	20586	.usize,
13001	.{ .register = tmp_reg },	20587	.{ .register = tmp_reg },
13002	if (src_info) \|info\| .{	20588	if (src_info) \|info\| .{
13003	.indirect = .{ .reg = info.addr_reg, .off = off },	20589	.indirect = .{ .reg = info.addr_reg, .off = off },
13004	} else switch (resolved_src_mcv) {	20590	} else switch (resolved_src_mcv) {
13005	.register_pair => \|src_regs\| .{	20591	inline .register_pair,
13006	.register = src_regs[limb_i],	20592	.register_triple,
13007	},	20593	.register_quadruple,
		20594	=> \|src_regs\| .{ .register = src_regs[limb_i] },
13008	.memory => \|src_addr\| .{	20595	.memory => \|src_addr\| .{
13009	.memory = @bitCast(@as(i64, @bitCast(src_addr)) + off),	20596	.memory = @bitCast(@as(i64, @bitCast(src_addr)) + off),
13010	},	20597	},
...	@@ -13062,12 +20649,12 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -13062,12 +20649,12 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
13062	const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);	20649	const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
13063	defer self.register_manager.unlockReg(tmp2_lock);	20650	defer self.register_manager.unlockReg(tmp2_lock);
13064		20651
13065	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	20652	if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
13066	.{ .vp_w, .insr },	20653	.{ .vp_w, .insr },
13067	tmp1_reg,	20654	tmp1_reg,
13068	dst_reg.to128(),	20655	dst_reg.to128(),
13069	try src_mcv.mem(self, .word),	20656	try src_mcv.mem(self, .{ .size = .word }),
13070	Immediate.u(1),	20657	.u(1),
13071	) else try self.asmRegisterRegisterRegister(	20658	) else try self.asmRegisterRegisterRegister(
13072	.{ .vp_, .unpcklwd },	20659	.{ .vp_, .unpcklwd },
13073	tmp1_reg,	20660	tmp1_reg,
...	@@ -13112,19 +20699,19 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -13112,19 +20699,19 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
13112	return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });	20699	return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
13113	}	20700	}
13114		20701
13115	fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {	20702	fn airCmpVector(self: *CodeGen, inst: Air.Inst.Index) !void {
13116	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	20703	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13117	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;	20704	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
13118	const dst_mcv = try self.genBinOp(	20705	const dst_mcv = try self.genBinOp(
13119	inst,	20706	inst,
13120	Air.Inst.Tag.fromCmpOp(extra.compareOperator(), false),	20707	.fromCmpOp(extra.compareOperator(), false),
13121	extra.lhs,	20708	extra.lhs,
13122	extra.rhs,	20709	extra.rhs,
13123	);	20710	);
13124	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });	20711	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
13125	}	20712	}
13126		20713
13127	fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {	20714	fn airCmpLtErrorsLen(self: *CodeGen, inst: Air.Inst.Index) !void {
13128	const pt = self.pt;	20715	const pt = self.pt;
13129	const zcu = pt.zcu;	20716	const zcu = pt.zcu;
13130	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	20717	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
...	@@ -13149,7 +20736,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13149,7 +20736,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
13149	registerAlias(dst_reg, op_abi_size),	20736	registerAlias(dst_reg, op_abi_size),
13150	.{	20737	.{
13151	.base = .{ .reg = addr_reg },	20738	.base = .{ .reg = addr_reg },
13152	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(op_abi_size) } },	20739	.mod = .{ .rm = .{ .size = .fromSize(op_abi_size) } },
13153	},	20740	},
13154	);	20741	);
13155		20742
...	@@ -13157,7 +20744,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13157,7 +20744,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
13157	return self.finishAir(inst, .{ .eflags = .b }, .{ un_op, .none, .none });	20744	return self.finishAir(inst, .{ .eflags = .b }, .{ un_op, .none, .none });
13158	}	20745	}
13159		20746
13160	fn airTry(self: *Self, inst: Air.Inst.Index) !void {	20747	fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
13161	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	20748	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
13162	const extra = self.air.extraData(Air.Try, pl_op.payload);	20749	const extra = self.air.extraData(Air.Try, pl_op.payload);
13163	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	20750	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...	@@ -13166,7 +20753,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13166,7 +20753,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
13166	return self.finishAir(inst, result, .{ .none, .none, .none });	20753	return self.finishAir(inst, result, .{ .none, .none, .none });
13167	}	20754	}
13168		20755
13169	fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {	20756	fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
13170	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	20757	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13171	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);	20758	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
13172	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	20759	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...	@@ -13176,7 +20763,7 @@ fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13176,7 +20763,7 @@ fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
13176	}	20763	}
13177		20764
13178	fn genTry(	20765	fn genTry(
13179	self: *Self,	20766	self: *CodeGen,
13180	inst: Air.Inst.Index,	20767	inst: Air.Inst.Index,
13181	operand: Air.Inst.Ref,	20768	operand: Air.Inst.Ref,
13182	body: []const Air.Inst.Index,	20769	body: []const Air.Inst.Index,
...	@@ -13191,7 +20778,7 @@ fn genTry(	...	@@ -13191,7 +20778,7 @@ fn genTry(
13191	else	20778	else
13192	try self.isErr(null, operand_ty, operand_mcv);	20779	try self.isErr(null, operand_ty, operand_mcv);
13193		20780
13194	const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv);	20781	const reloc = try self.genCondBrMir(.anyerror, is_err_mcv);
13195		20782
13196	if (self.liveness.operandDies(inst, 0)) {	20783	if (self.liveness.operandDies(inst, 0)) {
13197	if (operand.toIndex()) \|operand_inst\| try self.processDeath(operand_inst);	20784	if (operand.toIndex()) \|operand_inst\| try self.processDeath(operand_inst);
...	@@ -13222,7 +20809,7 @@ fn genTry(	...	@@ -13222,7 +20809,7 @@ fn genTry(
13222	return result;	20809	return result;
13223	}	20810	}
13224		20811
13225	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {	20812	fn airDbgStmt(self: *CodeGen, inst: Air.Inst.Index) !void {
13226	const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;	20813	const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
13227	_ = try self.addInst(.{	20814	_ = try self.addInst(.{
13228	.tag = .pseudo,	20815	.tag = .pseudo,
...	@@ -13232,18 +20819,16 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13232,18 +20819,16 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
13232	.column = dbg_stmt.column,	20819	.column = dbg_stmt.column,
13233	} },	20820	} },
13234	});	20821	});
13235	self.finishAirBookkeeping();
13236	}	20822	}
13237		20823
13238	fn airDbgEmptyStmt(self: *Self) !void {	20824	fn airDbgEmptyStmt(self: *CodeGen) !void {
13239	if (self.mir_instructions.len > 0 and	20825	if (self.mir_instructions.len > 0 and
13240	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)	20826	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)
13241	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;	20827	self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;
13242	try self.asmOpOnly(.{ ._, .nop });	20828	try self.asmOpOnly(.{ ._, .nop });
13243	self.finishAirBookkeeping();
13244	}	20829	}
13245		20830
13246	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {	20831	fn airDbgInlineBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
13247	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	20832	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13248	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);	20833	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
13249	const old_inline_func = self.inline_func;	20834	const old_inline_func = self.inline_func;
...	@@ -13262,13 +20847,13 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13262,13 +20847,13 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
13262	});	20847	});
13263	}	20848	}
13264		20849
13265	fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {	20850	fn airDbgVar(self: *CodeGen, inst: Air.Inst.Index) !void {
13266	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	20851	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
13267	try self.genLocalDebugInfo(inst, try self.resolveInst(pl_op.operand));	20852	try self.genLocalDebugInfo(inst, try self.resolveInst(pl_op.operand));
13268	return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });	20853	return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
13269	}	20854	}
13270		20855
13271	fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {	20856	fn genCondBrMir(self: *CodeGen, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13272	const pt = self.pt;	20857	const pt = self.pt;
13273	const abi_size = ty.abiSize(pt.zcu);	20858	const abi_size = ty.abiSize(pt.zcu);
13274	switch (mcv) {	20859	switch (mcv) {
...	@@ -13278,7 +20863,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {	...	@@ -13278,7 +20863,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13278	},	20863	},
13279	.register => \|reg\| {	20864	.register => \|reg\| {
13280	try self.spillEflagsIfOccupied();	20865	try self.spillEflagsIfOccupied();
13281	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));	20866	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), .u(1));
13282	return self.asmJccReloc(.z, undefined);	20867	return self.asmJccReloc(.z, undefined);
13283	},	20868	},
13284	.immediate,	20869	.immediate,
...	@@ -13295,7 +20880,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {	...	@@ -13295,7 +20880,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13295	}	20880	}
13296	}	20881	}
13297		20882
13298	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	20883	fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void {
13299	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	20884	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
13300	const cond = try self.resolveInst(pl_op.operand);	20885	const cond = try self.resolveInst(pl_op.operand);
13301	const cond_ty = self.typeOf(pl_op.operand);	20886	const cond_ty = self.typeOf(pl_op.operand);
...	@@ -13338,10 +20923,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13338,10 +20923,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
13338	});	20923	});
13339		20924
13340	// We already took care of pl_op.operand earlier, so there's nothing left to do.	20925	// We already took care of pl_op.operand earlier, so there's nothing left to do.
13341	self.finishAirBookkeeping();
13342	}	20926	}
13343		20927
13344	fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {	20928	fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
13345	const pt = self.pt;	20929	const pt = self.pt;
13346	const zcu = pt.zcu;	20930	const zcu = pt.zcu;
13347	switch (opt_mcv) {	20931	switch (opt_mcv) {
...	@@ -13353,10 +20937,10 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13353,10 +20937,10 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13353		20937
13354	const pl_ty = opt_ty.optionalChild(zcu);	20938	const pl_ty = opt_ty.optionalChild(zcu);
13355		20939
13356	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))	20940	const some_info: struct { off: u31, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
13357	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }	20941	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
13358	else	20942	else
13359	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };	20943	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = .bool };
13360		20944
13361	self.eflags_inst = inst;	20945	self.eflags_inst = inst;
13362	switch (opt_mcv) {	20946	switch (opt_mcv) {
...	@@ -13366,9 +20950,11 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13366,9 +20950,11 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13366	.undef,	20950	.undef,
13367	.immediate,	20951	.immediate,
13368	.eflags,	20952	.eflags,
13369	.register_pair,	20953	.register_triple,
		20954	.register_quadruple,
13370	.register_offset,	20955	.register_offset,
13371	.register_overflow,	20956	.register_overflow,
		20957	.register_mask,
13372	.lea_direct,	20958	.lea_direct,
13373	.lea_got,	20959	.lea_got,
13374	.lea_tlv,	20960	.lea_tlv,
...	@@ -13396,7 +20982,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13396,7 +20982,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13396	try self.asmRegisterImmediate(	20982	try self.asmRegisterImmediate(
13397	.{ ._, .bt },	20983	.{ ._, .bt },
13398	registerAlias(opt_reg, opt_abi_size),	20984	registerAlias(opt_reg, opt_abi_size),
13399	Immediate.u(@as(u6, @intCast(some_info.off * 8))),	20985	.u(@as(u6, @intCast(some_info.off * 8))),
		20986	);
		20987	return .{ .eflags = .nc };
		20988	},
		20989
		20990	.register_pair => \|opt_regs\| {
		20991	if (some_info.off == 0) {
		20992	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));
		20993	const alias_reg = registerAlias(opt_regs[0], some_abi_size);
		20994	assert(some_abi_size * 8 == alias_reg.bitSize());
		20995	try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
		20996	return .{ .eflags = .z };
		20997	}
		20998	assert(some_info.ty.ip_index == .bool_type);
		20999	const opt_abi_size: u32 = @intCast(opt_ty.abiSize(zcu));
		21000	try self.asmRegisterImmediate(
		21001	.{ ._, .bt },
		21002	registerAlias(opt_regs[some_info.off / 8], opt_abi_size),
		21003	.u(@as(u6, @truncate(some_info.off * 8))),
13400	);	21004	);
13401	return .{ .eflags = .nc };	21005	return .{ .eflags = .nc };
13402	},	21006	},
...	@@ -13411,18 +21015,18 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13411,18 +21015,18 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13411	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	21015	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
13412	defer self.register_manager.unlockReg(addr_reg_lock);	21016	defer self.register_manager.unlockReg(addr_reg_lock);
13413		21017
13414	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address(), .{});	21018	try self.genSetReg(addr_reg, .usize, opt_mcv.address(), .{});
13415	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));	21019	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));
13416	try self.asmMemoryImmediate(	21020	try self.asmMemoryImmediate(
13417	.{ ._, .cmp },	21021	.{ ._, .cmp },
13418	.{	21022	.{
13419	.base = .{ .reg = addr_reg },	21023	.base = .{ .reg = addr_reg },
13420	.mod = .{ .rm = .{	21024	.mod = .{ .rm = .{
13421	.size = Memory.Size.fromSize(some_abi_size),	21025	.size = .fromSize(some_abi_size),
13422	.disp = some_info.off,	21026	.disp = some_info.off,
13423	} },	21027	} },
13424	},	21028	},
13425	Immediate.u(0),	21029	.u(0),
13426	);	21030	);
13427	return .{ .eflags = .e };	21031	return .{ .eflags = .e };
13428	},	21032	},
...	@@ -13435,27 +21039,27 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -13435,27 +21039,27 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13435	.indirect => \|reg_off\| .{	21039	.indirect => \|reg_off\| .{
13436	.base = .{ .reg = reg_off.reg },	21040	.base = .{ .reg = reg_off.reg },
13437	.mod = .{ .rm = .{	21041	.mod = .{ .rm = .{
13438	.size = Memory.Size.fromSize(some_abi_size),	21042	.size = .fromSize(some_abi_size),
13439	.disp = reg_off.off + some_info.off,	21043	.disp = reg_off.off + some_info.off,
13440	} },	21044	} },
13441	},	21045	},
13442	.load_frame => \|frame_addr\| .{	21046	.load_frame => \|frame_addr\| .{
13443	.base = .{ .frame = frame_addr.index },	21047	.base = .{ .frame = frame_addr.index },
13444	.mod = .{ .rm = .{	21048	.mod = .{ .rm = .{
13445	.size = Memory.Size.fromSize(some_abi_size),	21049	.size = .fromSize(some_abi_size),
13446	.disp = frame_addr.off + some_info.off,	21050	.disp = frame_addr.off + some_info.off,
13447	} },	21051	} },
13448	},	21052	},
13449	else => unreachable,	21053	else => unreachable,
13450	},	21054	},
13451	Immediate.u(0),	21055	.u(0),
13452	);	21056	);
13453	return .{ .eflags = .e };	21057	return .{ .eflags = .e };
13454	},	21058	},
13455	}	21059	}
13456	}	21060	}
13457		21061
13458	fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {	21062	fn isNullPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
13459	const pt = self.pt;	21063	const pt = self.pt;
13460	const zcu = pt.zcu;	21064	const zcu = pt.zcu;
13461	const opt_ty = ptr_ty.childType(zcu);	21065	const opt_ty = ptr_ty.childType(zcu);
...	@@ -13466,7 +21070,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)	...	@@ -13466,7 +21070,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
13466	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))	21070	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
13467	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }	21071	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
13468	else	21072	else
13469	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };	21073	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = .bool };
13470		21074
13471	const ptr_reg = switch (ptr_mcv) {	21075	const ptr_reg = switch (ptr_mcv) {
13472	.register => \|reg\| reg,	21076	.register => \|reg\| reg,
...	@@ -13481,18 +21085,18 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)	...	@@ -13481,18 +21085,18 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
13481	.{	21085	.{
13482	.base = .{ .reg = ptr_reg },	21086	.base = .{ .reg = ptr_reg },
13483	.mod = .{ .rm = .{	21087	.mod = .{ .rm = .{
13484	.size = Memory.Size.fromSize(some_abi_size),	21088	.size = .fromSize(some_abi_size),
13485	.disp = some_info.off,	21089	.disp = some_info.off,
13486	} },	21090	} },
13487	},	21091	},
13488	Immediate.u(0),	21092	.u(0),
13489	);	21093	);
13490		21094
13491	self.eflags_inst = inst;	21095	self.eflags_inst = inst;
13492	return .{ .eflags = .e };	21096	return .{ .eflags = .e };
13493	}	21097	}
13494		21098
13495	fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {	21099	fn isErr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
13496	const pt = self.pt;	21100	const pt = self.pt;
13497	const zcu = pt.zcu;	21101	const zcu = pt.zcu;
13498	const err_ty = eu_ty.errorUnionSet(zcu);	21102	const err_ty = eu_ty.errorUnionSet(zcu);
...	@@ -13500,7 +21104,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)	...	@@ -13500,7 +21104,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
13500		21104
13501	try self.spillEflagsIfOccupied();	21105	try self.spillEflagsIfOccupied();
13502		21106
13503	const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));	21107	const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
13504	switch (eu_mcv) {	21108	switch (eu_mcv) {
13505	.register => \|reg\| {	21109	.register => \|reg\| {
13506	const eu_lock = self.register_manager.lockReg(reg);	21110	const eu_lock = self.register_manager.lockReg(reg);
...	@@ -13512,22 +21116,17 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)	...	@@ -13512,22 +21116,17 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
13512	.{ ._r, .sh },	21116	.{ ._r, .sh },
13513	eu_ty,	21117	eu_ty,
13514	.{ .register = tmp_reg },	21118	.{ .register = tmp_reg },
13515	Type.u8,	21119	.u8,
13516	.{ .immediate = @as(u6, @intCast(err_off * 8)) },	21120	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
13517	);	21121	);
13518	} else {	21122	} else {
13519	try self.truncateRegister(Type.anyerror, tmp_reg);	21123	try self.truncateRegister(.anyerror, tmp_reg);
13520	}	21124	}
13521	try self.genBinOpMir(	21125	try self.genBinOpMir(.{ ._, .cmp }, .anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
13522	.{ ._, .cmp },
13523	Type.anyerror,
13524	.{ .register = tmp_reg },
13525	.{ .immediate = 0 },
13526	);
13527	},	21126	},
13528	.load_frame => \|frame_addr\| try self.genBinOpMir(	21127	.load_frame => \|frame_addr\| try self.genBinOpMir(
13529	.{ ._, .cmp },	21128	.{ ._, .cmp },
13530	Type.anyerror,	21129	.anyerror,
13531	.{ .load_frame = .{	21130	.{ .load_frame = .{
13532	.index = frame_addr.index,	21131	.index = frame_addr.index,
13533	.off = frame_addr.off + err_off,	21132	.off = frame_addr.off + err_off,
...	@@ -13541,7 +21140,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)	...	@@ -13541,7 +21140,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
13541	return MCValue{ .eflags = .a };	21140	return MCValue{ .eflags = .a };
13542	}	21141	}
13543		21142
13544	fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {	21143	fn isErrPtr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
13545	const pt = self.pt;	21144	const pt = self.pt;
13546	const zcu = pt.zcu;	21145	const zcu = pt.zcu;
13547	const eu_ty = ptr_ty.childType(zcu);	21146	const eu_ty = ptr_ty.childType(zcu);
...	@@ -13557,24 +21156,24 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV	...	@@ -13557,24 +21156,24 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
13557	const ptr_lock = self.register_manager.lockReg(ptr_reg);	21156	const ptr_lock = self.register_manager.lockReg(ptr_reg);
13558	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	21157	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
13559		21158
13560	const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));	21159	const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
13561	try self.asmMemoryImmediate(	21160	try self.asmMemoryImmediate(
13562	.{ ._, .cmp },	21161	.{ ._, .cmp },
13563	.{	21162	.{
13564	.base = .{ .reg = ptr_reg },	21163	.base = .{ .reg = ptr_reg },
13565	.mod = .{ .rm = .{	21164	.mod = .{ .rm = .{
13566	.size = self.memSize(Type.anyerror),	21165	.size = self.memSize(.anyerror),
13567	.disp = err_off,	21166	.disp = err_off,
13568	} },	21167	} },
13569	},	21168	},
13570	Immediate.u(0),	21169	.u(0),
13571	);	21170	);
13572		21171
13573	if (maybe_inst) \|inst\| self.eflags_inst = inst;	21172	if (maybe_inst) \|inst\| self.eflags_inst = inst;
13574	return MCValue{ .eflags = .a };	21173	return MCValue{ .eflags = .a };
13575	}	21174	}
13576		21175
13577	fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {	21176	fn isNonErr(self: *CodeGen, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
13578	const is_err_res = try self.isErr(inst, eu_ty, eu_mcv);	21177	const is_err_res = try self.isErr(inst, eu_ty, eu_mcv);
13579	switch (is_err_res) {	21178	switch (is_err_res) {
13580	.eflags => \|cc\| {	21179	.eflags => \|cc\| {
...	@@ -13589,7 +21188,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC	...	@@ -13589,7 +21188,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC
13589	}	21188	}
13590	}	21189	}
13591		21190
13592	fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {	21191	fn isNonErrPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
13593	const is_err_res = try self.isErrPtr(inst, ptr_ty, ptr_mcv);	21192	const is_err_res = try self.isErrPtr(inst, ptr_ty, ptr_mcv);
13594	switch (is_err_res) {	21193	switch (is_err_res) {
13595	.eflags => \|cc\| {	21194	.eflags => \|cc\| {
...	@@ -13604,7 +21203,7 @@ fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue	...	@@ -13604,7 +21203,7 @@ fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue
13604	}	21203	}
13605	}	21204	}
13606		21205
13607	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {	21206	fn airIsNull(self: *CodeGen, inst: Air.Inst.Index) !void {
13608	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21207	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13609	const operand = try self.resolveInst(un_op);	21208	const operand = try self.resolveInst(un_op);
13610	const ty = self.typeOf(un_op);	21209	const ty = self.typeOf(un_op);
...	@@ -13612,7 +21211,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13612,7 +21211,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
13612	return self.finishAir(inst, result, .{ un_op, .none, .none });	21211	return self.finishAir(inst, result, .{ un_op, .none, .none });
13613	}	21212	}
13614		21213
13615	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {	21214	fn airIsNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
13616	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21215	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13617	const operand = try self.resolveInst(un_op);	21216	const operand = try self.resolveInst(un_op);
13618	const ty = self.typeOf(un_op);	21217	const ty = self.typeOf(un_op);
...	@@ -13620,7 +21219,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13620,7 +21219,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
13620	return self.finishAir(inst, result, .{ un_op, .none, .none });	21219	return self.finishAir(inst, result, .{ un_op, .none, .none });
13621	}	21220	}
13622		21221
13623	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {	21222	fn airIsNonNull(self: *CodeGen, inst: Air.Inst.Index) !void {
13624	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21223	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13625	const operand = try self.resolveInst(un_op);	21224	const operand = try self.resolveInst(un_op);
13626	const ty = self.typeOf(un_op);	21225	const ty = self.typeOf(un_op);
...	@@ -13632,7 +21231,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13632,7 +21231,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
13632	return self.finishAir(inst, result, .{ un_op, .none, .none });	21231	return self.finishAir(inst, result, .{ un_op, .none, .none });
13633	}	21232	}
13634		21233
13635	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {	21234	fn airIsNonNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
13636	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21235	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13637	const operand = try self.resolveInst(un_op);	21236	const operand = try self.resolveInst(un_op);
13638	const ty = self.typeOf(un_op);	21237	const ty = self.typeOf(un_op);
...	@@ -13643,7 +21242,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13643,7 +21242,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
13643	return self.finishAir(inst, result, .{ un_op, .none, .none });	21242	return self.finishAir(inst, result, .{ un_op, .none, .none });
13644	}	21243	}
13645		21244
13646	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {	21245	fn airIsErr(self: *CodeGen, inst: Air.Inst.Index) !void {
13647	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21246	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13648	const operand = try self.resolveInst(un_op);	21247	const operand = try self.resolveInst(un_op);
13649	const ty = self.typeOf(un_op);	21248	const ty = self.typeOf(un_op);
...	@@ -13651,7 +21250,7 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13651,7 +21250,7 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
13651	return self.finishAir(inst, result, .{ un_op, .none, .none });	21250	return self.finishAir(inst, result, .{ un_op, .none, .none });
13652	}	21251	}
13653		21252
13654	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	21253	fn airIsErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
13655	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21254	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13656	const operand = try self.resolveInst(un_op);	21255	const operand = try self.resolveInst(un_op);
13657	const ty = self.typeOf(un_op);	21256	const ty = self.typeOf(un_op);
...	@@ -13659,7 +21258,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13659,7 +21258,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
13659	return self.finishAir(inst, result, .{ un_op, .none, .none });	21258	return self.finishAir(inst, result, .{ un_op, .none, .none });
13660	}	21259	}
13661		21260
13662	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {	21261	fn airIsNonErr(self: *CodeGen, inst: Air.Inst.Index) !void {
13663	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21262	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13664	const operand = try self.resolveInst(un_op);	21263	const operand = try self.resolveInst(un_op);
13665	const ty = self.typeOf(un_op);	21264	const ty = self.typeOf(un_op);
...	@@ -13667,7 +21266,7 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13667,7 +21266,7 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
13667	return self.finishAir(inst, result, .{ un_op, .none, .none });	21266	return self.finishAir(inst, result, .{ un_op, .none, .none });
13668	}	21267	}
13669		21268
13670	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {	21269	fn airIsNonErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
13671	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	21270	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13672	const operand = try self.resolveInst(un_op);	21271	const operand = try self.resolveInst(un_op);
13673	const ty = self.typeOf(un_op);	21272	const ty = self.typeOf(un_op);
...	@@ -13675,7 +21274,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13675,7 +21274,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
13675	return self.finishAir(inst, result, .{ un_op, .none, .none });	21274	return self.finishAir(inst, result, .{ un_op, .none, .none });
13676	}	21275	}
13677		21276
13678	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {	21277	fn airLoop(self: *CodeGen, inst: Air.Inst.Index) !void {
13679	// A loop is a setup to be able to jump back to the beginning.	21278	// A loop is a setup to be able to jump back to the beginning.
13680	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	21279	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13681	const loop = self.air.extraData(Air.Block, ty_pl.payload);	21280	const loop = self.air.extraData(Air.Block, ty_pl.payload);
...	@@ -13686,15 +21285,14 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13686,15 +21285,14 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
13686		21285
13687	try self.loops.putNoClobber(self.gpa, inst, .{	21286	try self.loops.putNoClobber(self.gpa, inst, .{
13688	.state = state,	21287	.state = state,
13689	.jmp_target = @intCast(self.mir_instructions.len),	21288	.target = @intCast(self.mir_instructions.len),
13690	});	21289	});
13691	defer assert(self.loops.remove(inst));	21290	defer assert(self.loops.remove(inst));
13692		21291
13693	try self.genBodyBlock(body);	21292	try self.genBodyBlock(body);
13694	self.finishAirBookkeeping();
13695	}	21293	}
13696		21294
13697	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {	21295	fn airBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
13698	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	21296	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13699	const extra = self.air.extraData(Air.Block, ty_pl.payload);	21297	const extra = self.air.extraData(Air.Block, ty_pl.payload);
13700	try self.asmPseudo(.pseudo_dbg_enter_block_none);	21298	try self.asmPseudo(.pseudo_dbg_enter_block_none);
...	@@ -13702,10 +21300,10 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13702,10 +21300,10 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
13702	try self.asmPseudo(.pseudo_dbg_leave_block_none);	21300	try self.asmPseudo(.pseudo_dbg_leave_block_none);
13703	}	21301	}
13704		21302
13705	fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {	21303	fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
13706	// A block is a setup to be able to jump to the end.	21304	// A block is a setup to be able to jump to the end.
13707	const inst_tracking_i = self.inst_tracking.count();	21305	const inst_tracking_i = self.inst_tracking.count();
13708	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));	21306	self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));
13709		21307
13710	self.scope_generation += 1;	21308	self.scope_generation += 1;
13711	try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });	21309	try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
...	@@ -13729,28 +21327,197 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !	...	@@ -13729,28 +21327,197 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
13729	const tracking = &self.inst_tracking.values()[inst_tracking_i];	21327	const tracking = &self.inst_tracking.values()[inst_tracking_i];
13730	if (self.liveness.isUnused(inst)) try tracking.die(self, inst);	21328	if (self.liveness.isUnused(inst)) try tracking.die(self, inst);
13731	self.getValueIfFree(tracking.short, inst);	21329	self.getValueIfFree(tracking.short, inst);
13732	self.finishAirBookkeeping();
13733	}	21330	}
13734		21331
13735	fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {	21332	fn lowerSwitchBr(
		21333	self: *CodeGen,
		21334	inst: Air.Inst.Index,
		21335	switch_br: Air.UnwrappedSwitch,
		21336	condition: MCValue,
		21337	condition_dies: bool,
		21338	is_loop: bool,
		21339	) !void {
13736	const zcu = self.pt.zcu;	21340	const zcu = self.pt.zcu;
13737	const condition_ty = self.typeOf(switch_br.operand);	21341	const condition_ty = self.typeOf(switch_br.operand);
13738	const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1);
13739	defer self.gpa.free(liveness.deaths);
13740		21342
13741	const signedness = switch (condition_ty.zigTypeTag(zcu)) {	21343	const ExpectedContents = extern struct {
13742	.bool, .pointer => .unsigned,	21344	liveness_deaths: [1 << 8 \| 1]Air.Inst.Index,
13743	.int, .@"enum", .error_set => condition_ty.intInfo(zcu).signedness,	21345	bigint_limbs: [std.math.big.int.calcTwosCompLimbCount(1 << 8)]std.math.big.Limb,
13744	else => unreachable,	21346	relocs: [1 << 6]Mir.Inst.Index,
13745	};	21347	};
		21348	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
		21349	std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
		21350	const allocator = stack.get();
13746		21351
13747	self.scope_generation += 1;	21352	self.scope_generation += 1;
13748	const state = try self.saveState();	21353	const state = try self.saveState();
13749		21354
13750	var it = switch_br.iterateCases();	21355	const liveness = try self.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1);
13751	while (it.next()) \|case\| {	21356	defer allocator.free(liveness.deaths);
13752	var relocs = try self.gpa.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);	21357
13753	defer self.gpa.free(relocs);	21358	if (!self.mod.pic and self.target.ofmt == .elf) table: {
		21359	var prong_items: u32 = 0;
		21360	var min: ?Value = null;
		21361	var max: ?Value = null;
		21362	{
		21363	var cases_it = switch_br.iterateCases();
		21364	while (cases_it.next()) \|case\| {
		21365	prong_items += @intCast(case.items.len + case.ranges.len);
		21366	for (case.items) \|item\| {
		21367	const val = Value.fromInterned(item.toInterned().?);
		21368	if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val;
		21369	if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val;
		21370	}
		21371	for (case.ranges) \|range\| {
		21372	const low = Value.fromInterned(range[0].toInterned().?);
		21373	if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low;
		21374	const high = Value.fromInterned(range[1].toInterned().?);
		21375	if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high;
		21376	}
		21377	}
		21378	}
		21379	// This condition also triggers for switches with no non-else prongs and switches on bool.
		21380	if (prong_items < 1 << 2 or prong_items > 1 << 8) break :table;
		21381
		21382	var min_space: Value.BigIntSpace = undefined;
		21383	const min_bigint = min.?.toBigInt(&min_space, zcu);
		21384	var max_space: Value.BigIntSpace = undefined;
		21385	const max_bigint = max.?.toBigInt(&max_space, zcu);
		21386	const limbs = try allocator.alloc(
		21387	std.math.big.Limb,
		21388	@max(min_bigint.limbs.len, max_bigint.limbs.len) + 1,
		21389	);
		21390	defer allocator.free(limbs);
		21391	const table_len = table_len: {
		21392	var table_len_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
		21393	table_len_bigint.sub(max_bigint, min_bigint);
		21394	assert(table_len_bigint.positive); // min <= max
		21395	break :table_len @as(u11, table_len_bigint.toConst().to(u10) catch break :table) + 1; // no more than a 1024 entry table
		21396	};
		21397	assert(prong_items <= table_len); // each prong item introduces at least one unique integer to the range
		21398	if (prong_items < table_len >> 2) break :table; // no more than 75% waste
		21399
		21400	const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: {
		21401	const condition_index = try self.allocTempRegOrMem(condition_ty, true);
		21402	try self.genCopy(condition_ty, condition_index, condition, .{});
		21403	break :condition_index condition_index;
		21404	};
		21405	try self.spillEflagsIfOccupied();
		21406	if (min.?.orderAgainstZero(zcu).compare(.neq)) try self.genBinOpMir(
		21407	.{ ._, .sub },
		21408	condition_ty,
		21409	condition_index,
		21410	.{ .air_ref = Air.internedToRef(min.?.toIntern()) },
		21411	);
		21412	const else_reloc = if (switch_br.else_body_len > 0) else_reloc: {
		21413	try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table_len - 1 });
		21414	break :else_reloc try self.asmJccReloc(.a, undefined);
		21415	} else undefined;
		21416	const table_start: u31 = @intCast(self.mir_table.items.len);
		21417	{
		21418	const condition_index_reg = if (condition_index.isRegister())
		21419	condition_index.getReg().?
		21420	else
		21421	try self.copyToTmpRegister(.usize, condition_index);
		21422	const condition_index_lock = self.register_manager.lockReg(condition_index_reg);
		21423	defer if (condition_index_lock) \|lock\| self.register_manager.unlockReg(lock);
		21424	try self.truncateRegister(condition_ty, condition_index_reg);
		21425	const ptr_size = @divExact(self.target.ptrBitWidth(), 8);
		21426	try self.asmMemory(.{ ._, .jmp }, .{
		21427	.base = .table,
		21428	.mod = .{ .rm = .{
		21429	.size = .ptr,
		21430	.index = registerAlias(condition_index_reg, ptr_size),
		21431	.scale = .fromFactor(@intCast(ptr_size)),
		21432	.disp = table_start * ptr_size,
		21433	} },
		21434	});
		21435	}
		21436	const else_reloc_marker: u32 = 0;
		21437	assert(self.mir_instructions.len > else_reloc_marker);
		21438	try self.mir_table.appendNTimes(self.gpa, else_reloc_marker, table_len);
		21439	if (is_loop) try self.loop_switches.putNoClobber(self.gpa, inst, .{
		21440	.start = table_start,
		21441	.len = table_len,
		21442	.min = min.?,
		21443	.else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable",
		21444	});
		21445	defer if (is_loop) {
		21446	var loop_switch_data = self.loop_switches.fetchRemove(inst).?.value;
		21447	switch (loop_switch_data.else_relocs) {
		21448	.@"unreachable", .backward => {},
		21449	.forward => \|*else_relocs\| else_relocs.deinit(self.gpa),
		21450	}
		21451	};
		21452	var cases_it = switch_br.iterateCases();
		21453	while (cases_it.next()) \|case\| {
		21454	{
		21455	const table = self.mir_table.items[table_start..][0..table_len];
		21456	for (case.items) \|item\| {
		21457	const val = Value.fromInterned(item.toInterned().?);
		21458	var val_space: Value.BigIntSpace = undefined;
		21459	const val_bigint = val.toBigInt(&val_space, zcu);
		21460	var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
		21461	index_bigint.sub(val_bigint, min_bigint);
		21462	table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(self.mir_instructions.len);
		21463	}
		21464	for (case.ranges) \|range\| {
		21465	var low_space: Value.BigIntSpace = undefined;
		21466	const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu);
		21467	var high_space: Value.BigIntSpace = undefined;
		21468	const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu);
		21469	var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
		21470	index_bigint.sub(low_bigint, min_bigint);
		21471	const start = index_bigint.toConst().to(u10) catch unreachable;
		21472	index_bigint.sub(high_bigint, min_bigint);
		21473	const end = @as(u11, index_bigint.toConst().to(u10) catch unreachable) + 1;
		21474	@memset(table[start..end], @intCast(self.mir_instructions.len));
		21475	}
		21476	}
		21477
		21478	for (liveness.deaths[case.idx]) \|operand\| try self.processDeath(operand);
		21479
		21480	try self.genBodyBlock(case.body);
		21481	try self.restoreState(state, &.{}, .{
		21482	.emit_instructions = false,
		21483	.update_tracking = true,
		21484	.resurrect = true,
		21485	.close_scope = true,
		21486	});
		21487	}
		21488	if (switch_br.else_body_len > 0) {
		21489	const else_body = cases_it.elseBody();
		21490
		21491	const else_deaths = liveness.deaths.len - 1;
		21492	for (liveness.deaths[else_deaths]) \|operand\| try self.processDeath(operand);
		21493
		21494	self.performReloc(else_reloc);
		21495	if (is_loop) {
		21496	const loop_switch_data = self.loop_switches.getPtr(inst).?;
		21497	for (loop_switch_data.else_relocs.forward.items) \|reloc\| self.performReloc(reloc);
		21498	loop_switch_data.else_relocs.forward.deinit(self.gpa);
		21499	loop_switch_data.else_relocs = .{ .backward = @intCast(self.mir_instructions.len) };
		21500	}
		21501	for (self.mir_table.items[table_start..][0..table_len]) \|entry\| if (entry. == else_reloc_marker) {
		21502	entry.* = @intCast(self.mir_instructions.len);
		21503	};
		21504
		21505	try self.genBodyBlock(else_body);
		21506	try self.restoreState(state, &.{}, .{
		21507	.emit_instructions = false,
		21508	.update_tracking = true,
		21509	.resurrect = true,
		21510	.close_scope = true,
		21511	});
		21512	}
		21513	return;
		21514	}
		21515
		21516	const signedness = if (condition_ty.isAbiInt(zcu)) condition_ty.intInfo(zcu).signedness else .unsigned;
		21517	var cases_it = switch_br.iterateCases();
		21518	while (cases_it.next()) \|case\| {
		21519	var relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
		21520	defer allocator.free(relocs);
13754		21521
13755	try self.spillEflagsIfOccupied();	21522	try self.spillEflagsIfOccupied();
13756	for (case.items, relocs[0..case.items.len]) \|item, *reloc\| {	21523	for (case.items, relocs[0..case.items.len]) \|item, *reloc\| {
...	@@ -13833,9 +21600,8 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit	...	@@ -13833,9 +21600,8 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit
13833	// Relocate the "skip" branch to fall through to the next case.	21600	// Relocate the "skip" branch to fall through to the next case.
13834	self.performReloc(skip_case_reloc);	21601	self.performReloc(skip_case_reloc);
13835	}	21602	}
13836
13837	if (switch_br.else_body_len > 0) {	21603	if (switch_br.else_body_len > 0) {
13838	const else_body = it.elseBody();	21604	const else_body = cases_it.elseBody();
13839		21605
13840	const else_deaths = liveness.deaths.len - 1;	21606	const else_deaths = liveness.deaths.len - 1;
13841	for (liveness.deaths[else_deaths]) \|operand\| try self.processDeath(operand);	21607	for (liveness.deaths[else_deaths]) \|operand\| try self.processDeath(operand);
...	@@ -13850,24 +21616,23 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit	...	@@ -13850,24 +21616,23 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit
13850	}	21616	}
13851	}	21617	}
13852		21618
13853	fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	21619	fn airSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
13854	const switch_br = self.air.unwrapSwitch(inst);	21620	const switch_br = self.air.unwrapSwitch(inst);
13855	const condition = try self.resolveInst(switch_br.operand);	21621	const condition = try self.resolveInst(switch_br.operand);
13856		21622
13857	// If the condition dies here in this switch instruction, process	21623	// If the condition dies here in this switch instruction, process
13858	// that death now instead of later as this has an effect on	21624	// that death now instead of later as this has an effect on
13859	// whether it needs to be spilled in the branches	21625	// whether it needs to be spilled in the branches
13860	if (self.liveness.operandDies(inst, 0)) {	21626	const condition_dies = self.liveness.operandDies(inst, 0);
		21627	if (condition_dies) {
13861	if (switch_br.operand.toIndex()) \|op_inst\| try self.processDeath(op_inst);	21628	if (switch_br.operand.toIndex()) \|op_inst\| try self.processDeath(op_inst);
13862	}	21629	}
13863		21630	try self.lowerSwitchBr(inst, switch_br, condition, condition_dies, false);
13864	try self.lowerSwitchBr(inst, switch_br, condition);
13865		21631
13866	// We already took care of pl_op.operand earlier, so there's nothing left to do	21632	// We already took care of pl_op.operand earlier, so there's nothing left to do
13867	self.finishAirBookkeeping();
13868	}	21633	}
13869		21634
13870	fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	21635	fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
13871	const switch_br = self.air.unwrapSwitch(inst);	21636	const switch_br = self.air.unwrapSwitch(inst);
13872	const condition = try self.resolveInst(switch_br.operand);	21637	const condition = try self.resolveInst(switch_br.operand);
13873		21638
...	@@ -13879,7 +21644,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13879,7 +21644,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13879	try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{});	21644	try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{});
13880	break :mat_cond mat_cond;	21645	break :mat_cond mat_cond;
13881	};	21646	};
13882	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond));	21647	self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(mat_cond));
13883		21648
13884	// If the condition dies here in this switch instruction, process	21649	// If the condition dies here in this switch instruction, process
13885	// that death now instead of later as this has an effect on	21650	// that death now instead of later as this has an effect on
...	@@ -13893,25 +21658,83 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13893,25 +21658,83 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13893		21658
13894	try self.loops.putNoClobber(self.gpa, inst, .{	21659	try self.loops.putNoClobber(self.gpa, inst, .{
13895	.state = state,	21660	.state = state,
13896	.jmp_target = @intCast(self.mir_instructions.len),	21661	.target = @intCast(self.mir_instructions.len),
13897	});	21662	});
13898	defer assert(self.loops.remove(inst));	21663	defer assert(self.loops.remove(inst));
13899		21664
13900	// Stop tracking block result without forgetting tracking info	21665	// Stop tracking block result without forgetting tracking info
13901	try self.freeValue(mat_cond);	21666	try self.freeValue(mat_cond);
13902		21667
13903	try self.lowerSwitchBr(inst, switch_br, mat_cond);	21668	try self.lowerSwitchBr(inst, switch_br, mat_cond, true, true);
13904		21669
13905	try self.processDeath(inst);	21670	try self.processDeath(inst);
13906	self.finishAirBookkeeping();
13907	}	21671	}
13908		21672
13909	fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {	21673	fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void {
13910	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;	21674	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
13911		21675
13912	const block_ty = self.typeOfIndex(br.block_inst);	21676	const block_ty = self.typeOfIndex(br.block_inst);
13913	const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
13914	const loop_data = self.loops.getPtr(br.block_inst).?;	21677	const loop_data = self.loops.getPtr(br.block_inst).?;
		21678	if (self.loop_switches.getPtr(br.block_inst)) \|table\| {
		21679	// Process operand death so that it is properly accounted for in the State below.
		21680	const condition_dies = self.liveness.operandDies(inst, 0);
		21681
		21682	try self.restoreState(loop_data.state, &.{}, .{
		21683	.emit_instructions = true,
		21684	.update_tracking = false,
		21685	.resurrect = false,
		21686	.close_scope = false,
		21687	});
		21688
		21689	const condition_ty = self.typeOf(br.operand);
		21690	const condition = try self.resolveInst(br.operand);
		21691	const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: {
		21692	const condition_index = try self.allocTempRegOrMem(condition_ty, true);
		21693	try self.genCopy(condition_ty, condition_index, condition, .{});
		21694	break :condition_index condition_index;
		21695	};
		21696	try self.spillEflagsIfOccupied();
		21697	if (table.min.orderAgainstZero(self.pt.zcu).compare(.neq)) try self.genBinOpMir(
		21698	.{ ._, .sub },
		21699	condition_ty,
		21700	condition_index,
		21701	.{ .air_ref = Air.internedToRef(table.min.toIntern()) },
		21702	);
		21703	switch (table.else_relocs) {
		21704	.@"unreachable" => {},
		21705	.forward => \|*else_relocs\| {
		21706	try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table.len - 1 });
		21707	try else_relocs.append(self.gpa, try self.asmJccReloc(.a, undefined));
		21708	},
		21709	.backward => \|else_reloc\| {
		21710	try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table.len - 1 });
		21711	_ = try self.asmJccReloc(.a, else_reloc);
		21712	},
		21713	}
		21714	{
		21715	const condition_index_reg = if (condition_index.isRegister())
		21716	condition_index.getReg().?
		21717	else
		21718	try self.copyToTmpRegister(.usize, condition_index);
		21719	const condition_index_lock = self.register_manager.lockReg(condition_index_reg);
		21720	defer if (condition_index_lock) \|lock\| self.register_manager.unlockReg(lock);
		21721	try self.truncateRegister(condition_ty, condition_index_reg);
		21722	const ptr_size = @divExact(self.target.ptrBitWidth(), 8);
		21723	try self.asmMemory(.{ ._, .jmp }, .{
		21724	.base = .table,
		21725	.mod = .{ .rm = .{
		21726	.size = .ptr,
		21727	.index = registerAlias(condition_index_reg, ptr_size),
		21728	.scale = .fromFactor(@intCast(ptr_size)),
		21729	.disp = @intCast(table.start * ptr_size),
		21730	} },
		21731	});
		21732	}
		21733
		21734	return self.finishAir(inst, .none, .{ br.operand, .none, .none });
		21735	}
		21736
		21737	const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
13915	done: {	21738	done: {
13916	try self.getValue(block_tracking.short, null);	21739	try self.getValue(block_tracking.short, null);
13917	const src_mcv = try self.resolveInst(br.operand);	21740	const src_mcv = try self.resolveInst(br.operand);
...	@@ -13945,15 +21768,13 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13945,15 +21768,13 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
13945		21768
13946	// Emit a jump with a relocation. It will be patched up after the block ends.	21769	// Emit a jump with a relocation. It will be patched up after the block ends.
13947	// Leave the jump offset undefined	21770	// Leave the jump offset undefined
13948	_ = try self.asmJmpReloc(loop_data.jmp_target);	21771	_ = try self.asmJmpReloc(loop_data.target);
13949		21772
13950	// Stop tracking block result without forgetting tracking info	21773	// Stop tracking block result without forgetting tracking info
13951	try self.freeValue(block_tracking.short);	21774	try self.freeValue(block_tracking.short);
13952
13953	self.finishAirBookkeeping();
13954	}	21775	}
13955		21776
13956	fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {	21777	fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void {
13957	const next_inst: u32 = @intCast(self.mir_instructions.len);	21778	const next_inst: u32 = @intCast(self.mir_instructions.len);
13958	switch (self.mir_instructions.items(.tag)[reloc]) {	21779	switch (self.mir_instructions.items(.tag)[reloc]) {
13959	.j, .jmp => {},	21780	.j, .jmp => {},
...	@@ -13966,7 +21787,7 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {	...	@@ -13966,7 +21787,7 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
13966	self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;	21787	self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
13967	}	21788	}
13968		21789
13969	fn airBr(self: *Self, inst: Air.Inst.Index) !void {	21790	fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void {
13970	const zcu = self.pt.zcu;	21791	const zcu = self.pt.zcu;
13971	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;	21792	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
13972		21793
...	@@ -14007,7 +21828,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14007,7 +21828,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
14007	}	21828	}
14008		21829
14009	if (first_br) {	21830	if (first_br) {
14010	block_tracking.* = InstTracking.init(block_result);	21831	block_tracking.* = .init(block_result);
14011	try self.saveRetroactiveState(&block_data.state);	21832	try self.saveRetroactiveState(&block_data.state);
14012	} else try self.restoreState(block_data.state, &.{}, .{	21833	} else try self.restoreState(block_data.state, &.{}, .{
14013	.emit_instructions = true,	21834	.emit_instructions = true,
...	@@ -14023,11 +21844,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14023,11 +21844,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
14023		21844
14024	// Stop tracking block result without forgetting tracking info	21845	// Stop tracking block result without forgetting tracking info
14025	try self.freeValue(block_tracking.short);	21846	try self.freeValue(block_tracking.short);
14026
14027	self.finishAirBookkeeping();
14028	}	21847	}
14029		21848
14030	fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {	21849	fn airRepeat(self: *CodeGen, inst: Air.Inst.Index) !void {
14031	const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;	21850	const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;
14032	const repeat_info = self.loops.get(loop_inst).?;	21851	const repeat_info = self.loops.get(loop_inst).?;
14033	try self.restoreState(repeat_info.state, &.{}, .{	21852	try self.restoreState(repeat_info.state, &.{}, .{
...	@@ -14036,11 +21855,10 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14036,11 +21855,10 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
14036	.resurrect = false,	21855	.resurrect = false,
14037	.close_scope = true,	21856	.close_scope = true,
14038	});	21857	});
14039	_ = try self.asmJmpReloc(repeat_info.jmp_target);	21858	_ = try self.asmJmpReloc(repeat_info.target);
14040	self.finishAirBookkeeping();
14041	}	21859	}
14042		21860
14043	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	21861	fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
14044	const pt = self.pt;	21862	const pt = self.pt;
14045	const zcu = pt.zcu;	21863	const zcu = pt.zcu;
14046	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	21864	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -14054,7 +21872,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14054,7 +21872,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14054	extra_i += inputs.len;	21872	extra_i += inputs.len;
14055		21873
14056	var result: MCValue = .none;	21874	var result: MCValue = .none;
14057	var args = std.ArrayList(MCValue).init(self.gpa);	21875	var args: std.ArrayList(MCValue) = .init(self.gpa);
14058	try args.ensureTotalCapacity(outputs.len + inputs.len);	21876	try args.ensureTotalCapacity(outputs.len + inputs.len);
14059	defer {	21877	defer {
14060	for (args.items) \|arg\| if (arg.getReg()) \|reg\| self.register_manager.unlockReg(.{	21878	for (args.items) \|arg\| if (arg.getReg()) \|reg\| self.register_manager.unlockReg(.{
...	@@ -14062,15 +21880,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14062,15 +21880,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14062	});	21880	});
14063	args.deinit();	21881	args.deinit();
14064	}	21882	}
14065	var arg_map = std.StringHashMap(u8).init(self.gpa);	21883	var arg_map: std.StringHashMap(u8) = .init(self.gpa);
14066	try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));	21884	try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
14067	defer arg_map.deinit();	21885	defer arg_map.deinit();
14068		21886
14069	var outputs_extra_i = extra_i;	21887	var outputs_extra_i = extra_i;
14070	for (outputs) \|output\| {	21888	for (outputs) \|output\| {
14071	const extra_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);	21889	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
14072	const constraint = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	21890	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
14073	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	21891	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
14074	// This equation accounts for the fact that even if we have exactly 4 bytes	21892	// This equation accounts for the fact that even if we have exactly 4 bytes
14075	// for the string, we still use the next u32 for the null terminator.	21893	// for the string, we still use the next u32 for the null terminator.
14076	extra_i += (constraint.len + name.len + (2 + 3)) / 4;	21894	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	@@ -14097,8 +21915,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14097,8 +21915,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14097	const is_early_clobber = constraint[1] == '&';	21915	const is_early_clobber = constraint[1] == '&';
14098	const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];	21916	const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
14099	const arg_mcv: MCValue = arg_mcv: {	21917	const arg_mcv: MCValue = arg_mcv: {
14100	const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "r") or	21918	const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or
14101	mem.eql(u8, rest, "f") or mem.eql(u8, rest, "x"))	21919	std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x"))
14102	registerAlias(	21920	registerAlias(
14103	self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) {	21921	self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) {
14104	'r' => abi.RegisterClass.gp,	21922	'r' => abi.RegisterClass.gp,
...	@@ -14108,20 +21926,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14108,20 +21926,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14108	}) orelse return self.fail("ran out of registers lowering inline asm", .{}),	21926	}) orelse return self.fail("ran out of registers lowering inline asm", .{}),
14109	@intCast(ty.abiSize(zcu)),	21927	@intCast(ty.abiSize(zcu)),
14110	)	21928	)
14111	else if (mem.eql(u8, rest, "m"))	21929	else if (std.mem.eql(u8, rest, "m"))
14112	if (output != .none) null else return self.fail(	21930	if (output != .none) null else return self.fail(
14113	"memory constraint unsupported for asm result: '{s}'",	21931	"memory constraint unsupported for asm result: '{s}'",
14114	.{constraint},	21932	.{constraint},
14115	)	21933	)
14116	else if (mem.eql(u8, rest, "g") or	21934	else if (std.mem.eql(u8, rest, "g") or
14117	mem.eql(u8, rest, "rm") or mem.eql(u8, rest, "mr") or	21935	std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or
14118	mem.eql(u8, rest, "r,m") or mem.eql(u8, rest, "m,r"))	21936	std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r"))
14119	self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse	21937	self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse
14120	if (output != .none)	21938	if (output != .none)
14121	null	21939	null
14122	else	21940	else
14123	return self.fail("ran out of registers lowering inline asm", .{})	21941	return self.fail("ran out of registers lowering inline asm", .{})
14124	else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}"))	21942	else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}"))
14125	parseRegName(rest["{".len .. rest.len - "}".len]) orelse	21943	parseRegName(rest["{".len .. rest.len - "}".len]) orelse
14126	return self.fail("invalid register constraint: '{s}'", .{constraint})	21944	return self.fail("invalid register constraint: '{s}'", .{constraint})
14127	else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {	21945	else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {
...	@@ -14134,18 +21952,19 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14134,18 +21952,19 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14134	break :arg_mcv if (arg_maybe_reg) \|reg\| .{ .register = reg } else arg: {	21952	break :arg_mcv if (arg_maybe_reg) \|reg\| .{ .register = reg } else arg: {
14135	const ptr_mcv = try self.resolveInst(output);	21953	const ptr_mcv = try self.resolveInst(output);
14136	switch (ptr_mcv) {	21954	switch (ptr_mcv) {
14137	.immediate => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|	21955	.immediate => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14138	break :arg ptr_mcv.deref(),	21956	break :arg ptr_mcv.deref(),
14139	.register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),	21957	.register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),
14140	else => {},	21958	else => {},
14141	}	21959	}
14142	break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(Type.usize, ptr_mcv) } };	21960	break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(.usize, ptr_mcv) } };
14143	};	21961	};
14144	};	21962	};
14145	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {	21963	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|tracked_index\| {
14146	_ = self.register_manager.lockReg(reg);	21964	try self.register_manager.getRegIndex(tracked_index, if (output == .none) inst else null);
		21965	_ = self.register_manager.lockRegIndexAssumeUnused(tracked_index);
14147	};	21966	};
14148	if (!mem.eql(u8, name, "_"))	21967	if (!std.mem.eql(u8, name, "_"))
14149	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));	21968	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
14150	args.appendAssumeCapacity(arg_mcv);	21969	args.appendAssumeCapacity(arg_mcv);
14151	if (output == .none) result = arg_mcv;	21970	if (output == .none) result = arg_mcv;
...	@@ -14153,17 +21972,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14153,17 +21972,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14153	}	21972	}
14154		21973
14155	for (inputs) \|input\| {	21974	for (inputs) \|input\| {
14156	const input_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]);	21975	const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
14157	const constraint = mem.sliceTo(input_bytes, 0);	21976	const constraint = std.mem.sliceTo(input_bytes, 0);
14158	const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);	21977	const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
14159	// This equation accounts for the fact that even if we have exactly 4 bytes	21978	// This equation accounts for the fact that even if we have exactly 4 bytes
14160	// for the string, we still use the next u32 for the null terminator.	21979	// for the string, we still use the next u32 for the null terminator.
14161	extra_i += (constraint.len + name.len + (2 + 3)) / 4;	21980	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
14162		21981
14163	const ty = self.typeOf(input);	21982	const ty = self.typeOf(input);
14164	const input_mcv = try self.resolveInst(input);	21983	const input_mcv = try self.resolveInst(input);
14165	const arg_mcv: MCValue = if (mem.eql(u8, constraint, "r") or	21984	const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or
14166	mem.eql(u8, constraint, "f") or mem.eql(u8, constraint, "x"))	21985	std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x"))
14167	arg: {	21986	arg: {
14168	const rc = switch (constraint[0]) {	21987	const rc = switch (constraint[0]) {
14169	'r' => abi.RegisterClass.gp,	21988	'r' => abi.RegisterClass.gp,
...	@@ -14177,16 +21996,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14177,16 +21996,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14177	const reg = try self.register_manager.allocReg(null, rc);	21996	const reg = try self.register_manager.allocReg(null, rc);
14178	try self.genSetReg(reg, ty, input_mcv, .{});	21997	try self.genSetReg(reg, ty, input_mcv, .{});
14179	break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) };	21998	break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) };
14180	} else if (mem.eql(u8, constraint, "i") or mem.eql(u8, constraint, "n"))	21999	} else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n"))
14181	switch (input_mcv) {	22000	switch (input_mcv) {
14182	.immediate => \|imm\| .{ .immediate = imm },	22001	.immediate => \|imm\| .{ .immediate = imm },
14183	else => return self.fail("immediate operand requires comptime value: '{s}'", .{	22002	else => return self.fail("immediate operand requires comptime value: '{s}'", .{
14184	constraint,	22003	constraint,
14185	}),	22004	}),
14186	}	22005	}
14187	else if (mem.eql(u8, constraint, "m")) arg: {	22006	else if (std.mem.eql(u8, constraint, "m")) arg: {
14188	switch (input_mcv) {	22007	switch (input_mcv) {
14189	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|	22008	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14190	break :arg input_mcv,	22009	break :arg input_mcv,
14191	.indirect, .load_frame => break :arg input_mcv,	22010	.indirect, .load_frame => break :arg input_mcv,
14192	.load_symbol, .load_direct, .load_got, .load_tlv => {},	22011	.load_symbol, .load_direct, .load_got, .load_tlv => {},
...	@@ -14201,24 +22020,24 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14201,24 +22020,24 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14201	try self.genCopy(ty, temp_mcv, input_mcv, .{});	22020	try self.genCopy(ty, temp_mcv, input_mcv, .{});
14202	break :arg temp_mcv;	22021	break :arg temp_mcv;
14203	};	22022	};
14204	try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{});	22023	try self.genSetReg(addr_reg, .usize, input_mcv.address(), .{});
14205	break :arg .{ .indirect = .{ .reg = addr_reg } };	22024	break :arg .{ .indirect = .{ .reg = addr_reg } };
14206	} else if (mem.eql(u8, constraint, "g") or	22025	} else if (std.mem.eql(u8, constraint, "g") or
14207	mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or	22026	std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or
14208	mem.eql(u8, constraint, "r,m") or mem.eql(u8, constraint, "m,r"))	22027	std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r"))
14209	arg: {	22028	arg: {
14210	switch (input_mcv) {	22029	switch (input_mcv) {
14211	.register, .indirect, .load_frame => break :arg input_mcv,	22030	.register, .indirect, .load_frame => break :arg input_mcv,
14212	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|	22031	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
14213	break :arg input_mcv,	22032	break :arg input_mcv,
14214	else => {},	22033	else => {},
14215	}	22034	}
14216	const temp_mcv = try self.allocTempRegOrMem(ty, true);	22035	const temp_mcv = try self.allocTempRegOrMem(ty, true);
14217	try self.genCopy(ty, temp_mcv, input_mcv, .{});	22036	try self.genCopy(ty, temp_mcv, input_mcv, .{});
14218	break :arg temp_mcv;	22037	break :arg temp_mcv;
14219	} else if (mem.eql(u8, constraint, "X"))	22038	} else if (std.mem.eql(u8, constraint, "X"))
14220	input_mcv	22039	input_mcv
14221	else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: {	22040	else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: {
14222	const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse	22041	const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
14223	return self.fail("invalid register constraint: '{s}'", .{constraint});	22042	return self.fail("invalid register constraint: '{s}'", .{constraint});
14224	try self.register_manager.getReg(reg, null);	22043	try self.register_manager.getReg(reg, null);
...	@@ -14233,7 +22052,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14233,7 +22052,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14233	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {	22052	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
14234	_ = self.register_manager.lockReg(reg);	22053	_ = self.register_manager.lockReg(reg);
14235	};	22054	};
14236	if (!mem.eql(u8, name, "_"))	22055	if (!std.mem.eql(u8, name, "_"))
14237	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));	22056	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
14238	args.appendAssumeCapacity(arg_mcv);	22057	args.appendAssumeCapacity(arg_mcv);
14239	}	22058	}
...	@@ -14241,7 +22060,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14241,7 +22060,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14241	{	22060	{
14242	var clobber_i: u32 = 0;	22061	var clobber_i: u32 = 0;
14243	while (clobber_i < clobbers_len) : (clobber_i += 1) {	22062	while (clobber_i < clobbers_len) : (clobber_i += 1) {
14244	const clobber = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	22063	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
14245	// This equation accounts for the fact that even if we have exactly 4 bytes	22064	// This equation accounts for the fact that even if we have exactly 4 bytes
14246	// for the string, we still use the next u32 for the null terminator.	22065	// for the string, we still use the next u32 for the null terminator.
14247	extra_i += clobber.len / 4 + 1;	22066	extra_i += clobber.len / 4 + 1;
...	@@ -14294,20 +22113,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14294,20 +22113,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14294	labels.deinit(self.gpa);	22113	labels.deinit(self.gpa);
14295	}	22114	}
14296		22115
14297	const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];	22116	const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
14298	var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");	22117	var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;");
14299	next_line: while (line_it.next()) \|line\| {	22118	next_line: while (line_it.next()) \|line\| {
14300	var mnem_it = mem.tokenizeAny(u8, line, " \t");	22119	var mnem_it = std.mem.tokenizeAny(u8, line, " \t");
14301	var prefix: Instruction.Prefix = .none;	22120	var prefix: encoder.Instruction.Prefix = .none;
14302	const mnem_str = while (mnem_it.next()) \|mnem_str\| {	22121	const mnem_str = while (mnem_it.next()) \|mnem_str\| {
14303	if (mnem_str[0] == '#') continue :next_line;	22122	if (mnem_str[0] == '#') continue :next_line;
14304	if (mem.startsWith(u8, mnem_str, "//")) continue :next_line;	22123	if (std.mem.startsWith(u8, mnem_str, "//")) continue :next_line;
14305	if (std.meta.stringToEnum(Instruction.Prefix, mnem_str)) \|pre\| {	22124	if (std.meta.stringToEnum(encoder.Instruction.Prefix, mnem_str)) \|pre\| {
14306	if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});	22125	if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});
14307	prefix = pre;	22126	prefix = pre;
14308	continue;	22127	continue;
14309	}	22128	}
14310	if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;	22129	if (!std.mem.endsWith(u8, mnem_str, ":")) break mnem_str;
14311	const label_name = mnem_str[0 .. mnem_str.len - ":".len];	22130	const label_name = mnem_str[0 .. mnem_str.len - ":".len];
14312	if (!Label.isValid(.definition, label_name))	22131	if (!Label.isValid(.definition, label_name))
14313	return self.fail("invalid label: '{s}'", .{label_name});	22132	return self.fail("invalid label: '{s}'", .{label_name});
...	@@ -14332,21 +22151,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14332,21 +22151,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14332		22151
14333	var mnem_size: ?Memory.Size = if (prefix == .directive)	22152	var mnem_size: ?Memory.Size = if (prefix == .directive)
14334	null	22153	null
14335	else if (mem.endsWith(u8, mnem_str, "b"))	22154	else if (std.mem.endsWith(u8, mnem_str, "b"))
14336	.byte	22155	.byte
14337	else if (mem.endsWith(u8, mnem_str, "w"))	22156	else if (std.mem.endsWith(u8, mnem_str, "w"))
14338	.word	22157	.word
14339	else if (mem.endsWith(u8, mnem_str, "l"))	22158	else if (std.mem.endsWith(u8, mnem_str, "l"))
14340	.dword	22159	.dword
14341	else if (mem.endsWith(u8, mnem_str, "q") and	22160	else if (std.mem.endsWith(u8, mnem_str, "q") and
14342	(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))	22161	(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !std.mem.endsWith(u8, mnem_str, "dq")))
14343	.qword	22162	.qword
14344	else if (mem.endsWith(u8, mnem_str, "t"))	22163	else if (std.mem.endsWith(u8, mnem_str, "t"))
14345	.tbyte	22164	.tbyte
14346	else	22165	else
14347	null;	22166	null;
14348	const mnem_tag = while (true) break std.meta.stringToEnum(	22167	const mnem_tag = while (true) break std.meta.stringToEnum(
14349	Instruction.Mnemonic,	22168	encoder.Instruction.Mnemonic,
14350	mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],	22169	mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
14351	) orelse if (mnem_size) \|_\| {	22170	) orelse if (mnem_size) \|_\| {
14352	mnem_size = null;	22171	mnem_size = null;
...	@@ -14367,18 +22186,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14367,18 +22186,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14367	.{ ._, .pseudo }	22186	.{ ._, .pseudo }
14368	else for (std.enums.values(Mir.Inst.Fixes)) \|fixes\| {	22187	else for (std.enums.values(Mir.Inst.Fixes)) \|fixes\| {
14369	const fixes_name = @tagName(fixes);	22188	const fixes_name = @tagName(fixes);
14370	const space_i = mem.indexOfScalar(u8, fixes_name, ' ');	22189	const space_i = std.mem.indexOfScalar(u8, fixes_name, ' ');
14371	const fixes_prefix = if (space_i) \|i\|	22190	const fixes_prefix = if (space_i) \|i\|
14372	std.meta.stringToEnum(Instruction.Prefix, fixes_name[0..i]).?	22191	std.meta.stringToEnum(encoder.Instruction.Prefix, fixes_name[0..i]).?
14373	else	22192	else
14374	.none;	22193	.none;
14375	if (fixes_prefix != prefix) continue;	22194	if (fixes_prefix != prefix) continue;
14376	const pattern = fixes_name[if (space_i) \|i\| i + " ".len else 0..];	22195	const pattern = fixes_name[if (space_i) \|i\| i + " ".len else 0..];
14377	const wildcard_i = mem.indexOfScalar(u8, pattern, '_').?;	22196	const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
14378	const mnem_prefix = pattern[0..wildcard_i];	22197	const mnem_prefix = pattern[0..wildcard_i];
14379	const mnem_suffix = pattern[wildcard_i + "_".len ..];	22198	const mnem_suffix = pattern[wildcard_i + "_".len ..];
14380	if (!mem.startsWith(u8, mnem_name, mnem_prefix)) continue;	22199	if (!std.mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
14381	if (!mem.endsWith(u8, mnem_name, mnem_suffix)) continue;	22200	if (!std.mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
14382	break .{ fixes, std.meta.stringToEnum(	22201	break .{ fixes, std.meta.stringToEnum(
14383	Mir.Inst.Tag,	22202	Mir.Inst.Tag,
14384	mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],	22203	mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],
...	@@ -14390,31 +22209,25 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14390,31 +22209,25 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14390	});	22209	});
14391	};	22210	};
14392		22211
14393	const Operand = union(enum) {	22212	var ops: [4]Operand = @splat(.none);
14394	none,	22213	var ops_len: usize = 0;
14395	reg: Register,
14396	mem: Memory,
14397	imm: Immediate,
14398	inst: Mir.Inst.Index,
14399	};
14400	var ops: [4]Operand = .{.none} ** 4;
14401		22214
14402	var last_op = false;	22215	var last_op = false;
14403	var op_it = mem.splitScalar(u8, mnem_it.rest(), ',');	22216	var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ',');
14404	next_op: for (&ops) \|*op\| {	22217	next_op: for (&ops) \|*op\| {
14405	const op_str = while (!last_op) {	22218	const op_str = while (!last_op) {
14406	const full_str = op_it.next() orelse break :next_op;	22219	const full_str = op_it.next() orelse break :next_op;
14407	const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse	22220	const code_str = if (std.mem.indexOfScalar(u8, full_str, '#') orelse
14408	mem.indexOf(u8, full_str, "//")) \|comment\|	22221	std.mem.indexOf(u8, full_str, "//")) \|comment\|
14409	code: {	22222	code: {
14410	last_op = true;	22223	last_op = true;
14411	break :code full_str[0..comment];	22224	break :code full_str[0..comment];
14412	} else full_str;	22225	} else full_str;
14413	const trim_str = mem.trim(u8, code_str, " \t*");	22226	const trim_str = std.mem.trim(u8, code_str, " \t*");
14414	if (trim_str.len > 0) break trim_str;	22227	if (trim_str.len > 0) break trim_str;
14415	} else break;	22228	} else break;
14416	if (mem.startsWith(u8, op_str, "%%")) {	22229	if (std.mem.startsWith(u8, op_str, "%%")) {
14417	const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');	22230	const colon = std.mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
14418	const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse	22231	const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
14419	return self.fail("invalid register: '{s}'", .{op_str});	22232	return self.fail("invalid register: '{s}'", .{op_str});
14420	if (colon) \|colon_pos\| {	22233	if (colon) \|colon_pos\| {
...	@@ -14428,12 +22241,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14428,12 +22241,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14428	} },	22241	} },
14429	} };	22242	} };
14430	} else {	22243	} else {
14431	if (mnem_size) \|size\| if (reg.bitSize() != size.bitSize())	22244	if (mnem_size) \|size\| if (reg.bitSize() != size.bitSize(self.target))
14432	return self.fail("invalid register size: '{s}'", .{op_str});	22245	return self.fail("invalid register size: '{s}'", .{op_str});
14433	op.* = .{ .reg = reg };	22246	op.* = .{ .reg = reg };
14434	}	22247	}
14435	} else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {	22248	} else if (std.mem.startsWith(u8, op_str, "%[") and std.mem.endsWith(u8, op_str, "]")) {
14436	const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':');	22249	const colon = std.mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
14437	const modifier = if (colon) \|colon_pos\|	22250	const modifier = if (colon) \|colon_pos\|
14438	op_str[colon_pos + ":".len .. op_str.len - "]".len]	22251	op_str[colon_pos + ":".len .. op_str.len - "]".len]
14439	else	22252	else
...	@@ -14442,15 +22255,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14442,15 +22255,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14442	arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse	22255	arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse
14443	return self.fail("no matching constraint: '{s}'", .{op_str})	22256	return self.fail("no matching constraint: '{s}'", .{op_str})
14444	]) {	22257	]) {
14445	.immediate => \|imm\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "c"))	22258	.immediate => \|imm\| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "c"))
14446	.{ .imm = Immediate.u(imm) }	22259	.{ .imm = .u(imm) }
14447	else	22260	else
14448	return self.fail("invalid modifier: '{s}'", .{modifier}),	22261	return self.fail("invalid modifier: '{s}'", .{modifier}),
14449	.register => \|reg\| if (mem.eql(u8, modifier, ""))	22262	.register => \|reg\| if (std.mem.eql(u8, modifier, ""))
14450	.{ .reg = reg }	22263	.{ .reg = reg }
14451	else	22264	else
14452	return self.fail("invalid modifier: '{s}'", .{modifier}),	22265	return self.fail("invalid modifier: '{s}'", .{modifier}),
14453	.memory => \|addr\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "P"))	22266	.memory => \|addr\| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "P"))
14454	.{ .mem = .{	22267	.{ .mem = .{
14455	.base = .{ .reg = .ds },	22268	.base = .{ .reg = .ds },
14456	.mod = .{ .rm = .{	22269	.mod = .{ .rm = .{
...	@@ -14461,7 +22274,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14461,7 +22274,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14461	} }	22274	} }
14462	else	22275	else
14463	return self.fail("invalid modifier: '{s}'", .{modifier}),	22276	return self.fail("invalid modifier: '{s}'", .{modifier}),
14464	.indirect => \|reg_off\| if (mem.eql(u8, modifier, ""))	22277	.indirect => \|reg_off\| if (std.mem.eql(u8, modifier, ""))
14465	.{ .mem = .{	22278	.{ .mem = .{
14466	.base = .{ .reg = reg_off.reg },	22279	.base = .{ .reg = reg_off.reg },
14467	.mod = .{ .rm = .{	22280	.mod = .{ .rm = .{
...	@@ -14472,7 +22285,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14472,7 +22285,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14472	} }	22285	} }
14473	else	22286	else
14474	return self.fail("invalid modifier: '{s}'", .{modifier}),	22287	return self.fail("invalid modifier: '{s}'", .{modifier}),
14475	.load_frame => \|frame_addr\| if (mem.eql(u8, modifier, ""))	22288	.load_frame => \|frame_addr\| if (std.mem.eql(u8, modifier, ""))
14476	.{ .mem = .{	22289	.{ .mem = .{
14477	.base = .{ .frame = frame_addr.index },	22290	.base = .{ .frame = frame_addr.index },
14478	.mod = .{ .rm = .{	22291	.mod = .{ .rm = .{
...	@@ -14483,42 +22296,42 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14483,42 +22296,42 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14483	} }	22296	} }
14484	else	22297	else
14485	return self.fail("invalid modifier: '{s}'", .{modifier}),	22298	return self.fail("invalid modifier: '{s}'", .{modifier}),
14486	.lea_got => \|sym_index\| if (mem.eql(u8, modifier, "P"))	22299	.lea_got => \|sym_index\| if (std.mem.eql(u8, modifier, "P"))
14487	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }	22300	.{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_got = sym_index }) }
14488	else	22301	else
14489	return self.fail("invalid modifier: '{s}'", .{modifier}),	22302	return self.fail("invalid modifier: '{s}'", .{modifier}),
14490	.lea_symbol => \|sym_off\| if (mem.eql(u8, modifier, "P"))	22303	.lea_symbol => \|sym_off\| if (std.mem.eql(u8, modifier, "P"))
14491	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) }	22304	.{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_symbol = sym_off }) }
14492	else	22305	else
14493	return self.fail("invalid modifier: '{s}'", .{modifier}),	22306	return self.fail("invalid modifier: '{s}'", .{modifier}),
14494	else => return self.fail("invalid constraint: '{s}'", .{op_str}),	22307	else => return self.fail("invalid constraint: '{s}'", .{op_str}),
14495	};	22308	};
14496	} else if (mem.startsWith(u8, op_str, "$")) {	22309	} else if (std.mem.startsWith(u8, op_str, "$")) {
14497	if (std.fmt.parseInt(i32, op_str["$".len..], 0)) \|s\| {	22310	if (std.fmt.parseInt(i32, op_str["$".len..], 0)) \|s\| {
14498	if (mnem_size) \|size\| {	22311	if (mnem_size) \|size\| {
14499	const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));	22312	const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize(self.target) - 1));
14500	if ((if (s < 0) ~s else s) > max)	22313	if ((if (s < 0) ~s else s) > max)
14501	return self.fail("invalid immediate size: '{s}'", .{op_str});	22314	return self.fail("invalid immediate size: '{s}'", .{op_str});
14502	}	22315	}
14503	op.* = .{ .imm = Immediate.s(s) };	22316	op.* = .{ .imm = .s(s) };
14504	} else \|_\| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) \|u\| {	22317	} else \|_\| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) \|u\| {
14505	if (mnem_size) \|size\| {	22318	if (mnem_size) \|size\| {
14506	const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - size.bitSize());	22319	const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize(self.target));
14507	if (u > max)	22320	if (u > max)
14508	return self.fail("invalid immediate size: '{s}'", .{op_str});	22321	return self.fail("invalid immediate size: '{s}'", .{op_str});
14509	}	22322	}
14510	op.* = .{ .imm = Immediate.u(u) };	22323	op.* = .{ .imm = .u(u) };
14511	} else \|_\| return self.fail("invalid immediate: '{s}'", .{op_str});	22324	} else \|_\| return self.fail("invalid immediate: '{s}'", .{op_str});
14512	} else if (mem.endsWith(u8, op_str, ")")) {	22325	} else if (std.mem.endsWith(u8, op_str, ")")) {
14513	const open = mem.indexOfScalar(u8, op_str, '(') orelse	22326	const open = std.mem.indexOfScalar(u8, op_str, '(') orelse
14514	return self.fail("invalid operand: '{s}'", .{op_str});	22327	return self.fail("invalid operand: '{s}'", .{op_str});
14515	var sib_it = mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');	22328	var sib_it = std.mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
14516	const base_str = sib_it.next() orelse	22329	const base_str = sib_it.next() orelse
14517	return self.fail("invalid memory operand: '{s}'", .{op_str});	22330	return self.fail("invalid memory operand: '{s}'", .{op_str});
14518	if (base_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))	22331	if (base_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
14519	return self.fail("invalid memory operand: '{s}'", .{op_str});	22332	return self.fail("invalid memory operand: '{s}'", .{op_str});
14520	const index_str = sib_it.next() orelse "";	22333	const index_str = sib_it.next() orelse "";
14521	if (index_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))	22334	if (index_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%"))
14522	return self.fail("invalid memory operand: '{s}'", .{op_str});	22335	return self.fail("invalid memory operand: '{s}'", .{op_str});
14523	const scale_str = sib_it.next() orelse "";	22336	const scale_str = sib_it.next() orelse "";
14524	if (index_str.len == 0 and scale_str.len > 0)	22337	if (index_str.len == 0 and scale_str.len > 0)
...	@@ -14550,10 +22363,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14550,10 +22363,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14550	else	22363	else
14551	.none,	22364	.none,
14552	.scale = scale,	22365	.scale = scale,
14553	.disp = if (mem.startsWith(u8, op_str[0..open], "%[") and	22366	.disp = if (std.mem.startsWith(u8, op_str[0..open], "%[") and
14554	mem.endsWith(u8, op_str[0..open], "]"))	22367	std.mem.endsWith(u8, op_str[0..open], "]"))
14555	disp: {	22368	disp: {
14556	const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');	22369	const colon = std.mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
14557	const modifier = if (colon) \|colon_pos\|	22370	const modifier = if (colon) \|colon_pos\|
14558	op_str[colon_pos + ":".len .. open - "]".len]	22371	op_str[colon_pos + ":".len .. open - "]".len]
14559	else	22372	else
...	@@ -14562,9 +22375,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14562,9 +22375,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14562	arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse	22375	arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
14563	return self.fail("no matching constraint: '{s}'", .{op_str})	22376	return self.fail("no matching constraint: '{s}'", .{op_str})
14564	]) {	22377	]) {
14565	.immediate => \|imm\| if (mem.eql(u8, modifier, "") or	22378	.immediate => \|imm\| if (std.mem.eql(u8, modifier, "") or
14566	mem.eql(u8, modifier, "c"))	22379	std.mem.eql(u8, modifier, "c"))
14567	math.cast(i32, @as(i64, @bitCast(imm))) orelse	22380	std.math.cast(i32, @as(i64, @bitCast(imm))) orelse
14568	return self.fail("invalid displacement: '{s}'", .{op_str})	22381	return self.fail("invalid displacement: '{s}'", .{op_str})
14569	else	22382	else
14570	return self.fail("invalid modifier: '{s}'", .{modifier}),	22383	return self.fail("invalid modifier: '{s}'", .{modifier}),
...	@@ -14592,8 +22405,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14592,8 +22405,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14592	try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));	22405	try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));
14593	op.* = .{ .inst = label_gop.value_ptr.target };	22406	op.* = .{ .inst = label_gop.value_ptr.target };
14594	} else return self.fail("invalid operand: '{s}'", .{op_str});	22407	} else return self.fail("invalid operand: '{s}'", .{op_str});
		22408	ops_len += 1;
14595	} else if (op_it.next()) \|op_str\| return self.fail("extra operand: '{s}'", .{op_str});	22409	} else if (op_it.next()) \|op_str\| return self.fail("extra operand: '{s}'", .{op_str});
14596		22410
		22411	// convert from att syntax to intel syntax
		22412	std.mem.reverse(Operand, ops[0..ops_len]);
		22413
14597	(if (prefix == .directive) switch (mnem_tag) {	22414	(if (prefix == .directive) switch (mnem_tag) {
14598	.@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none)	22415	.@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none)
14599	self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm)	22416	self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm)
...	@@ -14649,68 +22466,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14649,68 +22466,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14649	error.InvalidInstruction,	22466	error.InvalidInstruction,
14650	.@".cfi_escape" => error.InvalidInstruction,	22467	.@".cfi_escape" => error.InvalidInstruction,
14651	else => unreachable,	22468	else => unreachable,
14652	} else switch (ops[0]) {	22469	} else self.asmOps(mnem_fixed_tag, ops)) catch \|err\| switch (err) {
14653	.none => self.asmOpOnly(mnem_fixed_tag),
14654	.reg => \|reg0\| switch (ops[1]) {
14655	.none => self.asmRegister(mnem_fixed_tag, reg0),
14656	.reg => \|reg1\| switch (ops[2]) {
14657	.none => self.asmRegisterRegister(mnem_fixed_tag, reg1, reg0),
14658	.reg => \|reg2\| switch (ops[3]) {
14659	.none => self.asmRegisterRegisterRegister(mnem_fixed_tag, reg2, reg1, reg0),
14660	else => error.InvalidInstruction,
14661	},
14662	.mem => \|mem2\| switch (ops[3]) {
14663	.none => self.asmMemoryRegisterRegister(mnem_fixed_tag, mem2, reg1, reg0),
14664	else => error.InvalidInstruction,
14665	},
14666	else => error.InvalidInstruction,
14667	},
14668	.mem => \|mem1\| switch (ops[2]) {
14669	.none => self.asmMemoryRegister(mnem_fixed_tag, mem1, reg0),
14670	else => error.InvalidInstruction,
14671	},
14672	else => error.InvalidInstruction,
14673	},
14674	.mem => \|mem0\| switch (ops[1]) {
14675	.none => self.asmMemory(mnem_fixed_tag, mem0),
14676	.reg => \|reg1\| switch (ops[2]) {
14677	.none => self.asmRegisterMemory(mnem_fixed_tag, reg1, mem0),
14678	else => error.InvalidInstruction,
14679	},
14680	else => error.InvalidInstruction,
14681	},
14682	.imm => \|imm0\| switch (ops[1]) {
14683	.none => self.asmImmediate(mnem_fixed_tag, imm0),
14684	.reg => \|reg1\| switch (ops[2]) {
14685	.none => self.asmRegisterImmediate(mnem_fixed_tag, reg1, imm0),
14686	.reg => \|reg2\| switch (ops[3]) {
14687	.none => self.asmRegisterRegisterImmediate(mnem_fixed_tag, reg2, reg1, imm0),
14688	.reg => \|reg3\| self.asmRegisterRegisterRegisterImmediate(
14689	mnem_fixed_tag,
14690	reg3,
14691	reg2,
14692	reg1,
14693	imm0,
14694	),
14695	else => error.InvalidInstruction,
14696	},
14697	.mem => \|mem2\| switch (ops[3]) {
14698	.none => self.asmMemoryRegisterImmediate(mnem_fixed_tag, mem2, reg1, imm0),
14699	else => error.InvalidInstruction,
14700	},
14701	else => error.InvalidInstruction,
14702	},
14703	.mem => \|mem1\| switch (ops[2]) {
14704	.none => self.asmMemoryImmediate(mnem_fixed_tag, mem1, imm0),
14705	else => error.InvalidInstruction,
14706	},
14707	else => error.InvalidInstruction,
14708	},
14709	.inst => \|inst0\| switch (ops[1]) {
14710	.none => self.asmReloc(mnem_fixed_tag, inst0),
14711	else => error.InvalidInstruction,
14712	},
14713	}) catch \|err\| switch (err) {
14714	error.InvalidInstruction => return self.fail(	22470	error.InvalidInstruction => return self.fail(
14715	"invalid instruction: '{s} {s} {s} {s} {s}'",	22471	"invalid instruction: '{s} {s} {s} {s} {s}'",
14716	.{	22472	.{
...	@@ -14730,10 +22486,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14730,10 +22486,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14730	return self.fail("undefined label: '{s}'", .{label.key_ptr.*});	22486	return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
14731		22487
14732	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {	22488	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {
14733	const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);	22489	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
14734	const constraint =	22490	const constraint =
14735	mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);	22491	std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
14736	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	22492	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
14737	// This equation accounts for the fact that even if we have exactly 4 bytes	22493	// This equation accounts for the fact that even if we have exactly 4 bytes
14738	// for the string, we still use the next u32 for the null terminator.	22494	// for the string, we still use the next u32 for the null terminator.
14739	outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;	22495	outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
...	@@ -14745,7 +22501,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -14745,7 +22501,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
14745	}	22501	}
14746		22502
14747	simple: {	22503	simple: {
14748	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);	22504	var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
14749	var buf_index: usize = 0;	22505	var buf_index: usize = 0;
14750	for (outputs) \|output\| {	22506	for (outputs) \|output\| {
14751	if (output == .none) continue;	22507	if (output == .none) continue;
...	@@ -14775,9 +22531,12 @@ const MoveStrategy = union(enum) {	...	@@ -14775,9 +22531,12 @@ const MoveStrategy = union(enum) {
14775	extract: Mir.Inst.FixedTag,	22531	extract: Mir.Inst.FixedTag,
14776	};	22532	};
14777		22533
14778	pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {	22534	pub fn read(strat: MoveStrategy, self: *CodeGen, dst_reg: Register, src_mem: Memory) !void {
14779	switch (strat) {	22535	switch (strat) {
14780	.move => \|tag\| try self.asmRegisterMemory(tag, dst_reg, src_mem),	22536	.move => \|tag\| try self.asmRegisterMemory(tag, switch (tag[1]) {
		22537	else => dst_reg,
		22538	.lea => if (dst_reg.bitSize() >= 32) dst_reg else dst_reg.to32(),
		22539	}, src_mem),
14781	.x87_load_store => {	22540	.x87_load_store => {
14782	try self.asmMemory(.{ .f_, .ld }, src_mem);	22541	try self.asmMemory(.{ .f_, .ld }, src_mem);
14783	assert(dst_reg != .st7);	22542	assert(dst_reg != .st7);
...	@@ -14787,34 +22546,50 @@ const MoveStrategy = union(enum) {	...	@@ -14787,34 +22546,50 @@ const MoveStrategy = union(enum) {
14787	ie.insert,	22546	ie.insert,
14788	dst_reg,	22547	dst_reg,
14789	src_mem,	22548	src_mem,
14790	Immediate.u(0),	22549	.u(0),
14791	),	22550	),
14792	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(	22551	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
14793	ie.insert,	22552	ie.insert,
14794	dst_reg,	22553	dst_reg,
14795	dst_reg,	22554	dst_reg,
14796	src_mem,	22555	src_mem,
14797	Immediate.u(0),	22556	.u(0),
14798	),	22557	),
14799	}	22558	}
14800	}	22559	}
14801	pub fn write(strat: MoveStrategy, self: *Self, dst_mem: Memory, src_reg: Register) !void {	22560	pub fn write(strat: MoveStrategy, self: *CodeGen, dst_mem: Memory, src_reg: Register) !void {
14802	switch (strat) {	22561	switch (strat) {
14803	.move => \|tag\| try self.asmMemoryRegister(tag, dst_mem, src_reg),	22562	.move => \|tag\| try self.asmMemoryRegister(tag, dst_mem, src_reg),
14804	.x87_load_store => {	22563	.x87_load_store => {
14805	try self.asmRegister(.{ .f_, .ld }, src_reg);	22564	try self.asmRegister(.{ .f_, .ld }, src_reg);
14806	try self.asmMemory(.{ .f_p, .st }, dst_mem);	22565	try self.asmMemory(.{ .f_p, .st }, dst_mem);
14807	},	22566	},
14808	.insert_extract, .vex_insert_extract => \|ie\| try self.asmMemoryRegisterImmediate(	22567	.insert_extract, .vex_insert_extract => \|ie\| if (ie.extract[0] != .p_w or self.hasFeature(.sse4_1))
14809	ie.extract,	22568	try self.asmMemoryRegisterImmediate(ie.extract, dst_mem, src_reg, .u(0))
14810	dst_mem,	22569	else if (self.hasFeature(.sse2)) {
14811	src_reg,	22570	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
14812	Immediate.u(0),	22571	try self.asmRegisterRegisterImmediate(ie.extract, tmp_reg.to32(), src_reg.to128(), .u(0));
14813	),	22572	try self.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16());
		22573	} else {
		22574	const tmp_frame_index = try self.allocFrameIndex(.init(.{
		22575	.size = 16,
		22576	.alignment = .@"16",
		22577	}));
		22578	try self.asmMemoryRegister(.{ ._ps, .mova }, .{
		22579	.base = .{ .frame = tmp_frame_index },
		22580	.mod = .{ .rm = .{ .size = .xword } },
		22581	}, src_reg.to128());
		22582	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
		22583	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg.to16(), .{
		22584	.base = .{ .frame = tmp_frame_index },
		22585	.mod = .{ .rm = .{ .size = .word } },
		22586	});
		22587	try self.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16());
		22588	},
14814	}	22589	}
14815	}	22590	}
14816	};	22591	};
14817	fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {	22592	fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {
14818	const pt = self.pt;	22593	const pt = self.pt;
14819	const zcu = pt.zcu;	22594	const zcu = pt.zcu;
14820	switch (class) {	22595	switch (class) {
...	@@ -14823,7 +22598,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14823,7 +22598,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14823	.mmx => {},	22598	.mmx => {},
14824	.sse => switch (ty.zigTypeTag(zcu)) {	22599	.sse => switch (ty.zigTypeTag(zcu)) {
14825	else => {	22600	else => {
14826	const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);	22601	const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
14827	assert(std.mem.indexOfNone(abi.Class, classes, &.{	22602	assert(std.mem.indexOfNone(abi.Class, classes, &.{
14828	.integer, .sse, .sseup, .memory, .float, .float_combine,	22603	.integer, .sse, .sseup, .memory, .float, .float_combine,
14829	}) == null);	22604	}) == null);
...	@@ -14853,10 +22628,13 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14853,10 +22628,13 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14853	else	22628	else
14854	.{ ._q, .mov } },	22629	.{ ._q, .mov } },
14855	9...16 => return .{ .move = if (self.hasFeature(.avx))	22630	9...16 => return .{ .move = if (self.hasFeature(.avx))
14856	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22631	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14857	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22632	else if (self.hasFeature(.sse2))
		22633	.{ if (aligned) ._dqa else ._dqu, .mov }
		22634	else
		22635	.{ ._ps, if (aligned) .mova else .movu } },
14858	17...32 => if (self.hasFeature(.avx))	22636	17...32 => if (self.hasFeature(.avx))
14859	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },	22637	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14860	else => {},	22638	else => {},
14861	} else switch (abi_size) {	22639	} else switch (abi_size) {
14862	4 => return .{ .move = if (self.hasFeature(.avx))	22640	4 => return .{ .move = if (self.hasFeature(.avx))
...	@@ -14865,15 +22643,18 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14865,15 +22643,18 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14865	.{ ._ss, .mov } },	22643	.{ ._ss, .mov } },
14866	5...8 => return .{ .move = if (self.hasFeature(.avx))	22644	5...8 => return .{ .move = if (self.hasFeature(.avx))
14867	.{ .v_sd, .mov }	22645	.{ .v_sd, .mov }
		22646	else if (self.hasFeature(.sse2))
		22647	.{ ._sd, .mov }
14868	else	22648	else
14869	.{ ._sd, .mov } },	22649	.{ ._ps, .movl } },
14870	9...16 => return .{ .move = if (self.hasFeature(.avx))	22650	9...16 => return .{ .move = if (self.hasFeature(.avx))
14871	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }	22651	.{ .v_pd, if (aligned) .mova else .movu }
14872	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },	22652	else if (self.hasFeature(.sse2))
14873	17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)	22653	.{ ._pd, if (aligned) .mova else .movu }
14874	.{ .v_pd, .mova }
14875	else	22654	else
14876	.{ .v_pd, .movu } },	22655	.{ ._ps, if (aligned) .mova else .movu } },
		22656	17...32 => if (self.hasFeature(.avx))
		22657	return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } },
14877	else => {},	22658	else => {},
14878	}	22659	}
14879	},	22660	},
...	@@ -14891,11 +22672,16 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14891,11 +22672,16 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14891	.{ ._ss, .mov } },	22672	.{ ._ss, .mov } },
14892	64 => return .{ .move = if (self.hasFeature(.avx))	22673	64 => return .{ .move = if (self.hasFeature(.avx))
14893	.{ .v_sd, .mov }	22674	.{ .v_sd, .mov }
		22675	else if (self.hasFeature(.sse2))
		22676	.{ ._sd, .mov }
14894	else	22677	else
14895	.{ ._sd, .mov } },	22678	.{ ._ps, .movl } },
14896	128 => return .{ .move = if (self.hasFeature(.avx))	22679	128 => return .{ .move = if (self.hasFeature(.avx))
14897	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22680	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14898	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22681	else if (self.hasFeature(.sse2))
		22682	.{ if (aligned) ._dqa else ._dqu, .mov }
		22683	else
		22684	.{ ._ps, if (aligned) .mova else .movu } },
14899	else => {},	22685	else => {},
14900	},	22686	},
14901	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {	22687	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
...	@@ -14909,65 +22695,62 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14909,65 +22695,62 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14909	.int => switch (ty.childType(zcu).intInfo(zcu).bits) {	22695	.int => switch (ty.childType(zcu).intInfo(zcu).bits) {
14910	1...8 => switch (ty.vectorLen(zcu)) {	22696	1...8 => switch (ty.vectorLen(zcu)) {
14911	1...16 => return .{ .move = if (self.hasFeature(.avx))	22697	1...16 => return .{ .move = if (self.hasFeature(.avx))
14912	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22698	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14913	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22699	else if (self.hasFeature(.sse2))
		22700	.{ if (aligned) ._dqa else ._dqu, .mov }
		22701	else
		22702	.{ ._ps, if (aligned) .mova else .movu } },
14914	17...32 => if (self.hasFeature(.avx))	22703	17...32 => if (self.hasFeature(.avx))
14915	return .{ .move = if (aligned)	22704	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14916	.{ .v_, .movdqa }
14917	else
14918	.{ .v_, .movdqu } },
14919	else => {},	22705	else => {},
14920	},	22706	},
14921	9...16 => switch (ty.vectorLen(zcu)) {	22707	9...16 => switch (ty.vectorLen(zcu)) {
14922	1...8 => return .{ .move = if (self.hasFeature(.avx))	22708	1...8 => return .{ .move = if (self.hasFeature(.avx))
14923	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22709	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14924	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22710	else if (self.hasFeature(.sse2))
		22711	.{ if (aligned) ._dqa else ._dqu, .mov }
		22712	else
		22713	.{ ._ps, if (aligned) .mova else .movu } },
14925	9...16 => if (self.hasFeature(.avx))	22714	9...16 => if (self.hasFeature(.avx))
14926	return .{ .move = if (aligned)	22715	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14927	.{ .v_, .movdqa }
14928	else
14929	.{ .v_, .movdqu } },
14930	else => {},	22716	else => {},
14931	},	22717	},
14932	17...32 => switch (ty.vectorLen(zcu)) {	22718	17...32 => switch (ty.vectorLen(zcu)) {
14933	1...4 => return .{ .move = if (self.hasFeature(.avx))	22719	1...4 => return .{ .move = if (self.hasFeature(.avx))
14934	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22720	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14935	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22721	else if (self.hasFeature(.sse2))
		22722	.{ if (aligned) ._dqa else ._dqu, .mov }
		22723	else
		22724	.{ ._ps, if (aligned) .mova else .movu } },
14936	5...8 => if (self.hasFeature(.avx))	22725	5...8 => if (self.hasFeature(.avx))
14937	return .{ .move = if (aligned)	22726	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14938	.{ .v_, .movdqa }
14939	else
14940	.{ .v_, .movdqu } },
14941	else => {},	22727	else => {},
14942	},	22728	},
14943	33...64 => switch (ty.vectorLen(zcu)) {	22729	33...64 => switch (ty.vectorLen(zcu)) {
14944	1...2 => return .{ .move = if (self.hasFeature(.avx))	22730	1...2 => return .{ .move = if (self.hasFeature(.avx))
14945	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22731	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14946	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22732	else if (self.hasFeature(.sse2))
		22733	.{ if (aligned) ._dqa else ._dqu, .mov }
		22734	else
		22735	.{ ._ps, if (aligned) .mova else .movu } },
14947	3...4 => if (self.hasFeature(.avx))	22736	3...4 => if (self.hasFeature(.avx))
14948	return .{ .move = if (aligned)	22737	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14949	.{ .v_, .movdqa }
14950	else
14951	.{ .v_, .movdqu } },
14952	else => {},	22738	else => {},
14953	},	22739	},
14954	65...128 => switch (ty.vectorLen(zcu)) {	22740	65...128 => switch (ty.vectorLen(zcu)) {
14955	1 => return .{ .move = if (self.hasFeature(.avx))	22741	1 => return .{ .move = if (self.hasFeature(.avx))
14956	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22742	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14957	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22743	else if (self.hasFeature(.sse2))
		22744	.{ if (aligned) ._dqa else ._dqu, .mov }
		22745	else
		22746	.{ ._ps, if (aligned) .mova else .movu } },
14958	2 => if (self.hasFeature(.avx))	22747	2 => if (self.hasFeature(.avx))
14959	return .{ .move = if (aligned)	22748	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14960	.{ .v_, .movdqa }
14961	else
14962	.{ .v_, .movdqu } },
14963	else => {},	22749	else => {},
14964	},	22750	},
14965	129...256 => switch (ty.vectorLen(zcu)) {	22751	129...256 => switch (ty.vectorLen(zcu)) {
14966	1 => if (self.hasFeature(.avx))	22752	1 => if (self.hasFeature(.avx))
14967	return .{ .move = if (aligned)	22753	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14968	.{ .v_, .movdqa }
14969	else
14970	.{ .v_, .movdqu } },
14971	else => {},	22754	else => {},
14972	},	22755	},
14973	else => {},	22756	else => {},
...	@@ -14975,13 +22758,13 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14975,13 +22758,13 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14975	.pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu))	22758	.pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu))
14976	switch (ty.vectorLen(zcu)) {	22759	switch (ty.vectorLen(zcu)) {
14977	1...2 => return .{ .move = if (self.hasFeature(.avx))	22760	1...2 => return .{ .move = if (self.hasFeature(.avx))
14978	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22761	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14979	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22762	else if (self.hasFeature(.sse2))
		22763	.{ if (aligned) ._dqa else ._dqu, .mov }
		22764	else
		22765	.{ ._ps, if (aligned) .mova else .movu } },
14980	3...4 => if (self.hasFeature(.avx))	22766	3...4 => if (self.hasFeature(.avx))
14981	return .{ .move = if (aligned)	22767	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14982	.{ .v_, .movdqa }
14983	else
14984	.{ .v_, .movdqu } },
14985	else => {},	22768	else => {},
14986	}	22769	}
14987	else	22770	else
...	@@ -14989,46 +22772,42 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14989,46 +22772,42 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14989	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {	22772	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {
14990	16 => switch (ty.vectorLen(zcu)) {	22773	16 => switch (ty.vectorLen(zcu)) {
14991	1...8 => return .{ .move = if (self.hasFeature(.avx))	22774	1...8 => return .{ .move = if (self.hasFeature(.avx))
14992	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22775	.{ if (aligned) .v_dqa else .v_dqu, .mov }
14993	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22776	else if (self.hasFeature(.sse2))
		22777	.{ if (aligned) ._dqa else ._dqu, .mov }
		22778	else
		22779	.{ ._ps, if (aligned) .mova else .movu } },
14994	9...16 => if (self.hasFeature(.avx))	22780	9...16 => if (self.hasFeature(.avx))
14995	return .{ .move = if (aligned)	22781	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
14996	.{ .v_, .movdqa }
14997	else
14998	.{ .v_, .movdqu } },
14999	else => {},	22782	else => {},
15000	},	22783	},
15001	32 => switch (ty.vectorLen(zcu)) {	22784	32 => switch (ty.vectorLen(zcu)) {
15002	1...4 => return .{ .move = if (self.hasFeature(.avx))	22785	1...4 => return .{ .move = if (self.hasFeature(.avx))
15003	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }	22786	.{ .v_ps, if (aligned) .mova else .movu }
15004	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },	22787	else
		22788	.{ ._ps, if (aligned) .mova else .movu } },
15005	5...8 => if (self.hasFeature(.avx))	22789	5...8 => if (self.hasFeature(.avx))
15006	return .{ .move = if (aligned)	22790	return .{ .move = .{ .v_ps, if (aligned) .mova else .movu } },
15007	.{ .v_ps, .mova }
15008	else
15009	.{ .v_ps, .movu } },
15010	else => {},	22791	else => {},
15011	},	22792	},
15012	64 => switch (ty.vectorLen(zcu)) {	22793	64 => switch (ty.vectorLen(zcu)) {
15013	1...2 => return .{ .move = if (self.hasFeature(.avx))	22794	1...2 => return .{ .move = if (self.hasFeature(.avx))
15014	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }	22795	.{ .v_pd, if (aligned) .mova else .movu }
15015	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },	22796	else
		22797	.{ ._pd, if (aligned) .mova else .movu } },
15016	3...4 => if (self.hasFeature(.avx))	22798	3...4 => if (self.hasFeature(.avx))
15017	return .{ .move = if (aligned)	22799	return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } },
15018	.{ .v_pd, .mova }
15019	else
15020	.{ .v_pd, .movu } },
15021	else => {},	22800	else => {},
15022	},	22801	},
15023	128 => switch (ty.vectorLen(zcu)) {	22802	128 => switch (ty.vectorLen(zcu)) {
15024	1 => return .{ .move = if (self.hasFeature(.avx))	22803	1 => return .{ .move = if (self.hasFeature(.avx))
15025	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	22804	.{ if (aligned) .v_dqa else .v_dqu, .mov }
15026	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	22805	else if (self.hasFeature(.sse2))
		22806	.{ if (aligned) ._dqa else ._dqu, .mov }
		22807	else
		22808	.{ ._ps, if (aligned) .mova else .movu } },
15027	2 => if (self.hasFeature(.avx))	22809	2 => if (self.hasFeature(.avx))
15028	return .{ .move = if (aligned)	22810	return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
15029	.{ .v_, .movdqa }
15030	else
15031	.{ .v_, .movdqu } },
15032	else => {},	22811	else => {},
15033	},	22812	},
15034	else => {},	22813	else => {},
...	@@ -15045,7 +22824,7 @@ const CopyOptions = struct {	...	@@ -15045,7 +22824,7 @@ const CopyOptions = struct {
15045	safety: bool = false,	22824	safety: bool = false,
15046	};	22825	};
15047		22826
15048	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {	22827	fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {
15049	const pt = self.pt;	22828	const pt = self.pt;
15050		22829
15051	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;	22830	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
...	@@ -15059,6 +22838,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy	...	@@ -15059,6 +22838,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
15059	.immediate,	22838	.immediate,
15060	.eflags,	22839	.eflags,
15061	.register_overflow,	22840	.register_overflow,
		22841	.register_mask,
15062	.lea_direct,	22842	.lea_direct,
15063	.lea_got,	22843	.lea_got,
15064	.lea_tlv,	22844	.lea_tlv,
...	@@ -15088,42 +22868,93 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy	...	@@ -15088,42 +22868,93 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
15088	.off = -dst_reg_off.off,	22868	.off = -dst_reg_off.off,
15089	} },	22869	} },
15090	}, opts),	22870	}, opts),
15091	.register_pair => \|dst_regs\| {	22871	inline .register_pair, .register_triple, .register_quadruple => \|dst_regs, dst_tag\| {
15092	const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) {	22872	const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = src_info: switch (src_mcv) {
15093	.register_pair, .memory, .indirect, .load_frame => null,	22873	.undef, .memory, .indirect, .load_frame => null,
15094	.load_symbol, .load_direct, .load_got, .load_tlv => src: {	22874	.register => \|src_reg\| switch (dst_regs[0].class()) {
		22875	.general_purpose => switch (src_reg.class()) {
		22876	else => unreachable,
		22877	.sse => if (ty.abiSize(pt.zcu) <= 16) {
		22878	if (self.hasFeature(.avx)) {
		22879	try self.asmRegisterRegister(.{ .v_q, .mov }, dst_regs[0].to64(), src_reg.to128());
		22880	try self.asmRegisterRegisterImmediate(.{ .vp_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1));
		22881	} else if (self.hasFeature(.sse4_1)) {
		22882	try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
		22883	try self.asmRegisterRegisterImmediate(.{ .p_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1));
		22884	} else {
		22885	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
		22886	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		22887	defer self.register_manager.unlockReg(tmp_lock);
		22888
		22889	try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
		22890	try self.asmRegisterRegister(.{ ._ps, .movhl }, tmp_reg.to128(), src_reg.to128());
		22891	try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[1].to64(), tmp_reg.to128());
		22892	}
		22893	return;
		22894	} else unreachable,
		22895	},
		22896	else => unreachable,
		22897	},
		22898	dst_tag => \|src_regs\| {
		22899	var hazard_regs = src_regs;
		22900	for (dst_regs, &hazard_regs, 1..) \|dst_reg, src_reg, hazard_index\| {
		22901	const dst_id = dst_reg.id();
		22902	if (dst_id == src_reg.id()) continue;
		22903	var mir_tag: Mir.Inst.Tag = .mov;
		22904	for (hazard_regs[hazard_index..]) \|*hazard_reg\| {
		22905	if (dst_id != hazard_reg.id()) continue;
		22906	mir_tag = .xchg;
		22907	hazard_reg.* = src_reg;
		22908	}
		22909	try self.asmRegisterRegister(.{ ._, mir_tag }, dst_reg.to64(), src_reg.to64());
		22910	}
		22911	return;
		22912	},
		22913	.load_symbol, .load_direct, .load_got, .load_tlv => {
15095	const src_addr_reg =	22914	const src_addr_reg =
15096	(try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64();	22915	(try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64();
15097	const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);	22916	const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
15098	errdefer self.register_manager.unlockReg(src_addr_lock);	22917	errdefer self.register_manager.unlockReg(src_addr_lock);
15099		22918
15100	try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address(), opts);	22919	try self.genSetReg(src_addr_reg, .usize, src_mcv.address(), opts);
15101	break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };	22920	break :src_info .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
15102	},	22921	},
15103	.air_ref => \|src_ref\| return self.genCopy(	22922	.air_ref => \|src_ref\| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref), opts),
15104	ty,
15105	dst_mcv,
15106	try self.resolveInst(src_ref),
15107	opts,
15108	),
15109	else => return self.fail("TODO implement genCopy for {s} of {}", .{	22923	else => return self.fail("TODO implement genCopy for {s} of {}", .{
15110	@tagName(src_mcv), ty.fmt(pt),	22924	@tagName(src_mcv), ty.fmt(pt),
15111	}),	22925	}),
15112	};	22926	};
15113	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);	22927	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
15114		22928
15115	var part_disp: i32 = 0;	22929	for ([_]bool{ false, true }) \|emit_hazard\| {
15116	for (dst_regs, try self.splitType(ty), 0..) \|dst_reg, dst_ty, part_i\| {	22930	var hazard_count: u3 = 0;
15117	try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {	22931	var part_disp: i32 = 0;
15118	.register_pair => \|src_regs\| .{ .register = src_regs[part_i] },	22932	for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) \|dst_reg, dst_ty, part_i\| {
15119	.memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),	22933	defer part_disp += @intCast(dst_ty.abiSize(pt.zcu));
15120	.load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{	22934	const is_hazard = if (src_mcv.getReg()) \|src_reg\|
15121	.reg = src_info.?.addr_reg,	22935	dst_reg.id() == src_reg.id()
15122	.off = part_disp,	22936	else if (src_info) \|info\|
15123	} },	22937	dst_reg.id() == info.addr_reg.id()
		22938	else
		22939	false;
		22940	if (is_hazard) hazard_count += 1;
		22941	if (is_hazard != emit_hazard) continue;
		22942	try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
		22943	.undef => if (opts.safety and part_i > 0) .{ .register = dst_regs[0] } else .undef,
		22944	dst_tag => \|src_regs\| .{ .register = src_regs[part_i] },
		22945	.memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
		22946	.load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
		22947	.reg = src_info.?.addr_reg,
		22948	.off = part_disp,
		22949	} },
		22950	else => unreachable,
		22951	}, opts);
		22952	}
		22953	switch (hazard_count) {
		22954	0 => break,
		22955	1 => continue,
15124	else => unreachable,	22956	else => unreachable,
15125	}, opts);	22957	}
15126	part_disp += @intCast(dst_ty.abiSize(pt.zcu));
15127	}	22958	}
15128	},	22959	},
15129	.indirect => \|reg_off\| try self.genSetMem(	22960	.indirect => \|reg_off\| try self.genSetMem(
...	@@ -15135,13 +22966,13 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy	...	@@ -15135,13 +22966,13 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
15135	),	22966	),
15136	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {	22967	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
15137	switch (dst_mcv) {	22968	switch (dst_mcv) {
15138	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|	22969	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
15139	return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts),	22970	return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts),
15140	.load_symbol, .load_direct, .load_got, .load_tlv => {},	22971	.load_symbol, .load_direct, .load_got, .load_tlv => {},
15141	else => unreachable,	22972	else => unreachable,
15142	}	22973	}
15143		22974
15144	const addr_reg = try self.copyToTmpRegister(Type.usize, dst_mcv.address());	22975	const addr_reg = try self.copyToTmpRegister(.usize, dst_mcv.address());
15145	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	22976	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
15146	defer self.register_manager.unlockReg(addr_lock);	22977	defer self.register_manager.unlockReg(addr_lock);
15147		22978
...	@@ -15158,7 +22989,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy	...	@@ -15158,7 +22989,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
15158	}	22989	}
15159		22990
15160	fn genSetReg(	22991	fn genSetReg(
15161	self: *Self,	22992	self: *CodeGen,
15162	dst_reg: Register,	22993	dst_reg: Register,
15163	ty: Type,	22994	ty: Type,
15164	src_mcv: MCValue,	22995	src_mcv: MCValue,
...	@@ -15179,17 +23010,17 @@ fn genSetReg(	...	@@ -15179,17 +23010,17 @@ fn genSetReg(
15179	=> unreachable,	23010	=> unreachable,
15180	.undef => if (opts.safety) switch (dst_reg.class()) {	23011	.undef => if (opts.safety) switch (dst_reg.class()) {
15181	.general_purpose => switch (abi_size) {	23012	.general_purpose => switch (abi_size) {
15182	1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), Immediate.u(0xAA)),	23013	1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xAA)),
15183	2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), Immediate.u(0xAAAA)),	23014	2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xAAAA)),
15184	3...4 => try self.asmRegisterImmediate(	23015	3...4 => try self.asmRegisterImmediate(
15185	.{ ._, .mov },	23016	.{ ._, .mov },
15186	dst_reg.to32(),	23017	dst_reg.to32(),
15187	Immediate.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),	23018	.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
15188	),	23019	),
15189	5...8 => try self.asmRegisterImmediate(	23020	5...8 => try self.asmRegisterImmediate(
15190	.{ ._, .mov },	23021	.{ ._, .mov },
15191	dst_reg.to64(),	23022	dst_reg.to64(),
15192	Immediate.u(0xAAAAAAAAAAAAAAAA),	23023	.u(0xAAAAAAAAAAAAAAAA),
15193	),	23024	),
15194	else => unreachable,	23025	else => unreachable,
15195	},	23026	},
...	@@ -15203,20 +23034,20 @@ fn genSetReg(	...	@@ -15203,20 +23034,20 @@ fn genSetReg(
15203	// register is the fastest way to zero a register.	23034	// register is the fastest way to zero a register.
15204	try self.spillEflagsIfOccupied();	23035	try self.spillEflagsIfOccupied();
15205	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());	23036	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
15206	} else if (abi_size > 4 and math.cast(u32, imm) != null) {	23037	} else if (abi_size > 4 and std.math.cast(u32, imm) != null) {
15207	// 32-bit moves zero-extend to 64-bit.	23038	// 32-bit moves zero-extend to 64-bit.
15208	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));	23039	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(imm));
15209	} else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) {	23040	} else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) {
15210	try self.asmRegisterImmediate(	23041	try self.asmRegisterImmediate(
15211	.{ ._, .mov },	23042	.{ ._, .mov },
15212	registerAlias(dst_reg, abi_size),	23043	registerAlias(dst_reg, abi_size),
15213	Immediate.s(@intCast(@as(i64, @bitCast(imm)))),	23044	.s(@intCast(@as(i64, @bitCast(imm)))),
15214	);	23045	);
15215	} else {	23046	} else {
15216	try self.asmRegisterImmediate(	23047	try self.asmRegisterImmediate(
15217	.{ ._, .mov },	23048	.{ ._, .mov },
15218	registerAlias(dst_reg, abi_size),	23049	registerAlias(dst_reg, abi_size),
15219	Immediate.u(imm),	23050	.u(imm),
15220	);	23051	);
15221	}	23052	}
15222	},	23053	},
...	@@ -15233,7 +23064,7 @@ fn genSetReg(	...	@@ -15233,7 +23064,7 @@ fn genSetReg(
15233	src_reg,	23064	src_reg,
15234	),	23065	),
15235	.x87, .mmx, .ip => unreachable,	23066	.x87, .mmx, .ip => unreachable,
15236	.sse => try self.asmRegisterRegister(	23067	.sse => if (self.hasFeature(.sse2)) try self.asmRegisterRegister(
15237	switch (abi_size) {	23068	switch (abi_size) {
15238	1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },	23069	1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
15239	5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },	23070	5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
...	@@ -15241,7 +23072,20 @@ fn genSetReg(	...	@@ -15241,7 +23072,20 @@ fn genSetReg(
15241	},	23072	},
15242	registerAlias(dst_reg, @max(abi_size, 4)),	23073	registerAlias(dst_reg, @max(abi_size, 4)),
15243	src_reg.to128(),	23074	src_reg.to128(),
15244	),	23075	) else {
		23076	const frame_index = try self.allocFrameIndex(.init(.{
		23077	.size = 4,
		23078	.alignment = .@"4",
		23079	}));
		23080	try self.asmMemoryRegister(.{ ._ss, .mov }, .{
		23081	.base = .{ .frame = frame_index },
		23082	.mod = .{ .rm = .{ .size = .dword } },
		23083	}, src_reg.to128());
		23084	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
		23085	.base = .{ .frame = frame_index },
		23086	.mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
		23087	});
		23088	},
15245	},	23089	},
15246	.segment => try self.asmRegisterRegister(	23090	.segment => try self.asmRegisterRegister(
15247	.{ ._, .mov },	23091	.{ ._, .mov },
...	@@ -15286,17 +23130,17 @@ fn genSetReg(	...	@@ -15286,17 +23130,17 @@ fn genSetReg(
15286	.sse => try self.asmRegisterRegister(	23130	.sse => try self.asmRegisterRegister(
15287	@as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) {	23131	@as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
15288	else => switch (abi_size) {	23132	else => switch (abi_size) {
15289	1...16 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else .{ ._, .movdqa },	23133	1...16 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else .{ ._dqa, .mov },
15290	17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,	23134	17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null,
15291	else => null,	23135	else => null,
15292	},	23136	},
15293	.float => switch (ty.scalarType(zcu).floatBits(self.target.*)) {	23137	.float => switch (ty.scalarType(zcu).floatBits(self.target.*)) {
15294	16, 128 => switch (abi_size) {	23138	16, 128 => switch (abi_size) {
15295	2...16 => if (self.hasFeature(.avx))	23139	2...16 => if (self.hasFeature(.avx))
15296	.{ .v_, .movdqa }	23140	.{ .v_dqa, .mov }
15297	else	23141	else
15298	.{ ._, .movdqa },	23142	.{ ._dqa, .mov },
15299	17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,	23143	17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null,
15300	else => null,	23144	else => null,
15301	},	23145	},
15302	32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova },	23146	32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova },
...	@@ -15311,7 +23155,42 @@ fn genSetReg(	...	@@ -15311,7 +23155,42 @@ fn genSetReg(
15311	},	23155	},
15312	.ip => unreachable,	23156	.ip => unreachable,
15313	},	23157	},
15314	.register_pair => \|src_regs\| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts),	23158	inline .register_pair,
		23159	.register_triple,
		23160	.register_quadruple,
		23161	=> \|src_regs\| switch (dst_reg.class()) {
		23162	.general_purpose => switch (src_regs[0].class()) {
		23163	.general_purpose => try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts),
		23164	else => unreachable,
		23165	},
		23166	.sse => switch (src_regs[0].class()) {
		23167	.general_purpose => if (abi_size <= 16) {
		23168	if (self.hasFeature(.avx)) {
		23169	try self.asmRegisterRegister(.{ .v_q, .mov }, dst_reg.to128(), src_regs[0].to64());
		23170	try self.asmRegisterRegisterRegisterImmediate(
		23171	.{ .vp_q, .insr },
		23172	dst_reg.to128(),
		23173	dst_reg.to128(),
		23174	src_regs[1].to64(),
		23175	.u(1),
		23176	);
		23177	} else if (self.hasFeature(.sse4_1)) {
		23178	try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
		23179	try self.asmRegisterRegisterImmediate(.{ .p_q, .insr }, dst_reg.to128(), src_regs[1].to64(), .u(1));
		23180	} else {
		23181	const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
		23182	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		23183	defer self.register_manager.unlockReg(tmp_lock);
		23184
		23185	try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
		23186	try self.asmRegisterRegister(.{ ._q, .mov }, tmp_reg.to128(), src_regs[1].to64());
		23187	try self.asmRegisterRegister(.{ ._ps, .movlh }, dst_reg.to128(), tmp_reg.to128());
		23188	}
		23189	} else unreachable,
		23190	else => unreachable,
		23191	},
		23192	else => unreachable,
		23193	},
15315	.register_offset,	23194	.register_offset,
15316	.indirect,	23195	.indirect,
15317	.load_frame,	23196	.load_frame,
...	@@ -15325,15 +23204,15 @@ fn genSetReg(	...	@@ -15325,15 +23204,15 @@ fn genSetReg(
15325	.load_frame => \|frame_addr\| try self.moveStrategy(	23204	.load_frame => \|frame_addr\| try self.moveStrategy(
15326	ty,	23205	ty,
15327	dst_reg.class(),	23206	dst_reg.class(),
15328	self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(	23207	self.getFrameAddrAlignment(frame_addr).compare(.gte, .fromLog2Units(
15329	math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),	23208	std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
15330	)),	23209	)),
15331	),	23210	),
15332	.lea_frame => .{ .move = .{ ._, .lea } },	23211	.lea_frame => .{ .move = .{ ._, .lea } },
15333	else => unreachable,	23212	else => unreachable,
15334	}).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {	23213	}).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {
15335	.register_offset, .indirect => \|reg_off\| .{	23214	.register_offset, .indirect => \|reg_off\| .{
15336	.base = .{ .reg = reg_off.reg },	23215	.base = .{ .reg = reg_off.reg.to64() },
15337	.mod = .{ .rm = .{	23216	.mod = .{ .rm = .{
15338	.size = self.memSize(ty),	23217	.size = self.memSize(ty),
15339	.disp = reg_off.off,	23218	.disp = reg_off.off,
...	@@ -15348,9 +23227,50 @@ fn genSetReg(	...	@@ -15348,9 +23227,50 @@ fn genSetReg(
15348	},	23227	},
15349	else => unreachable,	23228	else => unreachable,
15350	}),	23229	}),
		23230	.register_mask => \|src_reg_mask\| {
		23231	assert(src_reg_mask.reg.class() == .sse);
		23232	const has_avx = self.hasFeature(.avx);
		23233	const bits_reg = switch (dst_reg.class()) {
		23234	.general_purpose => dst_reg,
		23235	else => try self.register_manager.allocReg(null, abi.RegisterClass.gp),
		23236	};
		23237	const bits_lock = self.register_manager.lockReg(bits_reg);
		23238	defer if (bits_lock) \|lock\| self.register_manager.unlockReg(lock);
		23239
		23240	const pack_reg = switch (src_reg_mask.info.scalar) {
		23241	else => src_reg_mask.reg,
		23242	.word => try self.register_manager.allocReg(null, abi.RegisterClass.sse),
		23243	};
		23244	const pack_lock = self.register_manager.lockReg(pack_reg);
		23245	defer if (pack_lock) \|lock\| self.register_manager.unlockReg(lock);
		23246
		23247	var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(self.target), 8));
		23248	switch (src_reg_mask.info.scalar) {
		23249	else => {},
		23250	.word => {
		23251	const src_alias = registerAlias(src_reg_mask.reg, mask_size);
		23252	const pack_alias = registerAlias(pack_reg, mask_size);
		23253	if (has_avx) {
		23254	try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias);
		23255	} else {
		23256	try self.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias);
		23257	try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias);
		23258	}
		23259	mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable;
		23260	},
		23261	}
		23262	try self.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
		23263	.byte, .word => if (has_avx) .vp_b else .p_b,
		23264	.dword => if (has_avx) .v_ps else ._ps,
		23265	.qword => if (has_avx) .v_pd else ._pd,
		23266	else => unreachable,
		23267	}, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size));
		23268	if (src_reg_mask.info.inverted) try self.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size));
		23269	try self.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{});
		23270	},
15351	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {	23271	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
15352	switch (src_mcv) {	23272	switch (src_mcv) {
15353	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|	23273	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
15354	return (try self.moveStrategy(	23274	return (try self.moveStrategy(
15355	ty,	23275	ty,
15356	dst_reg.class(),	23276	dst_reg.class(),
...	@@ -15396,18 +23316,14 @@ fn genSetReg(	...	@@ -15396,18 +23316,14 @@ fn genSetReg(
15396	else => unreachable,	23316	else => unreachable,
15397	}	23317	}
15398		23318
15399	const addr_reg = try self.copyToTmpRegister(Type.usize, src_mcv.address());	23319	const addr_reg = try self.copyToTmpRegister(.usize, src_mcv.address());
15400	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	23320	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
15401	defer self.register_manager.unlockReg(addr_lock);	23321	defer self.register_manager.unlockReg(addr_lock);
15402		23322
15403	try (try self.moveStrategy(ty, dst_reg.class(), false)).read(	23323	try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, registerAlias(dst_reg, abi_size), .{
15404	self,	23324	.base = .{ .reg = addr_reg.to64() },
15405	registerAlias(dst_reg, abi_size),	23325	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
15406	.{	23326	});
15407	.base = .{ .reg = addr_reg },
15408	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
15409	},
15410	);
15411	},	23327	},
15412	.lea_symbol => \|sym_off\| switch (self.bin_file.tag) {	23328	.lea_symbol => \|sym_off\| switch (self.bin_file.tag) {
15413	.elf, .macho => try self.asmRegisterMemory(	23329	.elf, .macho => try self.asmRegisterMemory(
...	@@ -15447,7 +23363,7 @@ fn genSetReg(	...	@@ -15447,7 +23363,7 @@ fn genSetReg(
15447	}	23363	}
15448		23364
15449	fn genSetMem(	23365	fn genSetMem(
15450	self: *Self,	23366	self: *CodeGen,
15451	base: Memory.Base,	23367	base: Memory.Base,
15452	disp: i32,	23368	disp: i32,
15453	ty: Type,	23369	ty: Type,
...	@@ -15461,6 +23377,7 @@ fn genSetMem(	...	@@ -15461,6 +23377,7 @@ fn genSetMem(
15461	.none => .{ .immediate = @bitCast(@as(i64, disp)) },	23377	.none => .{ .immediate = @bitCast(@as(i64, disp)) },
15462	.reg => \|base_reg\| .{ .register_offset = .{ .reg = base_reg, .off = disp } },	23378	.reg => \|base_reg\| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
15463	.frame => \|base_frame_index\| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },	23379	.frame => \|base_frame_index\| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
		23380	.table => unreachable,
15464	.reloc => \|sym_index\| .{ .lea_symbol = .{ .sym_index = sym_index, .off = disp } },	23381	.reloc => \|sym_index\| .{ .lea_symbol = .{ .sym_index = sym_index, .off = disp } },
15465	};	23382	};
15466	switch (src_mcv) {	23383	switch (src_mcv) {
...	@@ -15478,31 +23395,31 @@ fn genSetMem(	...	@@ -15478,31 +23395,31 @@ fn genSetMem(
15478	),	23395	),
15479	.immediate => \|imm\| switch (abi_size) {	23396	.immediate => \|imm\| switch (abi_size) {
15480	1, 2, 4 => {	23397	1, 2, 4 => {
15481	const immediate = switch (if (ty.isAbiInt(zcu))	23398	const immediate: Immediate = switch (if (ty.isAbiInt(zcu))
15482	ty.intInfo(zcu).signedness	23399	ty.intInfo(zcu).signedness
15483	else	23400	else
15484	.unsigned) {	23401	.unsigned) {
15485	.signed => Immediate.s(@truncate(@as(i64, @bitCast(imm)))),	23402	.signed => .s(@truncate(@as(i64, @bitCast(imm)))),
15486	.unsigned => Immediate.u(@as(u32, @intCast(imm))),	23403	.unsigned => .u(@as(u32, @intCast(imm))),
15487	};	23404	};
15488	try self.asmMemoryImmediate(	23405	try self.asmMemoryImmediate(
15489	.{ ._, .mov },	23406	.{ ._, .mov },
15490	.{ .base = base, .mod = .{ .rm = .{	23407	.{ .base = base, .mod = .{ .rm = .{
15491	.size = Memory.Size.fromSize(abi_size),	23408	.size = .fromSize(abi_size),
15492	.disp = disp,	23409	.disp = disp,
15493	} } },	23410	} } },
15494	immediate,	23411	immediate,
15495	);	23412	);
15496	},	23413	},
15497	3, 5...7 => unreachable,	23414	3, 5...7 => unreachable,
15498	else => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {	23415	else => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {
15499	try self.asmMemoryImmediate(	23416	try self.asmMemoryImmediate(
15500	.{ ._, .mov },	23417	.{ ._, .mov },
15501	.{ .base = base, .mod = .{ .rm = .{	23418	.{ .base = base, .mod = .{ .rm = .{
15502	.size = Memory.Size.fromSize(abi_size),	23419	.size = .fromSize(abi_size),
15503	.disp = disp,	23420	.disp = disp,
15504	} } },	23421	} } },
15505	Immediate.s(small),	23422	.s(small),
15506	);	23423	);
15507	} else {	23424	} else {
15508	var offset: i32 = 0;	23425	var offset: i32 = 0;
...	@@ -15512,10 +23429,10 @@ fn genSetMem(	...	@@ -15512,10 +23429,10 @@ fn genSetMem(
15512	.size = .dword,	23429	.size = .dword,
15513	.disp = disp + offset,	23430	.disp = disp + offset,
15514	} } },	23431	} } },
15515	if (ty.isSignedInt(zcu)) Immediate.s(	23432	if (ty.isSignedInt(zcu)) .s(
15516	@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),	23433	@truncate(@as(i64, @bitCast(imm)) >> (std.math.cast(u6, offset * 8) orelse 63)),
15517	) else Immediate.u(	23434	) else .u(
15518	@as(u32, @truncate(if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),	23435	@as(u32, @truncate(if (std.math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),
15519	),	23436	),
15520	);	23437	);
15521	},	23438	},
...	@@ -15542,9 +23459,11 @@ fn genSetMem(	...	@@ -15542,9 +23459,11 @@ fn genSetMem(
15542	.general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),	23459	.general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8),
15543	.mmx, .sse => abi_size,	23460	.mmx, .sse => abi_size,
15544	});	23461	});
15545	const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size));	23462	const src_align: InternPool.Alignment = .fromNonzeroByteUnits(
		23463	std.math.ceilPowerOfTwoAssert(u32, src_size),
		23464	);
15546	if (src_size > mem_size) {	23465	if (src_size > mem_size) {
15547	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{	23466	const frame_index = try self.allocFrameIndex(.init(.{
15548	.size = src_size,	23467	.size = src_size,
15549	.alignment = src_align,	23468	.alignment = src_align,
15550	}));	23469	}));
...	@@ -15552,7 +23471,7 @@ fn genSetMem(	...	@@ -15552,7 +23471,7 @@ fn genSetMem(
15552	try (try self.moveStrategy(ty, src_alias.class(), true)).write(	23471	try (try self.moveStrategy(ty, src_alias.class(), true)).write(
15553	self,	23472	self,
15554	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{	23473	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
15555	.size = Memory.Size.fromSize(src_size),	23474	.size = .fromSize(src_size),
15556	} } },	23475	} } },
15557	src_alias,	23476	src_alias,
15558	);	23477	);
...	@@ -15568,19 +23487,23 @@ fn genSetMem(	...	@@ -15568,19 +23487,23 @@ fn genSetMem(
15568	.index = frame_index,	23487	.index = frame_index,
15569	.off = disp,	23488	.off = disp,
15570	}).compare(.gte, src_align),	23489	}).compare(.gte, src_align),
		23490	.table => unreachable,
15571	.reloc => false,	23491	.reloc => false,
15572	})).write(	23492	})).write(
15573	self,	23493	self,
15574	.{ .base = base, .mod = .{ .rm = .{	23494	.{ .base = base, .mod = .{ .rm = .{
15575	.size = Memory.Size.fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())),	23495	.size = .fromBitSize(@min(
		23496	self.memSize(ty).bitSize(self.target),
		23497	src_alias.bitSize(),
		23498	)),
15576	.disp = disp,	23499	.disp = disp,
15577	} } },	23500	} } },
15578	src_alias,	23501	src_alias,
15579	);	23502	);
15580	},	23503	},
15581	.register_pair => \|src_regs\| {	23504	inline .register_pair, .register_triple, .register_quadruple => \|src_regs\| {
15582	var part_disp: i32 = disp;	23505	var part_disp: i32 = disp;
15583	for (try self.splitType(ty), src_regs) \|src_ty, src_reg\| {	23506	for (try self.splitType(src_regs.len, ty), src_regs) \|src_ty, src_reg\| {
15584	try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts);	23507	try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts);
15585	part_disp += @intCast(src_ty.abiSize(zcu));	23508	part_disp += @intCast(src_ty.abiSize(zcu));
15586	}	23509	}
...	@@ -15609,7 +23532,7 @@ fn genSetMem(	...	@@ -15609,7 +23532,7 @@ fn genSetMem(
15609	try self.genSetMem(	23532	try self.genSetMem(
15610	base,	23533	base,
15611	disp + @as(i32, @intCast(child_ty.abiSize(zcu))),	23534	disp + @as(i32, @intCast(child_ty.abiSize(zcu))),
15612	Type.bool,	23535	.bool,
15613	.{ .eflags = ro.eflags },	23536	.{ .eflags = ro.eflags },
15614	opts,	23537	opts,
15615	);	23538	);
...	@@ -15645,6 +23568,13 @@ fn genSetMem(	...	@@ -15645,6 +23568,13 @@ fn genSetMem(
15645		23568
15646	try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);	23569	try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);
15647	},	23570	},
		23571	.register_mask => {
		23572	const src_reg = try self.copyToTmpRegister(ty, src_mcv);
		23573	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
		23574	defer self.register_manager.unlockReg(src_lock);
		23575
		23576	try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);
		23577	},
15648	.memory,	23578	.memory,
15649	.indirect,	23579	.indirect,
15650	.load_direct,	23580	.load_direct,
...	@@ -15666,51 +23596,104 @@ fn genSetMem(	...	@@ -15666,51 +23596,104 @@ fn genSetMem(
15666		23596
15667	try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);	23597	try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);
15668	},	23598	},
15669	else => try self.genInlineMemcpy(	23599	else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }, .{ .no_alias = true }),
15670	dst_ptr_mcv,
15671	src_mcv.address(),
15672	.{ .immediate = abi_size },
15673	),
15674	},	23600	},
15675	.air_ref => \|src_ref\| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts),	23601	.air_ref => \|src_ref\| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts),
15676	}	23602	}
15677	}	23603	}
15678		23604
15679	fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {	23605	fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct {
		23606	no_alias: bool,
		23607	}) InnerError!void {
		23608	if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) {
		23609	else => {},
		23610	.immediate => \|len_imm\| switch (len_imm) {
		23611	else => {},
		23612	1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) \|reg\| {
		23613	try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte }));
		23614	try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8());
		23615	return;
		23616	},
		23617	2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) \|reg\| {
		23618	try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word }));
		23619	try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16());
		23620	return;
		23621	},
		23622	4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) \|reg\| {
		23623	try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword }));
		23624	try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32());
		23625	return;
		23626	},
		23627	8 => if (self.target.cpu.arch == .x86_64) {
		23628	if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) \|reg\| {
		23629	try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword }));
		23630	try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64());
		23631	return;
		23632	}
		23633	},
		23634	16 => if (self.hasFeature(.avx)) {
		23635	if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) \|reg\| {
		23636	try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
		23637	try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
		23638	return;
		23639	}
		23640	} else if (self.hasFeature(.sse2)) {
		23641	if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) \|reg\| {
		23642	try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
		23643	try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
		23644	return;
		23645	}
		23646	} else if (self.hasFeature(.sse)) {
		23647	if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) \|reg\| {
		23648	try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
		23649	try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
		23650	return;
		23651	}
		23652	},
		23653	32 => if (self.hasFeature(.avx)) {
		23654	if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) \|reg\| {
		23655	try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword }));
		23656	try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256());
		23657	return;
		23658	}
		23659	},
		23660	},
		23661	};
15680	try self.spillRegisters(&.{ .rsi, .rdi, .rcx });	23662	try self.spillRegisters(&.{ .rsi, .rdi, .rcx });
15681	try self.genSetReg(.rsi, Type.usize, src_ptr, .{});	23663	try self.genSetReg(.rsi, .usize, src_ptr, .{});
15682	try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});	23664	try self.genSetReg(.rdi, .usize, dst_ptr, .{});
15683	try self.genSetReg(.rcx, Type.usize, len, .{});	23665	try self.genSetReg(.rcx, .usize, len, .{});
15684	try self.asmOpOnly(.{ .@"rep _sb", .mov });	23666	try self.asmOpOnly(.{ .@"rep _sb", .mov });
15685	}	23667	}
15686		23668
15687	fn genInlineMemset(	23669	fn genInlineMemset(
15688	self: *Self,	23670	self: *CodeGen,
15689	dst_ptr: MCValue,	23671	dst_ptr: MCValue,
15690	value: MCValue,	23672	value: MCValue,
15691	len: MCValue,	23673	len: MCValue,
15692	opts: CopyOptions,	23674	opts: CopyOptions,
15693	) InnerError!void {	23675	) InnerError!void {
15694	try self.spillRegisters(&.{ .rdi, .al, .rcx });	23676	try self.spillRegisters(&.{ .rdi, .al, .rcx });
15695	try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});	23677	try self.genSetReg(.rdi, .usize, dst_ptr, .{});
15696	try self.genSetReg(.al, Type.u8, value, opts);	23678	try self.genSetReg(.al, .u8, value, opts);
15697	try self.genSetReg(.rcx, Type.usize, len, .{});	23679	try self.genSetReg(.rcx, .usize, len, .{});
15698	try self.asmOpOnly(.{ .@"rep _sb", .sto });	23680	try self.asmOpOnly(.{ .@"rep _sb", .sto });
15699	}	23681	}
15700		23682
15701	fn genExternSymbolRef(	23683	fn genExternSymbolRef(
15702	self: *Self,	23684	self: *CodeGen,
15703	comptime tag: Mir.Inst.Tag,	23685	comptime tag: Mir.Inst.Tag,
15704	lib: ?[]const u8,	23686	lib: ?[]const u8,
15705	callee: []const u8,	23687	callee: []const u8,
15706	) InnerError!void {	23688	) InnerError!void {
15707	if (self.bin_file.cast(.coff)) \|coff_file\| {	23689	if (self.bin_file.cast(.coff)) \|coff_file\| {
15708	const global_index = try coff_file.getGlobalSymbol(callee, lib);	23690	const global_index = try coff_file.getGlobalSymbol(callee, lib);
		23691	const scratch_reg = abi.getCAbiLinkerScratchReg(self.fn_type.fnCallingConvention(self.pt.zcu));
15709	_ = try self.addInst(.{	23692	_ = try self.addInst(.{
15710	.tag = .mov,	23693	.tag = .mov,
15711	.ops = .import_reloc,	23694	.ops = .import_reloc,
15712	.data = .{ .rx = .{	23695	.data = .{ .rx = .{
15713	.r1 = .rax,	23696	.r1 = scratch_reg,
15714	.payload = try self.addExtra(bits.SymbolOffset{	23697	.payload = try self.addExtra(bits.SymbolOffset{
15715	.sym_index = link.File.Coff.global_symbol_bit \| global_index,	23698	.sym_index = link.File.Coff.global_symbol_bit \| global_index,
15716	}),	23699	}),
...	@@ -15718,14 +23701,14 @@ fn genExternSymbolRef(	...	@@ -15718,14 +23701,14 @@ fn genExternSymbolRef(
15718	});	23701	});
15719	switch (tag) {	23702	switch (tag) {
15720	.mov => {},	23703	.mov => {},
15721	.call => try self.asmRegister(.{ ._, .call }, .rax),	23704	.call => try self.asmRegister(.{ ._, .call }, scratch_reg),
15722	else => unreachable,	23705	else => unreachable,
15723	}	23706	}
15724	} else return self.fail("TODO implement calling extern functions", .{});	23707	} else return self.fail("TODO implement calling extern functions", .{});
15725	}	23708	}
15726		23709
15727	fn genLazySymbolRef(	23710	fn genLazySymbolRef(
15728	self: *Self,	23711	self: *CodeGen,
15729	comptime tag: Mir.Inst.Tag,	23712	comptime tag: Mir.Inst.Tag,
15730	reg: Register,	23713	reg: Register,
15731	lazy_sym: link.File.LazySymbol,	23714	lazy_sym: link.File.LazySymbol,
...	@@ -15737,10 +23720,10 @@ fn genLazySymbolRef(	...	@@ -15737,10 +23720,10 @@ fn genLazySymbolRef(
15737	return self.fail("{s} creating lazy symbol", .{@errorName(err)});	23720	return self.fail("{s} creating lazy symbol", .{@errorName(err)});
15738	if (self.mod.pic) {	23721	if (self.mod.pic) {
15739	switch (tag) {	23722	switch (tag) {
15740	.lea, .call => try self.genSetReg(reg, Type.usize, .{	23723	.lea, .call => try self.genSetReg(reg, .usize, .{
15741	.lea_symbol = .{ .sym_index = sym_index },	23724	.lea_symbol = .{ .sym_index = sym_index },
15742	}, .{}),	23725	}, .{}),
15743	.mov => try self.genSetReg(reg, Type.usize, .{	23726	.mov => try self.genSetReg(reg, .usize, .{
15744	.load_symbol = .{ .sym_index = sym_index },	23727	.load_symbol = .{ .sym_index = sym_index },
15745	}, .{}),	23728	}, .{}),
15746	else => unreachable,	23729	else => unreachable,
...	@@ -15755,7 +23738,7 @@ fn genLazySymbolRef(	...	@@ -15755,7 +23738,7 @@ fn genLazySymbolRef(
15755	.base = .{ .reloc = sym_index },	23738	.base = .{ .reloc = sym_index },
15756	.mod = .{ .rm = .{ .size = .qword } },	23739	.mod = .{ .rm = .{ .size = .qword } },
15757	}),	23740	}),
15758	.call => try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index })),	23741	.call => try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index })),
15759	else => unreachable,	23742	else => unreachable,
15760	}	23743	}
15761	} else if (self.bin_file.cast(.plan9)) \|p9_file\| {	23744	} else if (self.bin_file.cast(.plan9)) \|p9_file\| {
...	@@ -15781,7 +23764,7 @@ fn genLazySymbolRef(	...	@@ -15781,7 +23764,7 @@ fn genLazySymbolRef(
15781	.mov => try self.asmRegisterMemory(	23764	.mov => try self.asmRegisterMemory(
15782	.{ ._, tag },	23765	.{ ._, tag },
15783	reg.to64(),	23766	reg.to64(),
15784	Memory.initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),	23767	.initSib(.qword, .{ .base = .{ .reg = reg.to64() } }),
15785	),	23768	),
15786	else => unreachable,	23769	else => unreachable,
15787	}	23770	}
...	@@ -15790,8 +23773,8 @@ fn genLazySymbolRef(	...	@@ -15790,8 +23773,8 @@ fn genLazySymbolRef(
15790	return self.fail("{s} creating lazy symbol", .{@errorName(err)});	23773	return self.fail("{s} creating lazy symbol", .{@errorName(err)});
15791	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;	23774	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
15792	switch (tag) {	23775	switch (tag) {
15793	.lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym_index }, .{}),	23776	.lea, .call => try self.genSetReg(reg, .usize, .{ .lea_got = sym_index }, .{}),
15794	.mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym_index }, .{}),	23777	.mov => try self.genSetReg(reg, .usize, .{ .load_got = sym_index }, .{}),
15795	else => unreachable,	23778	else => unreachable,
15796	}	23779	}
15797	switch (tag) {	23780	switch (tag) {
...	@@ -15805,10 +23788,10 @@ fn genLazySymbolRef(	...	@@ -15805,10 +23788,10 @@ fn genLazySymbolRef(
15805	return self.fail("{s} creating lazy symbol", .{@errorName(err)});	23788	return self.fail("{s} creating lazy symbol", .{@errorName(err)});
15806	const sym = zo.symbols.items[sym_index];	23789	const sym = zo.symbols.items[sym_index];
15807	switch (tag) {	23790	switch (tag) {
15808	.lea, .call => try self.genSetReg(reg, Type.usize, .{	23791	.lea, .call => try self.genSetReg(reg, .usize, .{
15809	.lea_symbol = .{ .sym_index = sym.nlist_idx },	23792	.lea_symbol = .{ .sym_index = sym.nlist_idx },
15810	}, .{}),	23793	}, .{}),
15811	.mov => try self.genSetReg(reg, Type.usize, .{	23794	.mov => try self.genSetReg(reg, .usize, .{
15812	.load_symbol = .{ .sym_index = sym.nlist_idx },	23795	.load_symbol = .{ .sym_index = sym.nlist_idx },
15813	}, .{}),	23796	}, .{}),
15814	else => unreachable,	23797	else => unreachable,
...	@@ -15823,7 +23806,7 @@ fn genLazySymbolRef(	...	@@ -15823,7 +23806,7 @@ fn genLazySymbolRef(
15823	}	23806	}
15824	}	23807	}
15825		23808
15826	fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {	23809	fn airIntFromPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
15827	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	23810	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
15828	const result = result: {	23811	const result = result: {
15829	// TODO: handle case where the operand is a slice not a raw pointer	23812	// TODO: handle case where the operand is a slice not a raw pointer
...	@@ -15838,7 +23821,7 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15838,7 +23821,7 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
15838	return self.finishAir(inst, result, .{ un_op, .none, .none });	23821	return self.finishAir(inst, result, .{ un_op, .none, .none });
15839	}	23822	}
15840		23823
15841	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	23824	fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
15842	const pt = self.pt;	23825	const pt = self.pt;
15843	const zcu = pt.zcu;	23826	const zcu = pt.zcu;
15844	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	23827	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -15852,18 +23835,20 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15852,18 +23835,20 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
15852	else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv,	23835	else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv,
15853	};	23836	};
15854		23837
15855	const dst_rc = self.regClassForType(dst_ty);	23838	const dst_rc = self.regSetForType(dst_ty);
15856	const src_rc = self.regClassForType(src_ty);	23839	const src_rc = self.regSetForType(src_ty);
15857		23840
15858	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;	23841	const src_lock = if (src_mcv.getReg()) \|src_reg\| self.register_manager.lockReg(src_reg) else null;
15859	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);	23842	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
15860		23843
15861	const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and	23844	const dst_mcv = if ((if (src_mcv.getReg()) \|src_reg\| src_reg.class() == .general_purpose else true) and
		23845	dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and
		23846	dst_ty.abiAlignment(zcu).order(src_ty.abiAlignment(zcu)).compare(.lte) and
15862	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {	23847	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
15863	const dst_mcv = try self.allocRegOrMem(inst, true);	23848	const dst_mcv = try self.allocRegOrMem(inst, true);
15864	try self.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {	23849	try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
15865	.lt => dst_ty,	23850	.lt => dst_ty,
15866	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,	23851	.eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty,
15867	.gt => src_ty,	23852	.gt => src_ty,
15868	}, dst_mcv, src_mcv, .{});	23853	}, dst_mcv, src_mcv, .{});
15869	break :dst dst_mcv;	23854	break :dst dst_mcv;
...	@@ -15878,7 +23863,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15878,7 +23863,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
15878	const bit_size = dst_ty.bitSize(zcu);	23863	const bit_size = dst_ty.bitSize(zcu);
15879	if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;	23864	if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
15880		23865
15881	const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;	23866	const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
15882	const high_mcv: MCValue = switch (dst_mcv) {	23867	const high_mcv: MCValue = switch (dst_mcv) {
15883	.register => \|dst_reg\| .{ .register = dst_reg },	23868	.register => \|dst_reg\| .{ .register = dst_reg },
15884	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },	23869	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
...	@@ -15887,23 +23872,49 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15887,23 +23872,49 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
15887	const high_reg = if (high_mcv.isRegister())	23872	const high_reg = if (high_mcv.isRegister())
15888	high_mcv.getReg().?	23873	high_mcv.getReg().?
15889	else	23874	else
15890	try self.copyToTmpRegister(Type.usize, high_mcv);	23875	try self.copyToTmpRegister(.usize, high_mcv);
15891	const high_lock = self.register_manager.lockReg(high_reg);	23876	const high_lock = self.register_manager.lockReg(high_reg);
15892	defer if (high_lock) \|lock\| self.register_manager.unlockReg(lock);	23877	defer if (high_lock) \|lock\| self.register_manager.unlockReg(lock);
15893
15894	try self.truncateRegister(dst_ty, high_reg);	23878	try self.truncateRegister(dst_ty, high_reg);
15895	if (!high_mcv.isRegister()) try self.genCopy(	23879	if (!high_mcv.isRegister()) try self.genCopy(
15896	if (abi_size <= 8) dst_ty else Type.usize,	23880	if (abi_size <= 8) dst_ty else .usize,
15897	high_mcv,	23881	high_mcv,
15898	.{ .register = high_reg },	23882	.{ .register = high_reg },
15899	.{},	23883	.{},
15900	);	23884	);
		23885	var offset = dst_limbs_len * 8;
		23886	if (offset < abi_size) {
		23887	const dst_signedness: std.builtin.Signedness = if (dst_ty.isAbiInt(zcu))
		23888	dst_ty.intInfo(zcu).signedness
		23889	else
		23890	.unsigned;
		23891	const ext_mcv: MCValue = ext_mcv: switch (dst_signedness) {
		23892	.signed => {
		23893	try self.asmRegisterImmediate(.{ ._r, .sa }, high_reg, .u(63));
		23894	break :ext_mcv .{ .register = high_reg };
		23895	},
		23896	.unsigned => .{ .immediate = 0 },
		23897	};
		23898	while (offset < abi_size) : (offset += 8) {
		23899	const limb_mcv: MCValue = switch (dst_mcv) {
		23900	.register => \|dst_reg\| .{ .register = dst_reg },
		23901	.register_pair => \|dst_regs\| .{ .register = dst_regs[@divExact(offset, 8)] },
		23902	else => dst_mcv.address().offset(offset).deref(),
		23903	};
		23904	const limb_lock = if (limb_mcv.isRegister())
		23905	self.register_manager.lockReg(limb_mcv.getReg().?)
		23906	else
		23907	null;
		23908	defer if (limb_lock) \|lock\| self.register_manager.unlockReg(lock);
		23909	try self.genCopy(.usize, limb_mcv, ext_mcv, .{});
		23910	}
		23911	}
15901	break :result dst_mcv;	23912	break :result dst_mcv;
15902	};	23913	};
15903	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	23914	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
15904	}	23915	}
15905		23916
15906	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {	23917	fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
15907	const pt = self.pt;	23918	const pt = self.pt;
15908	const zcu = pt.zcu;	23919	const zcu = pt.zcu;
15909	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	23920	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -15914,12 +23925,12 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15914,12 +23925,12 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
15914	const array_ty = ptr_ty.childType(zcu);	23925	const array_ty = ptr_ty.childType(zcu);
15915	const array_len = array_ty.arrayLen(zcu);	23926	const array_len = array_ty.arrayLen(zcu);
15916		23927
15917	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));	23928	const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
15918	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});	23929	try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});
15919	try self.genSetMem(	23930	try self.genSetMem(
15920	.{ .frame = frame_index },	23931	.{ .frame = frame_index },
15921	@intCast(ptr_ty.abiSize(zcu)),	23932	@intCast(ptr_ty.abiSize(zcu)),
15922	Type.usize,	23933	.usize,
15923	.{ .immediate = array_len },	23934	.{ .immediate = array_len },
15924	.{},	23935	.{},
15925	);	23936	);
...	@@ -15928,7 +23939,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15928,7 +23939,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
15928	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	23939	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
15929	}	23940	}
15930		23941
15931	fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	23942	fn airFloatFromInt(self: *CodeGen, inst: Air.Inst.Index) !void {
15932	const pt = self.pt;	23943	const pt = self.pt;
15933	const zcu = pt.zcu;	23944	const zcu = pt.zcu;
15934	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	23945	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -15940,7 +23951,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15940,7 +23951,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
15940	const src_bits: u32 = @intCast(src_ty.bitSize(zcu));	23951	const src_bits: u32 = @intCast(src_ty.bitSize(zcu));
15941	const src_signedness =	23952	const src_signedness =
15942	if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;	23953	if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;
15943	const src_size = math.divCeil(u32, @max(switch (src_signedness) {	23954	const src_size = std.math.divCeil(u32, @max(switch (src_signedness) {
15944	.signed => src_bits,	23955	.signed => src_bits,
15945	.unsigned => src_bits + 1,	23956	.unsigned => src_bits + 1,
15946	}, 32), 8) catch unreachable;	23957	}, 32), 8) catch unreachable;
...	@@ -15967,7 +23978,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15967,7 +23978,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
15967	intCompilerRtAbiName(src_bits),	23978	intCompilerRtAbiName(src_bits),
15968	floatCompilerRtAbiName(dst_bits),	23979	floatCompilerRtAbiName(dst_bits),
15969	}) catch unreachable,	23980	}) catch unreachable,
15970	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});	23981	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
15971	}	23982	}
15972		23983
15973	const src_mcv = try self.resolveInst(ty_op.operand);	23984	const src_mcv = try self.resolveInst(ty_op.operand);
...	@@ -15980,7 +23991,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -15980,7 +23991,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
15980		23991
15981	if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);	23992	if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
15982		23993
15983	const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));	23994	const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
15984	const dst_mcv = MCValue{ .register = dst_reg };	23995	const dst_mcv = MCValue{ .register = dst_reg };
15985	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);	23996	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
15986	defer self.register_manager.unlockReg(dst_lock);	23997	defer self.register_manager.unlockReg(dst_lock);
...	@@ -16008,7 +24019,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16008,7 +24019,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
16008	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	24019	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
16009	}	24020	}
16010		24021
16011	fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {	24022	fn airIntFromFloat(self: *CodeGen, inst: Air.Inst.Index) !void {
16012	const pt = self.pt;	24023	const pt = self.pt;
16013	const zcu = pt.zcu;	24024	const zcu = pt.zcu;
16014	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	24025	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -16017,7 +24028,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16017,7 +24028,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16017	const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu));	24028	const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu));
16018	const dst_signedness =	24029	const dst_signedness =
16019	if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned;	24030	if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned;
16020	const dst_size = math.divCeil(u32, @max(switch (dst_signedness) {	24031	const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) {
16021	.signed => dst_bits,	24032	.signed => dst_bits,
16022	.unsigned => dst_bits + 1,	24033	.unsigned => dst_bits + 1,
16023	}, 32), 8) catch unreachable;	24034	}, 32), 8) catch unreachable;
...	@@ -16047,7 +24058,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16047,7 +24058,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16047	floatCompilerRtAbiName(src_bits),	24058	floatCompilerRtAbiName(src_bits),
16048	intCompilerRtAbiName(dst_bits),	24059	intCompilerRtAbiName(dst_bits),
16049	}) catch unreachable,	24060	}) catch unreachable,
16050	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});	24061	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
16051	}	24062	}
16052		24063
16053	const src_mcv = try self.resolveInst(ty_op.operand);	24064	const src_mcv = try self.resolveInst(ty_op.operand);
...	@@ -16058,7 +24069,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16058,7 +24069,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16058	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);	24069	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
16059	defer self.register_manager.unlockReg(src_lock);	24070	defer self.register_manager.unlockReg(src_lock);
16060		24071
16061	const dst_reg = try self.register_manager.allocReg(inst, self.regClassForType(dst_ty));	24072	const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
16062	const dst_mcv = MCValue{ .register = dst_reg };	24073	const dst_mcv = MCValue{ .register = dst_reg };
16063	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);	24074	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
16064	defer self.register_manager.unlockReg(dst_lock);	24075	defer self.register_manager.unlockReg(dst_lock);
...	@@ -16080,7 +24091,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16080,7 +24091,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16080	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	24091	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
16081	}	24092	}
16082		24093
16083	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {	24094	fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
16084	const pt = self.pt;	24095	const pt = self.pt;
16085	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	24096	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
16086	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;	24097	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
...	@@ -16097,37 +24108,37 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16097,37 +24108,37 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
16097	if (val_abi_size > 8) {	24108	if (val_abi_size > 8) {
16098	const exp_addr_mcv: MCValue = switch (exp_mcv) {	24109	const exp_addr_mcv: MCValue = switch (exp_mcv) {
16099	.memory, .indirect, .load_frame => exp_mcv.address(),	24110	.memory, .indirect, .load_frame => exp_mcv.address(),
16100	else => .{ .register = try self.copyToTmpRegister(Type.usize, exp_mcv.address()) },	24111	else => .{ .register = try self.copyToTmpRegister(.usize, exp_mcv.address()) },
16101	};	24112	};
16102	const exp_addr_lock =	24113	const exp_addr_lock =
16103	if (exp_addr_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;	24114	if (exp_addr_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
16104	defer if (exp_addr_lock) \|lock\| self.register_manager.unlockReg(lock);	24115	defer if (exp_addr_lock) \|lock\| self.register_manager.unlockReg(lock);
16105		24116
16106	try self.genSetReg(.rax, Type.usize, exp_addr_mcv.deref(), .{});	24117	try self.genSetReg(.rax, .usize, exp_addr_mcv.deref(), .{});
16107	try self.genSetReg(.rdx, Type.usize, exp_addr_mcv.offset(8).deref(), .{});	24118	try self.genSetReg(.rdx, .usize, exp_addr_mcv.offset(8).deref(), .{});
16108	} else try self.genSetReg(.rax, val_ty, exp_mcv, .{});	24119	} else try self.genSetReg(.rax, val_ty, exp_mcv, .{});
16109		24120
16110	const new_mcv = try self.resolveInst(extra.new_value);	24121	const new_mcv = try self.resolveInst(extra.new_value);
16111	const new_reg = if (val_abi_size > 8) new: {	24122	const new_reg = if (val_abi_size > 8) new: {
16112	const new_addr_mcv: MCValue = switch (new_mcv) {	24123	const new_addr_mcv: MCValue = switch (new_mcv) {
16113	.memory, .indirect, .load_frame => new_mcv.address(),	24124	.memory, .indirect, .load_frame => new_mcv.address(),
16114	else => .{ .register = try self.copyToTmpRegister(Type.usize, new_mcv.address()) },	24125	else => .{ .register = try self.copyToTmpRegister(.usize, new_mcv.address()) },
16115	};	24126	};
16116	const new_addr_lock =	24127	const new_addr_lock =
16117	if (new_addr_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;	24128	if (new_addr_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
16118	defer if (new_addr_lock) \|lock\| self.register_manager.unlockReg(lock);	24129	defer if (new_addr_lock) \|lock\| self.register_manager.unlockReg(lock);
16119		24130
16120	try self.genSetReg(.rbx, Type.usize, new_addr_mcv.deref(), .{});	24131	try self.genSetReg(.rbx, .usize, new_addr_mcv.deref(), .{});
16121	try self.genSetReg(.rcx, Type.usize, new_addr_mcv.offset(8).deref(), .{});	24132	try self.genSetReg(.rcx, .usize, new_addr_mcv.offset(8).deref(), .{});
16122	break :new null;	24133	break :new null;
16123	} else try self.copyToTmpRegister(val_ty, new_mcv);	24134	} else try self.copyToTmpRegister(val_ty, new_mcv);
16124	const new_lock = if (new_reg) \|reg\| self.register_manager.lockRegAssumeUnused(reg) else null;	24135	const new_lock = if (new_reg) \|reg\| self.register_manager.lockRegAssumeUnused(reg) else null;
16125	defer if (new_lock) \|lock\| self.register_manager.unlockReg(lock);	24136	defer if (new_lock) \|lock\| self.register_manager.unlockReg(lock);
16126		24137
16127	const ptr_mcv = try self.resolveInst(extra.ptr);	24138	const ptr_mcv = try self.resolveInst(extra.ptr);
16128	const mem_size = Memory.Size.fromSize(val_abi_size);	24139	const mem_size: Memory.Size = .fromSize(val_abi_size);
16129	const ptr_mem: Memory = switch (ptr_mcv) {	24140	const ptr_mem: Memory = switch (ptr_mcv) {
16130	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),	24141	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
16131	else => .{	24142	else => .{
16132	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },	24143	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
16133	.mod = .{ .rm = .{ .size = mem_size } },	24144	.mod = .{ .rm = .{ .size = mem_size } },
...	@@ -16140,6 +24151,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16140,6 +24151,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
16140	const ptr_lock = switch (ptr_mem.base) {	24151	const ptr_lock = switch (ptr_mem.base) {
16141	.none, .frame, .reloc => null,	24152	.none, .frame, .reloc => null,
16142	.reg => \|reg\| self.register_manager.lockReg(reg),	24153	.reg => \|reg\| self.register_manager.lockReg(reg),
		24154	.table => unreachable,
16143	};	24155	};
16144	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	24156	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
16145		24157
...	@@ -16159,16 +24171,16 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16159,16 +24171,16 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
16159	}	24171	}
16160		24172
16161	const dst_mcv = try self.allocRegOrMem(inst, false);	24173	const dst_mcv = try self.allocRegOrMem(inst, false);
16162	try self.genCopy(Type.usize, dst_mcv, .{ .register = .rax }, .{});	24174	try self.genCopy(.usize, dst_mcv, .{ .register = .rax }, .{});
16163	try self.genCopy(Type.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});	24175	try self.genCopy(.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});
16164	try self.genCopy(Type.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});	24176	try self.genCopy(.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});
16165	break :result dst_mcv;	24177	break :result dst_mcv;
16166	};	24178	};
16167	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });	24179	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
16168	}	24180	}
16169		24181
16170	fn atomicOp(	24182	fn atomicOp(
16171	self: *Self,	24183	self: *CodeGen,
16172	ptr_mcv: MCValue,	24184	ptr_mcv: MCValue,
16173	val_mcv: MCValue,	24185	val_mcv: MCValue,
16174	ptr_ty: Type,	24186	ptr_ty: Type,
...	@@ -16192,9 +24204,9 @@ fn atomicOp(	...	@@ -16192,9 +24204,9 @@ fn atomicOp(
16192	defer if (val_lock) \|lock\| self.register_manager.unlockReg(lock);	24204	defer if (val_lock) \|lock\| self.register_manager.unlockReg(lock);
16193		24205
16194	const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));	24206	const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
16195	const mem_size = Memory.Size.fromSize(val_abi_size);	24207	const mem_size: Memory.Size = .fromSize(val_abi_size);
16196	const ptr_mem: Memory = switch (ptr_mcv) {	24208	const ptr_mem: Memory = switch (ptr_mcv) {
16197	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),	24209	.immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
16198	else => .{	24210	else => .{
16199	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },	24211	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
16200	.mod = .{ .rm = .{ .size = mem_size } },	24212	.mod = .{ .rm = .{ .size = mem_size } },
...	@@ -16207,6 +24219,7 @@ fn atomicOp(	...	@@ -16207,6 +24219,7 @@ fn atomicOp(
16207	const mem_lock = switch (ptr_mem.base) {	24219	const mem_lock = switch (ptr_mem.base) {
16208	.none, .frame, .reloc => null,	24220	.none, .frame, .reloc => null,
16209	.reg => \|reg\| self.register_manager.lockReg(reg),	24221	.reg => \|reg\| self.register_manager.lockReg(reg),
		24222	.table => unreachable,
16210	};	24223	};
16211	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);	24224	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
16212		24225
...	@@ -16309,11 +24322,11 @@ fn atomicOp(	...	@@ -16309,11 +24322,11 @@ fn atomicOp(
16309	});	24322	});
16310	try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{});	24323	try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{});
16311	switch (mir_tag[0]) {	24324	switch (mir_tag[0]) {
16312	.v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory(	24325	.v_ss, .v_sd => if (val_mcv.isBase()) try self.asmRegisterRegisterMemory(
16313	mir_tag,	24326	mir_tag,
16314	sse_reg.to128(),	24327	sse_reg.to128(),
16315	sse_reg.to128(),	24328	sse_reg.to128(),
16316	try val_mcv.mem(self, self.memSize(val_ty)),	24329	try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
16317	) else try self.asmRegisterRegisterRegister(	24330	) else try self.asmRegisterRegisterRegister(
16318	mir_tag,	24331	mir_tag,
16319	sse_reg.to128(),	24332	sse_reg.to128(),
...	@@ -16323,10 +24336,10 @@ fn atomicOp(	...	@@ -16323,10 +24336,10 @@ fn atomicOp(
16323	else	24336	else
16324	try self.copyToTmpRegister(val_ty, val_mcv)).to128(),	24337	try self.copyToTmpRegister(val_ty, val_mcv)).to128(),
16325	),	24338	),
16326	._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory(	24339	._ss, ._sd => if (val_mcv.isBase()) try self.asmRegisterMemory(
16327	mir_tag,	24340	mir_tag,
16328	sse_reg.to128(),	24341	sse_reg.to128(),
16329	try val_mcv.mem(self, self.memSize(val_ty)),	24342	try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
16330	) else try self.asmRegisterRegister(	24343	) else try self.asmRegisterRegister(
16331	mir_tag,	24344	mir_tag,
16332	sse_reg.to128(),	24345	sse_reg.to128(),
...	@@ -16381,7 +24394,7 @@ fn atomicOp(	...	@@ -16381,7 +24394,7 @@ fn atomicOp(
16381	try self.asmCmovccRegisterMemory(	24394	try self.asmCmovccRegisterMemory(
16382	cc,	24395	cc,
16383	registerAlias(tmp_reg, cmov_abi_size),	24396	registerAlias(tmp_reg, cmov_abi_size),
16384	try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),	24397	try val_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
16385	);	24398	);
16386	},	24399	},
16387	else => {	24400	else => {
...	@@ -16434,11 +24447,11 @@ fn atomicOp(	...	@@ -16434,11 +24447,11 @@ fn atomicOp(
16434	const val_mem_mcv: MCValue = switch (val_mcv) {	24447	const val_mem_mcv: MCValue = switch (val_mcv) {
16435	.memory, .indirect, .load_frame => val_mcv,	24448	.memory, .indirect, .load_frame => val_mcv,
16436	else => .{ .indirect = .{	24449	else => .{ .indirect = .{
16437	.reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),	24450	.reg = try self.copyToTmpRegister(.usize, val_mcv.address()),
16438	} },	24451	} },
16439	};	24452	};
16440	const val_lo_mem = try val_mem_mcv.mem(self, .qword);	24453	const val_lo_mem = try val_mem_mcv.mem(self, .{ .size = .qword });
16441	const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);	24454	const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .{ .size = .qword });
16442	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {	24455	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
16443	try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);	24456	try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
16444	try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);	24457	try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
...	@@ -16491,7 +24504,7 @@ fn atomicOp(	...	@@ -16491,7 +24504,7 @@ fn atomicOp(
16491	},	24504	},
16492	};	24505	};
16493		24506
16494	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .register = .rcx });	24507	const tmp_reg = try self.copyToTmpRegister(.usize, .{ .register = .rcx });
16495	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	24508	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
16496	defer self.register_manager.unlockReg(tmp_lock);	24509	defer self.register_manager.unlockReg(tmp_lock);
16497		24510
...	@@ -16526,7 +24539,7 @@ fn atomicOp(	...	@@ -16526,7 +24539,7 @@ fn atomicOp(
16526	}	24539	}
16527	}	24540	}
16528		24541
16529	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {	24542	fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void {
16530	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	24543	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
16531	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;	24544	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
16532		24545
...	@@ -16547,7 +24560,7 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16547,7 +24560,7 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
16547	return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });	24560	return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
16548	}	24561	}
16549		24562
16550	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {	24563	fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
16551	const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;	24564	const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
16552		24565
16553	const ptr_ty = self.typeOf(atomic_load.ptr);	24566	const ptr_ty = self.typeOf(atomic_load.ptr);
...	@@ -16568,7 +24581,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16568,7 +24581,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
16568	return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });	24581	return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
16569	}	24582	}
16570		24583
16571	fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {	24584	fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
16572	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	24585	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
16573		24586
16574	const ptr_ty = self.typeOf(bin_op.lhs);	24587	const ptr_ty = self.typeOf(bin_op.lhs);
...	@@ -16581,7 +24594,7 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr	...	@@ -16581,7 +24594,7 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
16581	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	24594	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
16582	}	24595	}
16583		24596
16584	fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	24597	fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
16585	const pt = self.pt;	24598	const pt = self.pt;
16586	const zcu = pt.zcu;	24599	const zcu = pt.zcu;
16587	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	24600	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -16593,13 +24606,17 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16593,13 +24606,17 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16593	const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx });	24606	const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx });
16594	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);	24607	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
16595		24608
16596	const dst_ptr = try self.resolveInst(bin_op.lhs);	24609	const dst = try self.resolveInst(bin_op.lhs);
16597	const dst_ptr_ty = self.typeOf(bin_op.lhs);	24610	const dst_ty = self.typeOf(bin_op.lhs);
16598	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {	24611	const dst_locks: [2]?RegisterLock = switch (dst) {
16599	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	24612	.register => \|dst_reg\| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
16600	else => null,	24613	.register_pair => \|dst_regs\| .{
		24614	self.register_manager.lockRegAssumeUnused(dst_regs[0]),
		24615	self.register_manager.lockRegAssumeUnused(dst_regs[1]),
		24616	},
		24617	else => @splat(null),
16601	};	24618	};
16602	defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	24619	for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
16603		24620
16604	const src_val = try self.resolveInst(bin_op.rhs);	24621	const src_val = try self.resolveInst(bin_op.rhs);
16605	const elem_ty = self.typeOf(bin_op.rhs);	24622	const elem_ty = self.typeOf(bin_op.rhs);
...	@@ -16612,16 +24629,20 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16612,16 +24629,20 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16612	const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));	24629	const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));
16613		24630
16614	if (elem_abi_size == 1) {	24631	if (elem_abi_size == 1) {
16615	const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {	24632	const dst_ptr: MCValue = switch (dst_ty.ptrSize(zcu)) {
16616	// TODO: this only handles slices stored in the stack	24633	.slice => switch (dst) {
16617	.slice => dst_ptr,	24634	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
16618	.one => dst_ptr,	24635	else => dst,
		24636	},
		24637	.one => dst,
16619	.c, .many => unreachable,	24638	.c, .many => unreachable,
16620	};	24639	};
16621	const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {	24640	const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
16622	// TODO: this only handles slices stored in the stack	24641	.slice => switch (dst) {
16623	.slice => dst_ptr.address().offset(8).deref(),	24642	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
16624	.one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) },	24643	else => dst.address().offset(8).deref(),
		24644	},
		24645	.one => .{ .immediate = dst_ty.childType(zcu).arrayLen(zcu) },
16625	.c, .many => unreachable,	24646	.c, .many => unreachable,
16626	};	24647	};
16627	const len_lock: ?RegisterLock = switch (len) {	24648	const len_lock: ?RegisterLock = switch (len) {
...	@@ -16630,20 +24651,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16630,20 +24651,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16630	};	24651	};
16631	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);	24652	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
16632		24653
16633	try self.genInlineMemset(ptr, src_val, len, .{ .safety = safety });	24654	try self.genInlineMemset(dst_ptr, src_val, len, .{ .safety = safety });
16634	break :result;	24655	break :result;
16635	}	24656	}
16636		24657
16637	// Store the first element, and then rely on memcpy copying forwards.	24658	// Store the first element, and then rely on memcpy copying forwards.
16638	// Length zero requires a runtime check - so we handle arrays specially	24659	// Length zero requires a runtime check - so we handle arrays specially
16639	// here to elide it.	24660	// here to elide it.
16640	switch (dst_ptr_ty.ptrSize(zcu)) {	24661	switch (dst_ty.ptrSize(zcu)) {
16641	.slice => {	24662	.slice => {
16642	const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(zcu);	24663	const slice_ptr_ty = dst_ty.slicePtrFieldType(zcu);
16643		24664
16644	// TODO: this only handles slices stored in the stack	24665	const dst_ptr: MCValue = switch (dst) {
16645	const ptr = dst_ptr;	24666	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
16646	const len = dst_ptr.address().offset(8).deref();	24667	else => dst,
		24668	};
		24669	const len: MCValue = switch (dst) {
		24670	.register_pair => \|dst_regs\| .{ .register = dst_regs[1] },
		24671	else => dst.address().offset(8).deref(),
		24672	};
16647		24673
16648	// Used to store the number of elements for comparison.	24674	// Used to store the number of elements for comparison.
16649	// After comparison, updated to store number of bytes needed to copy.	24675	// After comparison, updated to store number of bytes needed to copy.
...	@@ -16652,11 +24678,11 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16652,11 +24678,11 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16652	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);	24678	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
16653	defer self.register_manager.unlockReg(len_lock);	24679	defer self.register_manager.unlockReg(len_lock);
16654		24680
16655	try self.genSetReg(len_reg, Type.usize, len, .{});	24681	try self.genSetReg(len_reg, .usize, len, .{});
16656	try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);	24682	try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);
16657		24683
16658	const skip_reloc = try self.asmJccReloc(.z, undefined);	24684	const skip_reloc = try self.asmJccReloc(.z, undefined);
16659	try self.store(slice_ptr_ty, ptr, src_val, .{ .safety = safety });	24685	try self.store(slice_ptr_ty, dst_ptr, src_val, .{ .safety = safety });
16660		24686
16661	const second_elem_ptr_reg =	24687	const second_elem_ptr_reg =
16662	try self.register_manager.allocReg(null, abi.RegisterClass.gp);	24688	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	@@ -16665,29 +24691,29 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16665,29 +24691,29 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16665	self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);	24691	self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
16666	defer self.register_manager.unlockReg(second_elem_ptr_lock);	24692	defer self.register_manager.unlockReg(second_elem_ptr_lock);
16667		24693
16668	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{	24694	try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{
16669	.reg = try self.copyToTmpRegister(Type.usize, ptr),	24695	.reg = try self.copyToTmpRegister(.usize, dst_ptr),
16670	.off = elem_abi_size,	24696	.off = elem_abi_size,
16671	} }, .{});	24697	} }, .{});
16672		24698
16673	try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });	24699	try self.genBinOpMir(.{ ._, .sub }, .usize, len_mcv, .{ .immediate = 1 });
16674	try self.asmRegisterRegisterImmediate(	24700	try self.asmRegisterRegisterImmediate(
16675	.{ .i_, .mul },	24701	.{ .i_, .mul },
16676	len_reg,	24702	len_reg,
16677	len_reg,	24703	len_reg,
16678	Immediate.s(elem_abi_size),	24704	.s(elem_abi_size),
16679	);	24705	);
16680	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);	24706	try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv, .{ .no_alias = false });
16681		24707
16682	self.performReloc(skip_reloc);	24708	self.performReloc(skip_reloc);
16683	},	24709	},
16684	.one => {	24710	.one => {
16685	const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);	24711	const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);
16686		24712
16687	const len = dst_ptr_ty.childType(zcu).arrayLen(zcu);	24713	const len = dst_ty.childType(zcu).arrayLen(zcu);
16688		24714
16689	assert(len != 0); // prevented by Sema	24715	assert(len != 0); // prevented by Sema
16690	try self.store(elem_ptr_ty, dst_ptr, src_val, .{ .safety = safety });	24716	try self.store(elem_ptr_ty, dst, src_val, .{ .safety = safety });
16691		24717
16692	const second_elem_ptr_reg =	24718	const second_elem_ptr_reg =
16693	try self.register_manager.allocReg(null, abi.RegisterClass.gp);	24719	try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...	@@ -16696,13 +24722,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16696,13 +24722,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16696	self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);	24722	self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
16697	defer self.register_manager.unlockReg(second_elem_ptr_lock);	24723	defer self.register_manager.unlockReg(second_elem_ptr_lock);
16698		24724
16699	try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{	24725	try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{
16700	.reg = try self.copyToTmpRegister(Type.usize, dst_ptr),	24726	.reg = try self.copyToTmpRegister(.usize, dst),
16701	.off = elem_abi_size,	24727	.off = elem_abi_size,
16702	} }, .{});	24728	} }, .{});
16703		24729
16704	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };	24730	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
16705	try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);	24731	try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy, .{ .no_alias = false });
16706	},	24732	},
16707	.c, .many => unreachable,	24733	.c, .many => unreachable,
16708	}	24734	}
...	@@ -16710,7 +24736,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -16710,7 +24736,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
16710	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });	24736	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
16711	}	24737	}
16712		24738
16713	fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	24739	fn airMemcpy(self: *CodeGen, inst: Air.Inst.Index) !void {
16714	const pt = self.pt;	24740	const pt = self.pt;
16715	const zcu = pt.zcu;	24741	const zcu = pt.zcu;
16716	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	24742	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	@@ -16719,64 +24745,87 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16719,64 +24745,87 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
16719	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });	24745	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
16720	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);	24746	defer for (reg_locks) \|lock\| self.register_manager.unlockReg(lock);
16721		24747
16722	const dst_ptr = try self.resolveInst(bin_op.lhs);	24748	const dst = try self.resolveInst(bin_op.lhs);
16723	const dst_ptr_ty = self.typeOf(bin_op.lhs);	24749	const dst_ty = self.typeOf(bin_op.lhs);
16724	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {	24750	const dst_locks: [2]?RegisterLock = switch (dst) {
16725	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	24751	.register => \|dst_reg\| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null },
16726	else => null,	24752	.register_pair => \|dst_regs\| .{
		24753	self.register_manager.lockRegAssumeUnused(dst_regs[0]),
		24754	self.register_manager.lockReg(dst_regs[1]),
		24755	},
		24756	else => @splat(null),
16727	};	24757	};
16728	defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	24758	for (dst_locks) \|dst_lock\| if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
16729		24759
16730	const src_ptr = try self.resolveInst(bin_op.rhs);	24760	const src = try self.resolveInst(bin_op.rhs);
16731	const src_ptr_lock: ?RegisterLock = switch (src_ptr) {	24761	const src_locks: [2]?RegisterLock = switch (src) {
16732	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	24762	.register => \|src_reg\| .{ self.register_manager.lockReg(src_reg), null },
16733	else => null,	24763	.register_pair => \|src_regs\| .{
		24764	self.register_manager.lockRegAssumeUnused(src_regs[0]),
		24765	self.register_manager.lockRegAssumeUnused(src_regs[1]),
		24766	},
		24767	else => @splat(null),
16734	};	24768	};
16735	defer if (src_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	24769	for (src_locks) \|src_lock\| if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
16736		24770
16737	const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {	24771	const len: MCValue = switch (dst_ty.ptrSize(zcu)) {
16738	.slice => len: {	24772	.slice => len: {
16739	const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);	24773	const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
16740	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);	24774	const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
16741	defer self.register_manager.unlockReg(len_lock);	24775	defer self.register_manager.unlockReg(len_lock);
16742		24776
16743	try self.asmRegisterMemoryImmediate(	24777	switch (dst) {
16744	.{ .i_, .mul },	24778	.register_pair => \|dst_regs\| try self.asmRegisterRegisterImmediate(
16745	len_reg,	24779	.{ .i_, .mul },
16746	try dst_ptr.address().offset(8).deref().mem(self, .qword),	24780	len_reg,
16747	Immediate.s(@intCast(dst_ptr_ty.childType(zcu).abiSize(zcu))),	24781	dst_regs[1],
16748	);	24782	.s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
		24783	),
		24784	else => try self.asmRegisterMemoryImmediate(
		24785	.{ .i_, .mul },
		24786	len_reg,
		24787	try dst.address().offset(8).deref().mem(self, .{ .size = .qword }),
		24788	.s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
		24789	),
		24790	}
16749	break :len .{ .register = len_reg };	24791	break :len .{ .register = len_reg };
16750	},	24792	},
16751	.one => len: {	24793	.one => len: {
16752	const array_ty = dst_ptr_ty.childType(zcu);	24794	const array_ty = dst_ty.childType(zcu);
16753	break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };	24795	break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };
16754	},	24796	},
16755	.c, .many => unreachable,	24797	.c, .many => unreachable,
16756	};	24798	};
16757	const len_lock: ?RegisterLock = switch (len) {	24799	const len_lock: ?RegisterLock = switch (len) {
16758	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	24800	.register => \|reg\| self.register_manager.lockReg(reg),
16759	else => null,	24801	else => null,
16760	};	24802	};
16761	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);	24803	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
16762		24804
16763	// TODO: dst_ptr and src_ptr could be slices rather than raw pointers	24805	const dst_ptr: MCValue = switch (dst) {
16764	try self.genInlineMemcpy(dst_ptr, src_ptr, len);	24806	.register_pair => \|dst_regs\| .{ .register = dst_regs[0] },
		24807	else => dst,
		24808	};
		24809	const src_ptr: MCValue = switch (src) {
		24810	.register_pair => \|src_regs\| .{ .register = src_regs[0] },
		24811	else => src,
		24812	};
		24813
		24814	try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{ .no_alias = true });
16765		24815
16766	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });	24816	return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
16767	}	24817	}
16768		24818
16769	fn airTagName(self: *Self, inst: Air.Inst.Index) !void {	24819	fn airTagName(self: *CodeGen, inst: Air.Inst.Index) !void {
16770	const pt = self.pt;	24820	const pt = self.pt;
16771	const zcu = pt.zcu;	24821	const zcu = pt.zcu;
16772	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	24822	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
16773	const inst_ty = self.typeOfIndex(inst);	24823	const inst_ty = self.typeOfIndex(inst);
16774	const enum_ty = self.typeOf(un_op);	24824	const enum_ty = self.typeOf(un_op);
16775	const resolved_cc = abi.resolveCallingConvention(.auto, self.target.*);
16776		24825
16777	// We need a properly aligned and sized call frame to be able to call this function.	24826	// We need a properly aligned and sized call frame to be able to call this function.
16778	{	24827	{
16779	const needed_call_frame = FrameAlloc.init(.{	24828	const needed_call_frame: FrameAlloc = .init(.{
16780	.size = inst_ty.abiSize(zcu),	24829	.size = inst_ty.abiSize(zcu),
16781	.alignment = inst_ty.abiAlignment(zcu),	24830	.alignment = inst_ty.abiAlignment(zcu),
16782	});	24831	});
...	@@ -16790,23 +24839,23 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16790,23 +24839,23 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
16790	}	24839	}
16791		24840
16792	try self.spillEflagsIfOccupied();	24841	try self.spillEflagsIfOccupied();
16793	try self.spillCallerPreservedRegs(resolved_cc);	24842	try self.spillCallerPreservedRegs(.auto);
16794		24843
16795	const param_regs = abi.getCAbiIntParamRegs(resolved_cc);	24844	const param_regs = abi.getCAbiIntParamRegs(.auto);
16796		24845
16797	const dst_mcv = try self.allocRegOrMem(inst, false);	24846	const dst_mcv = try self.allocRegOrMem(inst, false);
16798	try self.genSetReg(param_regs[0], Type.usize, dst_mcv.address(), .{});	24847	try self.genSetReg(param_regs[0], .usize, dst_mcv.address(), .{});
16799		24848
16800	const operand = try self.resolveInst(un_op);	24849	const operand = try self.resolveInst(un_op);
16801	try self.genSetReg(param_regs[1], enum_ty, operand, .{});	24850	try self.genSetReg(param_regs[1], enum_ty, operand, .{});
16802		24851
16803	const enum_lazy_sym: link.File.LazySymbol = .{ .kind = .code, .ty = enum_ty.toIntern() };	24852	const enum_lazy_sym: link.File.LazySymbol = .{ .kind = .code, .ty = enum_ty.toIntern() };
16804	try self.genLazySymbolRef(.call, .rax, enum_lazy_sym);	24853	try self.genLazySymbolRef(.call, abi.getCAbiLinkerScratchReg(self.fn_type.fnCallingConvention(zcu)), enum_lazy_sym);
16805		24854
16806	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });	24855	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
16807	}	24856	}
16808		24857
16809	fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	24858	fn airErrorName(self: *CodeGen, inst: Air.Inst.Index) !void {
16810	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	24859	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
16811		24860
16812	const err_ty = self.typeOf(un_op);	24861	const err_ty = self.typeOf(un_op);
...	@@ -16908,13 +24957,13 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16908,13 +24957,13 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
16908	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });	24957	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
16909	}	24958	}
16910		24959
16911	fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	24960	fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void {
16912	const pt = self.pt;	24961	const pt = self.pt;
16913	const zcu = pt.zcu;	24962	const zcu = pt.zcu;
16914	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	24963	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
16915	const vector_ty = self.typeOfIndex(inst);	24964	const vector_ty = self.typeOfIndex(inst);
16916	const vector_len = vector_ty.vectorLen(zcu);	24965	const vector_len = vector_ty.vectorLen(zcu);
16917	const dst_rc = self.regClassForType(vector_ty);	24966	const dst_rc = self.regSetForType(vector_ty);
16918	const scalar_ty = self.typeOf(ty_op.operand);	24967	const scalar_ty = self.typeOf(ty_op.operand);
16919		24968
16920	const result: MCValue = result: {	24969	const result: MCValue = result: {
...	@@ -16930,27 +24979,23 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16930,27 +24979,23 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16930	try self.genSetReg(	24979	try self.genSetReg(
16931	regs[1],	24980	regs[1],
16932	vector_ty,	24981	vector_ty,
16933	.{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },	24982	.{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) },
16934	.{},	24983	.{},
16935	);	24984	);
16936	const src_mcv = try self.resolveInst(ty_op.operand);	24985	const src_mcv = try self.resolveInst(ty_op.operand);
16937	const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);	24986	const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4);
16938	try self.asmCmovccRegisterRegister(	24987	try self.asmCmovccRegisterRegister(
16939	switch (src_mcv) {	24988	switch (src_mcv) {
16940	.eflags => \|cc\| cc,	24989	.eflags => \|cc\| cc,
16941	.register => \|src_reg\| cc: {	24990	.register => \|src_reg\| cc: {
16942	try self.asmRegisterImmediate(	24991	try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1));
16943	.{ ._, .@"test" },
16944	src_reg.to8(),
16945	Immediate.u(1),
16946	);
16947	break :cc .nz;	24992	break :cc .nz;
16948	},	24993	},
16949	else => cc: {	24994	else => cc: {
16950	try self.asmMemoryImmediate(	24995	try self.asmMemoryImmediate(
16951	.{ ._, .@"test" },	24996	.{ ._, .@"test" },
16952	try src_mcv.mem(self, .byte),	24997	try src_mcv.mem(self, .{ .size = .byte }),
16953	Immediate.u(1),	24998	.u(1),
16954	);	24999	);
16955	break :cc .nz;	25000	break :cc .nz;
16956	},	25001	},
...	@@ -16990,10 +25035,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16990,10 +25035,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16990	defer self.register_manager.unlockReg(dst_lock);	25035	defer self.register_manager.unlockReg(dst_lock);
16991		25036
16992	const src_mcv = try self.resolveInst(ty_op.operand);	25037	const src_mcv = try self.resolveInst(ty_op.operand);
16993	if (src_mcv.isMemory()) try self.asmRegisterMemory(	25038	if (src_mcv.isBase()) try self.asmRegisterMemory(
16994	mir_tag,	25039	mir_tag,
16995	registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),	25040	registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),
16996	try src_mcv.mem(self, self.memSize(scalar_ty)),	25041	try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }),
16997	) else {	25042	) else {
16998	if (mir_tag[0] == .v_i128) break :avx2;	25043	if (mir_tag[0] == .v_i128) break :avx2;
16999	try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});	25044	try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
...	@@ -17037,7 +25082,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17037,7 +25082,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17037	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },	25082	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
17038	dst_alias,	25083	dst_alias,
17039	dst_alias,	25084	dst_alias,
17040	Immediate.u(0b00_00_00_00),	25085	.u(0b00_00_00_00),
17041	);	25086	);
17042	if (switch (scalar_bits) {	25087	if (switch (scalar_bits) {
17043	1...8 => vector_len > 4,	25088	1...8 => vector_len > 4,
...	@@ -17049,7 +25094,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17049,7 +25094,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17049	.{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },	25094	.{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
17050	dst_alias,	25095	dst_alias,
17051	dst_alias,	25096	dst_alias,
17052	Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),	25097	.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
17053	);	25098	);
17054	break :result .{ .register = dst_reg };	25099	break :result .{ .register = dst_reg };
17055	},	25100	},
...	@@ -17066,10 +25111,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17066,10 +25111,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17066	const src_mcv = try self.resolveInst(ty_op.operand);	25111	const src_mcv = try self.resolveInst(ty_op.operand);
17067	if (self.hasFeature(.avx)) {	25112	if (self.hasFeature(.avx)) {
17068	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);	25113	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17069	if (src_mcv.isMemory()) try self.asmRegisterMemory(	25114	if (src_mcv.isBase()) try self.asmRegisterMemory(
17070	.{ .v_ss, .broadcast },	25115	.{ .v_ss, .broadcast },
17071	dst_reg.to128(),	25116	dst_reg.to128(),
17072	try src_mcv.mem(self, .dword),	25117	try src_mcv.mem(self, .{ .size = .dword }),
17073	) else {	25118	) else {
17074	const src_reg = if (src_mcv.isRegister())	25119	const src_reg = if (src_mcv.isRegister())
17075	src_mcv.getReg().?	25120	src_mcv.getReg().?
...	@@ -17080,7 +25125,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17080,7 +25125,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17080	dst_reg.to128(),	25125	dst_reg.to128(),
17081	src_reg.to128(),	25126	src_reg.to128(),
17082	src_reg.to128(),	25127	src_reg.to128(),
17083	Immediate.u(0),	25128	.u(0),
17084	);	25129	);
17085	}	25130	}
17086	break :result .{ .register = dst_reg };	25131	break :result .{ .register = dst_reg };
...	@@ -17095,7 +25140,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17095,7 +25140,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17095	.{ ._ps, .shuf },	25140	.{ ._ps, .shuf },
17096	dst_reg.to128(),	25141	dst_reg.to128(),
17097	dst_reg.to128(),	25142	dst_reg.to128(),
17098	Immediate.u(0),	25143	.u(0),
17099	);	25144	);
17100	break :result dst_mcv;	25145	break :result dst_mcv;
17101	}	25146	}
...	@@ -17103,10 +25148,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17103,10 +25148,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17103	5...8 => if (self.hasFeature(.avx)) {	25148	5...8 => if (self.hasFeature(.avx)) {
17104	const src_mcv = try self.resolveInst(ty_op.operand);	25149	const src_mcv = try self.resolveInst(ty_op.operand);
17105	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);	25150	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17106	if (src_mcv.isMemory()) try self.asmRegisterMemory(	25151	if (src_mcv.isBase()) try self.asmRegisterMemory(
17107	.{ .v_ss, .broadcast },	25152	.{ .v_ss, .broadcast },
17108	dst_reg.to256(),	25153	dst_reg.to256(),
17109	try src_mcv.mem(self, .dword),	25154	try src_mcv.mem(self, .{ .size = .dword }),
17110	) else {	25155	) else {
17111	const src_reg = if (src_mcv.isRegister())	25156	const src_reg = if (src_mcv.isRegister())
17112	src_mcv.getReg().?	25157	src_mcv.getReg().?
...	@@ -17122,14 +25167,14 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17122,14 +25167,14 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17122	dst_reg.to128(),	25167	dst_reg.to128(),
17123	src_reg.to128(),	25168	src_reg.to128(),
17124	src_reg.to128(),	25169	src_reg.to128(),
17125	Immediate.u(0),	25170	.u(0),
17126	);	25171	);
17127	try self.asmRegisterRegisterRegisterImmediate(	25172	try self.asmRegisterRegisterRegisterImmediate(
17128	.{ .v_f128, .insert },	25173	.{ .v_f128, .insert },
17129	dst_reg.to256(),	25174	dst_reg.to256(),
17130	dst_reg.to256(),	25175	dst_reg.to256(),
17131	dst_reg.to128(),	25176	dst_reg.to128(),
17132	Immediate.u(1),	25177	.u(1),
17133	);	25178	);
17134	}	25179	}
17135	}	25180	}
...	@@ -17149,10 +25194,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17149,10 +25194,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17149	const src_mcv = try self.resolveInst(ty_op.operand);	25194	const src_mcv = try self.resolveInst(ty_op.operand);
17150	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);	25195	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17151	if (self.hasFeature(.sse3)) {	25196	if (self.hasFeature(.sse3)) {
17152	if (src_mcv.isMemory()) try self.asmRegisterMemory(	25197	if (src_mcv.isBase()) try self.asmRegisterMemory(
17153	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },	25198	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
17154	dst_reg.to128(),	25199	dst_reg.to128(),
17155	try src_mcv.mem(self, .qword),	25200	try src_mcv.mem(self, .{ .size = .qword }),
17156	) else try self.asmRegisterRegister(	25201	) else try self.asmRegisterRegister(
17157	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },	25202	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
17158	dst_reg.to128(),	25203	dst_reg.to128(),
...	@@ -17174,10 +25219,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17174,10 +25219,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17174	3...4 => if (self.hasFeature(.avx)) {	25219	3...4 => if (self.hasFeature(.avx)) {
17175	const src_mcv = try self.resolveInst(ty_op.operand);	25220	const src_mcv = try self.resolveInst(ty_op.operand);
17176	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);	25221	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17177	if (src_mcv.isMemory()) try self.asmRegisterMemory(	25222	if (src_mcv.isBase()) try self.asmRegisterMemory(
17178	.{ .v_sd, .broadcast },	25223	.{ .v_sd, .broadcast },
17179	dst_reg.to256(),	25224	dst_reg.to256(),
17180	try src_mcv.mem(self, .qword),	25225	try src_mcv.mem(self, .{ .size = .qword }),
17181	) else {	25226	) else {
17182	const src_reg = if (src_mcv.isRegister())	25227	const src_reg = if (src_mcv.isRegister())
17183	src_mcv.getReg().?	25228	src_mcv.getReg().?
...	@@ -17198,7 +25243,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17198,7 +25243,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17198	dst_reg.to256(),	25243	dst_reg.to256(),
17199	dst_reg.to256(),	25244	dst_reg.to256(),
17200	dst_reg.to128(),	25245	dst_reg.to128(),
17201	Immediate.u(1),	25246	.u(1),
17202	);	25247	);
17203	}	25248	}
17204	}	25249	}
...	@@ -17217,10 +25262,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17217,10 +25262,10 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17217	2 => if (self.hasFeature(.avx)) {	25262	2 => if (self.hasFeature(.avx)) {
17218	const src_mcv = try self.resolveInst(ty_op.operand);	25263	const src_mcv = try self.resolveInst(ty_op.operand);
17219	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);	25264	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
17220	if (src_mcv.isMemory()) try self.asmRegisterMemory(	25265	if (src_mcv.isBase()) try self.asmRegisterMemory(
17221	.{ .v_f128, .broadcast },	25266	.{ .v_f128, .broadcast },
17222	dst_reg.to256(),	25267	dst_reg.to256(),
17223	try src_mcv.mem(self, .xword),	25268	try src_mcv.mem(self, .{ .size = .xword }),
17224	) else {	25269	) else {
17225	const src_reg = if (src_mcv.isRegister())	25270	const src_reg = if (src_mcv.isRegister())
17226	src_mcv.getReg().?	25271	src_mcv.getReg().?
...	@@ -17231,7 +25276,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17231,7 +25276,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17231	dst_reg.to256(),	25276	dst_reg.to256(),
17232	src_reg.to256(),	25277	src_reg.to256(),
17233	src_reg.to128(),	25278	src_reg.to128(),
17234	Immediate.u(1),	25279	.u(1),
17235	);	25280	);
17236	}	25281	}
17237	break :result .{ .register = dst_reg };	25282	break :result .{ .register = dst_reg };
...	@@ -17247,7 +25292,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17247,7 +25292,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17247	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	25292	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
17248	}	25293	}
17249		25294
17250	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	25295	fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
17251	const pt = self.pt;	25296	const pt = self.pt;
17252	const zcu = pt.zcu;	25297	const zcu = pt.zcu;
17253	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	25298	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
...	@@ -17308,7 +25353,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17308,7 +25353,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17308	mask_alias,	25353	mask_alias,
17309	mask_alias,	25354	mask_alias,
17310	mask_reg.to128(),	25355	mask_reg.to128(),
17311	Immediate.u(1),	25356	.u(1),
17312	);	25357	);
17313	break :broadcast;	25358	break :broadcast;
17314	},	25359	},
...	@@ -17326,9 +25371,9 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17326,9 +25371,9 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17326	else => unreachable,	25371	else => unreachable,
17327	}, .broadcast },	25372	}, .broadcast },
17328	mask_alias,	25373	mask_alias,
17329	if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{	25374	if (pred_mcv.isBase()) try pred_mcv.mem(self, .{ .size = .byte }) else .{
17330	.base = .{ .reg = (try self.copyToTmpRegister(	25375	.base = .{ .reg = (try self.copyToTmpRegister(
17331	Type.usize,	25376	.usize,
17332	pred_mcv.address(),	25377	pred_mcv.address(),
17333	)).to64() },	25378	)).to64() },
17334	.mod = .{ .rm = .{ .size = .byte } },	25379	.mod = .{ .rm = .{ .size = .byte } },
...	@@ -17362,7 +25407,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17362,7 +25407,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17362	.{ if (has_avx) .vp_w else .p_w, .shufl },	25407	.{ if (has_avx) .vp_w else .p_w, .shufl },
17363	mask_alias,	25408	mask_alias,
17364	mask_alias,	25409	mask_alias,
17365	Immediate.u(0b00_00_00_00),	25410	.u(0b00_00_00_00),
17366	);	25411	);
17367	if (abi_size <= 8) break :broadcast;	25412	if (abi_size <= 8) break :broadcast;
17368	}	25413	}
...	@@ -17370,7 +25415,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17370,7 +25415,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17370	.{ if (has_avx) .vp_d else .p_d, .shuf },	25415	.{ if (has_avx) .vp_d else .p_d, .shuf },
17371	mask_alias,	25416	mask_alias,
17372	mask_alias,	25417	mask_alias,
17373	Immediate.u(switch (elem_abi_size) {	25418	.u(switch (elem_abi_size) {
17374	1...2, 5...8 => 0b01_00_01_00,	25419	1...2, 5...8 => 0b01_00_01_00,
17375	3...4 => 0b00_00_00_00,	25420	3...4 => 0b00_00_00_00,
17376	else => unreachable,	25421	else => unreachable,
...	@@ -17386,12 +25431,12 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17386,12 +25431,12 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17386	.ty = mask_elem_ty.toIntern(),	25431	.ty = mask_elem_ty.toIntern(),
17387	.storage = .{ .u64 = bit / elem_bits },	25432	.storage = .{ .u64 = bit / elem_bits },
17388	} });	25433	} });
17389	const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{	25434	const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
17390	.ty = mask_ty.toIntern(),	25435	.ty = mask_ty.toIntern(),
17391	.storage = .{ .elems = mask_elems[0..vec_len] },	25436	.storage = .{ .elems = mask_elems[0..vec_len] },
17392	} })));	25437	} })));
17393	const mask_mem: Memory = .{	25438	const mask_mem: Memory = .{
17394	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },	25439	.base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) },
17395	.mod = .{ .rm = .{ .size = self.memSize(ty) } },	25440	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
17396	};	25441	};
17397	if (has_avx) try self.asmRegisterRegisterMemory(	25442	if (has_avx) try self.asmRegisterRegisterMemory(
...	@@ -17411,12 +25456,12 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17411,12 +25456,12 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17411	.ty = mask_elem_ty.toIntern(),	25456	.ty = mask_elem_ty.toIntern(),
17412	.storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },	25457	.storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
17413	} });	25458	} });
17414	const mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{	25459	const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
17415	.ty = mask_ty.toIntern(),	25460	.ty = mask_ty.toIntern(),
17416	.storage = .{ .elems = mask_elems[0..vec_len] },	25461	.storage = .{ .elems = mask_elems[0..vec_len] },
17417	} })));	25462	} })));
17418	const mask_mem: Memory = .{	25463	const mask_mem: Memory = .{
17419	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },	25464	.base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) },
17420	.mod = .{ .rm = .{ .size = self.memSize(ty) } },	25465	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
17421	};	25466	};
17422	if (has_avx) {	25467	if (has_avx) {
...	@@ -17520,11 +25565,11 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17520,11 +25565,11 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17520	try self.genSetReg(dst_reg, ty, rhs_mcv, .{});	25565	try self.genSetReg(dst_reg, ty, rhs_mcv, .{});
17521	break :rhs dst_alias;	25566	break :rhs dst_alias;
17522	};	25567	};
17523	if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(	25568	if (lhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister(
17524	mir_tag,	25569	mir_tag,
17525	dst_alias,	25570	dst_alias,
17526	rhs_alias,	25571	rhs_alias,
17527	try lhs_mcv.mem(self, self.memSize(ty)),	25572	try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
17528	mask_alias,	25573	mask_alias,
17529	) else try self.asmRegisterRegisterRegisterRegister(	25574	) else try self.asmRegisterRegisterRegisterRegister(
17530	mir_tag,	25575	mir_tag,
...	@@ -17536,10 +25581,10 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17536,10 +25581,10 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17536	try self.copyToTmpRegister(ty, lhs_mcv), abi_size),	25581	try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
17537	mask_alias,	25582	mask_alias,
17538	);	25583	);
17539	} else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(	25584	} else if (has_blend) if (lhs_mcv.isBase()) try self.asmRegisterMemoryRegister(
17540	mir_tag,	25585	mir_tag,
17541	dst_alias,	25586	dst_alias,
17542	try lhs_mcv.mem(self, self.memSize(ty)),	25587	try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
17543	mask_alias,	25588	mask_alias,
17544	) else try self.asmRegisterRegisterRegister(	25589	) else try self.asmRegisterRegisterRegister(
17545	mir_tag,	25590	mir_tag,
...	@@ -17561,10 +25606,10 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17561,10 +25606,10 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17561	},	25606	},
17562	}) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(pt)});	25607	}) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(pt)});
17563	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);	25608	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
17564	if (rhs_mcv.isMemory()) try self.asmRegisterMemory(	25609	if (rhs_mcv.isBase()) try self.asmRegisterMemory(
17565	.{ mir_fixes, .andn },	25610	.{ mir_fixes, .andn },
17566	mask_alias,	25611	mask_alias,
17567	try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),	25612	try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
17568	) else try self.asmRegisterRegister(	25613	) else try self.asmRegisterRegister(
17569	.{ mir_fixes, .andn },	25614	.{ mir_fixes, .andn },
17570	mask_alias,	25615	mask_alias,
...	@@ -17580,7 +25625,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17580,7 +25625,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17580	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });	25625	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
17581	}	25626	}
17582		25627
17583	fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	25628	fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {
17584	const pt = self.pt;	25629	const pt = self.pt;
17585	const zcu = pt.zcu;	25630	const zcu = pt.zcu;
17586	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	25631	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -17640,16 +25685,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17640,16 +25685,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17640		25685
17641	for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) \|variant\| unpck: {	25686	for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) \|variant\| unpck: {
17642	if (elem_abi_size > 8) break :unpck;	25687	if (elem_abi_size > 8) break :unpck;
17643	if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)	25688	if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck;
17644	has_avx
17645	else
17646	self.hasFeature(.avx2)) 32 else 16)) break :unpck;
17647		25689
17648	var sources = [1]?u1{null} ** 2;	25690	var sources: [2]?u1 = @splat(null);
17649	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {	25691	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17650	const mask_elem = maybe_mask_elem orelse continue;	25692	const mask_elem = maybe_mask_elem orelse continue;
17651	const mask_elem_index =	25693	const mask_elem_index =
17652	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;	25694	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
17653	const elem_byte = (elem_index >> 1) * elem_abi_size;	25695	const elem_byte = (elem_index >> 1) * elem_abi_size;
17654	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {	25696	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {
17655	.unpckl => 0b0000,	25697	.unpckl => 0b0000,
...	@@ -17701,11 +25743,11 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17701,11 +25743,11 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17701	},	25743	},
17702	else => unreachable,	25744	else => unreachable,
17703	} };	25745	} };
17704	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory(	25746	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory(
17705	mir_tag,	25747	mir_tag,
17706	dst_alias,	25748	dst_alias,
17707	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),	25749	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17708	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	25750	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
17709	) else try self.asmRegisterRegisterRegister(	25751	) else try self.asmRegisterRegisterRegister(
17710	mir_tag,	25752	mir_tag,
17711	dst_alias,	25753	dst_alias,
...	@@ -17714,10 +25756,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17714,10 +25756,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17714	rhs_mcv.getReg().?	25756	rhs_mcv.getReg().?
17715	else	25757	else
17716	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	25758	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17717	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(	25759	) else if (rhs_mcv.isBase()) try self.asmRegisterMemory(
17718	mir_tag,	25760	mir_tag,
17719	dst_alias,	25761	dst_alias,
17720	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	25762	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
17721	) else try self.asmRegisterRegister(	25763	) else try self.asmRegisterRegister(
17722	mir_tag,	25764	mir_tag,
17723	dst_alias,	25765	dst_alias,
...	@@ -17731,10 +25773,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17731,10 +25773,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17731		25773
17732	pshufd: {	25774	pshufd: {
17733	if (elem_abi_size != 4) break :pshufd;	25775	if (elem_abi_size != 4) break :pshufd;
17734	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;	25776	if (max_abi_size > self.vectorSize(.float)) break :pshufd;
17735		25777
17736	var control: u8 = 0b00_00_00_00;	25778	var control: u8 = 0b00_00_00_00;
17737	var sources = [1]?u1{null} ** 1;	25779	var sources: [1]?u1 = @splat(null);
17738	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {	25780	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17739	const mask_elem = maybe_mask_elem orelse continue;	25781	const mask_elem = maybe_mask_elem orelse continue;
17740	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);	25782	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
...	@@ -17746,10 +25788,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17746,10 +25788,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17746	} else sources[(elem_index & 0b010) >> 1] = source;	25788	} else sources[(elem_index & 0b010) >> 1] = source;
17747		25789
17748	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);	25790	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17749	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;	25791	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17750	if (elem_index & 0b100 == 0)	25792	if (elem_index & 0b100 == 0)
17751	control \|= select	25793	control \|= select_mask
17752	else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;	25794	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd;
17753	}	25795	}
17754		25796
17755	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };	25797	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
...	@@ -17763,11 +25805,11 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17763,11 +25805,11 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17763	try self.register_manager.allocReg(inst, abi.RegisterClass.sse);	25805	try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
17764	const dst_alias = registerAlias(dst_reg, max_abi_size);	25806	const dst_alias = registerAlias(dst_reg, max_abi_size);
17765		25807
17766	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	25808	if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
17767	.{ if (has_avx) .vp_d else .p_d, .shuf },	25809	.{ if (has_avx) .vp_d else .p_d, .shuf },
17768	dst_alias,	25810	dst_alias,
17769	try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	25811	try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
17770	Immediate.u(control),	25812	.u(control),
17771	) else try self.asmRegisterRegisterImmediate(	25813	) else try self.asmRegisterRegisterImmediate(
17772	.{ if (has_avx) .vp_d else .p_d, .shuf },	25814	.{ if (has_avx) .vp_d else .p_d, .shuf },
17773	dst_alias,	25815	dst_alias,
...	@@ -17775,17 +25817,17 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17775,17 +25817,17 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17775	src_mcv.getReg().?	25817	src_mcv.getReg().?
17776	else	25818	else
17777	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),	25819	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
17778	Immediate.u(control),	25820	.u(control),
17779	);	25821	);
17780	break :result .{ .register = dst_reg };	25822	break :result .{ .register = dst_reg };
17781	}	25823	}
17782		25824
17783	shufps: {	25825	shufps: {
17784	if (elem_abi_size != 4) break :shufps;	25826	if (elem_abi_size != 4) break :shufps;
17785	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;	25827	if (max_abi_size > self.vectorSize(.float)) break :shufps;
17786		25828
17787	var control: u8 = 0b00_00_00_00;	25829	var control: u8 = 0b00_00_00_00;
17788	var sources = [1]?u1{null} ** 2;	25830	var sources: [2]?u1 = @splat(null);
17789	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {	25831	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17790	const mask_elem = maybe_mask_elem orelse continue;	25832	const mask_elem = maybe_mask_elem orelse continue;
17791	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);	25833	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
...	@@ -17797,10 +25839,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17797,10 +25839,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17797	} else sources[(elem_index & 0b010) >> 1] = source;	25839	} else sources[(elem_index & 0b010) >> 1] = source;
17798		25840
17799	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);	25841	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17800	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;	25842	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17801	if (elem_index & 0b100 == 0)	25843	if (elem_index & 0b100 == 0)
17802	control \|= select	25844	control \|= select_mask
17803	else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;	25845	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps;
17804	}	25846	}
17805	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;	25847	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
17806		25848
...	@@ -17819,12 +25861,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17819,12 +25861,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17819	const dst_reg = dst_mcv.getReg().?;	25861	const dst_reg = dst_mcv.getReg().?;
17820	const dst_alias = registerAlias(dst_reg, max_abi_size);	25862	const dst_alias = registerAlias(dst_reg, max_abi_size);
17821		25863
17822	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	25864	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
17823	.{ .v_ps, .shuf },	25865	.{ .v_ps, .shuf },
17824	dst_alias,	25866	dst_alias,
17825	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),	25867	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17826	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	25868	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
17827	Immediate.u(control),	25869	.u(control),
17828	) else try self.asmRegisterRegisterRegisterImmediate(	25870	) else try self.asmRegisterRegisterRegisterImmediate(
17829	.{ .v_ps, .shuf },	25871	.{ .v_ps, .shuf },
17830	dst_alias,	25872	dst_alias,
...	@@ -17833,12 +25875,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17833,12 +25875,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17833	rhs_mcv.getReg().?	25875	rhs_mcv.getReg().?
17834	else	25876	else
17835	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	25877	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17836	Immediate.u(control),	25878	.u(control),
17837	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	25879	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
17838	.{ ._ps, .shuf },	25880	.{ ._ps, .shuf },
17839	dst_alias,	25881	dst_alias,
17840	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	25882	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
17841	Immediate.u(control),	25883	.u(control),
17842	) else try self.asmRegisterRegisterImmediate(	25884	) else try self.asmRegisterRegisterImmediate(
17843	.{ ._ps, .shuf },	25885	.{ ._ps, .shuf },
17844	dst_alias,	25886	dst_alias,
...	@@ -17846,17 +25888,17 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17846,17 +25888,17 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17846	rhs_mcv.getReg().?	25888	rhs_mcv.getReg().?
17847	else	25889	else
17848	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	25890	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17849	Immediate.u(control),	25891	.u(control),
17850	);	25892	);
17851	break :result dst_mcv;	25893	break :result dst_mcv;
17852	}	25894	}
17853		25895
17854	shufpd: {	25896	shufpd: {
17855	if (elem_abi_size != 8) break :shufpd;	25897	if (elem_abi_size != 8) break :shufpd;
17856	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;	25898	if (max_abi_size > self.vectorSize(.float)) break :shufpd;
17857		25899
17858	var control: u4 = 0b0_0_0_0;	25900	var control: u4 = 0b0_0_0_0;
17859	var sources = [1]?u1{null} ** 2;	25901	var sources: [2]?u1 = @splat(null);
17860	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {	25902	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17861	const mask_elem = maybe_mask_elem orelse continue;	25903	const mask_elem = maybe_mask_elem orelse continue;
17862	const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);	25904	const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
...	@@ -17886,12 +25928,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17886,12 +25928,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17886	const dst_reg = dst_mcv.getReg().?;	25928	const dst_reg = dst_mcv.getReg().?;
17887	const dst_alias = registerAlias(dst_reg, max_abi_size);	25929	const dst_alias = registerAlias(dst_reg, max_abi_size);
17888		25930
17889	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	25931	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
17890	.{ .v_pd, .shuf },	25932	.{ .v_pd, .shuf },
17891	dst_alias,	25933	dst_alias,
17892	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),	25934	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17893	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	25935	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
17894	Immediate.u(control),	25936	.u(control),
17895	) else try self.asmRegisterRegisterRegisterImmediate(	25937	) else try self.asmRegisterRegisterRegisterImmediate(
17896	.{ .v_pd, .shuf },	25938	.{ .v_pd, .shuf },
17897	dst_alias,	25939	dst_alias,
...	@@ -17900,12 +25942,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17900,12 +25942,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17900	rhs_mcv.getReg().?	25942	rhs_mcv.getReg().?
17901	else	25943	else
17902	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	25944	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17903	Immediate.u(control),	25945	.u(control),
17904	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	25946	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
17905	.{ ._pd, .shuf },	25947	.{ ._pd, .shuf },
17906	dst_alias,	25948	dst_alias,
17907	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),	25949	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
17908	Immediate.u(control),	25950	.u(control),
17909	) else try self.asmRegisterRegisterImmediate(	25951	) else try self.asmRegisterRegisterImmediate(
17910	.{ ._pd, .shuf },	25952	.{ ._pd, .shuf },
17911	dst_alias,	25953	dst_alias,
...	@@ -17913,27 +25955,27 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17913,27 +25955,27 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17913	rhs_mcv.getReg().?	25955	rhs_mcv.getReg().?
17914	else	25956	else
17915	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),	25957	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17916	Immediate.u(control),	25958	.u(control),
17917	);	25959	);
17918	break :result dst_mcv;	25960	break :result dst_mcv;
17919	}	25961	}
17920		25962
17921	blend: {	25963	blend: {
17922	if (elem_abi_size < 2) break :blend;	25964	if (elem_abi_size < 2) break :blend;
17923	if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend;	25965	if (dst_abi_size > self.vectorSize(.float)) break :blend;
17924	if (!self.hasFeature(.sse4_1)) break :blend;	25966	if (!self.hasFeature(.sse4_1)) break :blend;
17925		25967
17926	var control: u8 = 0b0_0_0_0_0_0_0_0;	25968	var control: u8 = 0b0_0_0_0_0_0_0_0;
17927	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {	25969	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
17928	const mask_elem = maybe_mask_elem orelse continue;	25970	const mask_elem = maybe_mask_elem orelse continue;
17929	const mask_elem_index =	25971	const mask_elem_index =
17930	math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;	25972	std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
17931	if (mask_elem_index != elem_index) break :blend;	25973	if (mask_elem_index != elem_index) break :blend;
17932		25974
17933	const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);	25975	const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
17934	if (elem_index & 0b1000 == 0)	25976	if (elem_index & 0b1000 == 0)
17935	control \|= select	25977	control \|= select_mask
17936	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;	25978	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend;
17937	}	25979	}
17938		25980
17939	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {	25981	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
...	@@ -17956,12 +25998,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17956,12 +25998,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17956		25998
17957	const rhs_mcv = try self.resolveInst(extra.b);	25999	const rhs_mcv = try self.resolveInst(extra.b);
17958	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);	26000	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
17959	if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	26001	if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
17960	.{ .vp_d, .blend },	26002	.{ .vp_d, .blend },
17961	registerAlias(dst_reg, dst_abi_size),	26003	registerAlias(dst_reg, dst_abi_size),
17962	registerAlias(lhs_reg, dst_abi_size),	26004	registerAlias(lhs_reg, dst_abi_size),
17963	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	26005	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
17964	Immediate.u(expanded_control),	26006	.u(expanded_control),
17965	) else try self.asmRegisterRegisterRegisterImmediate(	26007	) else try self.asmRegisterRegisterRegisterImmediate(
17966	.{ .vp_d, .blend },	26008	.{ .vp_d, .blend },
17967	registerAlias(dst_reg, dst_abi_size),	26009	registerAlias(dst_reg, dst_abi_size),
...	@@ -17970,7 +26012,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -17970,7 +26012,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17970	rhs_mcv.getReg().?	26012	rhs_mcv.getReg().?
17971	else	26013	else
17972	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	26014	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17973	Immediate.u(expanded_control),	26015	.u(expanded_control),
17974	);	26016	);
17975	break :result .{ .register = dst_reg };	26017	break :result .{ .register = dst_reg };
17976	}	26018	}
...	@@ -18008,1810 +26050,4076 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -18008,1810 +26050,4076 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18008	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);	26050	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
18009	const dst_reg = dst_mcv.getReg().?;	26051	const dst_reg = dst_mcv.getReg().?;
18010		26052
18011	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	26053	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
18012	.{ .vp_w, .blend },	26054	.{ .vp_w, .blend },
18013	registerAlias(dst_reg, dst_abi_size),	26055	registerAlias(dst_reg, dst_abi_size),
18014	registerAlias(if (lhs_mcv.isRegister())	26056	registerAlias(if (lhs_mcv.isRegister())
18015	lhs_mcv.getReg().?	26057	lhs_mcv.getReg().?
18016	else	26058	else
18017	dst_reg, dst_abi_size),	26059	dst_reg, dst_abi_size),
18018	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	26060	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18019	Immediate.u(expanded_control),	26061	.u(expanded_control),
18020	) else try self.asmRegisterRegisterRegisterImmediate(	26062	) else try self.asmRegisterRegisterRegisterImmediate(
18021	.{ .vp_w, .blend },	26063	.{ .vp_w, .blend },
18022	registerAlias(dst_reg, dst_abi_size),	26064	registerAlias(dst_reg, dst_abi_size),
18023	registerAlias(if (lhs_mcv.isRegister())	26065	registerAlias(if (lhs_mcv.isRegister())
18024	lhs_mcv.getReg().?	26066	lhs_mcv.getReg().?
18025	else	26067	else
18026	dst_reg, dst_abi_size),	26068	dst_reg, dst_abi_size),
		26069	registerAlias(if (rhs_mcv.isRegister())
		26070	rhs_mcv.getReg().?
		26071	else
		26072	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		26073	.u(expanded_control),
		26074	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
		26075	.{ .p_w, .blend },
		26076	registerAlias(dst_reg, dst_abi_size),
		26077	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
		26078	.u(expanded_control),
		26079	) else try self.asmRegisterRegisterImmediate(
		26080	.{ .p_w, .blend },
		26081	registerAlias(dst_reg, dst_abi_size),
		26082	registerAlias(if (rhs_mcv.isRegister())
		26083	rhs_mcv.getReg().?
		26084	else
		26085	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		26086	.u(expanded_control),
		26087	);
		26088	break :result .{ .register = dst_reg };
		26089	}
		26090
		26091	const expanded_control = switch (elem_abi_size) {
		26092	4, 8 => control,
		26093	16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) \|
		26094	@as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
		26095	else => unreachable,
		26096	};
		26097
		26098	const lhs_mcv = try self.resolveInst(extra.a);
		26099	const rhs_mcv = try self.resolveInst(extra.b);
		26100
		26101	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		26102	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
		26103	lhs_mcv
		26104	else if (has_avx and lhs_mcv.isRegister())
		26105	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		26106	else
		26107	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
		26108	const dst_reg = dst_mcv.getReg().?;
		26109
		26110	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
		26111	switch (elem_abi_size) {
		26112	4 => .{ .v_ps, .blend },
		26113	8, 16 => .{ .v_pd, .blend },
		26114	else => unreachable,
		26115	},
		26116	registerAlias(dst_reg, dst_abi_size),
		26117	registerAlias(if (lhs_mcv.isRegister())
		26118	lhs_mcv.getReg().?
		26119	else
		26120	dst_reg, dst_abi_size),
		26121	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
		26122	.u(expanded_control),
		26123	) else try self.asmRegisterRegisterRegisterImmediate(
		26124	switch (elem_abi_size) {
		26125	4 => .{ .v_ps, .blend },
		26126	8, 16 => .{ .v_pd, .blend },
		26127	else => unreachable,
		26128	},
		26129	registerAlias(dst_reg, dst_abi_size),
		26130	registerAlias(if (lhs_mcv.isRegister())
		26131	lhs_mcv.getReg().?
		26132	else
		26133	dst_reg, dst_abi_size),
		26134	registerAlias(if (rhs_mcv.isRegister())
		26135	rhs_mcv.getReg().?
		26136	else
		26137	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		26138	.u(expanded_control),
		26139	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
		26140	switch (elem_abi_size) {
		26141	4 => .{ ._ps, .blend },
		26142	8, 16 => .{ ._pd, .blend },
		26143	else => unreachable,
		26144	},
		26145	registerAlias(dst_reg, dst_abi_size),
		26146	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
		26147	.u(expanded_control),
		26148	) else try self.asmRegisterRegisterImmediate(
		26149	switch (elem_abi_size) {
		26150	4 => .{ ._ps, .blend },
		26151	8, 16 => .{ ._pd, .blend },
		26152	else => unreachable,
		26153	},
		26154	registerAlias(dst_reg, dst_abi_size),
		26155	registerAlias(if (rhs_mcv.isRegister())
		26156	rhs_mcv.getReg().?
		26157	else
		26158	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		26159	.u(expanded_control),
		26160	);
		26161	break :result .{ .register = dst_reg };
		26162	}
		26163
		26164	blendv: {
		26165	if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv;
		26166
		26167	const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8);
		26168	const select_mask_ty = try pt.vectorType(.{
		26169	.len = @intCast(mask_elems.len),
		26170	.child = select_mask_elem_ty.toIntern(),
		26171	});
		26172	var select_mask_elems: [32]InternPool.Index = undefined;
		26173	for (
		26174	select_mask_elems[0..mask_elems.len],
		26175	mask_elems,
		26176	0..,
		26177	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {
		26178	const mask_elem = maybe_mask_elem orelse continue;
		26179	const mask_elem_index =
		26180	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
		26181	if (mask_elem_index != elem_index) break :blendv;
		26182
		26183	select_mask_elem.* = (if (mask_elem < 0)
		26184	try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty)
		26185	else
		26186	try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();
		26187	}
		26188	const select_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
		26189	.ty = select_mask_ty.toIntern(),
		26190	.storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
		26191	} })));
		26192
		26193	if (self.hasFeature(.sse4_1)) {
		26194	const mir_tag: Mir.Inst.FixedTag = .{
		26195	if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
		26196	(dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
		26197	4 => if (has_avx) .v_ps else ._ps,
		26198	8 => if (has_avx) .v_pd else ._pd,
		26199	else => unreachable,
		26200	} else if (has_avx) .vp_b else .p_b,
		26201	.blendv,
		26202	};
		26203
		26204	const select_mask_reg = if (!has_avx) reg: {
		26205	try self.register_manager.getKnownReg(.xmm0, null);
		26206	try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
		26207	break :reg .xmm0;
		26208	} else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
		26209	const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
		26210	const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
		26211	defer self.register_manager.unlockReg(select_mask_lock);
		26212
		26213	const lhs_mcv = try self.resolveInst(extra.a);
		26214	const rhs_mcv = try self.resolveInst(extra.b);
		26215
		26216	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		26217	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
		26218	lhs_mcv
		26219	else if (has_avx and lhs_mcv.isRegister())
		26220	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		26221	else
		26222	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
		26223	const dst_reg = dst_mcv.getReg().?;
		26224	const dst_alias = registerAlias(dst_reg, dst_abi_size);
		26225
		26226	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister(
		26227	mir_tag,
		26228	dst_alias,
		26229	if (lhs_mcv.isRegister())
		26230	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
		26231	else
		26232	dst_alias,
		26233	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
		26234	select_mask_alias,
		26235	) else try self.asmRegisterRegisterRegisterRegister(
		26236	mir_tag,
		26237	dst_alias,
		26238	if (lhs_mcv.isRegister())
		26239	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
		26240	else
		26241	dst_alias,
18027	registerAlias(if (rhs_mcv.isRegister())	26242	registerAlias(if (rhs_mcv.isRegister())
18028	rhs_mcv.getReg().?	26243	rhs_mcv.getReg().?
18029	else	26244	else
18030	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	26245	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18031	Immediate.u(expanded_control),	26246	select_mask_alias,
18032	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	26247	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister(
18033	.{ .p_w, .blend },	26248	mir_tag,
18034	registerAlias(dst_reg, dst_abi_size),	26249	dst_alias,
18035	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	26250	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18036	Immediate.u(expanded_control),	26251	select_mask_alias,
18037	) else try self.asmRegisterRegisterImmediate(	26252	) else try self.asmRegisterRegisterRegister(
18038	.{ .p_w, .blend },	26253	mir_tag,
18039	registerAlias(dst_reg, dst_abi_size),	26254	dst_alias,
18040	registerAlias(if (rhs_mcv.isRegister())	26255	registerAlias(if (rhs_mcv.isRegister())
18041	rhs_mcv.getReg().?	26256	rhs_mcv.getReg().?
18042	else	26257	else
18043	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	26258	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18044	Immediate.u(expanded_control),	26259	select_mask_alias,
18045	);	26260	);
18046	break :result .{ .register = dst_reg };	26261	break :result dst_mcv;
18047	}	26262	}
18048		26263
18049	const expanded_control = switch (elem_abi_size) {
18050	4, 8 => control,
18051	16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) \|
18052	@as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
18053	else => unreachable,
18054	};
18055
18056	const lhs_mcv = try self.resolveInst(extra.a);	26264	const lhs_mcv = try self.resolveInst(extra.a);
18057	const rhs_mcv = try self.resolveInst(extra.b);	26265	const rhs_mcv = try self.resolveInst(extra.b);
18058		26266
18059	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and	26267	const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
18060	self.reuseOperand(inst, extra.a, 0, lhs_mcv))	26268	self.reuseOperand(inst, extra.b, 1, rhs_mcv))
18061	lhs_mcv	26269	rhs_mcv
18062	else if (has_avx and lhs_mcv.isRegister())
18063	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
18064	else	26270	else
18065	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);	26271	try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
18066	const dst_reg = dst_mcv.getReg().?;	26272	const dst_reg = dst_mcv.getReg().?;
		26273	const dst_alias = registerAlias(dst_reg, dst_abi_size);
18067		26274
18068	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(	26275	const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
18069	switch (elem_abi_size) {	26276	const mask_alias = registerAlias(mask_reg, dst_abi_size);
18070	4 => .{ .v_ps, .blend },	26277	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
18071	8, 16 => .{ .v_pd, .blend },	26278	defer self.register_manager.unlockReg(mask_lock);
18072	else => unreachable,	26279
18073	},	26280	const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
18074	registerAlias(dst_reg, dst_abi_size),	26281	switch (elem_ty.floatBits(self.target.*)) {
18075	registerAlias(if (lhs_mcv.isRegister())	26282	16, 80, 128 => .p_,
18076	lhs_mcv.getReg().?	26283	32 => ._ps,
18077	else	26284	64 => ._pd,
18078	dst_reg, dst_abi_size),
18079	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18080	Immediate.u(expanded_control),
18081	) else try self.asmRegisterRegisterRegisterImmediate(
18082	switch (elem_abi_size) {
18083	4 => .{ .v_ps, .blend },
18084	8, 16 => .{ .v_pd, .blend },
18085	else => unreachable,	26285	else => unreachable,
18086	},	26286	}
18087	registerAlias(dst_reg, dst_abi_size),	26287	else
18088	registerAlias(if (lhs_mcv.isRegister())	26288	.p_;
		26289	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
		26290	if (lhs_mcv.isBase()) try self.asmRegisterMemory(
		26291	.{ mir_fixes, .andn },
		26292	mask_alias,
		26293	try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
		26294	) else try self.asmRegisterRegister(
		26295	.{ mir_fixes, .andn },
		26296	mask_alias,
		26297	if (lhs_mcv.isRegister())
18089	lhs_mcv.getReg().?	26298	lhs_mcv.getReg().?
18090	else	26299	else
18091	dst_reg, dst_abi_size),	26300	try self.copyToTmpRegister(dst_ty, lhs_mcv),
18092	registerAlias(if (rhs_mcv.isRegister())	26301	);
18093	rhs_mcv.getReg().?	26302	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
18094	else	26303	break :result dst_mcv;
18095	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	26304	}
18096	Immediate.u(expanded_control),	26305
18097	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(	26306	pshufb: {
18098	switch (elem_abi_size) {	26307	if (max_abi_size > 16) break :pshufb;
18099	4 => .{ ._ps, .blend },	26308	if (!self.hasFeature(.ssse3)) break :pshufb;
18100	8, 16 => .{ ._pd, .blend },	26309
		26310	const temp_regs =
		26311	try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
		26312	const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
		26313	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
		26314
		26315	const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
		26316	try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
		26317
		26318	const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
		26319	try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
		26320
		26321	var lhs_mask_elems: [16]InternPool.Index = undefined;
		26322	for (lhs_mask_elems[0..max_abi_size], 0..) \|*lhs_mask_elem, byte_index\| {
		26323	const elem_index = byte_index / elem_abi_size;
		26324	lhs_mask_elem.* = try pt.intern(.{ .int = .{
		26325	.ty = .u8_type,
		26326	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
		26327	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
		26328	if (mask_elem < 0) break :elem 0b1_00_00000;
		26329	const mask_elem_index: u31 = @intCast(mask_elem);
		26330	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
		26331	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
		26332	} },
		26333	} });
		26334	}
		26335	const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
		26336	const lhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
		26337	.ty = lhs_mask_ty.toIntern(),
		26338	.storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
		26339	} })));
		26340	const lhs_mask_mem: Memory = .{
		26341	.base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) },
		26342	.mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
		26343	};
		26344	if (has_avx) try self.asmRegisterRegisterMemory(
		26345	.{ .vp_b, .shuf },
		26346	lhs_temp_alias,
		26347	lhs_temp_alias,
		26348	lhs_mask_mem,
		26349	) else try self.asmRegisterMemory(
		26350	.{ .p_b, .shuf },
		26351	lhs_temp_alias,
		26352	lhs_mask_mem,
		26353	);
		26354
		26355	var rhs_mask_elems: [16]InternPool.Index = undefined;
		26356	for (rhs_mask_elems[0..max_abi_size], 0..) \|*rhs_mask_elem, byte_index\| {
		26357	const elem_index = byte_index / elem_abi_size;
		26358	rhs_mask_elem.* = try pt.intern(.{ .int = .{
		26359	.ty = .u8_type,
		26360	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
		26361	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
		26362	if (mask_elem >= 0) break :elem 0b1_00_00000;
		26363	const mask_elem_index: u31 = @intCast(~mask_elem);
		26364	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
		26365	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
		26366	} },
		26367	} });
		26368	}
		26369	const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
		26370	const rhs_mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
		26371	.ty = rhs_mask_ty.toIntern(),
		26372	.storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
		26373	} })));
		26374	const rhs_mask_mem: Memory = .{
		26375	.base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) },
		26376	.mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
		26377	};
		26378	if (has_avx) try self.asmRegisterRegisterMemory(
		26379	.{ .vp_b, .shuf },
		26380	rhs_temp_alias,
		26381	rhs_temp_alias,
		26382	rhs_mask_mem,
		26383	) else try self.asmRegisterMemory(
		26384	.{ .p_b, .shuf },
		26385	rhs_temp_alias,
		26386	rhs_mask_mem,
		26387	);
		26388
		26389	if (has_avx) try self.asmRegisterRegisterRegister(
		26390	.{ switch (elem_ty.zigTypeTag(zcu)) {
		26391	else => break :result null,
		26392	.int => .vp_,
		26393	.float => switch (elem_ty.floatBits(self.target.*)) {
		26394	32 => .v_ps,
		26395	64 => .v_pd,
		26396	16, 80, 128 => break :result null,
		26397	else => unreachable,
		26398	},
		26399	}, .@"or" },
		26400	lhs_temp_alias,
		26401	lhs_temp_alias,
		26402	rhs_temp_alias,
		26403	) else try self.asmRegisterRegister(
		26404	.{ switch (elem_ty.zigTypeTag(zcu)) {
		26405	else => break :result null,
		26406	.int => .p_,
		26407	.float => switch (elem_ty.floatBits(self.target.*)) {
		26408	32 => ._ps,
		26409	64 => ._pd,
		26410	16, 80, 128 => break :result null,
		26411	else => unreachable,
		26412	},
		26413	}, .@"or" },
		26414	lhs_temp_alias,
		26415	rhs_temp_alias,
		26416	);
		26417	break :result .{ .register = temp_regs[0] };
		26418	}
		26419
		26420	break :result null;
		26421	}) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
		26422	lhs_ty.fmt(pt),
		26423	rhs_ty.fmt(pt),
		26424	dst_ty.fmt(pt),
		26425	Value.fromInterned(extra.mask).fmtValue(pt),
		26426	});
		26427	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
		26428	}
		26429
		26430	fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void {
		26431	const pt = self.pt;
		26432	const zcu = pt.zcu;
		26433	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
		26434
		26435	const result: MCValue = result: {
		26436	const operand_ty = self.typeOf(reduce.operand);
		26437	if (operand_ty.isVector(zcu) and operand_ty.childType(zcu).toIntern() == .bool_type) {
		26438	try self.spillEflagsIfOccupied();
		26439
		26440	const abi_size: u32 = @intCast(operand_ty.abiSize(zcu));
		26441	const operand_mcv = try self.resolveInst(reduce.operand);
		26442	const mask_len = operand_ty.vectorLen(zcu);
		26443	const mask_len_minus_one = (std.math.cast(u6, mask_len - 1) orelse {
		26444	const acc_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
		26445	const acc_lock = self.register_manager.lockRegAssumeUnused(acc_reg);
		26446	defer self.register_manager.unlockReg(acc_lock);
		26447	var limb_offset: i31 = 0;
		26448	while (limb_offset < abi_size) : (limb_offset += 8) {
		26449	try self.asmRegisterMemory(
		26450	.{ ._, if (limb_offset == 0) .mov else switch (reduce.operation) {
		26451	.Or => .@"or",
		26452	.And => .@"and",
		26453	else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}),
		26454	} },
		26455	acc_reg.to64(),
		26456	try operand_mcv.mem(self, .{
		26457	.size = .qword,
		26458	.disp = limb_offset,
		26459	}),
		26460	);
		26461	}
		26462	switch (reduce.operation) {
		26463	.Or => {
		26464	try self.asmRegisterRegister(.{ ._, .@"test" }, acc_reg.to64(), acc_reg.to64());
		26465	break :result .{ .eflags = .nz };
		26466	},
		26467	.And => {
		26468	try self.asmRegisterImmediate(.{ ._, .cmp }, acc_reg.to64(), .s(-1));
		26469	break :result .{ .eflags = .z };
		26470	},
18101	else => unreachable,	26471	else => unreachable,
		26472	}
		26473	});
		26474	const mask = @as(u64, std.math.maxInt(u64)) >> ~mask_len_minus_one;
		26475	switch (reduce.operation) {
		26476	.Or => {
		26477	if (operand_mcv.isBase()) try self.asmMemoryImmediate(
		26478	.{ ._, .@"test" },
		26479	try operand_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
		26480	if (mask_len < abi_size * 8)
		26481	.u(mask)
		26482	else
		26483	.s(-1),
		26484	) else {
		26485	const operand_reg = registerAlias(operand_reg: {
		26486	if (operand_mcv.isRegister()) {
		26487	const operand_reg = operand_mcv.getReg().?;
		26488	if (operand_reg.class() == .general_purpose) break :operand_reg operand_reg;
		26489	}
		26490	break :operand_reg try self.copyToTmpRegister(operand_ty, operand_mcv);
		26491	}, abi_size);
		26492	const operand_lock = self.register_manager.lockReg(operand_reg);
		26493	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
		26494
		26495	if (mask_len < abi_size * 8) try self.asmRegisterImmediate(
		26496	.{ ._, .@"test" },
		26497	operand_reg,
		26498	.u(mask),
		26499	) else try self.asmRegisterRegister(
		26500	.{ ._, .@"test" },
		26501	operand_reg,
		26502	operand_reg,
		26503	);
		26504	}
		26505	break :result .{ .eflags = .nz };
18102	},	26506	},
18103	registerAlias(dst_reg, dst_abi_size),	26507	.And => {
18104	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	26508	const tmp_reg = registerAlias(
18105	Immediate.u(expanded_control),	26509	try self.copyToTmpRegister(operand_ty, operand_mcv),
18106	) else try self.asmRegisterRegisterImmediate(	26510	abi_size,
18107	switch (elem_abi_size) {	26511	);
18108	4 => .{ ._ps, .blend },	26512	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
18109	8, 16 => .{ ._pd, .blend },	26513	defer self.register_manager.unlockReg(tmp_lock);
18110	else => unreachable,	26514
		26515	try self.asmRegister(.{ ._, .not }, tmp_reg);
		26516	if (mask_len < abi_size * 8)
		26517	try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, .u(mask))
		26518	else
		26519	try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg);
		26520	break :result .{ .eflags = .z };
18111	},	26521	},
18112	registerAlias(dst_reg, dst_abi_size),	26522	else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}),
18113	registerAlias(if (rhs_mcv.isRegister())	26523	}
18114	rhs_mcv.getReg().?
18115	else
18116	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
18117	Immediate.u(expanded_control),
18118	);
18119	break :result .{ .register = dst_reg };
18120	}	26524	}
		26525	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)});
		26526	};
		26527	return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
		26528	}
18121		26529
18122	blendv: {	26530	fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
18123	if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)	26531	const pt = self.pt;
18124	has_avx	26532	const zcu = pt.zcu;
18125	else	26533	const result_ty = self.typeOfIndex(inst);
18126	self.hasFeature(.avx2)) 32 else 16)) break :blendv;	26534	const len: usize = @intCast(result_ty.arrayLen(zcu));
		26535	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
		26536	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
		26537	const result: MCValue = result: {
		26538	switch (result_ty.zigTypeTag(zcu)) {
		26539	.@"struct" => {
		26540	const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
		26541	if (result_ty.containerLayout(zcu) == .@"packed") {
		26542	const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern());
		26543	try self.genInlineMemset(
		26544	.{ .lea_frame = .{ .index = frame_index } },
		26545	.{ .immediate = 0 },
		26546	.{ .immediate = result_ty.abiSize(zcu) },
		26547	.{},
		26548	);
		26549	for (elements, 0..) \|elem, elem_i_usize\| {
		26550	const elem_i: u32 = @intCast(elem_i_usize);
		26551	if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
18127		26552
18128	const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8);	26553	const elem_ty = result_ty.fieldType(elem_i, zcu);
18129	const select_mask_ty = try pt.vectorType(.{	26554	const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
18130	.len = @intCast(mask_elems.len),	26555	if (elem_bit_size > 64) {
18131	.child = select_mask_elem_ty.toIntern(),	26556	return self.fail(
18132	});	26557	"TODO airAggregateInit implement packed structs with large fields",
18133	var select_mask_elems: [32]InternPool.Index = undefined;	26558	.{},
18134	for (	26559	);
18135	select_mask_elems[0..mask_elems.len],	26560	}
18136	mask_elems,	26561	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
18137	0..,	26562	const elem_abi_bits = elem_abi_size * 8;
18138	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {	26563	const elem_off = pt.structPackedFieldBitOffset(loaded_struct, elem_i);
18139	const mask_elem = maybe_mask_elem orelse continue;	26564	const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
18140	const mask_elem_index =	26565	const elem_bit_off = elem_off % elem_abi_bits;
18141	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;	26566	const elem_mcv = try self.resolveInst(elem);
18142	if (mask_elem_index != elem_index) break :blendv;	26567	const mat_elem_mcv = switch (elem_mcv) {
		26568	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
		26569	else => elem_mcv,
		26570	};
		26571	const elem_lock = switch (mat_elem_mcv) {
		26572	.register => \|reg\| self.register_manager.lockReg(reg),
		26573	.immediate => \|imm\| lock: {
		26574	if (imm == 0) continue;
		26575	break :lock null;
		26576	},
		26577	else => null,
		26578	};
		26579	defer if (elem_lock) \|lock\| self.register_manager.unlockReg(lock);
18143		26580
18144	select_mask_elem.* = (if (mask_elem < 0)	26581	const elem_extra_bits = self.regExtraBits(elem_ty);
18145	try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty)	26582	{
18146	else	26583	const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);
18147	try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();	26584	const temp_alias = registerAlias(temp_reg, elem_abi_size);
18148	}	26585	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
18149	const select_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{	26586	defer self.register_manager.unlockReg(temp_lock);
18150	.ty = select_mask_ty.toIntern(),
18151	.storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
18152	} })));
18153		26587
18154	if (self.hasFeature(.sse4_1)) {	26588	if (elem_bit_off < elem_extra_bits) {
18155	const mir_tag: Mir.Inst.FixedTag = .{	26589	try self.truncateRegister(elem_ty, temp_alias);
18156	if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or	26590	}
18157	(dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {	26591	if (elem_bit_off > 0) try self.genShiftBinOpMir(
18158	4 => if (has_avx) .v_ps else ._ps,	26592	.{ ._l, .sh },
18159	8 => if (has_avx) .v_pd else ._pd,	26593	elem_ty,
18160	else => unreachable,	26594	.{ .register = temp_alias },
18161	} else if (has_avx) .vp_b else .p_b,	26595	.u8,
18162	.blendv,	26596	.{ .immediate = elem_bit_off },
18163	};	26597	);
		26598	try self.genBinOpMir(
		26599	.{ ._, .@"or" },
		26600	elem_ty,
		26601	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
		26602	.{ .register = temp_alias },
		26603	);
		26604	}
		26605	if (elem_bit_off > elem_extra_bits) {
		26606	const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);
		26607	const temp_alias = registerAlias(temp_reg, elem_abi_size);
		26608	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
		26609	defer self.register_manager.unlockReg(temp_lock);
18164		26610
18165	const select_mask_reg = if (!has_avx) reg: {	26611	if (elem_extra_bits > 0) {
18166	try self.register_manager.getKnownReg(.xmm0, null);	26612	try self.truncateRegister(elem_ty, temp_alias);
18167	try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});	26613	}
18168	break :reg .xmm0;	26614	try self.genShiftBinOpMir(
18169	} else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);	26615	.{ ._r, .sh },
18170	const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);	26616	elem_ty,
18171	const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);	26617	.{ .register = temp_reg },
18172	defer self.register_manager.unlockReg(select_mask_lock);	26618	.u8,
		26619	.{ .immediate = elem_abi_bits - elem_bit_off },
		26620	);
		26621	try self.genBinOpMir(
		26622	.{ ._, .@"or" },
		26623	elem_ty,
		26624	.{ .load_frame = .{
		26625	.index = frame_index,
		26626	.off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
		26627	} },
		26628	.{ .register = temp_alias },
		26629	);
		26630	}
		26631	}
		26632	} else for (elements, 0..) \|elem, elem_i\| {
		26633	if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
18173		26634
18174	const lhs_mcv = try self.resolveInst(extra.a);	26635	const elem_ty = result_ty.fieldType(elem_i, zcu);
18175	const rhs_mcv = try self.resolveInst(extra.b);	26636	const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu));
		26637	const elem_mcv = try self.resolveInst(elem);
		26638	const mat_elem_mcv = switch (elem_mcv) {
		26639	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
		26640	else => elem_mcv,
		26641	};
		26642	try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv, .{});
		26643	}
		26644	break :result .{ .load_frame = .{ .index = frame_index } };
		26645	},
		26646	.array, .vector => {
		26647	const elem_ty = result_ty.childType(zcu);
		26648	if (result_ty.isVector(zcu) and elem_ty.toIntern() == .bool_type) {
		26649	const result_size: u32 = @intCast(result_ty.abiSize(zcu));
		26650	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
		26651	try self.asmRegisterRegister(
		26652	.{ ._, .xor },
		26653	registerAlias(dst_reg, @min(result_size, 4)),
		26654	registerAlias(dst_reg, @min(result_size, 4)),
		26655	);
18176		26656
18177	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and	26657	for (elements, 0..) \|elem, elem_i\| {
18178	self.reuseOperand(inst, extra.a, 0, lhs_mcv))	26658	const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });
18179	lhs_mcv	26659	const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
18180	else if (has_avx and lhs_mcv.isRegister())	26660	defer self.register_manager.unlockReg(elem_lock);
18181	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
18182	else
18183	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
18184	const dst_reg = dst_mcv.getReg().?;
18185	const dst_alias = registerAlias(dst_reg, dst_abi_size);
18186		26661
18187	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(	26662	try self.asmRegisterImmediate(
18188	mir_tag,	26663	.{ ._, .@"and" },
18189	dst_alias,	26664	registerAlias(elem_reg, @min(result_size, 4)),
18190	if (lhs_mcv.isRegister())	26665	.u(1),
18191	registerAlias(lhs_mcv.getReg().?, dst_abi_size)	26666	);
18192	else	26667	if (elem_i > 0) try self.asmRegisterImmediate(
18193	dst_alias,	26668	.{ ._l, .sh },
18194	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	26669	registerAlias(elem_reg, result_size),
18195	select_mask_alias,	26670	.u(@intCast(elem_i)),
18196	) else try self.asmRegisterRegisterRegisterRegister(	26671	);
18197	mir_tag,	26672	try self.asmRegisterRegister(
18198	dst_alias,	26673	.{ ._, .@"or" },
18199	if (lhs_mcv.isRegister())	26674	registerAlias(dst_reg, result_size),
18200	registerAlias(lhs_mcv.getReg().?, dst_abi_size)	26675	registerAlias(elem_reg, result_size),
18201	else	26676	);
18202	dst_alias,	26677	}
18203	registerAlias(if (rhs_mcv.isRegister())	26678	break :result .{ .register = dst_reg };
18204	rhs_mcv.getReg().?	26679	} else {
18205	else	26680	const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
18206	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	26681	const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
18207	select_mask_alias,	26682
18208	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(	26683	for (elements, 0..) \|elem, elem_i\| {
18209	mir_tag,	26684	const elem_mcv = try self.resolveInst(elem);
18210	dst_alias,	26685	const mat_elem_mcv = switch (elem_mcv) {
18211	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),	26686	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
18212	select_mask_alias,	26687	else => elem_mcv,
18213	) else try self.asmRegisterRegisterRegister(	26688	};
18214	mir_tag,	26689	const elem_off: i32 = @intCast(elem_size * elem_i);
18215	dst_alias,	26690	try self.genSetMem(
18216	registerAlias(if (rhs_mcv.isRegister())	26691	.{ .frame = frame_index },
18217	rhs_mcv.getReg().?	26692	elem_off,
18218	else	26693	elem_ty,
18219	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),	26694	mat_elem_mcv,
18220	select_mask_alias,	26695	.{},
18221	);	26696	);
18222	break :result dst_mcv;	26697	}
18223	}	26698	if (result_ty.sentinel(zcu)) \|sentinel\| try self.genSetMem(
		26699	.{ .frame = frame_index },
		26700	@intCast(elem_size * elements.len),
		26701	elem_ty,
		26702	try self.genTypedValue(sentinel),
		26703	.{},
		26704	);
		26705	break :result .{ .load_frame = .{ .index = frame_index } };
		26706	}
		26707	},
		26708	else => unreachable,
		26709	}
		26710	};
18224		26711
18225	const lhs_mcv = try self.resolveInst(extra.a);	26712	if (elements.len <= Liveness.bpi - 1) {
18226	const rhs_mcv = try self.resolveInst(extra.b);	26713	var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
		26714	@memcpy(buf[0..elements.len], elements);
		26715	return self.finishAir(inst, result, buf);
		26716	}
		26717	var bt = self.liveness.iterateBigTomb(inst);
		26718	for (elements) \|elem\| try self.feed(&bt, elem);
		26719	return self.finishAirResult(inst, result);
		26720	}
18227		26721
18228	const dst_mcv: MCValue = if (rhs_mcv.isRegister() and	26722	fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void {
18229	self.reuseOperand(inst, extra.b, 1, rhs_mcv))	26723	const pt = self.pt;
18230	rhs_mcv	26724	const zcu = pt.zcu;
18231	else	26725	const ip = &zcu.intern_pool;
18232	try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);	26726	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
18233	const dst_reg = dst_mcv.getReg().?;	26727	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
18234	const dst_alias = registerAlias(dst_reg, dst_abi_size);	26728	const result: MCValue = result: {
		26729	const union_ty = self.typeOfIndex(inst);
		26730	const layout = union_ty.unionGetLayout(zcu);
18235		26731
18236	const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);	26732	const src_ty = self.typeOf(extra.init);
18237	const mask_alias = registerAlias(mask_reg, dst_abi_size);	26733	const src_mcv = try self.resolveInst(extra.init);
18238	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);	26734	if (layout.tag_size == 0) {
18239	defer self.register_manager.unlockReg(mask_lock);	26735	if (layout.abi_size <= src_ty.abiSize(zcu) and
		26736	self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv;
18240		26737
18241	const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())	26738	const dst_mcv = try self.allocRegOrMem(inst, true);
18242	switch (elem_ty.floatBits(self.target.*)) {	26739	try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
18243	16, 80, 128 => .p_,
18244	32 => ._ps,
18245	64 => ._pd,
18246	else => unreachable,
18247	}
18248	else
18249	.p_;
18250	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
18251	if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
18252	.{ mir_fixes, .andn },
18253	mask_alias,
18254	try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
18255	) else try self.asmRegisterRegister(
18256	.{ mir_fixes, .andn },
18257	mask_alias,
18258	if (lhs_mcv.isRegister())
18259	lhs_mcv.getReg().?
18260	else
18261	try self.copyToTmpRegister(dst_ty, lhs_mcv),
18262	);
18263	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
18264	break :result dst_mcv;	26740	break :result dst_mcv;
18265	}	26741	}
18266		26742
18267	pshufb: {	26743	const dst_mcv = try self.allocRegOrMem(inst, false);
18268	if (max_abi_size > 16) break :pshufb;
18269	if (!self.hasFeature(.ssse3)) break :pshufb;
18270		26744
18271	const temp_regs =	26745	const loaded_union = zcu.typeToUnion(union_ty).?;
18272	try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);	26746	const field_name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index];
18273	const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);	26747	const tag_ty: Type = .fromInterned(loaded_union.enum_tag_ty);
18274	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);	26748	const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
		26749	const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);
		26750	const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);
		26751	const tag_int = tag_int_val.toUnsignedInt(zcu);
		26752	const tag_off: i32 = @intCast(layout.tagOffset());
		26753	try self.genCopy(
		26754	tag_ty,
		26755	dst_mcv.address().offset(tag_off).deref(),
		26756	.{ .immediate = tag_int },
		26757	.{},
		26758	);
18275		26759
18276	const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);	26760	const pl_off: i32 = @intCast(layout.payloadOffset());
18277	try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});	26761	try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{});
18278		26762
18279	const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);	26763	break :result dst_mcv;
18280	try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});	26764	};
		26765	return self.finishAir(inst, result, .{ extra.init, .none, .none });
		26766	}
18281		26767
18282	var lhs_mask_elems: [16]InternPool.Index = undefined;	26768	fn airPrefetch(self: *CodeGen, inst: Air.Inst.Index) !void {
18283	for (lhs_mask_elems[0..max_abi_size], 0..) \|*lhs_mask_elem, byte_index\| {	26769	const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
18284	const elem_index = byte_index / elem_abi_size;	26770	return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
18285	lhs_mask_elem.* = try pt.intern(.{ .int = .{	26771	}
18286	.ty = .u8_type,
18287	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
18288	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
18289	if (mask_elem < 0) break :elem 0b1_00_00000;
18290	const mask_elem_index: u31 = @intCast(mask_elem);
18291	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
18292	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
18293	} },
18294	} });
18295	}
18296	const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
18297	const lhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{
18298	.ty = lhs_mask_ty.toIntern(),
18299	.storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
18300	} })));
18301	const lhs_mask_mem: Memory = .{
18302	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
18303	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
18304	};
18305	if (has_avx) try self.asmRegisterRegisterMemory(
18306	.{ .vp_b, .shuf },
18307	lhs_temp_alias,
18308	lhs_temp_alias,
18309	lhs_mask_mem,
18310	) else try self.asmRegisterMemory(
18311	.{ .p_b, .shuf },
18312	lhs_temp_alias,
18313	lhs_mask_mem,
18314	);
18315		26772
18316	var rhs_mask_elems: [16]InternPool.Index = undefined;	26773	fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void {
18317	for (rhs_mask_elems[0..max_abi_size], 0..) \|*rhs_mask_elem, byte_index\| {	26774	const pt = self.pt;
18318	const elem_index = byte_index / elem_abi_size;	26775	const zcu = pt.zcu;
18319	rhs_mask_elem.* = try pt.intern(.{ .int = .{	26776	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
18320	.ty = .u8_type,	26777	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
18321	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {	26778	const ty = self.typeOfIndex(inst);
18322	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;	26779
18323	if (mask_elem >= 0) break :elem 0b1_00_00000;	26780	const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
18324	const mask_elem_index: u31 = @intCast(~mask_elem);	26781	const result = result: {
18325	const byte_off: u32 = @intCast(byte_index % elem_abi_size);	26782	if (switch (ty.scalarType(zcu).floatBits(self.target.*)) {
18326	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);	26783	16, 80, 128 => true,
18327	} },	26784	32, 64 => !self.hasFeature(.fma),
18328	} });	26785	else => unreachable,
		26786	}) {
		26787	if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement airMulAdd for {}", .{
		26788	ty.fmt(pt),
		26789	});
		26790
		26791	var callee_buf: ["__fma?".len]u8 = undefined;
		26792	break :result try self.genCall(.{ .lib = .{
		26793	.return_type = ty.toIntern(),
		26794	.param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() },
		26795	.callee = std.fmt.bufPrint(&callee_buf, "{s}fma{s}", .{
		26796	floatLibcAbiPrefix(ty),
		26797	floatLibcAbiSuffix(ty),
		26798	}) catch unreachable,
		26799	} }, &.{ ty, ty, ty }, &.{
		26800	.{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand },
		26801	}, .{});
		26802	}
		26803
		26804	var mcvs: [3]MCValue = undefined;
		26805	var locks: [3]?RegisterManager.RegisterLock = @splat(null);
		26806	defer for (locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
		26807	var order: [3]u2 = @splat(0);
		26808	var unused: std.StaticBitSet(3) = .initFull();
		26809	for (ops, &mcvs, &locks, 0..) \|op, mcv, lock, op_i\| {
		26810	const op_index: u2 = @intCast(op_i);
		26811	mcv.* = try self.resolveInst(op);
		26812	if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
		26813	order[op_index] = 1;
		26814	unused.unset(0);
		26815	} else if (unused.isSet(2) and mcv.isBase()) {
		26816	order[op_index] = 3;
		26817	unused.unset(2);
18329	}	26818	}
18330	const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });	26819	switch (mcv.*) {
18331	const rhs_mask_mcv = try self.genTypedValue(Value.fromInterned(try pt.intern(.{ .aggregate = .{	26820	.register => \|reg\| lock.* = self.register_manager.lockReg(reg),
18332	.ty = rhs_mask_ty.toIntern(),	26821	else => {},
18333	.storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },	26822	}
18334	} })));	26823	}
18335	const rhs_mask_mem: Memory = .{	26824	for (&order, &mcvs, &locks) \|mop_index, mcv, *lock\| {
18336	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },	26825	if (mop_index.* != 0) continue;
18337	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },	26826	mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?));
18338	};	26827	if (mop_index.* > 1 and mcv.isRegister()) continue;
18339	if (has_avx) try self.asmRegisterRegisterMemory(	26828	const reg = try self.copyToTmpRegister(ty, mcv.*);
18340	.{ .vp_b, .shuf },	26829	mcv.* = .{ .register = reg };
18341	rhs_temp_alias,	26830	if (lock.*) \|old_lock\| self.register_manager.unlockReg(old_lock);
18342	rhs_temp_alias,	26831	lock.* = self.register_manager.lockRegAssumeUnused(reg);
18343	rhs_mask_mem,	26832	}
18344	) else try self.asmRegisterMemory(
18345	.{ .p_b, .shuf },
18346	rhs_temp_alias,
18347	rhs_mask_mem,
18348	);
18349		26833
18350	if (has_avx) try self.asmRegisterRegisterRegister(	26834	const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or
18351	.{ switch (elem_ty.zigTypeTag(zcu)) {	26835	std.mem.eql(u2, &order, &.{ 3, 1, 2 }))
18352	else => break :result null,	26836	switch (ty.zigTypeTag(zcu)) {
18353	.int => .vp_,	26837	.float => switch (ty.floatBits(self.target.*)) {
18354	.float => switch (elem_ty.floatBits(self.target.*)) {	26838	32 => .{ .v_ss, .fmadd132 },
18355	32 => .v_ps,	26839	64 => .{ .v_sd, .fmadd132 },
18356	64 => .v_pd,	26840	16, 80, 128 => null,
18357	16, 80, 128 => break :result null,	26841	else => unreachable,
		26842	},
		26843	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
		26844	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {
		26845	32 => switch (ty.vectorLen(zcu)) {
		26846	1 => .{ .v_ss, .fmadd132 },
		26847	2...8 => .{ .v_ps, .fmadd132 },
		26848	else => null,
		26849	},
		26850	64 => switch (ty.vectorLen(zcu)) {
		26851	1 => .{ .v_sd, .fmadd132 },
		26852	2...4 => .{ .v_pd, .fmadd132 },
		26853	else => null,
		26854	},
		26855	16, 80, 128 => null,
		26856	else => unreachable,
		26857	},
		26858	else => unreachable,
		26859	},
		26860	else => unreachable,
		26861	}
		26862	else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 }))
		26863	switch (ty.zigTypeTag(zcu)) {
		26864	.float => switch (ty.floatBits(self.target.*)) {
		26865	32 => .{ .v_ss, .fmadd213 },
		26866	64 => .{ .v_sd, .fmadd213 },
		26867	16, 80, 128 => null,
		26868	else => unreachable,
		26869	},
		26870	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
		26871	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {
		26872	32 => switch (ty.vectorLen(zcu)) {
		26873	1 => .{ .v_ss, .fmadd213 },
		26874	2...8 => .{ .v_ps, .fmadd213 },
		26875	else => null,
		26876	},
		26877	64 => switch (ty.vectorLen(zcu)) {
		26878	1 => .{ .v_sd, .fmadd213 },
		26879	2...4 => .{ .v_pd, .fmadd213 },
		26880	else => null,
		26881	},
		26882	16, 80, 128 => null,
18358	else => unreachable,	26883	else => unreachable,
18359	},	26884	},
18360	}, .@"or" },	26885	else => unreachable,
18361	lhs_temp_alias,	26886	},
18362	lhs_temp_alias,	26887	else => unreachable,
18363	rhs_temp_alias,	26888	}
18364	) else try self.asmRegisterRegister(	26889	else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 }))
18365	.{ switch (elem_ty.zigTypeTag(zcu)) {	26890	switch (ty.zigTypeTag(zcu)) {
18366	else => break :result null,	26891	.float => switch (ty.floatBits(self.target.*)) {
18367	.int => .p_,	26892	32 => .{ .v_ss, .fmadd231 },
18368	.float => switch (elem_ty.floatBits(self.target.*)) {	26893	64 => .{ .v_sd, .fmadd231 },
18369	32 => ._ps,	26894	16, 80, 128 => null,
18370	64 => ._pd,	26895	else => unreachable,
18371	16, 80, 128 => break :result null,	26896	},
		26897	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
		26898	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {
		26899	32 => switch (ty.vectorLen(zcu)) {
		26900	1 => .{ .v_ss, .fmadd231 },
		26901	2...8 => .{ .v_ps, .fmadd231 },
		26902	else => null,
		26903	},
		26904	64 => switch (ty.vectorLen(zcu)) {
		26905	1 => .{ .v_sd, .fmadd231 },
		26906	2...4 => .{ .v_pd, .fmadd231 },
		26907	else => null,
		26908	},
		26909	16, 80, 128 => null,
18372	else => unreachable,	26910	else => unreachable,
18373	},	26911	},
18374	}, .@"or" },	26912	else => unreachable,
18375	lhs_temp_alias,	26913	},
18376	rhs_temp_alias,	26914	else => unreachable,
18377	);	26915	}
18378	break :result .{ .register = temp_regs[0] };	26916	else
18379	}	26917	unreachable) orelse return self.fail("TODO implement airMulAdd for {}", .{ty.fmt(pt)});
18380		26918
18381	break :result null;	26919	var mops: [3]MCValue = undefined;
18382	}) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{	26920	for (order, mcvs) \|mop_index, mcv\| mops[mop_index - 1] = mcv;
18383	lhs_ty.fmt(pt), rhs_ty.fmt(pt), dst_ty.fmt(pt),	26921
18384	Value.fromInterned(extra.mask).fmtValue(pt),	26922	const abi_size: u32 = @intCast(ty.abiSize(zcu));
18385	});	26923	const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
18386	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });	26924	const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
		26925	if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
		26926	mir_tag,
		26927	mop1_reg,
		26928	mop2_reg,
		26929	registerAlias(mops[2].getReg().?, abi_size),
		26930	) else try self.asmRegisterRegisterMemory(
		26931	mir_tag,
		26932	mop1_reg,
		26933	mop2_reg,
		26934	try mops[2].mem(self, .{ .size = .fromSize(abi_size) }),
		26935	);
		26936	break :result mops[0];
		26937	};
		26938	return self.finishAir(inst, result, ops);
18387	}	26939	}
18388		26940
18389	fn airReduce(self: *Self, inst: Air.Inst.Index) !void {	26941	fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
18390	const pt = self.pt;	26942	const pt = self.pt;
18391	const zcu = pt.zcu;	26943	const zcu = pt.zcu;
18392	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;	26944	const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;
		26945	const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque);
18393		26946
18394	const result: MCValue = result: {	26947	const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) {
18395	const operand_ty = self.typeOf(reduce.operand);	26948	.x86_64_sysv => result: {
18396	if (operand_ty.isVector(zcu) and operand_ty.childType(zcu).toIntern() == .bool_type) {	26949	const info = self.va_info.sysv;
		26950	const dst_fi = try self.allocFrameIndex(.initSpill(va_list_ty, zcu));
		26951	var field_off: u31 = 0;
		26952	// gp_offset: c_uint,
		26953	try self.genSetMem(
		26954	.{ .frame = dst_fi },
		26955	field_off,
		26956	.c_uint,
		26957	.{ .immediate = info.gp_count * 8 },
		26958	.{},
		26959	);
		26960	field_off += @intCast(Type.c_uint.abiSize(zcu));
		26961	// fp_offset: c_uint,
		26962	try self.genSetMem(
		26963	.{ .frame = dst_fi },
		26964	field_off,
		26965	.c_uint,
		26966	.{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 },
		26967	.{},
		26968	);
		26969	field_off += @intCast(Type.c_uint.abiSize(zcu));
		26970	// overflow_arg_area: *anyopaque,
		26971	try self.genSetMem(
		26972	.{ .frame = dst_fi },
		26973	field_off,
		26974	ptr_anyopaque_ty,
		26975	.{ .lea_frame = info.overflow_arg_area },
		26976	.{},
		26977	);
		26978	field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
		26979	// reg_save_area: *anyopaque,
		26980	try self.genSetMem(
		26981	.{ .frame = dst_fi },
		26982	field_off,
		26983	ptr_anyopaque_ty,
		26984	.{ .lea_frame = info.reg_save_area },
		26985	.{},
		26986	);
		26987	field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
		26988	break :result .{ .load_frame = .{ .index = dst_fi } };
		26989	},
		26990	.x86_64_win => return self.fail("TODO implement c_va_start for Win64", .{}),
		26991	else => \|cc\| return self.fail("{s} does not support var args", .{@tagName(cc)}),
		26992	};
		26993	return self.finishAir(inst, result, .{ .none, .none, .none });
		26994	}
		26995
		26996	fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
		26997	const pt = self.pt;
		26998	const zcu = pt.zcu;
		26999	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
		27000	const ty = self.typeOfIndex(inst);
		27001	const promote_ty = self.promoteVarArg(ty);
		27002	const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque);
		27003	const unused = self.liveness.isUnused(inst);
		27004
		27005	const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) {
		27006	.x86_64_sysv => result: {
18397	try self.spillEflagsIfOccupied();	27007	try self.spillEflagsIfOccupied();
18398		27008
18399	const operand_mcv = try self.resolveInst(reduce.operand);	27009	const tmp_regs =
18400	const mask_len = (math.cast(u6, operand_ty.vectorLen(zcu)) orelse	27010	try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
18401	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}));	27011	const offset_reg = tmp_regs[0].to32();
18402	const mask = (@as(u64, 1) << mask_len) - 1;	27012	const addr_reg = tmp_regs[1].to64();
18403	const abi_size: u32 = @intCast(operand_ty.abiSize(zcu));	27013	const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
18404	switch (reduce.operation) {	27014	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);
18405	.Or => {	27015
18406	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(	27016	const promote_mcv = try self.allocTempRegOrMem(promote_ty, true);
18407	.{ ._, .@"test" },	27017	const promote_lock = switch (promote_mcv) {
18408	try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),	27018	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
18409	Immediate.u(mask),	27019	else => null,
18410	) else {	27020	};
18411	const operand_reg = registerAlias(if (operand_mcv.isRegister())	27021	defer if (promote_lock) \|lock\| self.register_manager.unlockReg(lock);
18412	operand_mcv.getReg().?	27022
18413	else	27023	const ptr_arg_list_reg =
18414	try self.copyToTmpRegister(operand_ty, operand_mcv), abi_size);	27024	try self.copyToTmpRegister(self.typeOf(ty_op.operand), .{ .air_ref = ty_op.operand });
18415	if (mask_len < abi_size * 8) try self.asmRegisterImmediate(	27025	const ptr_arg_list_lock = self.register_manager.lockRegAssumeUnused(ptr_arg_list_reg);
18416	.{ ._, .@"test" },	27026	defer self.register_manager.unlockReg(ptr_arg_list_lock);
18417	operand_reg,	27027
18418	Immediate.u(mask),	27028	const gp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 0 } };
18419	) else try self.asmRegisterRegister(	27029	const fp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 4 } };
18420	.{ ._, .@"test" },	27030	const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
18421	operand_reg,	27031	const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
18422	operand_reg,	27032
18423	);	27033	const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
18424	}	27034	switch (classes[0]) {
18425	break :result .{ .eflags = .nz };	27035	.integer => {
		27036	assert(classes.len == 1);
		27037
		27038	try self.genSetReg(offset_reg, .c_uint, gp_offset, .{});
		27039	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
		27040	abi.SysV.c_abi_int_param_regs.len * 8,
		27041	));
		27042	const mem_reloc = try self.asmJccReloc(.ae, undefined);
		27043
		27044	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
		27045	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
		27046	.base = .{ .reg = addr_reg },
		27047	.mod = .{ .rm = .{
		27048	.size = .qword,
		27049	.index = offset_reg.to64(),
		27050	} },
		27051	});
		27052	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
		27053	.base = .{ .reg = offset_reg.to64() },
		27054	.mod = .{ .rm = .{
		27055	.size = .qword,
		27056	.disp = 8,
		27057	} },
		27058	});
		27059	try self.genCopy(.c_uint, gp_offset, .{ .register = offset_reg }, .{});
		27060	const done_reloc = try self.asmJmpReloc(undefined);
		27061
		27062	self.performReloc(mem_reloc);
		27063	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
		27064	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
		27065	.base = .{ .reg = addr_reg },
		27066	.mod = .{ .rm = .{
		27067	.size = .qword,
		27068	.disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),
		27069	} },
		27070	});
		27071	try self.genCopy(
		27072	ptr_anyopaque_ty,
		27073	overflow_arg_area,
		27074	.{ .register = offset_reg.to64() },
		27075	.{},
		27076	);
		27077
		27078	self.performReloc(done_reloc);
		27079	if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
		27080	.indirect = .{ .reg = addr_reg },
		27081	}, .{});
18426	},	27082	},
18427	.And => {	27083	.sse => {
18428	const tmp_reg = try self.copyToTmpRegister(operand_ty, operand_mcv);	27084	assert(classes.len == 1);
18429	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	27085
18430	defer self.register_manager.unlockReg(tmp_lock);	27086	try self.genSetReg(offset_reg, .c_uint, fp_offset, .{});
		27087	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
		27088	abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
		27089	));
		27090	const mem_reloc = try self.asmJccReloc(.ae, undefined);
		27091
		27092	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
		27093	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
		27094	.base = .{ .reg = addr_reg },
		27095	.mod = .{ .rm = .{
		27096	.size = .qword,
		27097	.index = offset_reg.to64(),
		27098	} },
		27099	});
		27100	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
		27101	.base = .{ .reg = offset_reg.to64() },
		27102	.mod = .{ .rm = .{
		27103	.size = .qword,
		27104	.disp = 16,
		27105	} },
		27106	});
		27107	try self.genCopy(.c_uint, fp_offset, .{ .register = offset_reg }, .{});
		27108	const done_reloc = try self.asmJmpReloc(undefined);
18431		27109
18432	try self.asmRegister(.{ ._, .not }, tmp_reg);	27110	self.performReloc(mem_reloc);
18433	if (mask_len < abi_size * 8)	27111	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
18434	try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, Immediate.u(mask))	27112	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
18435	else	27113	.base = .{ .reg = addr_reg },
18436	try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg);	27114	.mod = .{ .rm = .{
18437	break :result .{ .eflags = .z };	27115	.size = .qword,
		27116	.disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),
		27117	} },
		27118	});
		27119	try self.genCopy(
		27120	ptr_anyopaque_ty,
		27121	overflow_arg_area,
		27122	.{ .register = offset_reg.to64() },
		27123	.{},
		27124	);
		27125
		27126	self.performReloc(done_reloc);
		27127	if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
		27128	.indirect = .{ .reg = addr_reg },
		27129	}, .{});
18438	},	27130	},
18439	else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}),	27131	.memory => {
		27132	assert(classes.len == 1);
		27133	unreachable;
		27134	},
		27135	else => return self.fail("TODO implement c_va_arg for {} on SysV", .{promote_ty.fmt(pt)}),
18440	}	27136	}
18441	}
18442	return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)});
18443	};
18444	return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
18445	}
18446		27137
18447	fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {	27138	if (unused) break :result .unreach;
18448	const pt = self.pt;	27139	if (ty.toIntern() == promote_ty.toIntern()) break :result promote_mcv;
18449	const zcu = pt.zcu;
18450	const result_ty = self.typeOfIndex(inst);
18451	const len: usize = @intCast(result_ty.arrayLen(zcu));
18452	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
18453	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
18454	const result: MCValue = result: {
18455	switch (result_ty.zigTypeTag(zcu)) {
18456	.@"struct" => {
18457	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
18458	if (result_ty.containerLayout(zcu) == .@"packed") {
18459	const struct_obj = zcu.typeToStruct(result_ty).?;
18460	try self.genInlineMemset(
18461	.{ .lea_frame = .{ .index = frame_index } },
18462	.{ .immediate = 0 },
18463	.{ .immediate = result_ty.abiSize(zcu) },
18464	.{},
18465	);
18466	for (elements, 0..) \|elem, elem_i_usize\| {
18467	const elem_i: u32 = @intCast(elem_i_usize);
18468	if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
18469		27140
18470	const elem_ty = result_ty.fieldType(elem_i, zcu);	27141	if (!promote_ty.isRuntimeFloat()) {
18471	const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));	27142	const dst_mcv = try self.allocRegOrMem(inst, true);
18472	if (elem_bit_size > 64) {	27143	try self.genCopy(ty, dst_mcv, promote_mcv, .{});
18473	return self.fail(	27144	break :result dst_mcv;
18474	"TODO airAggregateInit implement packed structs with large fields",	27145	}
18475	.{},
18476	);
18477	}
18478	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
18479	const elem_abi_bits = elem_abi_size * 8;
18480	const elem_off = pt.structPackedFieldBitOffset(struct_obj, elem_i);
18481	const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
18482	const elem_bit_off = elem_off % elem_abi_bits;
18483	const elem_mcv = try self.resolveInst(elem);
18484	const mat_elem_mcv = switch (elem_mcv) {
18485	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
18486	else => elem_mcv,
18487	};
18488	const elem_lock = switch (mat_elem_mcv) {
18489	.register => \|reg\| self.register_manager.lockReg(reg),
18490	.immediate => \|imm\| lock: {
18491	if (imm == 0) continue;
18492	break :lock null;
18493	},
18494	else => null,
18495	};
18496	defer if (elem_lock) \|lock\| self.register_manager.unlockReg(lock);
18497		27146
18498	const elem_extra_bits = self.regExtraBits(elem_ty);	27147	assert(ty.toIntern() == .f32_type and promote_ty.toIntern() == .f64_type);
18499	{	27148	const dst_mcv = if (promote_mcv.isRegister())
18500	const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);	27149	promote_mcv
18501	const temp_alias = registerAlias(temp_reg, elem_abi_size);	27150	else
18502	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);	27151	try self.copyToRegisterWithInstTracking(inst, ty, promote_mcv);
18503	defer self.register_manager.unlockReg(temp_lock);	27152	const dst_reg = dst_mcv.getReg().?.to128();
		27153	const dst_lock = self.register_manager.lockReg(dst_reg);
		27154	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
18504		27155
18505	if (elem_bit_off < elem_extra_bits) {	27156	if (self.hasFeature(.avx)) if (promote_mcv.isBase()) try self.asmRegisterRegisterMemory(
18506	try self.truncateRegister(elem_ty, temp_alias);	27157	.{ .v_ss, .cvtsd2 },
18507	}	27158	dst_reg,
18508	if (elem_bit_off > 0) try self.genShiftBinOpMir(	27159	dst_reg,
18509	.{ ._l, .sh },	27160	try promote_mcv.mem(self, .{ .size = .qword }),
18510	elem_ty,	27161	) else try self.asmRegisterRegisterRegister(
18511	.{ .register = temp_alias },	27162	.{ .v_ss, .cvtsd2 },
18512	Type.u8,	27163	dst_reg,
18513	.{ .immediate = elem_bit_off },	27164	dst_reg,
18514	);	27165	(if (promote_mcv.isRegister())
18515	try self.genBinOpMir(	27166	promote_mcv.getReg().?
18516	.{ ._, .@"or" },	27167	else
18517	elem_ty,	27168	try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
18518	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },	27169	) else if (promote_mcv.isBase()) try self.asmRegisterMemory(
18519	.{ .register = temp_alias },	27170	.{ ._ss, .cvtsd2 },
18520	);	27171	dst_reg,
18521	}	27172	try promote_mcv.mem(self, .{ .size = .qword }),
18522	if (elem_bit_off > elem_extra_bits) {	27173	) else try self.asmRegisterRegister(
18523	const temp_reg = try self.copyToTmpRegister(elem_ty, mat_elem_mcv);	27174	.{ ._ss, .cvtsd2 },
18524	const temp_alias = registerAlias(temp_reg, elem_abi_size);	27175	dst_reg,
18525	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);	27176	(if (promote_mcv.isRegister())
18526	defer self.register_manager.unlockReg(temp_lock);	27177	promote_mcv.getReg().?
		27178	else
		27179	try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
		27180	);
		27181	break :result promote_mcv;
		27182	},
		27183	.x86_64_win => return self.fail("TODO implement c_va_arg for Win64", .{}),
		27184	else => \|cc\| return self.fail("{s} does not support var args", .{@tagName(cc)}),
		27185	};
		27186	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
		27187	}
18527		27188
18528	if (elem_extra_bits > 0) {	27189	fn airVaCopy(self: *CodeGen, inst: Air.Inst.Index) !void {
18529	try self.truncateRegister(elem_ty, temp_alias);	27190	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
18530	}	27191	const ptr_va_list_ty = self.typeOf(ty_op.operand);
18531	try self.genShiftBinOpMir(
18532	.{ ._r, .sh },
18533	elem_ty,
18534	.{ .register = temp_reg },
18535	Type.u8,
18536	.{ .immediate = elem_abi_bits - elem_bit_off },
18537	);
18538	try self.genBinOpMir(
18539	.{ ._, .@"or" },
18540	elem_ty,
18541	.{ .load_frame = .{
18542	.index = frame_index,
18543	.off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
18544	} },
18545	.{ .register = temp_alias },
18546	);
18547	}
18548	}
18549	} else for (elements, 0..) \|elem, elem_i\| {
18550	if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
18551		27192
18552	const elem_ty = result_ty.fieldType(elem_i, zcu);	27193	const dst_mcv = try self.allocRegOrMem(inst, true);
18553	const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu));	27194	try self.load(dst_mcv, ptr_va_list_ty, .{ .air_ref = ty_op.operand });
18554	const elem_mcv = try self.resolveInst(elem);	27195	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
18555	const mat_elem_mcv = switch (elem_mcv) {	27196	}
18556	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },
18557	else => elem_mcv,
18558	};
18559	try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv, .{});
18560	}
18561	break :result .{ .load_frame = .{ .index = frame_index } };
18562	},
18563	.array, .vector => {
18564	const elem_ty = result_ty.childType(zcu);
18565	if (result_ty.isVector(zcu) and elem_ty.toIntern() == .bool_type) {
18566	const result_size: u32 = @intCast(result_ty.abiSize(zcu));
18567	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
18568	try self.asmRegisterRegister(
18569	.{ ._, .xor },
18570	registerAlias(dst_reg, @min(result_size, 4)),
18571	registerAlias(dst_reg, @min(result_size, 4)),
18572	);
18573		27197
18574	for (elements, 0..) \|elem, elem_i\| {	27198	fn airVaEnd(self: *CodeGen, inst: Air.Inst.Index) !void {
18575	const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });	27199	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
18576	const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);	27200	return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
18577	defer self.register_manager.unlockReg(elem_lock);	27201	}
18578		27202
18579	try self.asmRegisterImmediate(	27203	fn resolveInst(self: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue {
18580	.{ ._, .@"and" },	27204	const zcu = self.pt.zcu;
18581	registerAlias(elem_reg, @min(result_size, 4)),	27205	const ty = self.typeOf(ref);
18582	Immediate.u(1),
18583	);
18584	if (elem_i > 0) try self.asmRegisterImmediate(
18585	.{ ._l, .sh },
18586	registerAlias(elem_reg, result_size),
18587	Immediate.u(@intCast(elem_i)),
18588	);
18589	try self.asmRegisterRegister(
18590	.{ ._, .@"or" },
18591	registerAlias(dst_reg, result_size),
18592	registerAlias(elem_reg, result_size),
18593	);
18594	}
18595	break :result .{ .register = dst_reg };
18596	} else {
18597	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
18598	const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
18599		27206
18600	for (elements, 0..) \|elem, elem_i\| {	27207	// If the type has no codegen bits, no need to store it.
18601	const elem_mcv = try self.resolveInst(elem);	27208	if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;
18602	const mat_elem_mcv = switch (elem_mcv) {	27209
18603	.load_tlv => \|sym_index\| MCValue{ .lea_tlv = sym_index },	27210	const mcv = if (ref.toIndex()) \|inst\| mcv: {
18604	else => elem_mcv,	27211	break :mcv self.inst_tracking.getPtr(inst).?.short;
18605	};	27212	} else mcv: {
18606	const elem_off: i32 = @intCast(elem_size * elem_i);	27213	const ip_index = ref.toInterned().?;
18607	try self.genSetMem(	27214	const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);
18608	.{ .frame = frame_index },	27215	if (!gop.found_existing) gop.value_ptr.* = .init(init: {
18609	elem_off,	27216	const const_mcv = try self.genTypedValue(.fromInterned(ip_index));
18610	elem_ty,	27217	switch (const_mcv) {
18611	mat_elem_mcv,	27218	.lea_tlv => \|tlv_sym\| switch (self.bin_file.tag) {
		27219	.elf, .macho => {
		27220	if (self.mod.pic) {
		27221	try self.spillRegisters(&.{ .rdi, .rax });
		27222	} else {
		27223	try self.spillRegisters(&.{.rax});
		27224	}
		27225	const frame_index = try self.allocFrameIndex(.init(.{
		27226	.size = 8,
		27227	.alignment = .@"8",
		27228	}));
		27229	try self.genSetMem(
		27230	.{ .frame = frame_index },
		27231	0,
		27232	.usize,
		27233	.{ .lea_symbol = .{ .sym_index = tlv_sym } },
18612	.{},	27234	.{},
18613	);	27235	);
18614	}	27236	break :init .{ .load_frame = .{ .index = frame_index } };
18615	if (result_ty.sentinel(zcu)) \|sentinel\| try self.genSetMem(	27237	},
18616	.{ .frame = frame_index },	27238	else => break :init const_mcv,
18617	@intCast(elem_size * elements.len),	27239	},
18618	elem_ty,	27240	else => break :init const_mcv,
18619	try self.genTypedValue(sentinel),	27241	}
18620	.{},	27242	});
18621	);	27243	break :mcv gop.value_ptr.short;
18622	break :result .{ .load_frame = .{ .index = frame_index } };
18623	}
18624	},
18625	else => unreachable,
18626	}
18627	};	27244	};
18628		27245
18629	if (elements.len <= Liveness.bpi - 1) {	27246	switch (mcv) {
18630	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);	27247	.none, .unreach, .dead => unreachable,
18631	@memcpy(buf[0..elements.len], elements);	27248	else => return mcv,
18632	return self.finishAir(inst, result, buf);
18633	}	27249	}
18634	var bt = self.liveness.iterateBigTomb(inst);
18635	for (elements) \|elem\| try self.feed(&bt, elem);
18636	return self.finishAirResult(inst, result);
18637	}	27250	}
18638		27251
18639	fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {	27252	fn getResolvedInstValue(self: CodeGen, inst: Air.Inst.Index) InstTracking {
18640	const pt = self.pt;	27253	const tracking = self.inst_tracking.getPtr(inst).?;
18641	const zcu = pt.zcu;	27254	return switch (tracking.short) {
18642	const ip = &zcu.intern_pool;	27255	.none, .unreach, .dead => unreachable,
18643	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	27256	else => tracking,
18644	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;	27257	};
18645	const result: MCValue = result: {	27258	}
18646	const union_ty = self.typeOfIndex(inst);
18647	const layout = union_ty.unionGetLayout(zcu);
18648
18649	const src_ty = self.typeOf(extra.init);
18650	const src_mcv = try self.resolveInst(extra.init);
18651	if (layout.tag_size == 0) {
18652	if (layout.abi_size <= src_ty.abiSize(zcu) and
18653	self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv;
18654
18655	const dst_mcv = try self.allocRegOrMem(inst, true);
18656	try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
18657	break :result dst_mcv;
18658	}
18659
18660	const dst_mcv = try self.allocRegOrMem(inst, false);
18661
18662	const union_obj = zcu.typeToUnion(union_ty).?;
18663	const field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index];
18664	const tag_ty = Type.fromInterned(union_obj.enum_tag_ty);
18665	const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
18666	const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);
18667	const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);
18668	const tag_int = tag_int_val.toUnsignedInt(zcu);
18669	const tag_off: i32 = @intCast(layout.tagOffset());
18670	try self.genCopy(
18671	tag_ty,
18672	dst_mcv.address().offset(tag_off).deref(),
18673	.{ .immediate = tag_int },
18674	.{},
18675	);
18676		27259
18677	const pl_off: i32 = @intCast(layout.payloadOffset());	27260	/// If the MCValue is an immediate, and it does not fit within this type,
18678	try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{});	27261	/// we put it in a register.
		27262	/// A potential opportunity for future optimization here would be keeping track
		27263	/// of the fact that the instruction is available both as an immediate
		27264	/// and as a register.
		27265	fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !MCValue {
		27266	const mcv = try self.resolveInst(operand);
		27267	const ti = @typeInfo(T).int;
		27268	switch (mcv) {
		27269	.immediate => \|imm\| {
		27270	// This immediate is unsigned.
		27271	const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));
		27272	if (imm >= std.math.maxInt(U)) {
		27273	return MCValue{ .register = try self.copyToTmpRegister(.usize, mcv) };
		27274	}
		27275	},
		27276	else => {},
		27277	}
		27278	return mcv;
		27279	}
18679		27280
18680	break :result dst_mcv;	27281	fn genTypedValue(self: *CodeGen, val: Value) InnerError!MCValue {
		27282	const pt = self.pt;
		27283	return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) {
		27284	.mcv => \|mcv\| switch (mcv) {
		27285	.none => .none,
		27286	.undef => .undef,
		27287	.immediate => \|imm\| .{ .immediate = imm },
		27288	.memory => \|addr\| .{ .memory = addr },
		27289	.load_symbol => \|sym_index\| .{ .load_symbol = .{ .sym_index = sym_index } },
		27290	.lea_symbol => \|sym_index\| .{ .lea_symbol = .{ .sym_index = sym_index } },
		27291	.load_direct => \|sym_index\| .{ .load_direct = sym_index },
		27292	.lea_direct => \|sym_index\| .{ .lea_direct = sym_index },
		27293	.load_got => \|sym_index\| .{ .lea_got = sym_index },
		27294	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
		27295	},
		27296	.fail => \|msg\| return self.failMsg(msg),
18681	};	27297	};
18682	return self.finishAir(inst, result, .{ extra.init, .none, .none });
18683	}	27298	}
18684		27299
18685	fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {	27300	const CallMCValues = struct {
18686	const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;	27301	args: []MCValue,
18687	return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });	27302	return_value: InstTracking,
18688	}	27303	stack_byte_count: u31,
		27304	stack_align: InternPool.Alignment,
		27305	gp_count: u32,
		27306	fp_count: u32,
		27307
		27308	fn deinit(self: CallMCValues, func: CodeGen) void {
		27309	func.gpa.free(self.args);
		27310	self.* = undefined;
		27311	}
		27312	};
18689		27313
18690	fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {	27314	/// Caller must call `CallMCValues.deinit`.
		27315	fn resolveCallingConventionValues(
		27316	self: *CodeGen,
		27317	fn_info: InternPool.Key.FuncType,
		27318	var_args: []const Type,
		27319	stack_frame_base: FrameIndex,
		27320	) !CallMCValues {
18691	const pt = self.pt;	27321	const pt = self.pt;
18692	const zcu = pt.zcu;	27322	const zcu = pt.zcu;
18693	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	27323	const ip = &zcu.intern_pool;
18694	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;	27324	const cc = fn_info.cc;
18695	const ty = self.typeOfIndex(inst);	27325	const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
		27326	defer self.gpa.free(param_types);
18696		27327
18697	const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };	27328	for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) \|*dest, src\|
18698	const result = result: {	27329	dest.* = .fromInterned(src);
18699	if (switch (ty.scalarType(zcu).floatBits(self.target.*)) {	27330	for (param_types[fn_info.param_types.len..], var_args) \|*param_ty, arg_ty\|
18700	16, 80, 128 => true,	27331	param_ty.* = self.promoteVarArg(arg_ty);
18701	32, 64 => !self.hasFeature(.fma),
18702	else => unreachable,
18703	}) {
18704	if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement airMulAdd for {}", .{
18705	ty.fmt(pt),
18706	});
18707		27332
18708	var callee_buf: ["__fma?".len]u8 = undefined;	27333	var result: CallMCValues = .{
18709	break :result try self.genCall(.{ .lib = .{	27334	.args = try self.gpa.alloc(MCValue, param_types.len),
18710	.return_type = ty.toIntern(),	27335	// These undefined values must be populated before returning from this function.
18711	.param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() },	27336	.return_value = undefined,
18712	.callee = std.fmt.bufPrint(&callee_buf, "{s}fma{s}", .{	27337	.stack_byte_count = 0,
18713	floatLibcAbiPrefix(ty),	27338	.stack_align = undefined,
18714	floatLibcAbiSuffix(ty),	27339	.gp_count = 0,
18715	}) catch unreachable,	27340	.fp_count = 0,
18716	} }, &.{ ty, ty, ty }, &.{	27341	};
18717	.{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand },	27342	errdefer self.gpa.free(result.args);
18718	});
18719	}
18720		27343
18721	var mcvs: [3]MCValue = undefined;	27344	const ret_ty: Type = .fromInterned(fn_info.return_type);
18722	var locks = [1]?RegisterManager.RegisterLock{null} ** 3;	27345	switch (cc) {
18723	defer for (locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);	27346	.naked => {
18724	var order = [1]u2{0} ** 3;	27347	assert(result.args.len == 0);
18725	var unused = std.StaticBitSet(3).initFull();	27348	result.return_value = .init(.unreach);
18726	for (ops, &mcvs, &locks, 0..) \|op, mcv, lock, op_i\| {	27349	result.stack_align = switch (self.target.cpu.arch) {
18727	const op_index: u2 = @intCast(op_i);	27350	else => unreachable,
18728	mcv.* = try self.resolveInst(op);	27351	.x86 => .@"4",
18729	if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {	27352	.x86_64 => .@"8",
18730	order[op_index] = 1;	27353	};
18731	unused.unset(0);	27354	},
18732	} else if (unused.isSet(2) and mcv.isMemory()) {	27355	.x86_64_sysv, .x86_64_win => \|cc_opts\| {
18733	order[op_index] = 3;	27356	var ret_int_reg_i: u32 = 0;
18734	unused.unset(2);	27357	var ret_sse_reg_i: u32 = 0;
18735	}	27358	var param_int_reg_i: u32 = 0;
18736	switch (mcv.*) {	27359	var param_sse_reg_i: u32 = 0;
18737	.register => \|reg\| lock.* = self.register_manager.lockReg(reg),	27360	result.stack_align = .fromByteUnits(cc_opts.incoming_stack_alignment orelse 16);
18738	else => {},	27361
		27362	switch (cc) {
		27363	.x86_64_sysv => {},
		27364	.x86_64_win => result.stack_byte_count += @intCast(4 * 8),
		27365	else => unreachable,
18739	}	27366	}
18740	}
18741	for (&order, &mcvs, &locks) \|mop_index, mcv, *lock\| {
18742	if (mop_index.* != 0) continue;
18743	mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?));
18744	if (mop_index.* > 1 and mcv.isRegister()) continue;
18745	const reg = try self.copyToTmpRegister(ty, mcv.*);
18746	mcv.* = .{ .register = reg };
18747	if (lock.*) \|old_lock\| self.register_manager.unlockReg(old_lock);
18748	lock.* = self.register_manager.lockRegAssumeUnused(reg);
18749	}
18750		27367
18751	const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or	27368	// Return values
18752	mem.eql(u2, &order, &.{ 3, 1, 2 }))	27369	if (ret_ty.isNoReturn(zcu)) {
18753	switch (ty.zigTypeTag(zcu)) {	27370	result.return_value = .init(.unreach);
18754	.float => switch (ty.floatBits(self.target.*)) {	27371	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
18755	32 => .{ .v_ss, .fmadd132 },	27372	// TODO: is this even possible for C calling convention?
18756	64 => .{ .v_sd, .fmadd132 },	27373	result.return_value = .init(.none);
18757	16, 80, 128 => null,	27374	} else {
		27375	var ret_tracking: [4]InstTracking = undefined;
		27376	var ret_tracking_i: usize = 0;
		27377
		27378	const classes = switch (cc) {
		27379	.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),
		27380	.x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},
18758	else => unreachable,	27381	else => unreachable,
18759	},	27382	};
18760	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {	27383	for (classes) \|class\| switch (class) {
18761	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {	27384	.integer => {
18762	32 => switch (ty.vectorLen(zcu)) {	27385	const ret_int_reg = registerAlias(
18763	1 => .{ .v_ss, .fmadd132 },	27386	abi.getCAbiIntReturnRegs(cc)[ret_int_reg_i],
18764	2...8 => .{ .v_ps, .fmadd132 },	27387	@intCast(@min(ret_ty.abiSize(zcu), 8)),
18765	else => null,	27388	);
18766	},	27389	ret_int_reg_i += 1;
18767	64 => switch (ty.vectorLen(zcu)) {	27390
18768	1 => .{ .v_sd, .fmadd132 },	27391	ret_tracking[ret_tracking_i] = .init(.{ .register = ret_int_reg });
18769	2...4 => .{ .v_pd, .fmadd132 },	27392	ret_tracking_i += 1;
18770	else => null,	27393	},
18771	},	27394	.sse, .float, .float_combine, .win_i128 => {
18772	16, 80, 128 => null,	27395	const ret_sse_regs = abi.getCAbiSseReturnRegs(cc);
18773	else => unreachable,	27396	const abi_size: u32 = @intCast(ret_ty.abiSize(zcu));
		27397	const reg_size = @min(abi_size, self.vectorSize(.float));
		27398	var byte_offset: u32 = 0;
		27399	while (byte_offset < abi_size) : (byte_offset += reg_size) {
		27400	const ret_sse_reg = registerAlias(ret_sse_regs[ret_sse_reg_i], reg_size);
		27401	ret_sse_reg_i += 1;
		27402
		27403	ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg });
		27404	ret_tracking_i += 1;
		27405	}
		27406	},
		27407	.sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse),
		27408	.x87 => {
		27409	ret_tracking[ret_tracking_i] = .init(.{ .register = abi.getCAbiX87ReturnRegs(cc)[0] });
		27410	ret_tracking_i += 1;
		27411	},
		27412	.x87up => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .x87),
		27413	.complex_x87 => {
		27414	ret_tracking[ret_tracking_i] = .init(.{ .register_pair = abi.getCAbiX87ReturnRegs(cc)[0..2].* });
		27415	ret_tracking_i += 1;
18774	},	27416	},
		27417	.memory => {
		27418	const ret_int_reg = abi.getCAbiIntReturnRegs(cc)[ret_int_reg_i].to64();
		27419	ret_int_reg_i += 1;
		27420	const ret_indirect_reg = abi.getCAbiIntParamRegs(cc)[param_int_reg_i];
		27421	param_int_reg_i += 1;
		27422
		27423	ret_tracking[ret_tracking_i] = .{
		27424	.short = .{ .indirect = .{ .reg = ret_int_reg } },
		27425	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
		27426	};
		27427	ret_tracking_i += 1;
		27428	},
		27429	.none, .integer_per_element => unreachable,
		27430	};
		27431	result.return_value = switch (ret_tracking_i) {
18775	else => unreachable,	27432	else => unreachable,
18776	},	27433	1 => ret_tracking[0],
18777	else => unreachable,	27434	2 => .init(.{ .register_pair = .{
		27435	ret_tracking[0].short.register,
		27436	ret_tracking[1].short.register,
		27437	} }),
		27438	3 => .init(.{ .register_triple = .{
		27439	ret_tracking[0].short.register,
		27440	ret_tracking[1].short.register,
		27441	ret_tracking[2].short.register,
		27442	} }),
		27443	4 => .init(.{ .register_quadruple = .{
		27444	ret_tracking[0].short.register,
		27445	ret_tracking[1].short.register,
		27446	ret_tracking[2].short.register,
		27447	ret_tracking[3].short.register,
		27448	} }),
		27449	};
18778	}	27450	}
18779	else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))	27451
18780	switch (ty.zigTypeTag(zcu)) {	27452	// Input params
18781	.float => switch (ty.floatBits(self.target.*)) {	27453	for (param_types, result.args) \|ty, *arg\| {
18782	32 => .{ .v_ss, .fmadd213 },	27454	assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
18783	64 => .{ .v_sd, .fmadd213 },	27455	switch (cc) {
18784	16, 80, 128 => null,	27456	.x86_64_sysv => {},
		27457	.x86_64_win => {
		27458	param_int_reg_i = @max(param_int_reg_i, param_sse_reg_i);
		27459	param_sse_reg_i = param_int_reg_i;
		27460	},
18785	else => unreachable,	27461	else => unreachable,
18786	},	27462	}
18787	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {	27463
18788	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {	27464	var arg_mcv: [4]MCValue = undefined;
18789	32 => switch (ty.vectorLen(zcu)) {	27465	var arg_mcv_i: usize = 0;
18790	1 => .{ .v_ss, .fmadd213 },	27466
18791	2...8 => .{ .v_ps, .fmadd213 },	27467	const classes = switch (cc) {
18792	else => null,	27468	.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),
18793	},	27469	.x86_64_win => &.{abi.classifyWindows(ty, zcu)},
18794	64 => switch (ty.vectorLen(zcu)) {	27470	else => unreachable,
18795	1 => .{ .v_sd, .fmadd213 },	27471	};
18796	2...4 => .{ .v_pd, .fmadd213 },	27472	classes: for (classes) \|class\| switch (class) {
18797	else => null,	27473	.integer => {
		27474	const param_int_regs = abi.getCAbiIntParamRegs(cc);
		27475	if (param_int_reg_i >= param_int_regs.len) break;
		27476
		27477	const param_int_reg =
		27478	registerAlias(param_int_regs[param_int_reg_i], @intCast(@min(ty.abiSize(zcu), 8)));
		27479	param_int_reg_i += 1;
		27480
		27481	arg_mcv[arg_mcv_i] = .{ .register = param_int_reg };
		27482	arg_mcv_i += 1;
		27483	},
		27484	.sse, .float, .float_combine => {
		27485	const param_sse_regs = abi.getCAbiSseParamRegs(cc);
		27486	const abi_size: u32 = @intCast(ty.abiSize(zcu));
		27487	const reg_size = @min(abi_size, self.vectorSize(.float));
		27488	var byte_offset: u32 = 0;
		27489	while (byte_offset < abi_size) : (byte_offset += reg_size) {
		27490	if (param_sse_reg_i >= param_sse_regs.len) break :classes;
		27491
		27492	const param_sse_reg = registerAlias(param_sse_regs[param_sse_reg_i], reg_size);
		27493	param_sse_reg_i += 1;
		27494
		27495	arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg };
		27496	arg_mcv_i += 1;
		27497	}
		27498	},
		27499	.sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse),
		27500	.x87, .x87up, .complex_x87, .memory, .win_i128 => switch (cc) {
		27501	.x86_64_sysv => switch (class) {
		27502	.x87, .x87up, .complex_x87, .memory => break,
		27503	else => unreachable,
18798	},	27504	},
18799	16, 80, 128 => null,	27505	.x86_64_win => if (ty.abiSize(zcu) > 8) {
		27506	const param_int_reg = abi.getCAbiIntParamRegs(cc)[param_int_reg_i].to64();
		27507	param_int_reg_i += 1;
		27508
		27509	arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } };
		27510	arg_mcv_i += 1;
		27511	} else break,
18800	else => unreachable,	27512	else => unreachable,
18801	},	27513	},
18802	else => unreachable,	27514	.none => unreachable,
18803	},	27515	.integer_per_element => {
18804	else => unreachable,	27516	const param_int_regs_len: u32 =
		27517	@intCast(abi.getCAbiIntParamRegs(cc).len);
		27518	const remaining_param_int_regs: u3 =
		27519	@intCast(param_int_regs_len - param_int_reg_i);
		27520	param_int_reg_i = param_int_regs_len;
		27521
		27522	const frame_elem_align = 8;
		27523	const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs;
		27524	const frame_elem_size = std.mem.alignForward(
		27525	u64,
		27526	ty.childType(zcu).abiSize(zcu),
		27527	frame_elem_align,
		27528	);
		27529	const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size);
		27530
		27531	result.stack_byte_count =
		27532	std.mem.alignForward(u31, result.stack_byte_count, frame_elem_align);
		27533	arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{
		27534	.regs = remaining_param_int_regs,
		27535	.frame_off = @intCast(result.stack_byte_count),
		27536	.frame_index = stack_frame_base,
		27537	} };
		27538	arg_mcv_i += 1;
		27539	result.stack_byte_count += frame_size;
		27540	},
		27541	} else {
		27542	arg.* = switch (arg_mcv_i) {
		27543	else => unreachable,
		27544	1 => arg_mcv[0],
		27545	2 => .{ .register_pair = .{
		27546	arg_mcv[0].register,
		27547	arg_mcv[1].register,
		27548	} },
		27549	3 => .{ .register_triple = .{
		27550	arg_mcv[0].register,
		27551	arg_mcv[1].register,
		27552	arg_mcv[2].register,
		27553	} },
		27554	4 => .{ .register_quadruple = .{
		27555	arg_mcv[0].register,
		27556	arg_mcv[1].register,
		27557	arg_mcv[2].register,
		27558	arg_mcv[3].register,
		27559	} },
		27560	};
		27561	continue;
		27562	}
		27563
		27564	const param_align = ty.abiAlignment(zcu).max(.@"8");
		27565	result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
		27566	result.stack_align = result.stack_align.max(param_align);
		27567	arg.* = .{ .load_frame = .{
		27568	.index = stack_frame_base,
		27569	.off = result.stack_byte_count,
		27570	} };
		27571	result.stack_byte_count += @intCast(ty.abiSize(zcu));
18805	}	27572	}
18806	else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))	27573	assert(param_int_reg_i <= 6);
18807	switch (ty.zigTypeTag(zcu)) {	27574	result.gp_count = param_int_reg_i;
18808	.float => switch (ty.floatBits(self.target.*)) {	27575	assert(param_sse_reg_i <= 16);
18809	32 => .{ .v_ss, .fmadd231 },	27576	result.fp_count = param_sse_reg_i;
18810	64 => .{ .v_sd, .fmadd231 },	27577	},
18811	16, 80, 128 => null,	27578	.auto => {
18812	else => unreachable,	27579	result.stack_align = abi.zigcc.stack_align orelse .fromByteUnits(self.vectorSize(.float));
18813	},	27580
18814	.vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {	27581	var param_gpr = abi.getCAbiIntParamRegs(cc);
18815	.float => switch (ty.childType(zcu).floatBits(self.target.*)) {	27582	var param_x87 = abi.getCAbiX87ParamRegs(cc);
18816	32 => switch (ty.vectorLen(zcu)) {	27583	var param_sse = abi.getCAbiSseParamRegs(cc);
18817	1 => .{ .v_ss, .fmadd231 },	27584
18818	2...8 => .{ .v_ps, .fmadd231 },	27585	// Return values
18819	else => null,	27586	result.return_value = if (ret_ty.isNoReturn(zcu))
18820	},	27587	.init(.unreach)
18821	64 => switch (ty.vectorLen(zcu)) {	27588	else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu))
18822	1 => .{ .v_sd, .fmadd231 },	27589	.init(.none)
18823	2...4 => .{ .v_pd, .fmadd231 },	27590	else return_value: {
18824	else => null,	27591	const ret_gpr = abi.getCAbiIntReturnRegs(cc);
18825	},	27592	const ret_size: u31 = @intCast(ret_ty.abiSize(zcu));
18826	16, 80, 128 => null,	27593	if (abi.zigcc.return_in_regs) switch (self.regClassForType(ret_ty)) {
		27594	.general_purpose => if (ret_size <= @as(u4, switch (self.target.cpu.arch) {
		27595	else => unreachable,
		27596	.x86 => 4,
		27597	.x86_64 => 8,
		27598	}))
		27599	break :return_value .init(.{ .register = registerAlias(ret_gpr[0], ret_size) })
		27600	else if (ret_gpr.len >= 2 and ret_ty.isSliceAtRuntime(zcu))
		27601	break :return_value .init(.{ .register_pair = ret_gpr[0..2].* }),
		27602	.segment, .mmx, .ip => unreachable,
		27603	.x87 => break :return_value .init(.{ .register = .st0 }),
		27604	.sse => if (ret_size <= self.vectorSize(.float)) break :return_value .init(.{
		27605	.register = registerAlias(abi.getCAbiSseReturnRegs(cc)[0], @max(ret_size, 16)),
		27606	}),
		27607	};
		27608	const ret_indirect_reg = param_gpr[0];
		27609	param_gpr = param_gpr[1..];
		27610	break :return_value .{
		27611	.short = .{ .indirect = .{ .reg = ret_gpr[0] } },
		27612	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
		27613	};
		27614	};
		27615
		27616	// Input params
		27617	for (param_types, result.args) \|param_ty, *arg\| {
		27618	if (!param_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
		27619	arg.* = .none;
		27620	continue;
		27621	}
		27622	const param_size: u31 = @intCast(param_ty.abiSize(zcu));
		27623	if (abi.zigcc.params_in_regs) switch (self.regClassForType(param_ty)) {
		27624	.general_purpose => if (param_gpr.len >= 1 and param_size <= @as(u4, switch (self.target.cpu.arch) {
18827	else => unreachable,	27625	else => unreachable,
		27626	.x86 => 4,
		27627	.x86_64 => 8,
		27628	})) {
		27629	arg.* = .{ .register = registerAlias(param_gpr[0], param_size) };
		27630	param_gpr = param_gpr[1..];
		27631	continue;
		27632	} else if (param_gpr.len >= 2 and param_ty.isSliceAtRuntime(zcu)) {
		27633	arg.* = .{ .register_pair = param_gpr[0..2].* };
		27634	param_gpr = param_gpr[2..];
		27635	continue;
18828	},	27636	},
18829	else => unreachable,	27637	.segment, .mmx, .ip => unreachable,
18830	},	27638	.x87 => if (param_x87.len >= 1) {
18831	else => unreachable,	27639	arg.* = .{ .register = param_x87[0] };
		27640	param_x87 = param_x87[1..];
		27641	continue;
		27642	},
		27643	.sse => if (param_sse.len >= 1 and param_size <= self.vectorSize(.float)) {
		27644	arg.* = .{
		27645	.register = registerAlias(param_sse[0], @max(param_size, 16)),
		27646	};
		27647	param_sse = param_sse[1..];
		27648	continue;
		27649	},
		27650	};
		27651	const param_align = param_ty.abiAlignment(zcu);
		27652	result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count));
		27653	result.stack_align = result.stack_align.max(param_align);
		27654	arg.* = .{ .load_frame = .{
		27655	.index = stack_frame_base,
		27656	.off = result.stack_byte_count,
		27657	} };
		27658	result.stack_byte_count += param_size;
18832	}	27659	}
18833	else	27660	},
18834	unreachable) orelse return self.fail("TODO implement airMulAdd for {}", .{ty.fmt(pt)});	27661	else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
		27662	}
18835		27663
18836	var mops: [3]MCValue = undefined;	27664	result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count));
18837	for (order, mcvs) \|mop_index, mcv\| mops[mop_index - 1] = mcv;	27665	return result;
		27666	}
18838		27667
18839	const abi_size: u32 = @intCast(ty.abiSize(zcu));	27668	fn fail(self: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
18840	const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);	27669	@branchHint(.cold);
18841	const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);	27670	const zcu = self.pt.zcu;
18842	if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(	27671	switch (self.owner) {
18843	mir_tag,	27672	.nav_index => \|i\| return zcu.codegenFail(i, format, args),
18844	mop1_reg,	27673	.lazy_sym => \|s\| return zcu.codegenFailType(s.ty, format, args),
18845	mop2_reg,	27674	}
18846	registerAlias(mops[2].getReg().?, abi_size),	27675	return error.CodegenFail;
18847	) else try self.asmRegisterRegisterMemory(	27676	}
18848	mir_tag,	27677
18849	mop1_reg,	27678	fn failMsg(self: CodeGen, msg: Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
18850	mop2_reg,	27679	@branchHint(.cold);
18851	try mops[2].mem(self, Memory.Size.fromSize(abi_size)),	27680	const zcu = self.pt.zcu;
18852	);	27681	switch (self.owner) {
18853	break :result mops[0];	27682	.nav_index => \|i\| return zcu.codegenFailMsg(i, msg),
		27683	.lazy_sym => \|s\| return zcu.codegenFailTypeMsg(s.ty, msg),
		27684	}
		27685	return error.CodegenFail;
		27686	}
		27687
		27688	fn parseRegName(name: []const u8) ?Register {
		27689	if (@hasDecl(Register, "parseRegName")) {
		27690	return Register.parseRegName(name);
		27691	}
		27692	return std.meta.stringToEnum(Register, name);
		27693	}
		27694
		27695	/// Returns register wide enough to hold at least `size_bytes`.
		27696	fn registerAlias(reg: Register, size_bytes: u32) Register {
		27697	return switch (reg.class()) {
		27698	.general_purpose => if (size_bytes == 0)
		27699	unreachable // should be comptime-known
		27700	else if (size_bytes <= 1)
		27701	reg.to8()
		27702	else if (size_bytes <= 2)
		27703	reg.to16()
		27704	else if (size_bytes <= 4)
		27705	reg.to32()
		27706	else if (size_bytes <= 8)
		27707	reg.to64()
		27708	else
		27709	unreachable,
		27710	.segment => if (size_bytes <= 2)
		27711	reg
		27712	else
		27713	unreachable,
		27714	.x87 => if (size_bytes >= 10 and size_bytes <= 16)
		27715	reg
		27716	else
		27717	unreachable,
		27718	.mmx => if (size_bytes <= 8)
		27719	reg
		27720	else
		27721	unreachable,
		27722	.sse => if (size_bytes <= 16)
		27723	reg.to128()
		27724	else if (size_bytes <= 32)
		27725	reg.to256()
		27726	else
		27727	unreachable,
		27728	.ip => if (size_bytes <= 2)
		27729	.ip
		27730	else if (size_bytes <= 4)
		27731	.eip
		27732	else if (size_bytes <= 8)
		27733	.rip
		27734	else
		27735	unreachable,
		27736	};
		27737	}
		27738
		27739	fn memSize(self: *CodeGen, ty: Type) Memory.Size {
		27740	const zcu = self.pt.zcu;
		27741	return switch (ty.zigTypeTag(zcu)) {
		27742	.float => .fromBitSize(ty.floatBits(self.target.*)),
		27743	else => .fromSize(@intCast(ty.abiSize(zcu))),
18854	};	27744	};
18855	return self.finishAir(inst, result, ops);
18856	}	27745	}
18857		27746
18858	fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {	27747	fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Type {
18859	const pt = self.pt;	27748	const pt = self.pt;
18860	const zcu = pt.zcu;	27749	const zcu = pt.zcu;
18861	const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;	27750	var parts: [parts_len]Type = undefined;
18862	const ptr_anyopaque_ty = try pt.singleMutPtrType(Type.anyopaque);	27751	if (ty.isVector(zcu)) if (std.math.divExact(u32, ty.vectorLen(zcu), parts_len)) \|vec_len\| return .{
18863		27752	try pt.vectorType(.{ .len = vec_len, .child = ty.scalarType(zcu).toIntern() }),
18864	const result: MCValue = switch (abi.resolveCallingConvention(	27753	} ** parts_len else \|err\| switch (err) {
18865	self.fn_type.fnCallingConvention(zcu),	27754	error.DivisionByZero => unreachable,
18866	self.target.*,	27755	error.UnexpectedRemainder => {},
18867	)) {
18868	.x86_64_sysv => result: {
18869	const info = self.va_info.sysv;
18870	const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, zcu));
18871	var field_off: u31 = 0;
18872	// gp_offset: c_uint,
18873	try self.genSetMem(
18874	.{ .frame = dst_fi },
18875	field_off,
18876	Type.c_uint,
18877	.{ .immediate = info.gp_count * 8 },
18878	.{},
18879	);
18880	field_off += @intCast(Type.c_uint.abiSize(zcu));
18881	// fp_offset: c_uint,
18882	try self.genSetMem(
18883	.{ .frame = dst_fi },
18884	field_off,
18885	Type.c_uint,
18886	.{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 },
18887	.{},
18888	);
18889	field_off += @intCast(Type.c_uint.abiSize(zcu));
18890	// overflow_arg_area: *anyopaque,
18891	try self.genSetMem(
18892	.{ .frame = dst_fi },
18893	field_off,
18894	ptr_anyopaque_ty,
18895	.{ .lea_frame = info.overflow_arg_area },
18896	.{},
18897	);
18898	field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
18899	// reg_save_area: *anyopaque,
18900	try self.genSetMem(
18901	.{ .frame = dst_fi },
18902	field_off,
18903	ptr_anyopaque_ty,
18904	.{ .lea_frame = info.reg_save_area },
18905	.{},
18906	);
18907	field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu));
18908	break :result .{ .load_frame = .{ .index = dst_fi } };
18909	},
18910	.x86_64_win => return self.fail("TODO implement c_va_start for Win64", .{}),
18911	else => unreachable,
18912	};	27756	};
18913	return self.finishAir(inst, result, .{ .none, .none, .none });	27757	const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
		27758	if (classes.len == parts_len) for (&parts, classes, 0..) \|*part, class, part_i\| {
		27759	part.* = switch (class) {
		27760	.integer => if (part_i < parts_len - 1)
		27761	.u64
		27762	else part: {
		27763	const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;
		27764	const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8));
		27765	break :part switch (@divExact(ty.abiSize(zcu) - part_i * 8, elem_size)) {
		27766	1 => elem_ty,
		27767	else => \|array_len\| try pt.arrayType(.{ .len = array_len, .child = elem_ty.toIntern() }),
		27768	};
		27769	},
		27770	.float => .f32,
		27771	.float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }),
		27772	.sse => .f64,
		27773	else => break,
		27774	};
		27775	} else {
		27776	var part_sizes: u64 = 0;
		27777	for (parts) \|part\| part_sizes += part.abiSize(zcu);
		27778	if (part_sizes == ty.abiSize(zcu)) return parts;
		27779	};
		27780	return self.fail("TODO implement splitType({d}, {})", .{ parts_len, ty.fmt(pt) });
18914	}	27781	}
18915		27782
18916	fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {	27783	/// Truncates the value in the register in place.
		27784	/// Clobbers any remaining bits.
		27785	fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
18917	const pt = self.pt;	27786	const pt = self.pt;
18918	const zcu = pt.zcu;	27787	const zcu = pt.zcu;
18919	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	27788	const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
18920	const ty = self.typeOfIndex(inst);	27789	.signedness = .unsigned,
18921	const promote_ty = self.promoteVarArg(ty);	27790	.bits = @intCast(ty.bitSize(zcu)),
18922	const ptr_anyopaque_ty = try pt.singleMutPtrType(Type.anyopaque);	27791	};
18923	const unused = self.liveness.isUnused(inst);	27792	const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
18924		27793	try self.spillEflagsIfOccupied();
18925	const result: MCValue = switch (abi.resolveCallingConvention(	27794	switch (int_info.signedness) {
18926	self.fn_type.fnCallingConvention(zcu),	27795	.signed => {
18927	self.target.*,	27796	try self.genShiftBinOpMir(.{ ._l, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift });
18928	)) {	27797	try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift });
18929	.x86_64_sysv => result: {	27798	},
18930	try self.spillEflagsIfOccupied();	27799	.unsigned => {
18931		27800	const mask = ~@as(u64, 0) >> shift;
18932	const tmp_regs =	27801	if (int_info.bits <= 32) {
18933	try self.register_manager.allocRegs(2, .{null} ** 2, abi.RegisterClass.gp);	27802	try self.genBinOpMir(.{ ._, .@"and" }, .u32, .{ .register = reg }, .{ .immediate = mask });
18934	const offset_reg = tmp_regs[0].to32();	27803	} else {
18935	const addr_reg = tmp_regs[1].to64();	27804	const tmp_reg = try self.copyToTmpRegister(.usize, .{ .immediate = mask });
18936	const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);	27805	try self.genBinOpMir(.{ ._, .@"and" }, .usize, .{ .register = reg }, .{ .register = tmp_reg });
18937	defer for (tmp_locks) \|lock\| self.register_manager.unlockReg(lock);	27806	}
18938		27807	},
18939	const promote_mcv = try self.allocTempRegOrMem(promote_ty, true);	27808	}
18940	const promote_lock = switch (promote_mcv) {	27809	}
18941	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
18942	else => null,
18943	};
18944	defer if (promote_lock) \|lock\| self.register_manager.unlockReg(lock);
18945
18946	const ptr_arg_list_reg =
18947	try self.copyToTmpRegister(self.typeOf(ty_op.operand), .{ .air_ref = ty_op.operand });
18948	const ptr_arg_list_lock = self.register_manager.lockRegAssumeUnused(ptr_arg_list_reg);
18949	defer self.register_manager.unlockReg(ptr_arg_list_lock);
18950
18951	const gp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 0 } };
18952	const fp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 4 } };
18953	const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
18954	const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
18955
18956	const classes = mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
18957	switch (classes[0]) {
18958	.integer => {
18959	assert(classes.len == 1);
18960
18961	try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{});
18962	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
18963	abi.SysV.c_abi_int_param_regs.len * 8,
18964	));
18965	const mem_reloc = try self.asmJccReloc(.ae, undefined);
18966
18967	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
18968	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
18969	.base = .{ .reg = addr_reg },
18970	.mod = .{ .rm = .{
18971	.size = .qword,
18972	.index = offset_reg.to64(),
18973	} },
18974	});
18975	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
18976	.base = .{ .reg = offset_reg.to64() },
18977	.mod = .{ .rm = .{
18978	.size = .qword,
18979	.disp = 8,
18980	} },
18981	});
18982	try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg }, .{});
18983	const done_reloc = try self.asmJmpReloc(undefined);
18984		27810
18985	self.performReloc(mem_reloc);	27811	fn regBitSize(self: *CodeGen, ty: Type) u64 {
18986	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});	27812	const zcu = self.pt.zcu;
18987	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{	27813	const abi_size = ty.abiSize(zcu);
18988	.base = .{ .reg = addr_reg },	27814	return switch (ty.zigTypeTag(zcu)) {
18989	.mod = .{ .rm = .{	27815	else => switch (abi_size) {
18990	.size = .qword,	27816	1 => 8,
18991	.disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),	27817	2 => 16,
18992	} },	27818	3...4 => 32,
18993	});	27819	5...8 => 64,
18994	try self.genCopy(	27820	else => unreachable,
18995	ptr_anyopaque_ty,	27821	},
18996	overflow_arg_area,	27822	.float => switch (abi_size) {
18997	.{ .register = offset_reg.to64() },	27823	1...16 => 128,
18998	.{},	27824	17...32 => 256,
18999	);	27825	else => unreachable,
		27826	},
		27827	};
		27828	}
19000		27829
19001	self.performReloc(done_reloc);	27830	fn regExtraBits(self: *CodeGen, ty: Type) u64 {
19002	if (!unused) try self.genCopy(promote_ty, promote_mcv, .{	27831	return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
19003	.indirect = .{ .reg = addr_reg },	27832	}
19004	}, .{});
19005	},
19006	.sse => {
19007	assert(classes.len == 1);
19008		27833
19009	try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{});	27834	fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
19010	try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(	27835	return switch (feature) {
19011	abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,	27836	.@"64bit" => switch (cg.target.cpu.arch) {
19012	));	27837	else => unreachable,
19013	const mem_reloc = try self.asmJccReloc(.ae, undefined);	27838	.x86 => false,
		27839	.x86_64 => true,
		27840	},
		27841	.false_deps_getmant,
		27842	.false_deps_lzcnt_tzcnt,
		27843	.false_deps_mulc,
		27844	.false_deps_mullq,
		27845	.false_deps_perm,
		27846	.false_deps_popcnt,
		27847	.false_deps_range,
		27848	.slow_3ops_lea,
		27849	.slow_incdec,
		27850	.slow_lea,
		27851	.slow_pmaddwd,
		27852	.slow_pmulld,
		27853	.slow_shld,
		27854	.slow_two_mem_ops,
		27855	.slow_unaligned_mem_16,
		27856	.slow_unaligned_mem_32,
		27857	=> switch (cg.mod.optimize_mode) {
		27858	.Debug, .ReleaseSafe, .ReleaseFast => null,
		27859	.ReleaseSmall => false,
		27860	},
		27861	.fast_11bytenop,
		27862	.fast_15bytenop,
		27863	.fast_7bytenop,
		27864	.fast_bextr,
		27865	.fast_dpwssd,
		27866	.fast_gather,
		27867	.fast_hops,
		27868	.fast_imm16,
		27869	.fast_lzcnt,
		27870	.fast_movbe,
		27871	.fast_scalar_fsqrt,
		27872	.fast_scalar_shift_masks,
		27873	.fast_shld_rotate,
		27874	.fast_variable_crosslane_shuffle,
		27875	.fast_variable_perlane_shuffle,
		27876	.fast_vector_fsqrt,
		27877	.fast_vector_shift_masks,
		27878	=> switch (cg.mod.optimize_mode) {
		27879	.Debug, .ReleaseSafe, .ReleaseFast => null,
		27880	.ReleaseSmall => true,
		27881	},
		27882	.mmx => false,
		27883	else => null,
		27884	} orelse std.Target.x86.featureSetHas(cg.target.cpu.features, feature);
		27885	}
19014		27886
19015	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});	27887	fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type {
19016	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{	27888	const pt = self.pt;
19017	.base = .{ .reg = addr_reg },	27889	const zcu = pt.zcu;
19018	.mod = .{ .rm = .{	27890	return self.air.typeOf(inst, &zcu.intern_pool);
19019	.size = .qword,	27891	}
19020	.index = offset_reg.to64(),
19021	} },
19022	});
19023	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
19024	.base = .{ .reg = offset_reg.to64() },
19025	.mod = .{ .rm = .{
19026	.size = .qword,
19027	.disp = 16,
19028	} },
19029	});
19030	try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg }, .{});
19031	const done_reloc = try self.asmJmpReloc(undefined);
19032		27892
19033	self.performReloc(mem_reloc);	27893	fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type {
19034	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});	27894	const pt = self.pt;
19035	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{	27895	const zcu = pt.zcu;
19036	.base = .{ .reg = addr_reg },	27896	const temp: Temp = .{ .index = inst };
19037	.mod = .{ .rm = .{	27897	return switch (temp.unwrap(self)) {
19038	.size = .qword,	27898	.ref => switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {
19039	.disp = @intCast(@max(promote_ty.abiSize(zcu), 8)),	27899	.loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),
19040	} },	27900	else => self.air.typeOfIndex(inst, &zcu.intern_pool),
19041	});	27901	},
19042	try self.genCopy(	27902	.temp => temp.typeOf(self),
19043	ptr_anyopaque_ty,	27903	};
19044	overflow_arg_area,	27904	}
19045	.{ .register = offset_reg.to64() },
19046	.{},
19047	);
19048		27905
19049	self.performReloc(done_reloc);	27906	fn intCompilerRtAbiName(int_bits: u32) u8 {
19050	if (!unused) try self.genCopy(promote_ty, promote_mcv, .{	27907	return switch (int_bits) {
19051	.indirect = .{ .reg = addr_reg },	27908	1...32 => 's',
19052	}, .{});	27909	33...64 => 'd',
19053	},	27910	65...128 => 't',
19054	.memory => {	27911	else => unreachable,
19055	assert(classes.len == 1);	27912	};
19056	unreachable;	27913	}
19057	},
19058	else => return self.fail("TODO implement c_va_arg for {} on SysV", .{
19059	promote_ty.fmt(pt),
19060	}),
19061	}
19062		27914
19063	if (unused) break :result .unreach;	27915	fn floatCompilerRtAbiName(float_bits: u32) u8 {
19064	if (ty.toIntern() == promote_ty.toIntern()) break :result promote_mcv;	27916	return switch (float_bits) {
		27917	16 => 'h',
		27918	32 => 's',
		27919	64 => 'd',
		27920	80 => 'x',
		27921	128 => 't',
		27922	else => unreachable,
		27923	};
		27924	}
19065		27925
19066	if (!promote_ty.isRuntimeFloat()) {	27926	fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type {
19067	const dst_mcv = try self.allocRegOrMem(inst, true);	27927	if (ty.toIntern() == .f16_type and
19068	try self.genCopy(ty, dst_mcv, promote_mcv, .{});	27928	(other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and
19069	break :result dst_mcv;	27929	self.target.isDarwin()) return .u16;
19070	}	27930	return ty;
		27931	}
19071		27932
19072	assert(ty.toIntern() == .f32_type and promote_ty.toIntern() == .f64_type);	27933	fn floatLibcAbiPrefix(ty: Type) []const u8 {
19073	const dst_mcv = if (promote_mcv.isRegister())	27934	return switch (ty.toIntern()) {
19074	promote_mcv	27935	.f16_type, .f80_type => "__",
19075	else	27936	.f32_type, .f64_type, .f128_type, .c_longdouble_type => "",
19076	try self.copyToRegisterWithInstTracking(inst, ty, promote_mcv);	27937	else => unreachable,
19077	const dst_reg = dst_mcv.getReg().?.to128();	27938	};
19078	const dst_lock = self.register_manager.lockReg(dst_reg);	27939	}
19079	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
19080		27940
19081	if (self.hasFeature(.avx)) if (promote_mcv.isMemory()) try self.asmRegisterRegisterMemory(	27941	fn floatLibcAbiSuffix(ty: Type) []const u8 {
19082	.{ .v_ss, .cvtsd2 },	27942	return switch (ty.toIntern()) {
19083	dst_reg,	27943	.f16_type => "h",
19084	dst_reg,	27944	.f32_type => "f",
19085	try promote_mcv.mem(self, .qword),	27945	.f64_type => "",
19086	) else try self.asmRegisterRegisterRegister(	27946	.f80_type => "x",
19087	.{ .v_ss, .cvtsd2 },	27947	.f128_type => "q",
19088	dst_reg,	27948	.c_longdouble_type => "l",
19089	dst_reg,
19090	(if (promote_mcv.isRegister())
19091	promote_mcv.getReg().?
19092	else
19093	try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
19094	) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
19095	.{ ._ss, .cvtsd2 },
19096	dst_reg,
19097	try promote_mcv.mem(self, .qword),
19098	) else try self.asmRegisterRegister(
19099	.{ ._ss, .cvtsd2 },
19100	dst_reg,
19101	(if (promote_mcv.isRegister())
19102	promote_mcv.getReg().?
19103	else
19104	try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(),
19105	);
19106	break :result promote_mcv;
19107	},
19108	.x86_64_win => return self.fail("TODO implement c_va_arg for Win64", .{}),
19109	else => unreachable,	27949	else => unreachable,
19110	};	27950	};
19111	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
19112	}	27951	}
19113		27952
19114	fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {	27953	fn promoteInt(self: *CodeGen, ty: Type) Type {
19115	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	27954	const pt = self.pt;
19116	const ptr_va_list_ty = self.typeOf(ty_op.operand);	27955	const zcu = pt.zcu;
		27956	const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
		27957	.bool_type => .{ .signedness = .unsigned, .bits = 1 },
		27958	else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
		27959	};
		27960	for ([_]Type{
		27961	.c_int, .c_uint,
		27962	.c_long, .c_ulong,
		27963	.c_longlong, .c_ulonglong,
		27964	}) \|promote_ty\| {
		27965	const promote_info = promote_ty.intInfo(zcu);
		27966	if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
		27967	if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
		27968	promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
		27969	}
		27970	return ty;
		27971	}
19117		27972
19118	const dst_mcv = try self.allocRegOrMem(inst, true);	27973	fn promoteVarArg(self: *CodeGen, ty: Type) Type {
19119	try self.load(dst_mcv, ptr_va_list_ty, .{ .air_ref = ty_op.operand });	27974	if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
19120	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	27975	switch (ty.floatBits(self.target.*)) {
		27976	32, 64 => return .f64,
		27977	else => \|float_bits\| {
		27978	assert(float_bits == self.target.cTypeBitSize(.longdouble));
		27979	return .c_longdouble;
		27980	},
		27981	}
19121	}	27982	}
19122		27983
19123	fn airVaEnd(self: *Self, inst: Air.Inst.Index) !void {	27984	fn intInfo(ty: Type, cg: *CodeGen) ?std.builtin.Type.Int {
19124	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	27985	const zcu = cg.pt.zcu;
19125	return self.finishAir(inst, .unreach, .{ un_op, .none, .none });	27986	const ip = &zcu.intern_pool;
		27987	var ty_index = ty.ip_index;
		27988	while (true) switch (ip.indexToKey(ty_index)) {
		27989	.int_type => \|int_type\| return int_type,
		27990	.ptr_type => \|ptr_type\| return switch (ptr_type.flags.size) {
		27991	.one, .many, .c => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
		27992	.slice => null,
		27993	},
		27994	.opt_type => \|opt_child\| return if (!Type.fromInterned(opt_child).hasRuntimeBitsIgnoreComptime(zcu))
		27995	.{ .signedness = .unsigned, .bits = 1 }
		27996	else switch (ip.indexToKey(opt_child)) {
		27997	.ptr_type => \|ptr_type\| switch (ptr_type.flags.size) {
		27998	.one, .many => switch (ptr_type.flags.is_allowzero) {
		27999	false => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
		28000	true => null,
		28001	},
		28002	.slice, .c => null,
		28003	},
		28004	else => null,
		28005	},
		28006	.error_union_type => \|error_union_type\| return if (!Type.fromInterned(error_union_type.payload_type)
		28007	.hasRuntimeBitsIgnoreComptime(zcu)) .{ .signedness = .unsigned, .bits = zcu.errorSetBits() } else null,
		28008	.simple_type => \|simple_type\| return switch (simple_type) {
		28009	.bool => .{ .signedness = .unsigned, .bits = 1 },
		28010	.anyerror => .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
		28011	.isize => .{ .signedness = .signed, .bits = cg.target.ptrBitWidth() },
		28012	.usize => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
		28013	.c_char => .{ .signedness = cg.target.charSignedness(), .bits = cg.target.cTypeBitSize(.char) },
		28014	.c_short => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.short) },
		28015	.c_ushort => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.short) },
		28016	.c_int => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.int) },
		28017	.c_uint => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.int) },
		28018	.c_long => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.long) },
		28019	.c_ulong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.long) },
		28020	.c_longlong => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.longlong) },
		28021	.c_ulonglong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.longlong) },
		28022	.f16, .f32, .f64, .f80, .f128, .c_longdouble => null,
		28023	.anyopaque,
		28024	.void,
		28025	.type,
		28026	.comptime_int,
		28027	.comptime_float,
		28028	.noreturn,
		28029	.null,
		28030	.undefined,
		28031	.enum_literal,
		28032	.adhoc_inferred_error_set,
		28033	.generic_poison,
		28034	=> unreachable,
		28035	},
		28036	.struct_type => {
		28037	const loaded_struct = ip.loadStructType(ty_index);
		28038	switch (loaded_struct.layout) {
		28039	.auto, .@"extern" => return null,
		28040	.@"packed" => ty_index = loaded_struct.backingIntTypeUnordered(ip),
		28041	}
		28042	},
		28043	.union_type => return switch (ip.loadUnionType(ty_index).flagsUnordered(ip).layout) {
		28044	.auto, .@"extern" => null,
		28045	.@"packed" => .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) },
		28046	},
		28047	.enum_type => ty_index = ip.loadEnumType(ty_index).tag_ty,
		28048	.error_set_type, .inferred_error_set_type => return .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
		28049	else => return null,
		28050	};
19126	}	28051	}
19127		28052
19128	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {	28053	const Temp = struct {
19129	const zcu = self.pt.zcu;	28054	index: Air.Inst.Index,
19130	const ty = self.typeOf(ref);
19131		28055
19132	// If the type has no codegen bits, no need to store it.	28056	fn unwrap(temp: Temp, cg: *CodeGen) union(enum) {
19133	if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;	28057	ref: Air.Inst.Ref,
		28058	temp: Index,
		28059	} {
		28060	switch (temp.index.unwrap()) {
		28061	.ref => \|ref\| return .{ .ref = ref },
		28062	.target => \|target_index\| {
		28063	const temp_index: Index = @enumFromInt(target_index);
		28064	assert(temp_index.isValid(cg));
		28065	return .{ .temp = temp_index };
		28066	},
		28067	}
		28068	}
19134		28069
19135	const mcv = if (ref.toIndex()) \|inst\| mcv: {	28070	fn typeOf(temp: Temp, cg: *CodeGen) Type {
19136	break :mcv self.inst_tracking.getPtr(inst).?.short;	28071	return switch (temp.unwrap(cg)) {
19137	} else mcv: {	28072	.ref => \|ref\| cg.typeOf(ref),
19138	const ip_index = ref.toInterned().?;	28073	.temp => \|temp_index\| temp_index.typeOf(cg),
19139	const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);	28074	};
19140	if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: {	28075	}
19141	const const_mcv = try self.genTypedValue(Value.fromInterned(ip_index));	28076
19142	switch (const_mcv) {	28077	fn isMut(temp: Temp, cg: *CodeGen) bool {
19143	.lea_tlv => \|tlv_sym\| switch (self.bin_file.tag) {	28078	return switch (temp.unwrap(cg)) {
19144	.elf, .macho => {	28079	.ref => false,
19145	if (self.mod.pic) {	28080	.temp => \|temp_index\| switch (temp_index.tracking(cg).short) {
19146	try self.spillRegisters(&.{ .rdi, .rax });	28081	.none,
19147	} else {	28082	.unreach,
19148	try self.spillRegisters(&.{.rax});	28083	.dead,
		28084	.undef,
		28085	.immediate,
		28086	.eflags,
		28087	.register_offset,
		28088	.register_mask,
		28089	.memory,
		28090	.load_symbol,
		28091	.lea_symbol,
		28092	.indirect,
		28093	.load_direct,
		28094	.lea_direct,
		28095	.load_got,
		28096	.lea_got,
		28097	.load_tlv,
		28098	.lea_tlv,
		28099	.lea_frame,
		28100	.elementwise_regs_then_frame,
		28101	.reserved_frame,
		28102	.air_ref,
		28103	=> false,
		28104	.register,
		28105	.register_pair,
		28106	.register_triple,
		28107	.register_quadruple,
		28108	.register_overflow,
		28109	=> true,
		28110	.load_frame => \|frame_addr\| !frame_addr.index.isNamed(),
		28111	},
		28112	};
		28113	}
		28114
		28115	fn tracking(temp: Temp, cg: *CodeGen) InstTracking {
		28116	return cg.inst_tracking.get(temp.index).?;
		28117	}
		28118
		28119	fn getOffset(temp: Temp, off: i32, cg: *CodeGen) !Temp {
		28120	const new_temp_index = cg.next_temp_index;
		28121	cg.temp_type[@intFromEnum(new_temp_index)] = .usize;
		28122	cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		28123	switch (temp.tracking(cg).short) {
		28124	else => \|mcv\| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
		28125	.register => \|reg\| {
		28126	const new_reg =
		28127	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28128	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28129	try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
		28130	.base = .{ .reg = reg.to64() },
		28131	.mod = .{ .rm = .{
		28132	.size = .qword,
		28133	.disp = off,
		28134	} },
		28135	});
		28136	},
		28137	.register_offset => \|reg_off\| {
		28138	const new_reg =
		28139	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28140	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28141	try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
		28142	.base = .{ .reg = reg_off.reg.to64() },
		28143	.mod = .{ .rm = .{
		28144	.size = .qword,
		28145	.disp = reg_off.off + off,
		28146	} },
		28147	});
		28148	},
		28149	.lea_symbol => \|sym_off\| new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = .{
		28150	.sym_index = sym_off.sym_index,
		28151	.off = sym_off.off + off,
		28152	} }),
		28153	.load_frame => \|frame_addr\| {
		28154	const new_reg =
		28155	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28156	new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{
		28157	.reg = new_reg,
		28158	.off = off,
		28159	} });
		28160	try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
		28161	.base = .{ .frame = frame_addr.index },
		28162	.mod = .{ .rm = .{
		28163	.size = .qword,
		28164	.disp = frame_addr.off,
		28165	} },
		28166	});
		28167	},
		28168	.lea_frame => \|frame_addr\| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{
		28169	.index = frame_addr.index,
		28170	.off = frame_addr.off + off,
		28171	} }),
		28172	}
		28173	return .{ .index = new_temp_index.toIndex() };
		28174	}
		28175
		28176	fn toOffset(temp: Temp, off: i32, cg: CodeGen) !void {
		28177	if (off == 0) return;
		28178	switch (temp.unwrap(cg)) {
		28179	.ref => {},
		28180	.temp => \|temp_index\| {
		28181	const temp_tracking = temp_index.tracking(cg);
		28182	switch (temp_tracking.short) {
		28183	else => {},
		28184	.register => \|reg\| {
		28185	try cg.freeValue(temp_tracking.long);
		28186	temp_tracking.* = .init(.{ .register_offset = .{
		28187	.reg = reg,
		28188	.off = off,
		28189	} });
		28190	return;
		28191	},
		28192	.register_offset => \|reg_off\| {
		28193	try cg.freeValue(temp_tracking.long);
		28194	temp_tracking.* = .init(.{ .register_offset = .{
		28195	.reg = reg_off.reg,
		28196	.off = reg_off.off + off,
		28197	} });
		28198	return;
		28199	},
		28200	.lea_symbol => \|sym_off\| {
		28201	assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
		28202	temp_tracking.* = .init(.{ .lea_symbol = .{
		28203	.sym_index = sym_off.sym_index,
		28204	.off = sym_off.off + off,
		28205	} });
		28206	return;
		28207	},
		28208	.lea_frame => \|frame_addr\| {
		28209	assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
		28210	temp_tracking.* = .init(.{ .lea_frame = .{
		28211	.index = frame_addr.index,
		28212	.off = frame_addr.off + off,
		28213	} });
		28214	return;
		28215	},
		28216	}
		28217	},
		28218	}
		28219	const new_temp = try temp.getOffset(off, cg);
		28220	try temp.die(cg);
		28221	temp.* = new_temp;
		28222	}
		28223
		28224	fn getLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) !Temp {
		28225	const new_temp_index = cg.next_temp_index;
		28226	cg.temp_type[@intFromEnum(new_temp_index)] = limb_ty;
		28227	switch (temp.tracking(cg).short) {
		28228	else => \|mcv\| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
		28229	.immediate => \|imm\| {
		28230	assert(limb_index == 0);
		28231	new_temp_index.tracking(cg).* = .init(.{ .immediate = imm });
		28232	},
		28233	.register => \|reg\| {
		28234	assert(limb_index == 0);
		28235	const new_reg =
		28236	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28237	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28238	try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
		28239	},
		28240	.register_pair => \|regs\| {
		28241	const new_reg =
		28242	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28243	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28244	try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
		28245	},
		28246	.register_offset => \|reg_off\| {
		28247	assert(limb_index == 0);
		28248	const new_reg =
		28249	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28250	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28251	try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
		28252	.base = .{ .reg = reg_off.reg.to64() },
		28253	.mod = .{ .rm = .{
		28254	.size = .qword,
		28255	.disp = reg_off.off + @as(u31, limb_index) * 8,
		28256	} },
		28257	});
		28258	},
		28259	.load_symbol => \|sym_off\| {
		28260	const new_reg =
		28261	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28262	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28263	try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
		28264	.base = .{ .reloc = sym_off.sym_index },
		28265	.mod = .{ .rm = .{
		28266	.size = .qword,
		28267	.disp = sym_off.off + @as(u31, limb_index) * 8,
		28268	} },
		28269	});
		28270	},
		28271	.lea_symbol => \|sym_off\| {
		28272	assert(limb_index == 0);
		28273	new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = sym_off });
		28274	},
		28275	.load_frame => \|frame_addr\| {
		28276	const new_reg =
		28277	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28278	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28279	try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
		28280	.base = .{ .frame = frame_addr.index },
		28281	.mod = .{ .rm = .{
		28282	.size = .qword,
		28283	.disp = frame_addr.off + @as(u31, limb_index) * 8,
		28284	} },
		28285	});
		28286	},
		28287	.lea_frame => \|frame_addr\| {
		28288	assert(limb_index == 0);
		28289	new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr });
		28290	},
		28291	}
		28292	cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		28293	return .{ .index = new_temp_index.toIndex() };
		28294	}
		28295
		28296	fn toLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: CodeGen) !void {
		28297	switch (temp.unwrap(cg)) {
		28298	.ref => {},
		28299	.temp => \|temp_index\| {
		28300	const temp_tracking = temp_index.tracking(cg);
		28301	switch (temp_tracking.short) {
		28302	else => {},
		28303	.register, .lea_symbol, .lea_frame => {
		28304	assert(limb_index == 0);
		28305	cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
		28306	return;
		28307	},
		28308	.register_pair => \|regs\| {
		28309	switch (temp_tracking.long) {
		28310	.none, .reserved_frame => {},
		28311	else => temp_tracking.long =
		28312	temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
19149	}	28313	}
19150	const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{	28314	for (regs, 0..) \|reg, reg_index\| if (reg_index != limb_index)
19151	.size = 8,	28315	cg.register_manager.freeReg(reg);
19152	.alignment = .@"8",	28316	temp_tracking.* = .init(.{ .register = regs[limb_index] });
19153	}));	28317	cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
19154	try self.genSetMem(	28318	return;
19155	.{ .frame = frame_index },
19156	0,
19157	Type.usize,
19158	.{ .lea_symbol = .{ .sym_index = tlv_sym } },
19159	.{},
19160	);
19161	break :init .{ .load_frame = .{ .index = frame_index } };
19162	},	28319	},
19163	else => break :init const_mcv,	28320	.load_symbol => \|sym_off\| {
19164	},	28321	assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
19165	else => break :init const_mcv,	28322	temp_tracking.* = .init(.{ .load_symbol = .{
19166	}	28323	.sym_index = sym_off.sym_index,
19167	});	28324	.off = sym_off.off + @as(u31, limb_index) * 8,
19168	break :mcv gop.value_ptr.short;	28325	} });
19169	};	28326	cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
		28327	return;
		28328	},
		28329	.load_frame => \|frame_addr\| if (!frame_addr.index.isNamed()) {
		28330	assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
		28331	temp_tracking.* = .init(.{ .load_frame = .{
		28332	.index = frame_addr.index,
		28333	.off = frame_addr.off + @as(u31, limb_index) * 8,
		28334	} });
		28335	cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
		28336	return;
		28337	},
		28338	}
		28339	},
		28340	}
		28341	const new_temp = try temp.getLimb(limb_ty, limb_index, cg);
		28342	try temp.die(cg);
		28343	temp.* = new_temp;
		28344	}
19170		28345
19171	switch (mcv) {	28346	fn toSlicePtr(temp: Temp, cg: CodeGen) !void {
19172	.none, .unreach, .dead => unreachable,	28347	const temp_ty = temp.typeOf(cg);
19173	else => return mcv,	28348	if (temp_ty.isSlice(cg.pt.zcu)) try temp.toLimb(temp_ty.slicePtrFieldType(cg.pt.zcu), 0, cg);
		28349	}
		28350
		28351	fn toSliceLen(temp: Temp, cg: CodeGen) !void {
		28352	try temp.toLimb(.usize, 1, cg);
19174	}	28353	}
19175	}
19176		28354
19177	fn getResolvedInstValue(self: Self, inst: Air.Inst.Index) InstTracking {	28355	fn toReg(temp: Temp, new_reg: Register, cg: CodeGen) !bool {
19178	const tracking = self.inst_tracking.getPtr(inst).?;	28356	const val, const ty = val_ty: switch (temp.unwrap(cg)) {
19179	return switch (tracking.short) {	28357	.ref => \|ref\| .{ temp.tracking(cg).short, cg.typeOf(ref) },
19180	.none, .unreach, .dead => unreachable,	28358	.temp => \|temp_index\| {
19181	else => tracking,	28359	const temp_tracking = temp_index.tracking(cg);
19182	};	28360	if (temp_tracking.short == .register and
19183	}	28361	temp_tracking.short.register == new_reg) return false;
		28362	break :val_ty .{ temp_tracking.short, temp_index.typeOf(cg) };
		28363	},
		28364	};
		28365	const new_temp_index = cg.next_temp_index;
		28366	try cg.register_manager.getReg(new_reg, new_temp_index.toIndex());
		28367	cg.temp_type[@intFromEnum(new_temp_index)] = ty;
		28368	try cg.genSetReg(new_reg, ty, val, .{});
		28369	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
		28370	try temp.die(cg);
		28371	cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		28372	temp.* = .{ .index = new_temp_index.toIndex() };
		28373	return true;
		28374	}
19184		28375
19185	/// If the MCValue is an immediate, and it does not fit within this type,	28376	fn toRegClass(temp: Temp, mut: bool, rc: Register.Class, cg: CodeGen) !bool {
19186	/// we put it in a register.	28377	const val = temp.tracking(cg).short;
19187	/// A potential opportunity for future optimization here would be keeping track	28378	if (!mut or temp.isMut(cg)) switch (val) {
19188	/// of the fact that the instruction is available both as an immediate	28379	else => {},
19189	/// and as a register.	28380	.register => \|reg\| if (reg.class() == rc) return false,
19190	fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {	28381	.register_offset => \|reg_off\| if (reg_off.reg.class() == rc and reg_off.off == 0) return false,
19191	const mcv = try self.resolveInst(operand);	28382	};
19192	const ti = @typeInfo(T).int;	28383	const ty = temp.typeOf(cg);
19193	switch (mcv) {	28384	const new_temp_index = cg.next_temp_index;
19194	.immediate => \|imm\| {	28385	cg.temp_type[@intFromEnum(new_temp_index)] = ty;
19195	// This immediate is unsigned.	28386	const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
19196	const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));	28387	try cg.genSetReg(new_reg, ty, val, .{});
19197	if (imm >= math.maxInt(U)) {	28388	new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
19198	return MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) };	28389	try temp.die(cg);
19199	}	28390	cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
19200	},	28391	temp.* = .{ .index = new_temp_index.toIndex() };
19201	else => {},	28392	return true;
19202	}	28393	}
19203	return mcv;
19204	}
19205		28394
19206	fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {	28395	fn toPair(first_temp: Temp, second_temp: Temp, cg: *CodeGen) !void {
19207	const pt = self.pt;	28396	while (true) for ([_]*Temp{ first_temp, second_temp }) \|part_temp\| {
19208	return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) {	28397	if (try part_temp.toRegClass(true, .general_purpose, cg)) break;
19209	.mcv => \|mcv\| switch (mcv) {	28398	} else break;
19210	.none => .none,	28399	const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
19211	.undef => .undef,	28400	const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
19212	.immediate => \|imm\| .{ .immediate = imm },	28401	const result: MCValue = .{ .register_pair = .{
19213	.memory => \|addr\| .{ .memory = addr },	28402	first_temp_tracking.short.register,
19214	.load_symbol => \|sym_index\| .{ .load_symbol = .{ .sym_index = sym_index } },	28403	second_temp_tracking.short.register,
19215	.lea_symbol => \|sym_index\| .{ .lea_symbol = .{ .sym_index = sym_index } },	28404	} };
19216	.load_direct => \|sym_index\| .{ .load_direct = sym_index },	28405	const result_temp_index = cg.next_temp_index;
19217	.lea_direct => \|sym_index\| .{ .lea_direct = sym_index },	28406	const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
19218	.load_got => \|sym_index\| .{ .lea_got = sym_index },	28407	assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
19219	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },	28408	assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
19220	},	28409	cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8;
19221	.fail => \|msg\| return self.failMsg(msg),	28410	result_temp_index.tracking(cg).* = .init(result);
19222	};	28411	first_temp.* = result_temp;
19223	}	28412	}
19224		28413
19225	const CallMCValues = struct {	28414	fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void {
19226	args: []MCValue,	28415	assert(info.scalar != .none);
19227	return_value: InstTracking,	28416	const mcv = &temp.unwrap(cg).temp.tracking(cg).short;
19228	stack_byte_count: u31,	28417	const reg = mcv.register;
19229	stack_align: Alignment,	28418	mcv.* = .{ .register_mask = .{ .reg = reg, .info = info } };
19230	gp_count: u32,	28419	}
19231	fp_count: u32,
19232		28420
19233	fn deinit(self: CallMCValues, func: Self) void {	28421	fn toLea(temp: Temp, cg: CodeGen) !bool {
19234	func.gpa.free(self.args);	28422	switch (temp.tracking(cg).short) {
19235	self.* = undefined;	28423	.none,
		28424	.unreach,
		28425	.dead,
		28426	.undef,
		28427	.eflags,
		28428	.register_pair,
		28429	.register_triple,
		28430	.register_quadruple,
		28431	.register_overflow,
		28432	.register_mask,
		28433	.elementwise_regs_then_frame,
		28434	.reserved_frame,
		28435	.air_ref,
		28436	=> unreachable, // not a valid pointer
		28437	.immediate,
		28438	.register,
		28439	.register_offset,
		28440	.lea_direct,
		28441	.lea_got,
		28442	.lea_tlv,
		28443	.lea_frame,
		28444	=> return false,
		28445	.memory,
		28446	.indirect,
		28447	.load_symbol,
		28448	.load_direct,
		28449	.load_got,
		28450	.load_tlv,
		28451	.load_frame,
		28452	=> return temp.toRegClass(true, .general_purpose, cg),
		28453	.lea_symbol => \|sym_off\| {
		28454	const off = sym_off.off;
		28455	if (off == 0) return false;
		28456	try temp.toOffset(-off, cg);
		28457	while (try temp.toRegClass(true, .general_purpose, cg)) {}
		28458	try temp.toOffset(off, cg);
		28459	return true;
		28460	},
		28461	}
19236	}	28462	}
19237	};
19238		28463
19239	/// Caller must call `CallMCValues.deinit`.	28464	fn toMemory(temp: Temp, cg: CodeGen) !bool {
19240	fn resolveCallingConventionValues(	28465	const temp_tracking = temp.tracking(cg);
19241	self: *Self,	28466	if (temp_tracking.short.isMemory()) return false;
19242	fn_info: InternPool.Key.FuncType,	28467	const new_temp_index = cg.next_temp_index;
19243	var_args: []const Type,	28468	const ty = temp.typeOf(cg);
19244	stack_frame_base: FrameIndex,	28469	cg.temp_type[@intFromEnum(new_temp_index)] = ty;
19245	) !CallMCValues {	28470	const new_frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu));
19246	const pt = self.pt;	28471	try cg.genSetMem(.{ .frame = new_frame_index }, 0, ty, temp_tracking.short, .{});
19247	const zcu = pt.zcu;	28472	new_temp_index.tracking(cg).* = .init(.{ .load_frame = .{ .index = new_frame_index } });
19248	const ip = &zcu.intern_pool;	28473	try temp.die(cg);
19249	const cc = fn_info.cc;	28474	cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
19250	const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);	28475	temp.* = .{ .index = new_temp_index.toIndex() };
19251	defer self.gpa.free(param_types);	28476	return true;
		28477	}
19252		28478
19253	for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) \|*dest, src\| {	28479	// hack around linker relocation bugs
19254	dest.* = Type.fromInterned(src);	28480	fn toBase(temp: Temp, cg: CodeGen) !bool {
		28481	const temp_tracking = temp.tracking(cg);
		28482	if (temp_tracking.short.isBase()) return false;
		28483	if (try temp.toMemory(cg)) return true;
		28484	const new_temp_index = cg.next_temp_index;
		28485	cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg);
		28486	const new_reg =
		28487	try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
		28488	try cg.genSetReg(new_reg, .usize, temp_tracking.short.address(), .{});
		28489	new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } });
		28490	try temp.die(cg);
		28491	cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
		28492	temp.* = .{ .index = new_temp_index.toIndex() };
		28493	return true;
19255	}	28494	}
19256	for (param_types[fn_info.param_types.len..], var_args) \|*param_ty, arg_ty\|
19257	param_ty.* = self.promoteVarArg(arg_ty);
19258		28495
19259	var result: CallMCValues = .{	28496	const AccessOptions = struct {
19260	.args = try self.gpa.alloc(MCValue, param_types.len),	28497	disp: i32 = 0,
19261	// These undefined values must be populated before returning from this function.	28498	safe: bool = false,
19262	.return_value = undefined,
19263	.stack_byte_count = 0,
19264	.stack_align = undefined,
19265	.gp_count = 0,
19266	.fp_count = 0,
19267	};	28499	};
19268	errdefer self.gpa.free(result.args);
19269
19270	const ret_ty = Type.fromInterned(fn_info.return_type);
19271		28500
19272	const resolved_cc = abi.resolveCallingConvention(cc, self.target.*);	28501	fn load(ptr: Temp, val_ty: Type, opts: AccessOptions, cg: CodeGen) !Temp {
19273	switch (cc) {	28502	const val = try cg.tempAlloc(val_ty);
19274	.naked => {	28503	try ptr.toOffset(opts.disp, cg);
19275	assert(result.args.len == 0);	28504	while (try ptr.toLea(cg)) {}
19276	result.return_value = InstTracking.init(.unreach);	28505	const val_mcv = val.tracking(cg).short;
19277	result.stack_align = .@"8";	28506	switch (val_mcv) {
19278	},	28507	else => \|mcv\| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
19279	.x86_64_sysv, .x86_64_win => \|cc_opts\| {	28508	.register => \|val_reg\| try ptr.loadReg(val_ty, registerAlias(
19280	var ret_int_reg_i: u32 = 0;	28509	val_reg,
19281	var ret_sse_reg_i: u32 = 0;	28510	@intCast(val_ty.abiSize(cg.pt.zcu)),
19282	var param_int_reg_i: u32 = 0;	28511	), cg),
19283	var param_sse_reg_i: u32 = 0;	28512	inline .register_pair,
19284	result.stack_align = .fromByteUnits(cc_opts.incoming_stack_alignment orelse 16);	28513	.register_triple,
		28514	.register_quadruple,
		28515	=> \|val_regs\| for (val_regs) \|val_reg\| {
		28516	try ptr.loadReg(val_ty, val_reg, cg);
		28517	try ptr.toOffset(@divExact(val_reg.bitSize(), 8), cg);
		28518	while (try ptr.toLea(cg)) {}
		28519	},
		28520	.register_offset => \|val_reg_off\| switch (val_reg_off.off) {
		28521	0 => try ptr.loadReg(val_ty, registerAlias(
		28522	val_reg_off.reg,
		28523	@intCast(val_ty.abiSize(cg.pt.zcu)),
		28524	), cg),
		28525	else => unreachable,
		28526	},
		28527	.memory, .indirect, .load_frame, .load_symbol => {
		28528	var val_ptr = try cg.tempInit(.usize, val_mcv.address());
		28529	var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
		28530	try val_ptr.memcpy(ptr, &len, cg);
		28531	try val_ptr.die(cg);
		28532	try len.die(cg);
		28533	},
		28534	}
		28535	return val;
		28536	}
19285		28537
19286	switch (resolved_cc) {	28538	fn store(ptr: Temp, val: Temp, opts: AccessOptions, cg: *CodeGen) !void {
19287	.x86_64_sysv => {},	28539	const val_ty = val.typeOf(cg);
19288	.x86_64_win => {	28540	try ptr.toOffset(opts.disp, cg);
19289	// Align the stack to 16bytes before allocating shadow stack space (if any).	28541	while (try ptr.toLea(cg)) {}
19290	result.stack_byte_count += @intCast(4 * Type.usize.abiSize(zcu));	28542	val_to_gpr: while (true) : (while (try ptr.toLea(cg) or
		28543	try val.toRegClass(false, .general_purpose, cg))
		28544	{}) {
		28545	const val_mcv = val.tracking(cg).short;
		28546	switch (val_mcv) {
		28547	else => \|mcv\| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
		28548	.undef => if (opts.safe) {
		28549	var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
		28550	var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
		28551	try ptr.memset(&pat, &len, cg);
		28552	try pat.die(cg);
		28553	try len.die(cg);
		28554	},
		28555	.immediate => \|val_imm\| {
		28556	const val_op: Immediate = if (std.math.cast(u31, val_imm)) \|val_uimm31\|
		28557	.u(val_uimm31)
		28558	else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) \|val_simm32\|
		28559	.s(val_simm32)
		28560	else
		28561	continue :val_to_gpr;
		28562	// hack around linker relocation bugs
		28563	switch (ptr.tracking(cg).short) {
		28564	else => {},
		28565	.lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
		28566	}
		28567	try cg.asmMemoryImmediate(
		28568	.{ ._, .mov },
		28569	try ptr.tracking(cg).short.deref().mem(cg, .{
		28570	.size = cg.memSize(val_ty),
		28571	}),
		28572	val_op,
		28573	);
		28574	},
		28575	.eflags => \|cc\| {
		28576	// hack around linker relocation bugs
		28577	switch (ptr.tracking(cg).short) {
		28578	else => {},
		28579	.lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
		28580	}
		28581	try cg.asmSetccMemory(
		28582	cc,
		28583	try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
		28584	);
		28585	},
		28586	.register => \|val_reg\| try ptr.storeRegs(val_ty, &.{registerAlias(
		28587	val_reg,
		28588	@intCast(val_ty.abiSize(cg.pt.zcu)),
		28589	)}, cg),
		28590	inline .register_pair,
		28591	.register_triple,
		28592	.register_quadruple,
		28593	=> \|val_regs\| try ptr.storeRegs(val_ty, &val_regs, cg),
		28594	.register_offset => \|val_reg_off\| switch (val_reg_off.off) {
		28595	0 => try ptr.storeRegs(val_ty, &.{registerAlias(
		28596	val_reg_off.reg,
		28597	@intCast(val_ty.abiSize(cg.pt.zcu)),
		28598	)}, cg),
		28599	else => continue :val_to_gpr,
		28600	},
		28601	.register_overflow => \|val_reg_ov\| {
		28602	const ip = &cg.pt.zcu.intern_pool;
		28603	const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
		28604	.tuple_type => \|tuple_type\| {
		28605	const tuple_field_types = tuple_type.types.get(ip);
		28606	assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
		28607	break :first_ty tuple_field_types[0];
		28608	},
		28609	.opt_type => \|opt_child\| {
		28610	assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
		28611	break :first_ty opt_child;
		28612	},
		28613	else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
		28614	});
		28615	const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
		28616	try ptr.storeRegs(first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
		28617	try ptr.toOffset(first_size, cg);
		28618	try cg.asmSetccMemory(
		28619	val_reg_ov.eflags,
		28620	try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
		28621	);
		28622	},
		28623	.lea_frame, .lea_symbol => continue :val_to_gpr,
		28624	.memory, .indirect, .load_frame, .load_symbol => {
		28625	var val_ptr = try cg.tempInit(.usize, val_mcv.address());
		28626	var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
		28627	try ptr.memcpy(&val_ptr, &len, cg);
		28628	try val_ptr.die(cg);
		28629	try len.die(cg);
19291	},	28630	},
19292	else => unreachable,
19293	}	28631	}
		28632	break;
		28633	}
		28634	}
19294		28635
19295	// Return values	28636	fn read(src: Temp, val_ty: Type, opts: AccessOptions, cg: CodeGen) !Temp {
19296	if (ret_ty.zigTypeTag(zcu) == .noreturn) {	28637	var val = try cg.tempAlloc(val_ty);
19297	result.return_value = InstTracking.init(.unreach);	28638	while (try src.toBase(cg)) {}
19298	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {	28639	const val_mcv = val.tracking(cg).short;
19299	// TODO: is this even possible for C calling convention?	28640	switch (val_mcv) {
19300	result.return_value = InstTracking.init(.none);	28641	else => \|mcv\| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
19301	} else {	28642	.register => \|val_reg\| try src.readReg(opts.disp, val_ty, registerAlias(
19302	var ret_tracking: [2]InstTracking = undefined;	28643	val_reg,
19303	var ret_tracking_i: usize = 0;	28644	@intCast(val_ty.abiSize(cg.pt.zcu)),
		28645	), cg),
		28646	inline .register_pair, .register_triple, .register_quadruple => \|val_regs\| {
		28647	var disp = opts.disp;
		28648	for (val_regs) \|val_reg\| {
		28649	try src.readReg(disp, val_ty, val_reg, cg);
		28650	disp += @divExact(val_reg.bitSize(), 8);
		28651	}
		28652	},
		28653	.register_offset => \|val_reg_off\| switch (val_reg_off.off) {
		28654	0 => try src.readReg(opts.disp, val_ty, registerAlias(
		28655	val_reg_off.reg,
		28656	@intCast(val_ty.abiSize(cg.pt.zcu)),
		28657	), cg),
		28658	else => unreachable,
		28659	},
		28660	.memory, .indirect, .load_frame, .load_symbol => {
		28661	var val_ptr = try cg.tempInit(.usize, val_mcv.address());
		28662	var src_ptr =
		28663	try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));
		28664	var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
		28665	try val_ptr.memcpy(&src_ptr, &len, cg);
		28666	try val_ptr.die(cg);
		28667	try src_ptr.die(cg);
		28668	try len.die(cg);
		28669	},
		28670	}
		28671	return val;
		28672	}
19304		28673
19305	const classes = switch (resolved_cc) {	28674	fn write(dst: Temp, val: Temp, opts: AccessOptions, cg: *CodeGen) !void {
19306	.x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),	28675	const val_ty = val.typeOf(cg);
19307	.x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},	28676	while (try dst.toBase(cg)) {}
19308	else => unreachable,	28677	val_to_gpr: while (true) : (while (try dst.toBase(cg) or
19309	};	28678	try val.toRegClass(false, .general_purpose, cg))
19310	for (classes) \|class\| switch (class) {	28679	{}) {
19311	.integer => {	28680	const val_mcv = val.tracking(cg).short;
19312	const ret_int_reg = registerAlias(	28681	switch (val_mcv) {
19313	abi.getCAbiIntReturnRegs(resolved_cc)[ret_int_reg_i],	28682	else => \|mcv\| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
19314	@intCast(@min(ret_ty.abiSize(zcu), 8)),	28683	.undef => if (opts.safe) {
19315	);	28684	var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
19316	ret_int_reg_i += 1;	28685	var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
		28686	var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
		28687	try dst_ptr.memset(&pat, &len, cg);
		28688	try dst_ptr.die(cg);
		28689	try pat.die(cg);
		28690	try len.die(cg);
		28691	},
		28692	.immediate => \|val_imm\| {
		28693	const val_op: Immediate = if (std.math.cast(u31, val_imm)) \|val_uimm31\|
		28694	.u(val_uimm31)
		28695	else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) \|val_simm32\|
		28696	.s(val_simm32)
		28697	else
		28698	continue :val_to_gpr;
		28699	try cg.asmMemoryImmediate(
		28700	.{ ._, .mov },
		28701	try dst.tracking(cg).short.mem(cg, .{
		28702	.size = cg.memSize(val_ty),
		28703	.disp = opts.disp,
		28704	}),
		28705	val_op,
		28706	);
		28707	},
		28708	.eflags => \|cc\| try cg.asmSetccMemory(
		28709	cc,
		28710	try dst.tracking(cg).short.mem(cg, .{
		28711	.size = .byte,
		28712	.disp = opts.disp,
		28713	}),
		28714	),
		28715	.register => \|val_reg\| try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
		28716	val_reg,
		28717	@intCast(val_ty.abiSize(cg.pt.zcu)),
		28718	)}, cg),
		28719	inline .register_pair,
		28720	.register_triple,
		28721	.register_quadruple,
		28722	=> \|val_regs\| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg),
		28723	.register_offset => \|val_reg_off\| switch (val_reg_off.off) {
		28724	0 => try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
		28725	val_reg_off.reg,
		28726	@intCast(val_ty.abiSize(cg.pt.zcu)),
		28727	)}, cg),
		28728	else => continue :val_to_gpr,
		28729	},
		28730	.register_overflow => \|val_reg_ov\| {
		28731	const ip = &cg.pt.zcu.intern_pool;
		28732	const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
		28733	.tuple_type => \|tuple_type\| {
		28734	const tuple_field_types = tuple_type.types.get(ip);
		28735	assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
		28736	break :first_ty tuple_field_types[0];
		28737	},
		28738	.opt_type => \|opt_child\| {
		28739	assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
		28740	break :first_ty opt_child;
		28741	},
		28742	else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
		28743	});
		28744	const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
		28745	try dst.writeRegs(opts.disp, first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
		28746	try cg.asmSetccMemory(
		28747	val_reg_ov.eflags,
		28748	try dst.tracking(cg).short.mem(cg, .{
		28749	.size = .byte,
		28750	.disp = opts.disp + first_size,
		28751	}),
		28752	);
		28753	},
		28754	.lea_frame, .lea_symbol => continue :val_to_gpr,
		28755	.memory, .indirect, .load_frame, .load_symbol => {
		28756	var dst_ptr =
		28757	try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
		28758	var val_ptr = try cg.tempInit(.usize, val_mcv.address());
		28759	var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
		28760	try dst_ptr.memcpy(&val_ptr, &len, cg);
		28761	try dst_ptr.die(cg);
		28762	try val_ptr.die(cg);
		28763	try len.die(cg);
		28764	},
		28765	}
		28766	break;
		28767	}
		28768	}
19317		28769
19318	ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg });	28770	fn loadReg(ptr: Temp, dst_ty: Type, dst_reg: Register, cg: CodeGen) !void {
19319	ret_tracking_i += 1;	28771	const dst_rc = dst_reg.class();
19320	},	28772	const strat = try cg.moveStrategy(dst_ty, dst_rc, false);
19321	.sse, .float, .float_combine, .win_i128 => {	28773	// hack around linker relocation bugs
19322	const ret_sse_reg = registerAlias(	28774	switch (ptr.tracking(cg).short) {
19323	abi.getCAbiSseReturnRegs(resolved_cc)[ret_sse_reg_i],	28775	else => {},
19324	@intCast(ret_ty.abiSize(zcu)),	28776	.lea_symbol => \|sym_off\| if (dst_rc != .general_purpose or sym_off.off != 0)
19325	);	28777	while (try ptr.toRegClass(false, .general_purpose, cg)) {},
19326	ret_sse_reg_i += 1;	28778	}
		28779	try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{
		28780	.size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
		28781	}));
		28782	}
19327		28783
19328	ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_sse_reg });	28784	fn storeRegs(ptr: Temp, src_ty: Type, src_regs: []const Register, cg: CodeGen) !void {
19329	ret_tracking_i += 1;	28785	var part_disp: u31 = 0;
19330	},	28786	var deferred_disp: u31 = 0;
19331	.sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse),	28787	var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
19332	.x87 => {	28788	for (src_regs) \|src_reg\| {
19333	ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = .st0 });	28789	const src_rc = src_reg.class();
19334	ret_tracking_i += 1;	28790	const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
19335	},	28791	const part_size = @divExact(part_bit_size, 8);
19336	.x87up => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .x87),	28792	if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
19337	.complex_x87 => {	28793	// hack around linker relocation bugs
19338	ret_tracking[ret_tracking_i] =	28794	switch (ptr.tracking(cg).short) {
19339	InstTracking.init(.{ .register_pair = .{ .st0, .st1 } });	28795	else => {},
19340	ret_tracking_i += 1;	28796	.lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
19341	},	28797	}
19342	.memory => {	28798	const strat = try cg.moveStrategy(src_ty, src_rc, false);
19343	const ret_int_reg = abi.getCAbiIntReturnRegs(resolved_cc)[ret_int_reg_i].to64();	28799	try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{
19344	ret_int_reg_i += 1;	28800	.size = .fromBitSize(part_bit_size),
19345	const ret_indirect_reg = abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i];	28801	.disp = part_disp,
19346	param_int_reg_i += 1;	28802	}), registerAlias(src_reg, part_size));
		28803	} else {
		28804	const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
		28805	const frame_index = try cg.allocFrameIndex(.init(.{
		28806	.size = frame_size,
		28807	.alignment = .fromNonzeroByteUnits(frame_size),
		28808	}));
		28809	const strat = try cg.moveStrategy(src_ty, src_rc, true);
		28810	try strat.write(cg, .{
		28811	.base = .{ .frame = frame_index },
		28812	.mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
		28813	}, registerAlias(src_reg, frame_size));
		28814	try ptr.toOffset(deferred_disp, cg);
		28815	deferred_disp = 0;
		28816	var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
		28817	var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
		28818	try ptr.memcpy(&src_ptr, &len, cg);
		28819	try src_ptr.die(cg);
		28820	try len.die(cg);
		28821	}
		28822	part_disp += part_size;
		28823	deferred_disp += part_size;
		28824	src_abi_size -= part_size;
		28825	}
		28826	}
19347		28827
19348	ret_tracking[ret_tracking_i] = .{	28828	fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
19349	.short = .{ .indirect = .{ .reg = ret_int_reg } },	28829	const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false);
19350	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },	28830	try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{
19351	};	28831	.size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
19352	ret_tracking_i += 1;	28832	.disp = disp,
19353	},	28833	}));
19354	.none, .integer_per_element => unreachable,	28834	}
19355	};	28835
19356	result.return_value = switch (ret_tracking_i) {	28836	fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) !void {
19357	else => unreachable,	28837	var part_disp = disp;
19358	1 => ret_tracking[0],	28838	var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
19359	2 => InstTracking.init(.{ .register_pair = .{	28839	for (src_regs) \|src_reg\| {
19360	ret_tracking[0].short.register, ret_tracking[1].short.register,	28840	const src_rc = src_reg.class();
19361	} }),	28841	const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
19362	};	28842	const part_size = @divExact(part_bit_size, 8);
		28843	if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
		28844	const strat = try cg.moveStrategy(src_ty, src_rc, false);
		28845	try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{
		28846	.size = .fromBitSize(part_bit_size),
		28847	.disp = part_disp,
		28848	}), registerAlias(src_reg, part_size));
		28849	} else {
		28850	const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
		28851	const frame_index = try cg.allocFrameIndex(.init(.{
		28852	.size = frame_size,
		28853	.alignment = .fromNonzeroByteUnits(frame_size),
		28854	}));
		28855	const strat = try cg.moveStrategy(src_ty, src_rc, true);
		28856	try strat.write(cg, .{
		28857	.base = .{ .frame = frame_index },
		28858	.mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
		28859	}, registerAlias(src_reg, frame_size));
		28860	var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
		28861	try dst_ptr.toOffset(part_disp, cg);
		28862	var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
		28863	var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
		28864	try dst_ptr.memcpy(&src_ptr, &len, cg);
		28865	try dst_ptr.die(cg);
		28866	try src_ptr.die(cg);
		28867	try len.die(cg);
19363	}	28868	}
		28869	part_disp += part_size;
		28870	src_abi_size -= part_size;
		28871	}
		28872	}
19364		28873
19365	// Input params	28874	fn memcpy(dst: Temp, src: Temp, len: Temp, cg: CodeGen) !void {
19366	for (param_types, result.args) \|ty, *arg\| {	28875	while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) \|temp, reg\| {
19367	assert(ty.hasRuntimeBitsIgnoreComptime(zcu));	28876	if (try temp.toReg(reg, cg)) break;
19368	switch (resolved_cc) {	28877	} else break;
19369	.x86_64_sysv => {},	28878	try cg.asmOpOnly(.{ .@"rep _sb", .mov });
19370	.x86_64_win => {	28879	}
19371	param_int_reg_i = @max(param_int_reg_i, param_sse_reg_i);
19372	param_sse_reg_i = param_int_reg_i;
19373	},
19374	else => unreachable,
19375	}
19376		28880
19377	var arg_mcv: [2]MCValue = undefined;	28881	fn memset(dst: Temp, val: Temp, len: Temp, cg: CodeGen) !void {
19378	var arg_mcv_i: usize = 0;	28882	while (true) for ([_]*Temp{ dst, val, len }, [_]Register{ .rdi, .rax, .rcx }) \|temp, reg\| {
		28883	if (try temp.toReg(reg, cg)) break;
		28884	} else break;
		28885	try cg.asmOpOnly(.{ .@"rep _sb", .sto });
		28886	}
19379		28887
19380	const classes = switch (resolved_cc) {	28888	fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void {
19381	.x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),	28889	if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) {
19382	.x86_64_win => &.{abi.classifyWindows(ty, zcu)},	28890	.ref => {
19383	else => unreachable,	28891	const result = try cg.allocRegOrMem(inst, true);
19384	};	28892	try cg.genCopy(cg.typeOfIndex(inst), result, temp.tracking(cg).short, .{});
19385	for (classes) \|class\| switch (class) {	28893	tracking_log.debug("{} => {} (birth)", .{ inst, result });
19386	.integer => {	28894	cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
19387	const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc);	28895	},
19388	if (param_int_reg_i >= param_int_regs.len) break;	28896	.temp => \|temp_index\| {
		28897	const temp_tracking = temp_index.tracking(cg);
		28898	tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });
		28899	cg.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
		28900	assert(cg.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
		28901	},
		28902	}
		28903	}
19389		28904
19390	const param_int_reg = registerAlias(	28905	fn die(temp: Temp, cg: *CodeGen) !void {
19391	abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i],	28906	switch (temp.unwrap(cg)) {
19392	@intCast(@min(ty.abiSize(zcu), 8)),	28907	.ref => {},
19393	);	28908	.temp => \|temp_index\| try temp_index.tracking(cg).die(cg, temp_index.toIndex()),
19394	param_int_reg_i += 1;	28909	}
		28910	}
19395		28911
19396	arg_mcv[arg_mcv_i] = .{ .register = param_int_reg };	28912	const Index = enum(u4) {
19397	arg_mcv_i += 1;	28913	_,
19398	},
19399	.sse, .float, .float_combine => {
19400	const param_sse_regs = abi.getCAbiSseParamRegs(resolved_cc);
19401	if (param_sse_reg_i >= param_sse_regs.len) break;
19402		28914
19403	const param_sse_reg = registerAlias(	28915	fn toIndex(index: Index) Air.Inst.Index {
19404	abi.getCAbiSseParamRegs(resolved_cc)[param_sse_reg_i],	28916	return .fromTargetIndex(@intFromEnum(index));
19405	@intCast(ty.abiSize(zcu)),	28917	}
19406	);
19407	param_sse_reg_i += 1;
19408		28918
19409	arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg };	28919	fn fromIndex(index: Air.Inst.Index) Index {
19410	arg_mcv_i += 1;	28920	return @enumFromInt(index.toTargetIndex());
19411	},	28921	}
19412	.sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse),
19413	.x87, .x87up, .complex_x87, .memory, .win_i128 => switch (resolved_cc) {
19414	.x86_64_sysv => switch (class) {
19415	.x87, .x87up, .complex_x87, .memory => break,
19416	else => unreachable,
19417	},
19418	.x86_64_win => if (ty.abiSize(zcu) > 8) {
19419	const param_int_reg =
19420	abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i].to64();
19421	param_int_reg_i += 1;
19422		28922
19423	arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } };	28923	fn tracking(index: Index, cg: CodeGen) InstTracking {
19424	arg_mcv_i += 1;	28924	return &cg.inst_tracking.values()[@intFromEnum(index)];
19425	} else break,	28925	}
19426	else => unreachable,
19427	},
19428	.none => unreachable,
19429	.integer_per_element => {
19430	const param_int_regs_len: u32 =
19431	@intCast(abi.getCAbiIntParamRegs(resolved_cc).len);
19432	const remaining_param_int_regs: u3 =
19433	@intCast(param_int_regs_len - param_int_reg_i);
19434	param_int_reg_i = param_int_regs_len;
19435		28926
19436	const frame_elem_align = 8;	28927	fn isValid(index: Index, cg: *CodeGen) bool {
19437	const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs;	28928	return index.tracking(cg).short != .dead;
19438	const frame_elem_size = mem.alignForward(	28929	}
19439	u64,
19440	ty.childType(zcu).abiSize(zcu),
19441	frame_elem_align,
19442	);
19443	const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size);
19444		28930
19445	result.stack_byte_count =	28931	fn typeOf(index: Index, cg: *CodeGen) Type {
19446	mem.alignForward(u31, result.stack_byte_count, frame_elem_align);	28932	assert(index.isValid(cg));
19447	arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{	28933	return cg.temp_type[@intFromEnum(index)];
19448	.regs = remaining_param_int_regs,	28934	}
19449	.frame_off = @intCast(result.stack_byte_count),
19450	.frame_index = stack_frame_base,
19451	} };
19452	arg_mcv_i += 1;
19453	result.stack_byte_count += frame_size;
19454	},
19455	} else {
19456	arg.* = switch (arg_mcv_i) {
19457	else => unreachable,
19458	1 => arg_mcv[0],
19459	2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } },
19460	};
19461	continue;
19462	}
19463		28935
19464	const param_size: u31 = @intCast(ty.abiSize(zcu));	28936	const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type);
19465	const param_align = ty.abiAlignment(zcu).max(.@"8");	28937	const Set = std.StaticBitSet(max);
19466	result.stack_byte_count = mem.alignForward(	28938	const SafetySet = if (std.debug.runtime_safety) Set else struct {
19467	u31,	28939	inline fn initEmpty() @This() {
19468	result.stack_byte_count,	28940	return .{};
19469	@intCast(param_align.toByteUnits().?),
19470	);
19471	result.stack_align = result.stack_align.max(param_align);
19472	arg.* = .{ .load_frame = .{
19473	.index = stack_frame_base,
19474	.off = result.stack_byte_count,
19475	} };
19476	result.stack_byte_count += param_size;
19477	}	28941	}
19478	assert(param_int_reg_i <= 6);
19479	result.gp_count = param_int_reg_i;
19480	assert(param_sse_reg_i <= 16);
19481	result.fp_count = param_sse_reg_i;
19482	},
19483	.auto => {
19484	result.stack_align = .@"16";
19485		28942
19486	// Return values	28943	inline fn isSet(_: @This(), index: usize) bool {
19487	if (ret_ty.zigTypeTag(zcu) == .noreturn) {	28944	assert(index < max);
19488	result.return_value = InstTracking.init(.unreach);	28945	return true;
19489	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
19490	result.return_value = InstTracking.init(.none);
19491	} else {
19492	const ret_reg = abi.getCAbiIntReturnRegs(resolved_cc)[0];
19493	const ret_ty_size: u31 = @intCast(ret_ty.abiSize(zcu));
19494	if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
19495	const aliased_reg = registerAlias(ret_reg, ret_ty_size);
19496	result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
19497	} else {
19498	const ret_indirect_reg = abi.getCAbiIntParamRegs(resolved_cc)[0];
19499	result.return_value = .{
19500	.short = .{ .indirect = .{ .reg = ret_reg } },
19501	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
19502	};
19503	}
19504	}	28946	}
19505		28947
19506	// Input params	28948	inline fn set(_: @This(), index: usize) void {
19507	for (param_types, result.args) \|ty, *arg\| {	28949	assert(index < max);
19508	if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) {	28950	}
19509	arg.* = .none;	28951
19510	continue;	28952	inline fn eql(_: @This(), _: @This()) bool {
19511	}	28953	return true;
19512	const param_size: u31 = @intCast(ty.abiSize(zcu));	28954	}
19513	const param_align = ty.abiAlignment(zcu);	28955	};
19514	result.stack_byte_count = mem.alignForward(	28956	};
19515	u31,	28957	};
19516	result.stack_byte_count,	28958
19517	@intCast(param_align.toByteUnits().?),	28959	fn resetTemps(cg: *CodeGen) void {
19518	);	28960	for (0..@intFromEnum(cg.next_temp_index)) \|temp_index\| {
19519	result.stack_align = result.stack_align.max(param_align);	28961	const temp: Temp.Index = @enumFromInt(temp_index);
19520	arg.* = .{ .load_frame = .{	28962	assert(!temp.isValid(cg));
19521	.index = stack_frame_base,	28963	cg.temp_type[temp_index] = undefined;
19522	.off = result.stack_byte_count,	28964	}
19523	} };	28965	cg.next_temp_index = @enumFromInt(0);
19524	result.stack_byte_count += param_size;	28966	}
		28967
		28968	fn reuseTemp(
		28969	cg: *CodeGen,
		28970	new_inst: Air.Inst.Index,
		28971	old_inst: Air.Inst.Index,
		28972	tracking: *InstTracking,
		28973	) bool {
		28974	switch (tracking.short) {
		28975	.register,
		28976	.register_pair,
		28977	.register_offset,
		28978	.register_overflow,
		28979	.register_mask,
		28980	.indirect,
		28981	=> for (tracking.short.getRegs()) \|tracked_reg\| {
		28982	if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) \|tracked_index\| {
		28983	cg.register_manager.registers[tracked_index] = new_inst;
19525	}	28984	}
19526	},	28985	},
19527	else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),	28986	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
		28987	else => {},
		28988	}
		28989	switch (tracking.short) {
		28990	.eflags, .register_overflow => cg.eflags_inst = new_inst,
		28991	else => {},
19528	}	28992	}
		28993	tracking.reuse(cg, new_inst, old_inst);
		28994	return true;
		28995	}
19529		28996
19530	result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count));	28997	fn tempAlloc(cg: *CodeGen, ty: Type) !Temp {
19531	return result;	28998	const temp_index = cg.next_temp_index;
		28999	temp_index.tracking(cg).* = .init(
		29000	try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),
		29001	);
		29002	cg.temp_type[@intFromEnum(temp_index)] = ty;
		29003	cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
		29004	return .{ .index = temp_index.toIndex() };
19532	}	29005	}
19533		29006
19534	fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {	29007	fn tempAllocReg(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) !Temp {
19535	@branchHint(.cold);	29008	const temp_index = cg.next_temp_index;
19536	const zcu = self.pt.zcu;	29009	temp_index.tracking(cg).* = .init(
19537	switch (self.owner) {	29010	.{ .register = try cg.register_manager.allocReg(temp_index.toIndex(), rs) },
19538	.nav_index => \|i\| return zcu.codegenFail(i, format, args),	29011	);
19539	.lazy_sym => \|s\| return zcu.codegenFailType(s.ty, format, args),	29012	cg.temp_type[@intFromEnum(temp_index)] = ty;
19540	}	29013	cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
19541	return error.CodegenFail;	29014	return .{ .index = temp_index.toIndex() };
19542	}	29015	}
19543		29016
19544	fn failMsg(self: Self, msg: ErrorMsg) error{ OutOfMemory, CodegenFail } {	29017	fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) !Temp {
19545	@branchHint(.cold);	29018	const temp_index = cg.next_temp_index;
19546	const zcu = self.pt.zcu;	29019	temp_index.tracking(cg).* = .init(
19547	switch (self.owner) {	29020	.{ .register_pair = try cg.register_manager.allocRegs(2, temp_index.toIndex(), rs) },
19548	.nav_index => \|i\| return zcu.codegenFailMsg(i, msg),	29021	);
19549	.lazy_sym => \|s\| return zcu.codegenFailTypeMsg(s.ty, msg),	29022	cg.temp_type[@intFromEnum(temp_index)] = ty;
19550	}	29023	cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
19551	return error.CodegenFail;	29024	return .{ .index = temp_index.toIndex() };
19552	}	29025	}
19553		29026
19554	fn parseRegName(name: []const u8) ?Register {	29027	fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp {
19555	if (@hasDecl(Register, "parseRegName")) {	29028	const temp_index = cg.next_temp_index;
19556	return Register.parseRegName(name);	29029	temp_index.tracking(cg).* = .init(
		29030	try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false),
		29031	);
		29032	cg.temp_type[@intFromEnum(temp_index)] = ty;
		29033	cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
		29034	return .{ .index = temp_index.toIndex() };
		29035	}
		29036
		29037	fn tempInit(cg: *CodeGen, ty: Type, value: MCValue) !Temp {
		29038	const temp_index = cg.next_temp_index;
		29039	temp_index.tracking(cg).* = .init(value);
		29040	cg.temp_type[@intFromEnum(temp_index)] = ty;
		29041	try cg.getValue(value, temp_index.toIndex());
		29042	cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
		29043	return .{ .index = temp_index.toIndex() };
		29044	}
		29045
		29046	fn tempFromValue(cg: *CodeGen, value: Value) !Temp {
		29047	return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.genTypedValue(value));
		29048	}
		29049
		29050	fn tempFromOperand(
		29051	cg: *CodeGen,
		29052	inst: Air.Inst.Index,
		29053	op_index: Liveness.OperandInt,
		29054	op_ref: Air.Inst.Ref,
		29055	ignore_death: bool,
		29056	) !Temp {
		29057	const zcu = cg.pt.zcu;
		29058	const ip = &zcu.intern_pool;
		29059
		29060	if (ignore_death or !cg.liveness.operandDies(inst, op_index)) {
		29061	if (op_ref.toIndex()) \|op_inst\| return .{ .index = op_inst };
		29062	const val = op_ref.toInterned().?;
		29063	const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
		29064	if (!gop.found_existing) gop.value_ptr.* = .init(init: {
		29065	const const_mcv = try cg.genTypedValue(.fromInterned(val));
		29066	switch (const_mcv) {
		29067	.lea_tlv => \|tlv_sym\| switch (cg.bin_file.tag) {
		29068	.elf, .macho => {
		29069	if (cg.mod.pic) {
		29070	try cg.spillRegisters(&.{ .rdi, .rax });
		29071	} else {
		29072	try cg.spillRegisters(&.{.rax});
		29073	}
		29074	const frame_index = try cg.allocFrameIndex(.init(.{
		29075	.size = 8,
		29076	.alignment = .@"8",
		29077	}));
		29078	try cg.genSetMem(
		29079	.{ .frame = frame_index },
		29080	0,
		29081	.usize,
		29082	.{ .lea_symbol = .{ .sym_index = tlv_sym } },
		29083	.{},
		29084	);
		29085	break :init .{ .load_frame = .{ .index = frame_index } };
		29086	},
		29087	else => break :init const_mcv,
		29088	},
		29089	else => break :init const_mcv,
		29090	}
		29091	});
		29092	return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
19557	}	29093	}
19558	return std.meta.stringToEnum(Register, name);	29094
		29095	const temp_index = cg.next_temp_index;
		29096	const temp: Temp = .{ .index = temp_index.toIndex() };
		29097	const op_inst = op_ref.toIndex().?;
		29098	const tracking = cg.getResolvedInstValue(op_inst);
		29099	temp_index.tracking(cg).* = tracking.*;
		29100	if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
		29101	cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
		29102	cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
		29103	return temp;
		29104	}
		29105
		29106	inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
		29107	var temps: [op_refs.len]Temp = undefined;
		29108	inline for (&temps, 0.., op_refs) \|*temp, op_index, op_ref\| {
		29109	temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) \|prev_op_index\| {
		29110	if (op_ref == op_refs[prev_op_index]) break true;
		29111	} else false);
		29112	}
		29113	return temps;
19559	}	29114	}
19560		29115
19561	/// Returns register wide enough to hold at least `size_bytes`.	29116	const Operand = union(enum) {
19562	fn registerAlias(reg: Register, size_bytes: u32) Register {	29117	none,
19563	return switch (reg.class()) {	29118	reg: Register,
19564	.general_purpose => if (size_bytes == 0)	29119	mem: Memory,
19565	unreachable // should be comptime-known	29120	imm: Immediate,
19566	else if (size_bytes <= 1)	29121	inst: Mir.Inst.Index,
19567	reg.to8()	29122	};
19568	else if (size_bytes <= 2)	29123
19569	reg.to16()	29124	const Select = struct {
19570	else if (size_bytes <= 4)	29125	cg: *CodeGen,
19571	reg.to32()	29126	temps: [@intFromEnum(Select.Operand.Ref.none)]Temp,
19572	else if (size_bytes <= 8)	29127	labels: [@intFromEnum(Label._)]struct {
19573	reg.to64()	29128	backward: ?Mir.Inst.Index,
19574	else	29129	forward: [1]?Mir.Inst.Index,
19575	unreachable,	29130	},
19576	.segment => if (size_bytes <= 2)	29131
19577	reg	29132	fn emitLabel(s: *Select, label_index: Label) void {
19578	else	29133	if (label_index == ._) return;
19579	unreachable,	29134	const label = &s.labels[@intFromEnum(label_index)];
19580	.x87 => if (size_bytes == 16)	29135	for (&label.forward) \|*reloc\| {
19581	reg	29136	if (reloc.*) \|r\| s.cg.performReloc(r);
19582	else	29137	reloc.* = null;
19583	unreachable,	29138	}
19584	.mmx => if (size_bytes <= 8)	29139	label.backward = @intCast(s.cg.mir_instructions.len);
19585	reg	29140	}
19586	else	29141
19587	unreachable,	29142	fn emit(s: *Select, inst: Instruction) !void {
19588	.sse => if (size_bytes <= 16)	29143	s.emitLabel(inst[0]);
19589	reg.to128()	29144	const mir_tag: Mir.Inst.FixedTag = .{ inst[1], inst[2] };
19590	else if (size_bytes <= 32)	29145	var mir_ops: [4]CodeGen.Operand = undefined;
19591	reg.to256()	29146	inline for (&mir_ops, 3..) \|mir_op, inst_index\| mir_op. = try inst[inst_index].lower(s);
19592	else	29147	s.cg.asmOps(mir_tag, mir_ops) catch \|err\| switch (err) {
19593	unreachable,	29148	error.InvalidInstruction => {
19594	.ip => if (size_bytes <= 2)	29149	const fixes = @tagName(mir_tag[0]);
19595	.ip	29150	const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
19596	else if (size_bytes <= 4)	29151	return s.cg.fail(
19597	.eip	29152	"invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
19598	else if (size_bytes <= 8)	29153	.{
19599	.rip	29154	fixes[0..fixes_blank],
19600	else	29155	@tagName(mir_tag[1]),
19601	unreachable,	29156	fixes[fixes_blank + 1 ..],
		29157	@tagName(mir_ops[0]),
		29158	@tagName(mir_ops[1]),
		29159	@tagName(mir_ops[2]),
		29160	@tagName(mir_ops[3]),
		29161	},
		29162	);
		29163	},
		29164	else => \|e\| return e,
		29165	};
		29166	}
		29167
		29168	const Case = struct {
		29169	required_features: [4]?std.Target.x86.Feature = @splat(null),
		29170	dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any),
		29171	src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any),
		29172	patterns: []const Select.Pattern,
		29173	extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused),
		29174	dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused),
		29175	clobbers: struct { eflags: bool = false } = .{},
		29176	each: union(enum) {
		29177	once: []const Instruction,
		29178	},
19602	};	29179	};
19603	}
19604		29180
19605	fn memSize(self: *Self, ty: Type) Memory.Size {	29181	const Constraint = union(enum) {
19606	const pt = self.pt;	29182	any,
19607	const zcu = pt.zcu;	29183	any_bool_vec,
19608	return switch (ty.zigTypeTag(zcu)) {	29184	any_int,
19609	.float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),	29185	any_signed_int,
19610	else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))),	29186	any_float,
		29187	po2_any,
		29188	bool_vec: Memory.Size,
		29189	vec: Memory.Size,
		29190	signed_int_vec: Memory.Size,
		29191	signed_int_or_full_vec: Memory.Size,
		29192	unsigned_int_vec: Memory.Size,
		29193	size: Memory.Size,
		29194	multiple_size: Memory.Size,
		29195	int: Memory.Size,
		29196	scalar_int: Memory.Size,
		29197	scalar_signed_int: Memory.Size,
		29198	scalar_unsigned_int: Memory.Size,
		29199	scalar_remainder_int: struct { of: Memory.Size, is: Memory.Size },
		29200	exact_int: u16,
		29201	exact_signed_int: u16,
		29202	exact_unsigned_int: u16,
		29203	signed_or_exact_int: Memory.Size,
		29204	unsigned_or_exact_int: Memory.Size,
		29205	po2_int: Memory.Size,
		29206	signed_po2_int: Memory.Size,
		29207	unsigned_po2_or_exact_int: Memory.Size,
		29208	remainder_int: struct { of: Memory.Size, is: Memory.Size },
		29209	exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
		29210	signed_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
		29211	unsigned_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
		29212	signed_int: Memory.Size,
		29213	unsigned_int: Memory.Size,
		29214	elem_size_is: u8,
		29215	po2_elem_size,
		29216	elem_int: Memory.Size,
		29217
		29218	fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool {
		29219	const zcu = cg.pt.zcu;
		29220	return switch (constraint) {
		29221	.any => true,
		29222	.any_bool_vec => ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type,
		29223	.any_int => intInfo(ty, cg) != null,
		29224	.any_signed_int => if (intInfo(ty, cg)) \|int_info\| int_info.signedness == .signed else false,
		29225	.any_float => ty.isRuntimeFloat(),
		29226	.po2_any => std.math.isPowerOfTwo(ty.abiSize(zcu)),
		29227	.bool_vec => \|size\| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and
		29228	size.bitSize(cg.target) >= ty.vectorLen(zcu),
		29229	.vec => \|size\| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() != .bool_type and
		29230	size.bitSize(cg.target) >= ty.abiSize(zcu),
		29231	.signed_int_vec => \|size\| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
		29232	if (intInfo(ty.childType(zcu), cg)) \|int_info\| int_info.signedness == .signed else false,
		29233	.signed_int_or_full_vec => \|size\| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
		29234	if (intInfo(ty.childType(zcu), cg)) \|int_info\| switch (int_info.signedness) {
		29235	.signed => true,
		29236	.unsigned => int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits),
		29237	} else false,
		29238	.unsigned_int_vec => \|size\| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
		29239	if (intInfo(ty.childType(zcu), cg)) \|int_info\| int_info.signedness == .unsigned else false,
		29240	.size => \|size\| size.bitSize(cg.target) >= 8 * ty.abiSize(zcu),
		29241	.multiple_size => \|size\| size.bitSize(cg.target) % 8 * ty.abiSize(zcu) == 0,
		29242	.int => \|size\| if (intInfo(ty, cg)) \|int_info\| size.bitSize(cg.target) >= int_info.bits else false,
		29243	.scalar_int => \|size\| if (intInfo(ty.scalarType(zcu), cg)) \|int_info\|
		29244	size.bitSize(cg.target) >= int_info.bits
		29245	else
		29246	false,
		29247	.scalar_signed_int => \|size\| if (intInfo(ty.scalarType(zcu), cg)) \|int_info\| switch (int_info.signedness) {
		29248	.signed => size.bitSize(cg.target) >= int_info.bits,
		29249	.unsigned => false,
		29250	} else false,
		29251	.scalar_unsigned_int => \|size\| if (intInfo(ty.scalarType(zcu), cg)) \|int_info\| switch (int_info.signedness) {
		29252	.signed => false,
		29253	.unsigned => size.bitSize(cg.target) >= int_info.bits,
		29254	} else false,
		29255	.scalar_remainder_int => \|of_is\| if (intInfo(ty.scalarType(zcu), cg)) \|int_info\|
		29256	of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
		29257	else
		29258	false,
		29259	.exact_int => \|bit_size\| if (intInfo(ty, cg)) \|int_info\| bit_size == int_info.bits else false,
		29260	.exact_signed_int => \|bit_size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29261	.signed => bit_size == int_info.bits,
		29262	.unsigned => false,
		29263	} else false,
		29264	.exact_unsigned_int => \|bit_size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29265	.signed => false,
		29266	.unsigned => bit_size == int_info.bits,
		29267	} else false,
		29268	.signed_or_exact_int => \|size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29269	.signed => size.bitSize(cg.target) >= int_info.bits,
		29270	.unsigned => size.bitSize(cg.target) == int_info.bits,
		29271	} else false,
		29272	.unsigned_or_exact_int => \|size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29273	.signed => size.bitSize(cg.target) == int_info.bits,
		29274	.unsigned => size.bitSize(cg.target) >= int_info.bits,
		29275	} else false,
		29276	.po2_int => \|size\| if (intInfo(ty, cg)) \|int_info\|
		29277	std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits
		29278	else
		29279	false,
		29280	.signed_po2_int => \|size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29281	.signed => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits,
		29282	.unsigned => false,
		29283	} else false,
		29284	.unsigned_po2_or_exact_int => \|size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29285	.signed => size.bitSize(cg.target) == int_info.bits,
		29286	.unsigned => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits,
		29287	} else false,
		29288	.remainder_int => \|of_is\| if (intInfo(ty, cg)) \|int_info\|
		29289	of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
		29290	else
		29291	false,
		29292	.exact_remainder_int => \|of_is\| if (intInfo(ty, cg)) \|int_info\|
		29293	of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
		29294	else
		29295	false,
		29296	.signed_or_exact_remainder_int => \|of_is\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29297	.signed => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
		29298	.unsigned => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
		29299	} else false,
		29300	.unsigned_or_exact_remainder_int => \|of_is\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29301	.signed => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
		29302	.unsigned => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
		29303	} else false,
		29304	.signed_int => \|size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29305	.signed => size.bitSize(cg.target) >= int_info.bits,
		29306	.unsigned => false,
		29307	} else false,
		29308	.unsigned_int => \|size\| if (intInfo(ty, cg)) \|int_info\| switch (int_info.signedness) {
		29309	.signed => false,
		29310	.unsigned => size.bitSize(cg.target) >= int_info.bits,
		29311	} else false,
		29312	.elem_size_is => \|size\| size == ty.elemType2(zcu).abiSize(zcu),
		29313	.po2_elem_size => std.math.isPowerOfTwo(ty.elemType2(zcu).abiSize(zcu)),
		29314	.elem_int => \|size\| if (intInfo(ty.elemType2(zcu), cg)) \|elem_int_info\|
		29315	size.bitSize(cg.target) >= elem_int_info.bits
		29316	else
		29317	false,
		29318	};
		29319	}
		29320	};
		29321
		29322	const Pattern = struct {
		29323	src: [2]Src,
		29324	commute: struct { u8, u8 } = .{ 0, 0 },
		29325
		29326	const Src = enum {
		29327	none,
		29328	any,
		29329	imm8,
		29330	imm16,
		29331	imm32,
		29332	simm32,
		29333	mem,
		29334	to_mem,
		29335	mut_mem,
		29336	to_mut_mem,
		29337	gpr,
		29338	to_gpr,
		29339	mut_gpr,
		29340	to_mut_gpr,
		29341	mm,
		29342	to_mm,
		29343	mut_mm,
		29344	to_mut_mm,
		29345	xmm,
		29346	to_xmm,
		29347	mut_xmm,
		29348	to_mut_xmm,
		29349	ymm,
		29350	to_ymm,
		29351	mut_ymm,
		29352	to_mut_ymm,
		29353
		29354	fn matches(src: Src, temp: Temp, cg: *CodeGen) bool {
		29355	return switch (src) {
		29356	.none => unreachable,
		29357	.any => true,
		29358	.imm8 => switch (temp.tracking(cg).short) {
		29359	.immediate => \|imm\| std.math.cast(u8, imm) != null,
		29360	else => false,
		29361	},
		29362	.imm16 => switch (temp.tracking(cg).short) {
		29363	.immediate => \|imm\| std.math.cast(u16, imm) != null,
		29364	else => false,
		29365	},
		29366	.imm32 => switch (temp.tracking(cg).short) {
		29367	.immediate => \|imm\| std.math.cast(u32, imm) != null,
		29368	else => false,
		29369	},
		29370	.simm32 => switch (temp.tracking(cg).short) {
		29371	.immediate => \|imm\| std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
		29372	else => false,
		29373	},
		29374	.mem => temp.tracking(cg).short.isMemory(),
		29375	.to_mem, .to_mut_mem => true,
		29376	.mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(),
		29377	.gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) {
		29378	.register => \|reg\| reg.class() == .general_purpose,
		29379	.register_offset => \|reg_off\| reg_off.reg.class() == .general_purpose and reg_off.off == 0,
		29380	else => false,
		29381	},
		29382	.mut_gpr => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) {
		29383	.register => \|reg\| reg.class() == .general_purpose,
		29384	.register_offset => \|reg_off\| reg_off.reg.class() == .general_purpose and reg_off.off == 0,
		29385	else => false,
		29386	},
		29387	.to_gpr, .to_mut_gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8,
		29388	.mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
		29389	.register => \|reg\| reg.class() == .mmx,
		29390	.register_offset => \|reg_off\| reg_off.reg.class() == .mmx and reg_off.off == 0,
		29391	else => false,
		29392	},
		29393	.mut_mm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
		29394	.register => \|reg\| reg.class() == .mmx,
		29395	.register_offset => \|reg_off\| reg_off.reg.class() == .mmx and reg_off.off == 0,
		29396	else => false,
		29397	},
		29398	.to_mm, .to_mut_mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8,
		29399	.xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
		29400	.register => \|reg\| reg.class() == .sse,
		29401	.register_offset => \|reg_off\| reg_off.reg.class() == .sse and reg_off.off == 0,
		29402	else => false,
		29403	},
		29404	.mut_xmm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
		29405	.register => \|reg\| reg.class() == .sse,
		29406	.register_offset => \|reg_off\| reg_off.reg.class() == .sse and reg_off.off == 0,
		29407	else => false,
		29408	},
		29409	.to_xmm, .to_mut_xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16,
		29410	.ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
		29411	.register => \|reg\| reg.class() == .sse,
		29412	.register_offset => \|reg_off\| reg_off.reg.class() == .sse and reg_off.off == 0,
		29413	else => false,
		29414	},
		29415	.mut_ymm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
		29416	.register => \|reg\| reg.class() == .sse,
		29417	.register_offset => \|reg_off\| reg_off.reg.class() == .sse and reg_off.off == 0,
		29418	else => false,
		29419	},
		29420	.to_ymm, .to_mut_ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32,
		29421	};
		29422	}
		29423
		29424	fn convert(src: Src, temp: Temp, cg: CodeGen) !bool {
		29425	return switch (src) {
		29426	.none => unreachable,
		29427	.any, .imm8, .imm16, .imm32, .simm32 => false,
		29428	.mem, .to_mem, .mut_mem, .to_mut_mem => try temp.toBase(cg),
		29429	.gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg),
		29430	.mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg),
		29431	.mm, .to_mm => try temp.toRegClass(false, .mmx, cg),
		29432	.mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg),
		29433	.xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg),
		29434	.mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg),
		29435	};
		29436	}
		29437	};
19611	};	29438	};
19612	}
19613		29439
19614	fn splitType(self: *Self, ty: Type) ![2]Type {	29440	const TempSpec = struct {
19615	const pt = self.pt;	29441	type: Type = .noreturn,
19616	const zcu = pt.zcu;	29442	kind: Kind,
19617	const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);	29443
19618	var parts: [2]Type = undefined;	29444	const unused: TempSpec = .{ .kind = .unused };
19619	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {	29445
19620	part.* = switch (class) {	29446	const Kind = union(enum) {
19621	.integer => switch (part_i) {	29447	unused,
19622	0 => Type.u64,	29448	any,
19623	1 => part: {	29449	cc: Condition,
19624	const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;	29450	reg: Register,
19625	const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8));	29451	rc: Register.Class,
19626	break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) {	29452	rc_mask: struct { rc: Register.Class, info: MaskInfo },
19627	1 => elem_ty,	29453	mem,
19628	else => \|len\| try pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),	29454	smin_mem: Select.Operand.Ref,
		29455	smax_mem: Select.Operand.Ref,
		29456	umin_mem: Select.Operand.Ref,
		29457	umax_mem: Select.Operand.Ref,
		29458	ref: Select.Operand.Ref,
		29459	ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo },
		29460
		29461	fn finish(kind: Kind, temp: Temp, s: *const Select) void {
		29462	switch (kind) {
		29463	else => {},
		29464	inline .rc_mask, .ref_mask => \|mask\| temp.asMask(mask.info, s.cg),
		29465	}
		29466	}
		29467	};
		29468
		29469	fn create(spec: TempSpec, s: *Select) !?Temp {
		29470	const cg = s.cg;
		29471	return switch (spec.kind) {
		29472	.unused => null,
		29473	.any => try cg.tempAlloc(spec.type),
		29474	.cc => \|cc\| try cg.tempInit(spec.type, .{ .eflags = cc }),
		29475	.reg => \|reg\| try cg.tempInit(spec.type, .{ .register = reg }),
		29476	.rc => \|rc\| try cg.tempAllocReg(spec.type, regSetForRegClass(rc)),
		29477	.rc_mask => \|rc_mask\| try cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)),
		29478	.mem => try cg.tempAllocMem(spec.type),
		29479	.smin_mem, .smax_mem, .umin_mem, .umax_mem => \|ty_ref\| {
		29480	const pt = cg.pt;
		29481	const zcu = pt.zcu;
		29482	const ip = &zcu.intern_pool;
		29483	const ty = ty_ref.deref(s).typeOf(s.cg);
		29484	const vector_len, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) {
		29485	else => .{ null, ty },
		29486	.vector_type => \|vector_type\| .{ vector_type.len, .fromInterned(vector_type.child) },
19629	};	29487	};
		29488	const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) {
		29489	.bool_type => .{
		29490	scalar_ty,
		29491	.fromInterned(switch (spec.kind) {
		29492	else => unreachable,
		29493	.smin_mem, .umax_mem => .bool_true,
		29494	.smax_mem, .umin_mem => .bool_false,
		29495	}),
		29496	},
		29497	else => {
		29498	const scalar_info = intInfo(scalar_ty, cg).?;
		29499	const scalar_int_ty = try pt.intType(scalar_info.signedness, scalar_info.bits);
		29500	if (scalar_info.bits <= 64) {
		29501	const int_val: i64 = switch (spec.kind) {
		29502	else => unreachable,
		29503	.smin_mem => std.math.minInt(i64),
		29504	.smax_mem => std.math.maxInt(i64),
		29505	.umin_mem => 0,
		29506	.umax_mem => -1,
		29507	};
		29508	const shift: u6 = @intCast(64 - scalar_info.bits);
		29509	break :res_scalar .{ scalar_int_ty, switch (scalar_info.signedness) {
		29510	.signed => try pt.intValue_i64(scalar_int_ty, int_val >> shift),
		29511	.unsigned => try pt.intValue_u64(scalar_int_ty, @as(u64, @bitCast(int_val)) >> shift),
		29512	} };
		29513	}
		29514	var big_int: std.math.big.int.Managed = try .init(cg.gpa);
		29515	defer big_int.deinit();
		29516	try big_int.setTwosCompIntLimit(switch (spec.kind) {
		29517	else => unreachable,
		29518	.smin_mem, .umin_mem => .min,
		29519	.smax_mem, .umax_mem => .max,
		29520	}, switch (spec.kind) {
		29521	else => unreachable,
		29522	.smin_mem, .smax_mem => .signed,
		29523	.umin_mem, .umax_mem => .unsigned,
		29524	}, scalar_info.bits);
		29525	try big_int.truncate(&big_int, scalar_info.signedness, scalar_info.bits);
		29526	break :res_scalar .{ scalar_int_ty, try pt.intValue_big(scalar_int_ty, big_int.toConst()) };
		29527	},
		29528	};
		29529	const res_val: Value = if (vector_len) \|len\| .fromInterned(try pt.intern(.{ .aggregate = .{
		29530	.ty = (try pt.vectorType(.{
		29531	.len = len,
		29532	.child = res_scalar_ty.toIntern(),
		29533	})).toIntern(),
		29534	.storage = .{ .repeated_elem = res_scalar_val.toIntern() },
		29535	} })) else res_scalar_val;
		29536	return try cg.tempFromValue(res_val);
19630	},	29537	},
19631	else => unreachable,	29538	.ref => \|ref\| ref.deref(s),
		29539	.ref_mask => \|ref_mask\| ref_mask.ref.deref(s),
		29540	};
		29541	}
		29542	};
		29543
		29544	const Instruction = struct {
		29545	Label,
		29546	Mir.Inst.Fixes,
		29547	Mir.Inst.Tag,
		29548	Select.Operand,
		29549	Select.Operand,
		29550	Select.Operand,
		29551	Select.Operand,
		29552	};
		29553	const Label = enum { @"0:", @"1:", @"_" };
		29554	const Operand = struct {
		29555	tag: Tag,
		29556	base: Ref.Sized = .none,
		29557	index: packed struct(u6) {
		29558	ref: Ref,
		29559	scale: Memory.Scale,
		29560	} = .{ .ref = .none, .scale = .@"1" },
		29561	adjust: Adjust = .none,
		29562	imm: i32 = 0,
		29563
		29564	const Tag = enum {
		29565	none,
		29566	backward_label,
		29567	forward_label,
		29568	ref,
		29569	simm,
		29570	uimm,
		29571	lea,
		29572	mem,
		29573	};
		29574	const Adjust = packed struct(u8) {
		29575	factor: i2,
		29576	scale: Memory.Scale,
		29577	amount: enum(u4) {
		29578	none,
		29579	ptr_size,
		29580	ptr_bit_size,
		29581	size,
		29582	src0_size,
		29583	bit_size,
		29584	src0_bit_size,
		29585	len,
		29586	elem_limbs,
		29587	src0_elem_size,
		29588	src0_elem_size_times_src1,
		29589	log2_src0_elem_size,
		29590	smin,
		29591	smax,
		29592	umax,
19632	},	29593	},
19633	.float => Type.f32,	29594
19634	.float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }),	29595	const none: Adjust = .{ .factor = 0, .scale = .@"1", .amount = .none };
19635	.sse => Type.f64,	29596	const sub_ptr_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .ptr_size };
19636	else => break,	29597	const add_ptr_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .ptr_bit_size };
		29598	const add_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .size };
		29599	const sub_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .size };
		29600	const add_src0_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_size };
		29601	const sub_src0_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_size };
		29602	const add_2_bit_size: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .bit_size };
		29603	const add_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .bit_size };
		29604	const sub_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .bit_size };
		29605	const add_src0_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_bit_size };
		29606	const sub_src0_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_bit_size };
		29607	const add_8_len: Adjust = .{ .factor = 1, .scale = .@"8", .amount = .len };
		29608	const add_4_len: Adjust = .{ .factor = 1, .scale = .@"4", .amount = .len };
		29609	const add_3_len: Adjust = .{ .factor = 1, .scale = .@"3", .amount = .len };
		29610	const add_2_len: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .len };
		29611	const add_len: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .len };
		29612	const sub_len: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .len };
		29613	const add_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_elem_size };
		29614	const add_2_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .src0_elem_size };
		29615	const add_4_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"4", .amount = .src0_elem_size };
		29616	const add_8_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"8", .amount = .src0_elem_size };
		29617	const sub_src0_elem_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_elem_size };
		29618	const add_src0_elem_size_times_src1: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_elem_size_times_src1 };
		29619	const sub_src0_elem_size_times_src1: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_elem_size_times_src1 };
		29620	const add_log2_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .log2_src0_elem_size };
		29621	const add_elem_limbs: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .elem_limbs };
		29622	const add_umax: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .umax };
19637	};	29623	};
19638	} else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts;	29624	const Ref = enum(u4) {
19639	return self.fail("TODO implement splitType for {}", .{ty.fmt(pt)});	29625	tmp0,
19640	}	29626	tmp1,
		29627	tmp2,
		29628	tmp3,
		29629	tmp4,
		29630	tmp5,
		29631	dst0,
		29632	src0,
		29633	src1,
		29634	none,
19641		29635
19642	/// Truncates the value in the register in place.	29636	const Sized = packed struct(u8) {
19643	/// Clobbers any remaining bits.	29637	ref: Ref,
19644	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	29638	size: Memory.Size,
19645	const pt = self.pt;	29639
19646	const zcu = pt.zcu;	29640	const none: Sized = .{ .ref = .none, .size = .none };
19647	const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{	29641
19648	.signedness = .unsigned,	29642	const tmp0: Sized = .{ .ref = .tmp0, .size = .none };
19649	.bits = @intCast(ty.bitSize(zcu)),	29643	const tmp0b: Sized = .{ .ref = .tmp0, .size = .byte };
19650	};	29644	const tmp0w: Sized = .{ .ref = .tmp0, .size = .word };
19651	const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return;	29645	const tmp0d: Sized = .{ .ref = .tmp0, .size = .dword };
19652	try self.spillEflagsIfOccupied();	29646	const tmp0p: Sized = .{ .ref = .tmp0, .size = .ptr };
19653	switch (int_info.signedness) {	29647	const tmp0q: Sized = .{ .ref = .tmp0, .size = .qword };
19654	.signed => {	29648	const tmp0x: Sized = .{ .ref = .tmp0, .size = .xword };
19655	try self.genShiftBinOpMir(	29649	const tmp0y: Sized = .{ .ref = .tmp0, .size = .yword };
19656	.{ ._l, .sa },	29650
19657	Type.isize,	29651	const tmp1: Sized = .{ .ref = .tmp1, .size = .none };
19658	.{ .register = reg },	29652	const tmp1b: Sized = .{ .ref = .tmp1, .size = .byte };
19659	Type.u8,	29653	const tmp1w: Sized = .{ .ref = .tmp1, .size = .word };
19660	.{ .immediate = shift },	29654	const tmp1d: Sized = .{ .ref = .tmp1, .size = .dword };
19661	);	29655	const tmp1p: Sized = .{ .ref = .tmp1, .size = .ptr };
19662	try self.genShiftBinOpMir(	29656	const tmp1q: Sized = .{ .ref = .tmp1, .size = .qword };
19663	.{ ._r, .sa },	29657	const tmp1x: Sized = .{ .ref = .tmp1, .size = .xword };
19664	Type.isize,	29658	const tmp1y: Sized = .{ .ref = .tmp1, .size = .yword };
19665	.{ .register = reg },	29659
19666	Type.u8,	29660	const tmp2: Sized = .{ .ref = .tmp2, .size = .none };
19667	.{ .immediate = shift },	29661	const tmp2b: Sized = .{ .ref = .tmp2, .size = .byte };
19668	);	29662	const tmp2w: Sized = .{ .ref = .tmp2, .size = .word };
19669	},	29663	const tmp2d: Sized = .{ .ref = .tmp2, .size = .dword };
19670	.unsigned => {	29664	const tmp2p: Sized = .{ .ref = .tmp2, .size = .ptr };
19671	const mask = ~@as(u64, 0) >> shift;	29665	const tmp2q: Sized = .{ .ref = .tmp2, .size = .qword };
19672	if (int_info.bits <= 32) {	29666	const tmp2x: Sized = .{ .ref = .tmp2, .size = .xword };
19673	try self.genBinOpMir(	29667	const tmp2y: Sized = .{ .ref = .tmp2, .size = .yword };
19674	.{ ._, .@"and" },	29668
19675	Type.u32,	29669	const tmp3: Sized = .{ .ref = .tmp3, .size = .none };
19676	.{ .register = reg },	29670	const tmp3b: Sized = .{ .ref = .tmp3, .size = .byte };
19677	.{ .immediate = mask },	29671	const tmp3w: Sized = .{ .ref = .tmp3, .size = .word };
19678	);	29672	const tmp3d: Sized = .{ .ref = .tmp3, .size = .dword };
19679	} else {	29673	const tmp3p: Sized = .{ .ref = .tmp3, .size = .ptr };
19680	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });	29674	const tmp3q: Sized = .{ .ref = .tmp3, .size = .qword };
19681	try self.genBinOpMir(	29675	const tmp3x: Sized = .{ .ref = .tmp3, .size = .xword };
19682	.{ ._, .@"and" },	29676	const tmp3y: Sized = .{ .ref = .tmp3, .size = .yword };
19683	Type.usize,	29677
19684	.{ .register = reg },	29678	const tmp4: Sized = .{ .ref = .tmp4, .size = .none };
19685	.{ .register = tmp_reg },	29679	const tmp4b: Sized = .{ .ref = .tmp4, .size = .byte };
19686	);	29680	const tmp4w: Sized = .{ .ref = .tmp4, .size = .word };
19687	}	29681	const tmp4d: Sized = .{ .ref = .tmp4, .size = .dword };
19688	},	29682	const tmp4p: Sized = .{ .ref = .tmp4, .size = .ptr };
19689	}	29683	const tmp4q: Sized = .{ .ref = .tmp4, .size = .qword };
19690	}	29684	const tmp4x: Sized = .{ .ref = .tmp4, .size = .xword };
		29685	const tmp4y: Sized = .{ .ref = .tmp4, .size = .yword };
		29686
		29687	const tmp5: Sized = .{ .ref = .tmp5, .size = .none };
		29688	const tmp5b: Sized = .{ .ref = .tmp5, .size = .byte };
		29689	const tmp5w: Sized = .{ .ref = .tmp5, .size = .word };
		29690	const tmp5d: Sized = .{ .ref = .tmp5, .size = .dword };
		29691	const tmp5p: Sized = .{ .ref = .tmp5, .size = .ptr };
		29692	const tmp5q: Sized = .{ .ref = .tmp5, .size = .qword };
		29693	const tmp5x: Sized = .{ .ref = .tmp5, .size = .xword };
		29694	const tmp5y: Sized = .{ .ref = .tmp5, .size = .yword };
		29695
		29696	const dst0: Sized = .{ .ref = .dst0, .size = .none };
		29697	const dst0b: Sized = .{ .ref = .dst0, .size = .byte };
		29698	const dst0w: Sized = .{ .ref = .dst0, .size = .word };
		29699	const dst0d: Sized = .{ .ref = .dst0, .size = .dword };
		29700	const dst0p: Sized = .{ .ref = .dst0, .size = .ptr };
		29701	const dst0q: Sized = .{ .ref = .dst0, .size = .qword };
		29702	const dst0x: Sized = .{ .ref = .dst0, .size = .xword };
		29703	const dst0y: Sized = .{ .ref = .dst0, .size = .yword };
		29704
		29705	const src0: Sized = .{ .ref = .src0, .size = .none };
		29706	const src0b: Sized = .{ .ref = .src0, .size = .byte };
		29707	const src0w: Sized = .{ .ref = .src0, .size = .word };
		29708	const src0d: Sized = .{ .ref = .src0, .size = .dword };
		29709	const src0p: Sized = .{ .ref = .src0, .size = .ptr };
		29710	const src0q: Sized = .{ .ref = .src0, .size = .qword };
		29711	const src0x: Sized = .{ .ref = .src0, .size = .xword };
		29712	const src0y: Sized = .{ .ref = .src0, .size = .yword };
		29713
		29714	const src1: Sized = .{ .ref = .src1, .size = .none };
		29715	const src1b: Sized = .{ .ref = .src1, .size = .byte };
		29716	const src1w: Sized = .{ .ref = .src1, .size = .word };
		29717	const src1d: Sized = .{ .ref = .src1, .size = .dword };
		29718	const src1p: Sized = .{ .ref = .src1, .size = .ptr };
		29719	const src1q: Sized = .{ .ref = .src1, .size = .qword };
		29720	const src1x: Sized = .{ .ref = .src1, .size = .xword };
		29721	const src1y: Sized = .{ .ref = .src1, .size = .yword };
		29722	};
19691		29723
19692	fn regBitSize(self: *Self, ty: Type) u64 {	29724	fn deref(ref: Ref, s: *const Select) Temp {
19693	const pt = self.pt;	29725	return s.temps[@intFromEnum(ref)];
19694	const zcu = pt.zcu;	29726	}
19695	const abi_size = ty.abiSize(zcu);	29727	};
19696	return switch (ty.zigTypeTag(zcu)) {
19697	else => switch (abi_size) {
19698	1 => 8,
19699	2 => 16,
19700	3...4 => 32,
19701	5...8 => 64,
19702	else => unreachable,
19703	},
19704	.float => switch (abi_size) {
19705	1...16 => 128,
19706	17...32 => 256,
19707	else => unreachable,
19708	},
19709	};
19710	}
19711		29728
19712	fn regExtraBits(self: *Self, ty: Type) u64 {	29729	const @"_": Select.Operand = .{ .tag = .none };
19713	return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);	29730
19714	}	29731	const @"0b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp0, .size = .none } };
		29732	const @"0f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp0, .size = .none } };
		29733	const @"1b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp1, .size = .none } };
		29734	const @"1f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp1, .size = .none } };
		29735
		29736	const tmp0b: Select.Operand = .{ .tag = .ref, .base = .tmp0b };
		29737	const tmp0w: Select.Operand = .{ .tag = .ref, .base = .tmp0w };
		29738	const tmp0d: Select.Operand = .{ .tag = .ref, .base = .tmp0d };
		29739	const tmp0p: Select.Operand = .{ .tag = .ref, .base = .tmp0p };
		29740	const tmp0q: Select.Operand = .{ .tag = .ref, .base = .tmp0q };
		29741	const tmp0x: Select.Operand = .{ .tag = .ref, .base = .tmp0x };
		29742	const tmp0y: Select.Operand = .{ .tag = .ref, .base = .tmp0y };
		29743
		29744	const tmp1b: Select.Operand = .{ .tag = .ref, .base = .tmp1b };
		29745	const tmp1w: Select.Operand = .{ .tag = .ref, .base = .tmp1w };
		29746	const tmp1d: Select.Operand = .{ .tag = .ref, .base = .tmp1d };
		29747	const tmp1p: Select.Operand = .{ .tag = .ref, .base = .tmp1p };
		29748	const tmp1q: Select.Operand = .{ .tag = .ref, .base = .tmp1q };
		29749	const tmp1x: Select.Operand = .{ .tag = .ref, .base = .tmp1x };
		29750	const tmp1y: Select.Operand = .{ .tag = .ref, .base = .tmp1y };
		29751
		29752	const tmp2b: Select.Operand = .{ .tag = .ref, .base = .tmp2b };
		29753	const tmp2w: Select.Operand = .{ .tag = .ref, .base = .tmp2w };
		29754	const tmp2d: Select.Operand = .{ .tag = .ref, .base = .tmp2d };
		29755	const tmp2p: Select.Operand = .{ .tag = .ref, .base = .tmp2p };
		29756	const tmp2q: Select.Operand = .{ .tag = .ref, .base = .tmp2q };
		29757	const tmp2x: Select.Operand = .{ .tag = .ref, .base = .tmp2x };
		29758	const tmp2y: Select.Operand = .{ .tag = .ref, .base = .tmp2y };
		29759
		29760	const tmp3b: Select.Operand = .{ .tag = .ref, .base = .tmp3b };
		29761	const tmp3w: Select.Operand = .{ .tag = .ref, .base = .tmp3w };
		29762	const tmp3d: Select.Operand = .{ .tag = .ref, .base = .tmp3d };
		29763	const tmp3p: Select.Operand = .{ .tag = .ref, .base = .tmp3p };
		29764	const tmp3q: Select.Operand = .{ .tag = .ref, .base = .tmp3q };
		29765	const tmp3x: Select.Operand = .{ .tag = .ref, .base = .tmp3x };
		29766	const tmp3y: Select.Operand = .{ .tag = .ref, .base = .tmp3y };
		29767
		29768	const tmp4b: Select.Operand = .{ .tag = .ref, .base = .tmp4b };
		29769	const tmp4w: Select.Operand = .{ .tag = .ref, .base = .tmp4w };
		29770	const tmp4d: Select.Operand = .{ .tag = .ref, .base = .tmp4d };
		29771	const tmp4p: Select.Operand = .{ .tag = .ref, .base = .tmp4p };
		29772	const tmp4q: Select.Operand = .{ .tag = .ref, .base = .tmp4q };
		29773	const tmp4x: Select.Operand = .{ .tag = .ref, .base = .tmp4x };
		29774	const tmp4y: Select.Operand = .{ .tag = .ref, .base = .tmp4y };
		29775
		29776	const tmp5b: Select.Operand = .{ .tag = .ref, .base = .tmp5b };
		29777	const tmp5w: Select.Operand = .{ .tag = .ref, .base = .tmp5w };
		29778	const tmp5d: Select.Operand = .{ .tag = .ref, .base = .tmp5d };
		29779	const tmp5p: Select.Operand = .{ .tag = .ref, .base = .tmp5p };
		29780	const tmp5q: Select.Operand = .{ .tag = .ref, .base = .tmp5q };
		29781	const tmp5x: Select.Operand = .{ .tag = .ref, .base = .tmp5x };
		29782	const tmp5y: Select.Operand = .{ .tag = .ref, .base = .tmp5y };
		29783
		29784	const dst0b: Select.Operand = .{ .tag = .ref, .base = .dst0b };
		29785	const dst0w: Select.Operand = .{ .tag = .ref, .base = .dst0w };
		29786	const dst0d: Select.Operand = .{ .tag = .ref, .base = .dst0d };
		29787	const dst0p: Select.Operand = .{ .tag = .ref, .base = .dst0p };
		29788	const dst0q: Select.Operand = .{ .tag = .ref, .base = .dst0q };
		29789	const dst0x: Select.Operand = .{ .tag = .ref, .base = .dst0x };
		29790	const dst0y: Select.Operand = .{ .tag = .ref, .base = .dst0y };
		29791
		29792	const src0b: Select.Operand = .{ .tag = .ref, .base = .src0b };
		29793	const src0w: Select.Operand = .{ .tag = .ref, .base = .src0w };
		29794	const src0d: Select.Operand = .{ .tag = .ref, .base = .src0d };
		29795	const src0p: Select.Operand = .{ .tag = .ref, .base = .src0p };
		29796	const src0q: Select.Operand = .{ .tag = .ref, .base = .src0q };
		29797	const src0x: Select.Operand = .{ .tag = .ref, .base = .src0x };
		29798	const src0y: Select.Operand = .{ .tag = .ref, .base = .src0y };
		29799
		29800	const src1b: Select.Operand = .{ .tag = .ref, .base = .src1b };
		29801	const src1w: Select.Operand = .{ .tag = .ref, .base = .src1w };
		29802	const src1d: Select.Operand = .{ .tag = .ref, .base = .src1d };
		29803	const src1p: Select.Operand = .{ .tag = .ref, .base = .src1p };
		29804	const src1q: Select.Operand = .{ .tag = .ref, .base = .src1q };
		29805	const src1x: Select.Operand = .{ .tag = .ref, .base = .src1x };
		29806	const src1y: Select.Operand = .{ .tag = .ref, .base = .src1y };
		29807
		29808	fn si(imm: i32) Select.Operand {
		29809	return .{ .tag = .simm, .imm = imm };
		29810	}
		29811	fn sa(base: Ref.Sized, adjust: Adjust) Select.Operand {
		29812	return .{ .tag = .simm, .base = base, .adjust = adjust };
		29813	}
		29814	fn sia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
		29815	return .{ .tag = .simm, .base = base, .adjust = adjust, .imm = imm };
		29816	}
		29817	fn ui(imm: i32) Select.Operand {
		29818	return .{ .tag = .uimm, .imm = imm };
		29819	}
		29820	fn ua(base: Ref.Sized, adjust: Adjust) Select.Operand {
		29821	return .{ .tag = .uimm, .base = base, .adjust = adjust };
		29822	}
		29823	fn uia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
		29824	return .{ .tag = .uimm, .base = base, .adjust = adjust, .imm = imm };
		29825	}
19715		29826
19716	fn hasFeature(self: *Self, feature: Target.x86.Feature) bool {	29827	fn lea(size: Memory.Size, base: Ref) Select.Operand {
19717	return Target.x86.featureSetHas(self.target.cpu.features, feature);	29828	return .{
19718	}	29829	.tag = .lea,
19719	fn hasAnyFeatures(self: *Self, features: anytype) bool {	29830	.base = .{ .ref = base, .size = size },
19720	return Target.x86.featureSetHasAny(self.target.cpu.features, features);	29831	};
19721	}	29832	}
19722	fn hasAllFeatures(self: *Self, features: anytype) bool {	29833	fn leaa(size: Memory.Size, base: Ref, adjust: Adjust) Select.Operand {
19723	return Target.x86.featureSetHasAll(self.target.cpu.features, features);	29834	return .{
19724	}	29835	.tag = .lea,
		29836	.base = .{ .ref = base, .size = size },
		29837	.adjust = adjust,
		29838	};
		29839	}
		29840	fn lead(size: Memory.Size, base: Ref, disp: i32) Select.Operand {
		29841	return .{
		29842	.tag = .lea,
		29843	.base = .{ .ref = base, .size = size },
		29844	.imm = disp,
		29845	};
		29846	}
		29847	fn leai(size: Memory.Size, base: Ref, index: Ref) Select.Operand {
		29848	return .{
		29849	.tag = .lea,
		29850	.base = .{ .ref = base, .size = size },
		29851	.index = .{ .ref = index, .scale = .@"1" },
		29852	};
		29853	}
		29854	fn leaia(size: Memory.Size, base: Ref, index: Ref, adjust: Adjust) Select.Operand {
		29855	return .{
		29856	.tag = .lea,
		29857	.base = .{ .ref = base, .size = size },
		29858	.index = .{ .ref = index, .scale = .@"1" },
		29859	.adjust = adjust,
		29860	};
		29861	}
		29862	fn leaid(size: Memory.Size, base: Ref, index: Ref, disp: i32) Select.Operand {
		29863	return .{
		29864	.tag = .lea,
		29865	.base = .{ .ref = base, .size = size },
		29866	.index = .{ .ref = index, .scale = .@"1" },
		29867	.imm = disp,
		29868	};
		29869	}
		29870	fn leasi(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref) Select.Operand {
		29871	return .{
		29872	.tag = .lea,
		29873	.base = .{ .ref = base, .size = size },
		29874	.index = .{ .ref = index, .scale = scale },
		29875	};
		29876	}
		29877	fn leasid(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
		29878	return .{
		29879	.tag = .lea,
		29880	.base = .{ .ref = base, .size = size },
		29881	.index = .{ .ref = index, .scale = scale },
		29882	.imm = disp,
		29883	};
		29884	}
		29885	fn leasiad(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
		29886	return .{
		29887	.tag = .lea,
		29888	.base = .{ .ref = base, .size = size },
		29889	.index = .{ .ref = index, .scale = scale },
		29890	.adjust = adjust,
		29891	.imm = disp,
		29892	};
		29893	}
19725		29894
19726	fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {	29895	fn mem(base: Ref.Sized) Select.Operand {
19727	const pt = self.pt;	29896	return .{
19728	const zcu = pt.zcu;	29897	.tag = .mem,
19729	return self.air.typeOf(inst, &zcu.intern_pool);	29898	.base = base,
19730	}	29899	};
		29900	}
		29901	fn memd(base: Ref.Sized, disp: i32) Select.Operand {
		29902	return .{
		29903	.tag = .mem,
		29904	.base = base,
		29905	.imm = disp,
		29906	};
		29907	}
		29908	fn mema(base: Ref.Sized, adjust: Adjust) Select.Operand {
		29909	return .{
		29910	.tag = .mem,
		29911	.base = base,
		29912	.adjust = adjust,
		29913	};
		29914	}
		29915	fn memad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand {
		29916	return .{
		29917	.tag = .mem,
		29918	.base = base,
		29919	.adjust = adjust,
		29920	.imm = disp,
		29921	};
		29922	}
		29923	fn memi(base: Ref.Sized, index: Ref) Select.Operand {
		29924	return .{
		29925	.tag = .mem,
		29926	.base = base,
		29927	.index = .{ .ref = index, .scale = .@"1" },
		29928	};
		29929	}
		29930	fn memia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand {
		29931	return .{
		29932	.tag = .mem,
		29933	.base = base,
		29934	.index = .{ .ref = index, .scale = .@"1" },
		29935	.adjust = adjust,
		29936	};
		29937	}
		29938	fn memiad(base: Ref.Sized, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
		29939	return .{
		29940	.tag = .mem,
		29941	.base = base,
		29942	.index = .{ .ref = index, .scale = .@"1" },
		29943	.adjust = adjust,
		29944	.imm = disp,
		29945	};
		29946	}
		29947	fn memid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand {
		29948	return .{
		29949	.tag = .mem,
		29950	.base = base,
		29951	.index = .{ .ref = index, .scale = .@"1" },
		29952	.imm = disp,
		29953	};
		29954	}
		29955	fn memsi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand {
		29956	return .{
		29957	.tag = .mem,
		29958	.base = base,
		29959	.index = .{ .ref = index, .scale = scale },
		29960	};
		29961	}
		29962	fn memsia(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust) Select.Operand {
		29963	return .{
		29964	.tag = .mem,
		29965	.base = base,
		29966	.index = .{ .ref = index, .scale = scale },
		29967	.adjust = adjust,
		29968	};
		29969	}
		29970	fn memsid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
		29971	return .{
		29972	.tag = .mem,
		29973	.base = base,
		29974	.index = .{ .ref = index, .scale = scale },
		29975	.imm = disp,
		29976	};
		29977	}
		29978	fn memsiad(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
		29979	return .{
		29980	.tag = .mem,
		29981	.base = base,
		29982	.index = .{ .ref = index, .scale = scale },
		29983	.adjust = adjust,
		29984	.imm = disp,
		29985	};
		29986	}
19731		29987
19732	fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {	29988	fn adjustedImm(op: Select.Operand, comptime SignedImm: type, s: *const Select) SignedImm {
19733	const pt = self.pt;	29989	const UnsignedImm = @Type(.{
19734	const zcu = pt.zcu;	29990	.int = .{ .signedness = .unsigned, .bits = @typeInfo(SignedImm).int.bits },
19735	return switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) {	29991	});
19736	.loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand),	29992	return @as(i5, op.adjust.factor) * op.adjust.scale.toFactor() * @as(SignedImm, switch (op.adjust.amount) {
19737	else => self.air.typeOfIndex(inst, &zcu.intern_pool),	29993	.none => 0,
19738	};	29994	.ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8),
19739	}	29995	.ptr_bit_size => s.cg.target.ptrBitWidth(),
		29996	.size => @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)),
		29997	.src0_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)),
		29998	.bit_size => @intCast(op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
		29999	.src0_bit_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
		30000	.len => @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu)),
		30001	.elem_limbs => @intCast(@divExact(
		30002	op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
		30003	@divExact(op.base.size.bitSize(s.cg.target), 8),
		30004	)),
		30005	.src0_elem_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)),
		30006	.src0_elem_size_times_src1 => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) *
		30007	Select.Operand.Ref.src1.deref(s).tracking(s.cg).short.immediate),
		30008	.log2_src0_elem_size => @intCast(std.math.log2(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))),
		30009	.smin => @as(SignedImm, std.math.minInt(SignedImm)) >> @truncate(
		30010	-%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
		30011	),
		30012	.smax => @as(SignedImm, std.math.maxInt(SignedImm)) >> @truncate(
		30013	-%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
		30014	),
		30015	.umax => @bitCast(@as(UnsignedImm, std.math.maxInt(UnsignedImm)) >> @truncate(
		30016	-%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
		30017	)),
		30018	}) + op.imm;
		30019	}
19740		30020
19741	fn intCompilerRtAbiName(int_bits: u32) u8 {	30021	fn lower(op: Select.Operand, s: *Select) !CodeGen.Operand {
19742	return switch (int_bits) {	30022	return switch (op.tag) {
19743	1...32 => 's',	30023	.none => .none,
19744	33...64 => 'd',	30024	.backward_label => .{ .inst = s.labels[@intFromEnum(op.base.ref)].backward.? },
19745	65...128 => 't',	30025	.forward_label => for (&s.labels[@intFromEnum(op.base.ref)].forward) \|*label\| {
19746	else => unreachable,	30026	if (label.*) \|_\| continue;
		30027	label.* = @intCast(s.cg.mir_instructions.len);
		30028	break .{ .inst = undefined };
		30029	} else unreachable,
		30030	.ref => switch (op.base.ref.deref(s).tracking(s.cg).short) {
		30031	.immediate => \|imm\| .{ .imm = switch (op.base.size) {
		30032	.byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) \|simm\| .s(simm) else .u(@as(u8, @intCast(imm))),
		30033	.word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) \|simm\| .s(simm) else .u(@as(u16, @intCast(imm))),
		30034	.dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|simm\| .s(simm) else .u(@as(u32, @intCast(imm))),
		30035	.qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) \|simm\| .s(simm) else .u(imm),
		30036	else => unreachable,
		30037	} },
		30038	else => \|mcv\| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) },
		30039	.register => \|reg\| .{ .reg = registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))) },
		30040	},
		30041	.simm => .{ .imm = .s(op.adjustedImm(i32, s)) },
		30042	.uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) },
		30043	.lea => .{ .mem = .{
		30044	.base = .{ .reg = registerAlias(op.base.ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)) },
		30045	.mod = .{ .rm = .{
		30046	.size = op.base.size,
		30047	.index = switch (op.index.ref) {
		30048	else => \|ref\| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
		30049	.none => .none,
		30050	},
		30051	.scale = op.index.scale,
		30052	.disp = op.adjustedImm(i32, s),
		30053	} },
		30054	} },
		30055	.mem => .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, .{
		30056	.size = op.base.size,
		30057	.index = switch (op.index.ref) {
		30058	else => \|ref\| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
		30059	.none => .none,
		30060	},
		30061	.scale = op.index.scale,
		30062	.disp = op.adjustedImm(i32, s),
		30063	}) },
		30064	};
		30065	}
19747	};	30066	};
19748	}	30067	};
		30068	fn select(
		30069	cg: *CodeGen,
		30070	dst_temps: []Temp,
		30071	dst_tys: []const Type,
		30072	src_temps: []Temp,
		30073	cases: []const Select.Case,
		30074	) !void {
		30075	cases: for (cases) \|case\| {
		30076	for (case.required_features) \|required_feature\| if (required_feature) \|feature\| if (!cg.hasFeature(feature)) continue :cases;
		30077	for (case.dst_constraints[0..dst_temps.len], dst_tys) \|dst_constraint, dst_ty\| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases;
		30078	for (case.src_constraints[0..src_temps.len], src_temps) \|src_constraint, src_temp\| if (!src_constraint.accepts(src_temp.typeOf(cg), cg)) continue :cases;
		30079	if (std.debug.runtime_safety) {
		30080	for (case.dst_constraints[dst_temps.len..]) \|dst_constraint\| assert(dst_constraint == .any);
		30081	for (case.src_constraints[src_temps.len..]) \|src_constraint\| assert(src_constraint == .any);
		30082	}
		30083	patterns: for (case.patterns) \|pattern\| {
		30084	for (pattern.src[0..src_temps.len], src_temps) \|src_pattern, src_temp\| if (!src_pattern.matches(src_temp, cg)) continue :patterns;
		30085	if (std.debug.runtime_safety) for (pattern.src[src_temps.len..]) \|src_pattern\| assert(src_pattern == .none);
		30086
		30087	var s: Select = .{
		30088	.cg = cg,
		30089	.temps = undefined,
		30090	.labels = @splat(.{ .forward = @splat(null), .backward = null }),
		30091	};
		30092	const tmp_slots = s.temps[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)];
		30093	const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)];
		30094	const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)];
19749		30095
19750	fn floatCompilerRtAbiName(float_bits: u32) u8 {	30096	@memcpy(src_slots[0..src_temps.len], src_temps);
19751	return switch (float_bits) {	30097	std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
19752	16 => 'h',	30098	for (tmp_slots, case.extra_temps) \|slot, spec\| slot. = try spec.create(&s) orelse continue;
19753	32 => 's',
19754	64 => 'd',
19755	80 => 'x',
19756	128 => 't',
19757	else => unreachable,
19758	};
19759	}
19760		30099
19761	fn floatCompilerRtAbiType(self: *Self, ty: Type, other_ty: Type) Type {	30100	while (true) for (pattern.src[0..src_temps.len], src_temps) \|src_pattern, *src_temp\| {
19762	if (ty.toIntern() == .f16_type and	30101	if (try src_pattern.convert(src_temp, cg)) break;
19763	(other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and	30102	} else break;
19764	self.target.isDarwin()) return Type.u16;	30103	@memcpy(src_slots[0..src_temps.len], src_temps);
19765	return ty;	30104	std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
19766	}
19767		30105
19768	fn floatLibcAbiPrefix(ty: Type) []const u8 {	30106	if (case.clobbers.eflags or case.each != .once) try cg.spillEflagsIfOccupied();
19769	return switch (ty.toIntern()) {
19770	.f16_type, .f80_type => "__",
19771	.f32_type, .f64_type, .f128_type, .c_longdouble_type => "",
19772	else => unreachable,
19773	};
19774	}
19775		30107
19776	fn floatLibcAbiSuffix(ty: Type) []const u8 {	30108	for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) \|*dst_temp, dst_ty, dst_kind\|
19777	return switch (ty.toIntern()) {	30109	dst_temp.* = (try Select.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, &s)).?;
19778	.f16_type => "h",	30110	@memcpy(dst_slots[0..dst_temps.len], dst_temps);
19779	.f32_type => "f",
19780	.f64_type => "",
19781	.f80_type => "x",
19782	.f128_type => "q",
19783	.c_longdouble_type => "l",
19784	else => unreachable,
19785	};
19786	}
19787		30111
19788	fn promoteInt(self: *Self, ty: Type) Type {	30112	switch (case.each) {
19789	const pt = self.pt;	30113	.once => \|body\| {
19790	const zcu = pt.zcu;	30114	for (body) \|inst\| try s.emit(inst);
19791	const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {	30115	s.emitLabel(.@"0:");
19792	.bool_type => .{ .signedness = .unsigned, .bits = 1 },	30116	},
19793	else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,	30117	}
19794	};
19795	for ([_]Type{
19796	Type.c_int, Type.c_uint,
19797	Type.c_long, Type.c_ulong,
19798	Type.c_longlong, Type.c_ulonglong,
19799	}) \|promote_ty\| {
19800	const promote_info = promote_ty.intInfo(zcu);
19801	if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
19802	if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
19803	promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
19804	}
19805	return ty;
19806	}
19807		30118
19808	fn promoteVarArg(self: *Self, ty: Type) Type {	30119	for (dst_temps, case.dst_temps[0..dst_temps.len]) \|dst_temp, dst_kind\| dst_kind.finish(dst_temp, &s);
19809	if (!ty.isRuntimeFloat()) return self.promoteInt(ty);	30120	for (case.extra_temps, tmp_slots) \|spec, temp\| if (spec.kind != .unused) try temp.die(cg);
19810	switch (ty.floatBits(self.target.*)) {	30121	return;
19811	32, 64 => return Type.f64,	30122	}
19812	else => \|float_bits\| {
19813	assert(float_bits == self.target.cTypeBitSize(.longdouble));
19814	return Type.c_longdouble;
19815	},
19816	}	30123	}
		30124	return error.SelectFailed;
19817	}	30125	}

src/arch/x86_64/Disassembler.zig+34-16

 @@ -38,28 +38,46 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
     const enc = try dis.parseEncoding(prefixes) orelse return error.UnknownOpcode;
     switch (enc.data.op_en) {
-        .zo => return inst(enc, .{}),
+        .z => return inst(enc, .{}),
-        .d, .i => {
+        .o => {
-            const imm = try dis.parseImm(enc.data.ops[0]);
+            const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
             return inst(enc, .{
-                .op1 = .{ .imm = imm },
+                .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
             });
         },
-        .zi => {
+        .zo => {
+            const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
+            return inst(enc, .{
+                .op1 = .{ .reg = enc.data.ops[0].toReg() },
+                .op2 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[1].regBitSize()) },
+            });
+        },
+        .oz => {
+            const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
+            return inst(enc, .{
+                .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
+                .op2 = .{ .reg = enc.data.ops[1].toReg() },
+            });
+        },
+        .oi => {
+            const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
             const imm = try dis.parseImm(enc.data.ops[1]);
             return inst(enc, .{
-                .op1 = .{ .reg = Register.rax.toBitSize(enc.data.ops[0].regBitSize()) },
+                .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
                 .op2 = .{ .imm = imm },
             });
         },
-        .o, .oi => {
+        .i, .d => {
-            const reg_low_enc = @as(u3, @truncate(dis.code[dis.pos - 1]));
+            const imm = try dis.parseImm(enc.data.ops[0]);
-            const op2: Instruction.Operand = if (enc.data.op_en == .oi) .{
-                .imm = try dis.parseImm(enc.data.ops[1]),
-            } else .none;
             return inst(enc, .{
-                .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
+                .op1 = .{ .imm = imm },
-                .op2 = op2,
+            });
+        },
+        .zi => {
+            const imm = try dis.parseImm(enc.data.ops[1]);
+            return inst(enc, .{
+                .op1 = .{ .reg = enc.data.ops[0].toReg() },
+                .op2 = .{ .imm = imm },
             });
         },
         .m, .mi, .m1, .mc => {
 @@ -118,7 +136,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
             const seg = segmentRegister(prefixes.legacy);
             const offset = try dis.parseOffset();
             return inst(enc, .{
-                .op1 = .{ .reg = Register.rax.toBitSize(enc.data.ops[0].regBitSize()) },
+                .op1 = .{ .reg = enc.data.ops[0].toReg() },
                 .op2 = .{ .mem = Memory.initMoffs(seg, offset) },
             });
         },
 @@ -127,7 +145,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
             const offset = try dis.parseOffset();
             return inst(enc, .{
                 .op1 = .{ .mem = Memory.initMoffs(seg, offset) },
-                .op2 = .{ .reg = Register.rax.toBitSize(enc.data.ops[1].regBitSize()) },
+                .op2 = .{ .reg = enc.data.ops[1].toReg() },
             });
         },
         .mr, .mri, .mrc => {
 @@ -223,7 +241,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
                 .op3 = op3,
             });
         },
-        .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable, // TODO
+        .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable, // TODO
     }
 }

src/arch/x86_64/Emit.zig+58-34

 @@ -10,22 +10,21 @@ prev_di_loc: Loc,
 /// Relative to the beginning of `code`.
 prev_di_pc: usize,
-code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty,
-relocs: std.ArrayListUnmanaged(Reloc) = .empty,
 pub const Error = Lower.Error || error{
     EmitFail,
 } || link.File.UpdateDebugInfoError;
 pub fn emitMir(emit: *Emit) Error!void {
     const gpa = emit.lower.bin_file.comp.gpa;
+    const code_offset_mapping = try emit.lower.allocator.alloc(u32, emit.lower.mir.instructions.len);
+    defer emit.lower.allocator.free(code_offset_mapping);
+    var relocs: std.ArrayListUnmanaged(Reloc) = .empty;
+    defer relocs.deinit(emit.lower.allocator);
+    var table_relocs: std.ArrayListUnmanaged(TableReloc) = .empty;
+    defer table_relocs.deinit(emit.lower.allocator);
     for (0..emit.lower.mir.instructions.len) |mir_i| {
         const mir_index: Mir.Inst.Index = @intCast(mir_i);
-        try emit.code_offset_mapping.putNoClobber(
+        code_offset_mapping[mir_index] = @intCast(emit.code.items.len);
-            emit.lower.allocator,
-            mir_index,
-            @intCast(emit.code.items.len),
-        );
         const lowered = try emit.lower.lowerMir(mir_index);
         var lowered_relocs = lowered.relocs;
         for (lowered.insts, 0..) |lowered_inst, lowered_index| {
 @@ -89,13 +88,17 @@ pub fn emitMir(emit: *Emit) Error!void {
                 lowered_relocs[0].lowered_inst_index == lowered_index) : ({
                 lowered_relocs = lowered_relocs[1..];
             }) switch (lowered_relocs[0].target) {
-                .inst => |target| try emit.relocs.append(emit.lower.allocator, .{
+                .inst => |target| try relocs.append(emit.lower.allocator, .{
                     .source = start_offset,
                     .source_offset = end_offset - 4,
                     .target = target,
                     .target_offset = lowered_relocs[0].off,
                     .length = @intCast(end_offset - start_offset),
                 }),
+                .table => try table_relocs.append(emit.lower.allocator, .{
+                    .source_offset = end_offset - 4,
+                    .target_offset = lowered_relocs[0].off,
+                }),
                 .linker_extern_fn => |sym_index| if (emit.lower.bin_file.cast(.elf)) |elf_file| {
                     // Add relocation to the decl.
                     const zo = elf_file.zigObjectPtr().?;
 @@ -103,7 +106,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                     const r_type = @intFromEnum(std.elf.R_X86_64.PLT32);
                     try atom_ptr.addReloc(gpa, .{
                         .r_offset = end_offset - 4,
-                        .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
+                        .r_info = @as(u64, sym_index) << 32 | r_type,
                         .r_addend = lowered_relocs[0].off - 4,
                     }, zo);
                 } else if (emit.lower.bin_file.cast(.macho)) |macho_file| {
 @@ -150,7 +153,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                     const r_type = @intFromEnum(std.elf.R_X86_64.TLSLD);
                     try atom.addReloc(gpa, .{
                         .r_offset = end_offset - 4,
-                        .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
+                        .r_info = @as(u64, sym_index) << 32 | r_type,
                         .r_addend = lowered_relocs[0].off - 4,
                     }, zo);
                 },
 @@ -161,7 +164,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                     const r_type = @intFromEnum(std.elf.R_X86_64.DTPOFF32);
                     try atom.addReloc(gpa, .{
                         .r_offset = end_offset - 4,
-                        .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
+                        .r_info = @as(u64, sym_index) << 32 | r_type,
                         .r_addend = lowered_relocs[0].off,
                     }, zo);
                 },
 @@ -176,7 +179,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                             @intFromEnum(std.elf.R_X86_64.PC32);
                         try atom.addReloc(gpa, .{
                             .r_offset = end_offset - 4,
-                            .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
+                            .r_info = @as(u64, sym_index) << 32 | r_type,
                             .r_addend = lowered_relocs[0].off - 4,
                         }, zo);
                     } else {
 @@ -186,7 +189,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                             @intFromEnum(std.elf.R_X86_64.@"32");
                         try atom.addReloc(gpa, .{
                             .r_offset = end_offset - 4,
-                            .r_info = (@as(u64, @intCast(sym_index)) << 32) | r_type,
+                            .r_info = @as(u64, sym_index) << 32 | r_type,
                             .r_addend = lowered_relocs[0].off,
                         }, zo);
                     }
 @@ -412,7 +415,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                                             loc_buf[0] = switch (mem.base()) {
                                                 .none => .{ .constu = 0 },
                                                 .reg => |reg| .{ .breg = reg.dwarfNum() },
-                                                .frame => unreachable,
+                                                .frame, .table => unreachable,
                                                 .reloc => |sym_index| .{ .addr = .{ .sym = sym_index } },
                                             };
                                             break :base &loc_buf[0];
 @@ -463,13 +466,40 @@ pub fn emitMir(emit: *Emit) Error!void {
             }
         }
     }
-    try emit.fixupRelocs();
+    {
-}
+        // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
+        // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
+        // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
+        // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
+        for (relocs.items) |reloc| {
+            const target = code_offset_mapping[reloc.target];
+            const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.source + reloc.length)) + reloc.target_offset;
+            std.mem.writeInt(i32, emit.code.items[reloc.source_offset..][0..4], @intCast(disp), .little);
+        }
+    }
+    if (emit.lower.mir.table.len > 0) {
+        if (emit.lower.bin_file.cast(.elf)) |elf_file| {
+            const zo = elf_file.zigObjectPtr().?;
+            const atom = zo.symbol(emit.atom_index).atom(elf_file).?;
-pub fn deinit(emit: *Emit) void {
+            const ptr_size = @divExact(emit.lower.target.ptrBitWidth(), 8);
-    emit.relocs.deinit(emit.lower.allocator);
+            var table_offset = std.mem.alignForward(u32, @intCast(emit.code.items.len), ptr_size);
-    emit.code_offset_mapping.deinit(emit.lower.allocator);
+            for (table_relocs.items) |table_reloc| try atom.addReloc(gpa, .{
-    emit.* = undefined;
+                .r_offset = table_reloc.source_offset,
+                .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"32"),
+                .r_addend = @as(i64, table_offset) + table_reloc.target_offset,
+            }, zo);
+            for (emit.lower.mir.table) |entry| {
+                try atom.addReloc(gpa, .{
+                    .r_offset = table_offset,
+                    .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"64"),
+                    .r_addend = code_offset_mapping[entry],
+                }, zo);
+                table_offset += ptr_size;
+            }
+            try emit.code.appendNTimes(gpa, 0, table_offset - emit.code.items.len);
+        } else unreachable;
+    }
 }
 fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error {
 @@ -481,7 +511,7 @@ fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error {
 const Reloc = struct {
     /// Offset of the instruction.
-    source: usize,
+    source: u32,
     /// Offset of the relocation within the instruction.
     source_offset: u32,
     /// Target of the relocation.
 @@ -492,18 +522,12 @@ const Reloc = struct {
     length: u5,
 };
-fn fixupRelocs(emit: *Emit) Error!void {
+const TableReloc = struct {
-    // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
+    /// Offset of the relocation.
-    // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
+    source_offset: u32,
-    // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
+    /// Offset from the start of the table.
-    // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
+    target_offset: i32,
-    for (emit.relocs.items) |reloc| {
+};
-        const target = emit.code_offset_mapping.get(reloc.target) orelse
-            return emit.fail("JMP/CALL relocation target not found!", .{});
-        const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.source + reloc.length)) + reloc.target_offset;
-        std.mem.writeInt(i32, emit.code.items[reloc.source_offset..][0..4], @intCast(disp), .little);
-    }
-}
 const Loc = struct {
     line: u32,

src/arch/x86_64/Encoding.zig+65-23

...	@@ -30,9 +30,10 @@ pub fn findByMnemonic(	...	@@ -30,9 +30,10 @@ pub fn findByMnemonic(
30	prefix: Instruction.Prefix,	30	prefix: Instruction.Prefix,
31	mnemonic: Mnemonic,	31	mnemonic: Mnemonic,
32	ops: []const Instruction.Operand,	32	ops: []const Instruction.Operand,
		33	target: *const std.Target,
33	) !?Encoding {	34	) !?Encoding {
34	var input_ops = [1]Op{.none} ** 4;	35	var input_ops: [4]Op = @splat(.none);
35	for (input_ops[0..ops.len], ops) \|input_op, op\| input_op. = Op.fromOperand(op);	36	for (input_ops[0..ops.len], ops) \|input_op, op\| input_op. = Op.fromOperand(op, target);
36		37
37	const rex_required = for (ops) \|op\| switch (op) {	38	const rex_required = for (ops) \|op\| switch (op) {
38	.reg => \|r\| switch (r) {	39	.reg => \|r\| switch (r) {
...	@@ -57,6 +58,16 @@ pub fn findByMnemonic(	...	@@ -57,6 +58,16 @@ pub fn findByMnemonic(
57	var shortest_enc: ?Encoding = null;	58	var shortest_enc: ?Encoding = null;
58	var shortest_len: ?usize = null;	59	var shortest_len: ?usize = null;
59	next: for (mnemonic_to_encodings_map[@intFromEnum(mnemonic)]) \|data\| {	60	next: for (mnemonic_to_encodings_map[@intFromEnum(mnemonic)]) \|data\| {
		61	if (!switch (data.feature) {
		62	.none => true,
		63	inline else => \|tag\| has_features: {
		64	comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' ');
		65	comptime var features: []const std.Target.x86.Feature = &.{};
		66	inline while (comptime feature_it.next()) \|feature\| features = features ++ .{@field(std.Target.x86.Feature, feature)};
		67	break :has_features std.Target.x86.featureSetHasAll(target.cpu.features, features[0..].*);
		68	},
		69	}) continue;
		70
60	switch (data.mode) {	71	switch (data.mode) {
61	.none, .short => if (rex_required) continue,	72	.none, .short => if (rex_required) continue,
62	.rex, .rex_short => if (!rex_required) continue,	73	.rex, .rex_short => if (!rex_required) continue,
...	@@ -64,7 +75,7 @@ pub fn findByMnemonic(	...	@@ -64,7 +75,7 @@ pub fn findByMnemonic(
64	}	75	}
65	for (input_ops, data.ops) \|input_op, data_op\| if (!input_op.isSubset(data_op)) continue :next;	76	for (input_ops, data.ops) \|input_op, data_op\| if (!input_op.isSubset(data_op)) continue :next;
66		77
67	const enc = Encoding{ .mnemonic = mnemonic, .data = data };	78	const enc: Encoding = .{ .mnemonic = mnemonic, .data = data };
68	if (shortest_enc) \|previous_shortest_enc\| {	79	if (shortest_enc) \|previous_shortest_enc\| {
69	const len = estimateInstructionLength(prefix, enc, ops);	80	const len = estimateInstructionLength(prefix, enc, ops);
70	const previous_shortest_len = shortest_len orelse	81	const previous_shortest_len = shortest_len orelse
...	@@ -165,9 +176,14 @@ pub fn format(	...	@@ -165,9 +176,14 @@ pub fn format(
165	for (opc) \|byte\| try writer.print("{x:0>2} ", .{byte});	176	for (opc) \|byte\| try writer.print("{x:0>2} ", .{byte});
166		177
167	switch (encoding.data.op_en) {	178	switch (encoding.data.op_en) {
168	.zo, .fd, .td, .i, .zi, .d => {},	179	.z, .fd, .td, .i, .zi, .d => {},
169	.o, .oi => {	180	.o, .zo, .oz, .oi => {
170	const tag = switch (encoding.data.ops[0]) {	181	const op = switch (encoding.data.op_en) {
		182	.o, .oz, .oi => encoding.data.ops[0],
		183	.zo => encoding.data.ops[1],
		184	else => unreachable,
		185	};
		186	const tag = switch (op) {
171	.r8 => "rb",	187	.r8 => "rb",
172	.r16 => "rw",	188	.r16 => "rw",
173	.r32 => "rd",	189	.r32 => "rd",
...	@@ -177,7 +193,7 @@ pub fn format(	...	@@ -177,7 +193,7 @@ pub fn format(
177	try writer.print("+{s} ", .{tag});	193	try writer.print("+{s} ", .{tag});
178	},	194	},
179	.m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),	195	.m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),
180	.mr, .rm, .rmi, .mri, .mrc, .rm0, .rvm, .rvmr, .rvmi, .mvr => try writer.writeAll("/r "),	196	.mr, .rm, .rmi, .mri, .mrc, .rm0, .rvm, .rvmr, .rvmi, .mvr, .rmv => try writer.writeAll("/r "),
181	}	197	}
182		198
183	switch (encoding.data.op_en) {	199	switch (encoding.data.op_en) {
...	@@ -202,7 +218,7 @@ pub fn format(	...	@@ -202,7 +218,7 @@ pub fn format(
202	try writer.print("{s} ", .{tag});	218	try writer.print("{s} ", .{tag});
203	},	219	},
204	.rvmr => try writer.writeAll("/is4 "),	220	.rvmr => try writer.writeAll("/is4 "),
205	.zo, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .rvm, .mvr => {},	221	.z, .fd, .td, .o, .zo, .oz, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .rvm, .mvr, .rmv => {},
206	}	222	}
207		223
208	try writer.print("{s} ", .{@tagName(encoding.mnemonic)});	224	try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
...	@@ -239,7 +255,8 @@ pub const Mnemonic = enum {	...	@@ -239,7 +255,8 @@ pub const Mnemonic = enum {
239	// General-purpose	255	// General-purpose
240	adc, add, @"and",	256	adc, add, @"and",
241	bsf, bsr, bswap, bt, btc, btr, bts,	257	bsf, bsr, bswap, bt, btc, btr, bts,
242	call, cbw, cdq, cdqe, clflush,	258	call, cbw, cdq, cdqe,
		259	clac, clc, cld, clflush, cli, clts, clui,
243	cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,	260	cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
244	cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,	261	cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
245	cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,	262	cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
...	@@ -260,10 +277,12 @@ pub const Mnemonic = enum {	...	@@ -260,10 +277,12 @@ pub const Mnemonic = enum {
260	neg, nop, not,	277	neg, nop, not,
261	@"or",	278	@"or",
262	pause, pop, popcnt, popfq, push, pushfq,	279	pause, pop, popcnt, popfq, push, pushfq,
263	rcl, rcr, ret, rol, ror,	280	rcl, rcr, ret, rol, ror, rorx,
264	sal, sar, sbb,	281	sal, sar, sarx, sbb,
265	scas, scasb, scasd, scasq, scasw,	282	scas, scasb, scasd, scasq, scasw,
266	shl, shld, shr, shrd, sub, syscall,	283	shl, shld, shlx, shr, shrd, shrx,
		284	stac, stc, std, sti, stui,
		285	sub, syscall,
267	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,	286	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
268	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,	287	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
269	setnz, seto, setp, setpe, setpo, sets, setz,	288	setnz, seto, setp, setpe, setpo, sets, setz,
...	@@ -296,7 +315,7 @@ pub const Mnemonic = enum {	...	@@ -296,7 +315,7 @@ pub const Mnemonic = enum {
296	ldmxcsr,	315	ldmxcsr,
297	maxps, maxss,	316	maxps, maxss,
298	minps, minss,	317	minps, minss,
299	movaps, movhlps, movlhps,	318	movaps, movhlps, movhps, movlhps, movlps,
300	movmskps,	319	movmskps,
301	movss, movups,	320	movss, movups,
302	mulps, mulss,	321	mulps, mulss,
...	@@ -322,6 +341,7 @@ pub const Mnemonic = enum {	...	@@ -322,6 +341,7 @@ pub const Mnemonic = enum {
322	minpd, minsd,	341	minpd, minsd,
323	movapd,	342	movapd,
324	movdqa, movdqu,	343	movdqa, movdqu,
		344	movhpd, movlpd,
325	movmskpd,	345	movmskpd,
326	//movsd,	346	//movsd,
327	movupd,	347	movupd,
...	@@ -353,6 +373,7 @@ pub const Mnemonic = enum {	...	@@ -353,6 +373,7 @@ pub const Mnemonic = enum {
353	pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,	373	pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,
354	pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,	374	pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,
355	pmulld,	375	pmulld,
		376	ptest,
356	roundpd, roundps, roundsd, roundss,	377	roundpd, roundps, roundsd, roundss,
357	// SSE4.2	378	// SSE4.2
358	pcmpgtq,	379	pcmpgtq,
...	@@ -383,7 +404,7 @@ pub const Mnemonic = enum {	...	@@ -383,7 +404,7 @@ pub const Mnemonic = enum {
383	vmovd,	404	vmovd,
384	vmovddup,	405	vmovddup,
385	vmovdqa, vmovdqu,	406	vmovdqa, vmovdqu,
386	vmovhlps, vmovlhps,	407	vmovhlps, vmovhpd, vmovhps, vmovlhps, vmovlpd, vmovlps,
387	vmovmskpd, vmovmskps,	408	vmovmskpd, vmovmskps,
388	vmovq,	409	vmovq,
389	vmovsd,	410	vmovsd,
...	@@ -413,6 +434,7 @@ pub const Mnemonic = enum {	...	@@ -413,6 +434,7 @@ pub const Mnemonic = enum {
413	vpsrad, vpsraq, vpsraw,	434	vpsrad, vpsraq, vpsraw,
414	vpsrld, vpsrldq, vpsrlq, vpsrlw,	435	vpsrld, vpsrldq, vpsrlq, vpsrlw,
415	vpsubb, vpsubd, vpsubq, vpsubsb, vpsubsw, vpsubusb, vpsubusw, vpsubw,	436	vpsubb, vpsubd, vpsubq, vpsubsb, vpsubsw, vpsubusb, vpsubusw, vpsubw,
		437	vptest,
416	vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,	438	vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
417	vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,	439	vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
418	vpxor,	440	vpxor,
...	@@ -421,6 +443,7 @@ pub const Mnemonic = enum {	...	@@ -421,6 +443,7 @@ pub const Mnemonic = enum {
421	vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,	443	vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,
422	vstmxcsr,	444	vstmxcsr,
423	vsubpd, vsubps, vsubsd, vsubss,	445	vsubpd, vsubps, vsubsd, vsubss,
		446	vtestpd, vtestps,
424	vxorpd, vxorps,	447	vxorpd, vxorps,
425	// F16C	448	// F16C
426	vcvtph2ps, vcvtps2ph,	449	vcvtph2ps, vcvtps2ph,
...	@@ -437,14 +460,14 @@ pub const Mnemonic = enum {	...	@@ -437,14 +460,14 @@ pub const Mnemonic = enum {
437		460
438	pub const OpEn = enum {	461	pub const OpEn = enum {
439	// zig fmt: off	462	// zig fmt: off
440	zo,	463	z,
441	o, oi,	464	o, zo, oz, oi,
442	i, zi,	465	i, zi,
443	d, m,	466	d, m,
444	fd, td,	467	fd, td,
445	m1, mc, mi, mr, rm,	468	m1, mc, mi, mr, rm,
446	rmi, mri, mrc,	469	rmi, mri, mrc,
447	rm0, vmi, rvm, rvmr, rvmi, mvr,	470	rm0, vmi, rvm, rvmr, rvmi, mvr, rmv,
448	// zig fmt: on	471	// zig fmt: on
449	};	472	};
450		473
...	@@ -471,7 +494,7 @@ pub const Op = enum {	...	@@ -471,7 +494,7 @@ pub const Op = enum {
471	ymm, ymm_m256,	494	ymm, ymm_m256,
472	// zig fmt: on	495	// zig fmt: on
473		496
474	pub fn fromOperand(operand: Instruction.Operand) Op {	497	pub fn fromOperand(operand: Instruction.Operand, target: *const std.Target) Op {
475	return switch (operand) {	498	return switch (operand) {
476	.none => .none,	499	.none => .none,
477		500
...	@@ -513,7 +536,7 @@ pub const Op = enum {	...	@@ -513,7 +536,7 @@ pub const Op = enum {
513		536
514	.mem => \|mem\| switch (mem) {	537	.mem => \|mem\| switch (mem) {
515	.moffs => .moffs,	538	.moffs => .moffs,
516	.sib, .rip => switch (mem.bitSize()) {	539	.sib, .rip => switch (mem.bitSize(target)) {
517	0 => .m,	540	0 => .m,
518	8 => .m8,	541	8 => .m8,
519	16 => .m16,	542	16 => .m16,
...	@@ -557,6 +580,21 @@ pub const Op = enum {	...	@@ -557,6 +580,21 @@ pub const Op = enum {
557	};	580	};
558	}	581	}
559		582
		583	pub fn toReg(op: Op) Register {
		584	return switch (op) {
		585	else => .none,
		586	.al => .al,
		587	.ax => .ax,
		588	.eax => .eax,
		589	.rax => .rax,
		590	.cl => .cl,
		591	.rip => .rip,
		592	.eip => .eip,
		593	.ip => .ip,
		594	.xmm0 => .xmm0,
		595	};
		596	}
		597
560	pub fn immBitSize(op: Op) u64 {	598	pub fn immBitSize(op: Op) u64 {
561	return switch (op) {	599	return switch (op) {
562	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,	600	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
...	@@ -808,6 +846,8 @@ pub const Feature = enum {	...	@@ -808,6 +846,8 @@ pub const Feature = enum {
808	avx,	846	avx,
809	avx2,	847	avx2,
810	bmi,	848	bmi,
		849	bmi2,
		850	cmov,
811	f16c,	851	f16c,
812	fma,	852	fma,
813	lzcnt,	853	lzcnt,
...	@@ -815,6 +855,7 @@ pub const Feature = enum {	...	@@ -815,6 +855,7 @@ pub const Feature = enum {
815	pclmul,	855	pclmul,
816	@"pclmul avx",	856	@"pclmul avx",
817	popcnt,	857	popcnt,
		858	smap,
818	sse,	859	sse,
819	sse2,	860	sse2,
820	sse3,	861	sse3,
...	@@ -822,6 +863,7 @@ pub const Feature = enum {	...	@@ -822,6 +863,7 @@ pub const Feature = enum {
822	sse4_2,	863	sse4_2,
823	ssse3,	864	ssse3,
824	sha,	865	sha,
		866	uintr,
825	vaes,	867	vaes,
826	vpclmulqdq,	868	vpclmulqdq,
827	x87,	869	x87,
...	@@ -831,7 +873,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op	...	@@ -831,7 +873,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
831	var inst = Instruction{	873	var inst = Instruction{
832	.prefix = prefix,	874	.prefix = prefix,
833	.encoding = encoding,	875	.encoding = encoding,
834	.ops = [1]Operand{.none} ** 4,	876	.ops = @splat(.none),
835	};	877	};
836	@memcpy(inst.ops[0..ops.len], ops);	878	@memcpy(inst.ops[0..ops.len], ops);
837		879
...	@@ -846,7 +888,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op	...	@@ -846,7 +888,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
846	const mnemonic_to_encodings_map = init: {	888	const mnemonic_to_encodings_map = init: {
847	@setEvalBranchQuota(5_000);	889	@setEvalBranchQuota(5_000);
848	const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len;	890	const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len;
849	var mnemonic_map: [mnemonic_count][]Data = .{&.{}} ** mnemonic_count;	891	var mnemonic_map: [mnemonic_count][]Data = @splat(&.{});
850	const encodings = @import("encodings.zig");	892	const encodings = @import("encodings.zig");
851	for (encodings.table) \|entry\| mnemonic_map[@intFromEnum(entry[0])].len += 1;	893	for (encodings.table) \|entry\| mnemonic_map[@intFromEnum(entry[0])].len += 1;
852	var data_storage: [encodings.table.len]Data = undefined;	894	var data_storage: [encodings.table.len]Data = undefined;
...	@@ -855,7 +897,7 @@ const mnemonic_to_encodings_map = init: {	...	@@ -855,7 +897,7 @@ const mnemonic_to_encodings_map = init: {
855	value.ptr = data_storage[storage_i..].ptr;	897	value.ptr = data_storage[storage_i..].ptr;
856	storage_i += value.len;	898	storage_i += value.len;
857	}	899	}
858	var mnemonic_i: [mnemonic_count]usize = .{0} ** mnemonic_count;	900	var mnemonic_i: [mnemonic_count]usize = @splat(0);
859	const ops_len = @typeInfo(std.meta.FieldType(Data, .ops)).array.len;	901	const ops_len = @typeInfo(std.meta.FieldType(Data, .ops)).array.len;
860	const opc_len = @typeInfo(std.meta.FieldType(Data, .opc)).array.len;	902	const opc_len = @typeInfo(std.meta.FieldType(Data, .opc)).array.len;
861	for (encodings.table) \|entry\| {	903	for (encodings.table) \|entry\| {
...	@@ -872,7 +914,7 @@ const mnemonic_to_encodings_map = init: {	...	@@ -872,7 +914,7 @@ const mnemonic_to_encodings_map = init: {
872	i.* += 1;	914	i.* += 1;
873	}	915	}
874	const final_storage = data_storage;	916	const final_storage = data_storage;
875	var final_map: [mnemonic_count][]const Data = .{&.{}} ** mnemonic_count;	917	var final_map: [mnemonic_count][]const Data = @splat(&.{});
876	storage_i = 0;	918	storage_i = 0;
877	for (&final_map, mnemonic_map) \|*final_value, value\| {	919	for (&final_map, mnemonic_map) \|*final_value, value\| {
878	final_value.* = final_storage[storage_i..][0..value.len];	920	final_value.* = final_storage[storage_i..][0..value.len];

src/arch/x86_64/Lower.zig+39-37

 @@ -1,6 +1,7 @@
 //! This file contains the functionality for lowering x86_64 MIR to Instructions
 bin_file: *link.File,
+target: *const std.Target,
 output_mode: std.builtin.OutputMode,
 link_mode: std.builtin.LinkMode,
 pic: bool,
 @@ -56,6 +57,7 @@ pub const Reloc = struct {
     const Target = union(enum) {
         inst: Mir.Inst.Index,
+        table,
         linker_reloc: u32,
         linker_tlsld: u32,
         linker_dtpoff: u32,
 @@ -193,7 +195,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
             .pseudo_probe_align_ri_s => {
                 try lower.emit(.none, .@"test", &.{
                     .{ .reg = inst.data.ri.r1 },
-                    .{ .imm = Immediate.s(@bitCast(inst.data.ri.i)) },
+                    .{ .imm = .s(@bitCast(inst.data.ri.i)) },
                 });
                 try lower.emit(.none, .jz, &.{
                     .{ .imm = lower.reloc(.{ .inst = index + 1 }, 0) },
 @@ -229,14 +231,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
                 }
                 try lower.emit(.none, .sub, &.{
                     .{ .reg = inst.data.ri.r1 },
-                    .{ .imm = Immediate.s(@bitCast(inst.data.ri.i)) },
+                    .{ .imm = .s(@bitCast(inst.data.ri.i)) },
                 });
                 assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts);
             },
             .pseudo_probe_adjust_setup_rri_s => {
                 try lower.emit(.none, .mov, &.{
                     .{ .reg = inst.data.rri.r2.to32() },
-                    .{ .imm = Immediate.s(@bitCast(inst.data.rri.i)) },
+                    .{ .imm = .s(@bitCast(inst.data.rri.i)) },
                 });
                 try lower.emit(.none, .sub, &.{
                     .{ .reg = inst.data.rri.r1 },
 @@ -255,7 +257,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
                 });
                 try lower.emit(.none, .sub, &.{
                     .{ .reg = inst.data.rr.r2 },
-                    .{ .imm = Immediate.s(page_size) },
+                    .{ .imm = .s(page_size) },
                 });
                 try lower.emit(.none, .jae, &.{
                     .{ .imm = lower.reloc(.{ .inst = index }, 0) },
 @@ -347,7 +349,7 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
     return error.LowerFail;
 }
-pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
+pub fn imm(lower: *const Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
     return switch (ops) {
         .rri_s,
         .ri_s,
 @@ -355,7 +357,7 @@ pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .mi_s,
         .rmi_s,
         .pseudo_dbg_local_ai_s,
-        => Immediate.s(@bitCast(i)),
+        => .s(@bitCast(i)),
         .rrri,
         .rri_u,
 @@ -368,18 +370,26 @@ pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .rrm,
         .rrmi,
         .pseudo_dbg_local_ai_u,
-        => Immediate.u(i),
+        => .u(i),
         .ri_64,
         .pseudo_dbg_local_ai_64,
-        => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
+        => .u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
         else => unreachable,
     };
 }
-pub fn mem(lower: Lower, payload: u32) Memory {
+pub fn mem(lower: *Lower, payload: u32) Memory {
-    return lower.mir.resolveFrameLoc(lower.mir.extraData(Mir.Memory, payload).data).decode();
+    var m = lower.mir.resolveFrameLoc(lower.mir.extraData(Mir.Memory, payload).data).decode();
+    switch (m) {
+        .sib => |*sib| switch (sib.base) {
+            else => {},
+            .table => sib.disp = lower.reloc(.table, sib.disp).signed,
+        },
+        else => {},
+    }
+    return m;
 }
 fn reloc(lower: *Lower, target: Reloc.Target, off: i32) Immediate {
 @@ -389,7 +399,7 @@ fn reloc(lower: *Lower, target: Reloc.Target, off: i32) Immediate {
         .off = off,
     };
     lower.result_relocs_len += 1;
-    return Immediate.s(0);
+    return .s(0);
 }
 fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
 @@ -417,19 +427,17 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
                                 // Here, we currently assume local dynamic TLS vars, and so
                                 // we emit LD model.
                                 _ = lower.reloc(.{ .linker_tlsld = sym_index }, 0);
-                                lower.result_insts[lower.result_insts_len] =
+                                lower.result_insts[lower.result_insts_len] = try .new(.none, .lea, &.{
-                                    try Instruction.new(.none, .lea, &[_]Operand{
                                     .{ .reg = .rdi },
                                     .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) },
-                                });
+                                }, lower.target);
                                 lower.result_insts_len += 1;
                                 _ = lower.reloc(.{
                                     .linker_extern_fn = try elf_file.getGlobalSymbol("__tls_get_addr", null),
                                 }, 0);
-                                lower.result_insts[lower.result_insts_len] =
+                                lower.result_insts[lower.result_insts_len] = try .new(.none, .call, &.{
-                                    try Instruction.new(.none, .call, &[_]Operand{
+                                    .{ .imm = .s(0) },
-                                    .{ .imm = Immediate.s(0) },
+                                }, lower.target);
-                                });
                                 lower.result_insts_len += 1;
                                 _ = lower.reloc(.{ .linker_dtpoff = sym_index }, 0);
                                 emit_mnemonic = .lea;
 @@ -439,11 +447,10 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
                                 }) };
                             } else {
                                 // Since we are linking statically, we emit LE model directly.
-                                lower.result_insts[lower.result_insts_len] =
+                                lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
-                                    try Instruction.new(.none, .mov, &[_]Operand{
                                     .{ .reg = .rax },
                                     .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .fs } }) },
-                                });
+                                }, lower.target);
                                 lower.result_insts_len += 1;
                                 _ = lower.reloc(.{ .linker_reloc = sym_index }, 0);
                                 emit_mnemonic = .lea;
 @@ -463,11 +470,10 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
                             .mov => {
                                 if (elf_sym.flags.is_extern_ptr) {
                                     const reg = ops[0].reg;
-                                    lower.result_insts[lower.result_insts_len] =
+                                    lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
-                                        try Instruction.new(.none, .mov, &[_]Operand{
                                         .{ .reg = reg.to64() },
                                         .{ .mem = Memory.initRip(.qword, 0) },
-                                    });
+                                    }, lower.target);
                                     lower.result_insts_len += 1;
                                     break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{ .base = .{
                                         .reg = reg.to64(),
 @@ -482,7 +488,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
                             }) },
                             .lea => {
                                 emit_mnemonic = .mov;
-                                break :op .{ .imm = Immediate.s(0) };
+                                break :op .{ .imm = .s(0) };
                             },
                             .mov => break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{
                                 .base = .{ .reg = .ds },
 @@ -495,16 +501,14 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
                         if (macho_sym.flags.tlv) {
                             _ = lower.reloc(.{ .linker_reloc = sym_index }, 0);
-                            lower.result_insts[lower.result_insts_len] =
+                            lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
-                                try Instruction.new(.none, .mov, &[_]Operand{
                                 .{ .reg = .rdi },
                                 .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) },
-                            });
+                            }, lower.target);
                             lower.result_insts_len += 1;
-                            lower.result_insts[lower.result_insts_len] =
+                            lower.result_insts[lower.result_insts_len] = try .new(.none, .call, &.{
-                                try Instruction.new(.none, .call, &[_]Operand{
                                 .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .rdi } }) },
-                            });
+                            }, lower.target);
                             lower.result_insts_len += 1;
                             emit_mnemonic = .mov;
                             break :op .{ .reg = .rax };
 @@ -519,11 +523,10 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
                             .mov => {
                                 if (macho_sym.flags.is_extern_ptr) {
                                     const reg = ops[0].reg;
-                                    lower.result_insts[lower.result_insts_len] =
+                                    lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
-                                        try Instruction.new(.none, .mov, &[_]Operand{
                                         .{ .reg = reg.to64() },
                                         .{ .mem = Memory.initRip(.qword, 0) },
-                                    });
+                                    }, lower.target);
                                     lower.result_insts_len += 1;
                                     break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{ .base = .{
                                         .reg = reg.to64(),
 @@ -540,8 +543,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
             },
         };
     }
-    lower.result_insts[lower.result_insts_len] =
+    lower.result_insts[lower.result_insts_len] = try .new(emit_prefix, emit_mnemonic, emit_ops, lower.target);
-        try Instruction.new(emit_prefix, emit_mnemonic, emit_ops);
     lower.result_insts_len += 1;
 }
 @@ -743,7 +745,7 @@ fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Er
             while (it.next()) |i| {
                 try lower.emit(.directive, .@".cfi_rel_offset", &.{
                     .{ .reg = callee_preserved_regs[i] },
-                    .{ .imm = Immediate.s(off) },
+                    .{ .imm = .s(off) },
                 });
                 off += 8;
             }

src/arch/x86_64/Mir.zig+64-15

 @@ -9,6 +9,7 @@
 instructions: std.MultiArrayList(Inst).Slice,
 /// The meaning of this data is determined by `Inst.Tag` value.
 extra: []const u32,
+table: []const Inst.Index,
 frame_locs: std.MultiArrayList(FrameLoc).Slice,
 pub const Inst = struct {
 @@ -22,17 +23,26 @@ pub const Inst = struct {
         /// ___
         @"_",
-        /// Integer __
+        /// Integer ___
         i_,
         /// ___ Left
         _l,
         /// ___ Left Double
         _ld,
+        /// ___ Left Without Affecting Flags
+        _lx,
         /// ___ Right
         _r,
         /// ___ Right Double
         _rd,
+        /// ___ Right Without Affecting Flags
+        _rx,
+        /// ___ Forward
+        _f,
+        /// ___ Reverse
+        //_r,
         /// ___ Above
         _a,
 @@ -43,6 +53,7 @@ pub const Inst = struct {
         /// ___ Below Or Equal
         _be,
         /// ___ Carry
+        /// ___ Carry Flag
         _c,
         /// ___ Equal
         _e,
 @@ -94,6 +105,14 @@ pub const Inst = struct {
         _s,
         /// ___ Zero
         _z,
+        /// ___ Alignment Check Flag
+        _ac,
+        /// ___ Direction Flag
+        //_d,
+        /// ___ Interrupt Flag
+        _i,
+        /// ___ User Interrupt Flag
+        _ui,
         /// ___ Byte
         //_b,
 @@ -210,6 +229,10 @@ pub const Inst = struct {
         p_q,
         /// Packed ___ Double Quadword
         p_dq,
+        /// ___ Aligned Packed Integer Values
+        _dqa,
+        /// ___ Unaligned Packed Integer Values
+        _dqu,
         /// ___ Scalar Single-Precision Values
         _ss,
 @@ -230,6 +253,10 @@ pub const Inst = struct {
         v_d,
         /// VEX-Encoded ___ QuadWord
         v_q,
+        /// VEX-Encoded ___ Aligned Packed Integer Values
+        v_dqa,
+        /// VEX-Encoded ___ Unaligned Packed Integer Values
+        v_dqu,
         /// VEX-Encoded ___ Integer Data
         v_i128,
         /// VEX-Encoded Packed ___
 @@ -287,9 +314,8 @@ pub const Inst = struct {
         /// Bitwise logical and of packed double-precision floating-point values
         @"and",
         /// Bit scan forward
-        bsf,
         /// Bit scan reverse
-        bsr,
+        bs,
         /// Byte swap
         bswap,
         /// Bit test
 @@ -305,6 +331,10 @@ pub const Inst = struct {
         cdq,
         /// Convert doubleword to quadword
         cdqe,
+        /// Clear carry flag
+        /// Clear direction flag
+        /// Clear interrupt flag
+        cl,
         /// Flush cache line
         clflush,
         /// Conditional move
 @@ -358,6 +388,8 @@ pub const Inst = struct {
         /// Move scalar double-precision floating-point value
         /// Move doubleword
         /// Move quadword
+        /// Move aligned packed integer values
+        /// Move unaligned packed integer values
         mov,
         /// Move data after swapping bytes
         movbe,
 @@ -401,9 +433,11 @@ pub const Inst = struct {
         ret,
         /// Rotate left
         /// Rotate right
+        /// Rotate right logical without affecting flags
         ro,
         /// Arithmetic shift left
         /// Arithmetic shift right
+        /// Shift left arithmetic without affecting flags
         sa,
         /// Integer subtraction with borrow
         sbb,
 @@ -417,6 +451,8 @@ pub const Inst = struct {
         /// Double precision shift left
         /// Logical shift right
         /// Double precision shift right
+        /// Shift left logical without affecting flags
+        /// Shift right logical without affecting flags
         sh,
         /// Subtract
         /// Subtract packed integers
 @@ -425,6 +461,11 @@ pub const Inst = struct {
         /// Subtract packed double-precision floating-point values
         /// Subtract scalar double-precision floating-point values
         sub,
+        /// Set carry flag
+        /// Set direction flag
+        /// Set interrupt flag
+        /// Store floating-point value
+        st,
         /// Store string
         sto,
         /// Syscall
 @@ -460,8 +501,6 @@ pub const Inst = struct {
         ldenv,
         /// Store x87 FPU environment
         nstenv,
-        /// Store floating-point value
-        st,
         /// Store x87 FPU environment
         stenv,
 @@ -542,8 +581,14 @@ pub const Inst = struct {
         /// Move aligned packed single-precision floating-point values
         /// Move aligned packed double-precision floating-point values
         mova,
+        /// Move high packed single-precision floating-point values
+        /// Move high packed double-precision floating-point values
+        movh,
         /// Move packed single-precision floating-point values high to low
         movhl,
+        /// Move low packed single-precision floating-point values
+        /// Move low packed double-precision floating-point values
+        movl,
         /// Move packed single-precision floating-point values low to high
         movlh,
         /// Move unaligned packed single-precision floating-point values
 @@ -601,10 +646,6 @@ pub const Inst = struct {
         cvttps2dq,
         /// Convert with truncation scalar double-precision floating-point value to doubleword integer
         cvttsd2si,
-        /// Move aligned packed integer values
-        movdqa,
-        /// Move unaligned packed integer values
-        movdqu,
         /// Packed interleave shuffle of quadruplets of single-precision floating-point values
         /// Packed interleave shuffle of pairs of double-precision floating-point values
         /// Shuffle packed doublewords
 @@ -1127,11 +1168,13 @@ pub const AirOffset = struct { air_inst: Air.Inst.Index, off: i32 };
 /// Used in conjunction with payload to transfer a list of used registers in a compact manner.
 pub const RegisterList = struct {
-    bitset: BitSet = BitSet.initEmpty(),
+    bitset: BitSet,
     const BitSet = IntegerBitSet(32);
     const Self = @This();
+    pub const empty: RegisterList = .{ .bitset = .initEmpty() };
     fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
         for (registers, 0..) |cpreg, i| {
             if (reg.id() == cpreg.id()) return @intCast(i);
 @@ -1157,8 +1200,12 @@ pub const RegisterList = struct {
         return @intCast(self.bitset.count());
     }
-    pub fn size(self: Self) i32 {
+    pub fn size(self: Self, target: *const std.Target) i32 {
-        return @intCast(self.bitset.count() * 8);
+        return @intCast(self.bitset.count() * @as(u4, switch (target.cpu.arch) {
+            else => unreachable,
+            .x86 => 4,
+            .x86_64 => 8,
+        }));
     }
 };
 @@ -1197,7 +1244,7 @@ pub const Memory = struct {
         size: bits.Memory.Size,
         index: Register,
         scale: bits.Memory.Scale,
-        _: u16 = undefined,
+        _: u15 = undefined,
     };
     pub fn encode(mem: bits.Memory) Memory {
 @@ -1220,7 +1267,7 @@ pub const Memory = struct {
                 },
             },
             .base = switch (mem.base) {
-                .none => undefined,
+                .none, .table => undefined,
                 .reg => |reg| @intFromEnum(reg),
                 .frame => |frame_index| @intFromEnum(frame_index),
                 .reloc => |sym_index| sym_index,
 @@ -1249,6 +1296,7 @@ pub const Memory = struct {
                         .none => .none,
                         .reg => .{ .reg = @enumFromInt(mem.base) },
                         .frame => .{ .frame = @enumFromInt(mem.base) },
+                        .table => .table,
                         .reloc => .{ .reloc = mem.base },
                     },
                     .scale_index = switch (mem.info.index) {
 @@ -1277,6 +1325,7 @@ pub const Memory = struct {
 pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
     mir.instructions.deinit(gpa);
     gpa.free(mir.extra);
+    gpa.free(mir.table);
     mir.frame_locs.deinit(gpa);
     mir.* = undefined;
 }
 @@ -1312,7 +1361,7 @@ pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffse
 pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
     return switch (mem.info.base) {
-        .none, .reg, .reloc => mem,
+        .none, .reg, .table, .reloc => mem,
         .frame => if (mir.frame_locs.len > 0) .{
             .info = .{
                 .base = .reg,

src/arch/x86_64/abi.zig+78-26

 @@ -242,17 +242,20 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8
                 .sse,   .sseup, .sseup, .sseup,
                 .sseup, .sseup, .sseup, .none,
             };
-            // LLVM always returns vectors byval
+            if (bits <= 512 or (ctx == .ret and bits <= @as(u64, if (std.Target.x86.featureSetHas(target.cpu.features, .avx512f))
-            if (bits <= 512 or ctx == .ret) return .{
+                2048
+            else if (std.Target.x86.featureSetHas(target.cpu.features, .avx))
+                1024
+            else
+                512))) return .{
                 .sse,   .sseup, .sseup, .sseup,
                 .sseup, .sseup, .sseup, .sseup,
             };
             return memory_class;
         },
         .optional => {
-            if (ty.isPtrLikeOptional(zcu)) {
+            if (ty.optionalReprIsPayload(zcu)) {
-                result[0] = .integer;
+                return classifySystemV(ty.optionalChild(zcu), zcu, target, ctx);
-                return result;
             }
             return memory_class;
         },
 @@ -405,6 +408,31 @@ fn classifySystemVUnion(
     return starting_byte_offset + loaded_union.sizeUnordered(ip);
 }
+pub const zigcc = struct {
+    pub const stack_align: ?InternPool.Alignment = null;
+    pub const return_in_regs = true;
+    pub const params_in_regs = true;
+    const volatile_gpr = gp_regs.len - 5;
+    const volatile_x87 = x87_regs.len - 1;
+    const volatile_sse = sse_avx_regs.len;
+    /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them
+    /// for anything else but stack offset tracking therefore we exclude them from this set.
+    pub const callee_preserved_regs = gp_regs[volatile_gpr..] ++ x87_regs[volatile_x87 .. x87_regs.len - 1] ++ sse_avx_regs[volatile_sse..];
+    /// These registers need to be preserved (saved on the stack) and restored by the caller before
+    /// the caller relinquishes control to a subroutine via call instruction (or similar).
+    /// In other words, these registers are free to use by the callee.
+    pub const caller_preserved_regs = gp_regs[0..volatile_gpr] ++ x87_regs[0..volatile_x87] ++ sse_avx_regs[0..volatile_sse];
+    const int_param_regs = gp_regs[0 .. volatile_gpr - 1];
+    const x87_param_regs = x87_regs[0..volatile_x87];
+    const sse_param_regs = sse_avx_regs[0..volatile_sse];
+    const int_return_regs = gp_regs[0..volatile_gpr];
+    const x87_return_regs = x87_regs[0..volatile_x87];
+    const sse_return_regs = sse_avx_regs[0..volatile_gpr];
+};
 pub const SysV = struct {
     /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them
     /// for anything else but stack offset tracking therefore we exclude them from this set.
 @@ -415,9 +443,11 @@ pub const SysV = struct {
     pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
     pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
+    pub const c_abi_x87_param_regs = x87_regs[0..0].*;
     pub const c_abi_sse_param_regs = sse_avx_regs[0..8].*;
     pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
-    pub const c_abi_sse_return_regs = sse_avx_regs[0..2].*;
+    pub const c_abi_x87_return_regs = x87_regs[0..2].*;
+    pub const c_abi_sse_return_regs = sse_avx_regs[0..4].*;
 };
 pub const Win64 = struct {
 @@ -430,74 +460,96 @@ pub const Win64 = struct {
     pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
     pub const c_abi_int_param_regs = [_]Register{ .rcx, .rdx, .r8, .r9 };
+    pub const c_abi_x87_param_regs = x87_regs[0..0].*;
     pub const c_abi_sse_param_regs = sse_avx_regs[0..4].*;
     pub const c_abi_int_return_regs = [_]Register{.rax};
+    pub const c_abi_x87_return_regs = x87_regs[0..0].*;
     pub const c_abi_sse_return_regs = sse_avx_regs[0..1].*;
 };
-pub fn resolveCallingConvention(
+pub fn getCalleePreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
-    cc: std.builtin.CallingConvention,
-    target: std.Target,
-) std.builtin.CallingConvention {
-    return switch (cc) {
-        .auto => switch (target.os.tag) {
-            else => .{ .x86_64_sysv = .{} },
-            .windows => .{ .x86_64_win = .{} },
-        },
-        else => cc,
-    };
-}
-pub fn getCalleePreservedRegs(cc: std.builtin.CallingConvention) []const Register {
     return switch (cc) {
+        .auto => zigcc.callee_preserved_regs,
         .x86_64_sysv => &SysV.callee_preserved_regs,
         .x86_64_win => &Win64.callee_preserved_regs,
         else => unreachable,
     };
 }
-pub fn getCallerPreservedRegs(cc: std.builtin.CallingConvention) []const Register {
+pub fn getCallerPreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
     return switch (cc) {
+        .auto => zigcc.caller_preserved_regs,
         .x86_64_sysv => &SysV.caller_preserved_regs,
         .x86_64_win => &Win64.caller_preserved_regs,
         else => unreachable,
     };
 }
-pub fn getCAbiIntParamRegs(cc: std.builtin.CallingConvention) []const Register {
+pub fn getCAbiIntParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
     return switch (cc) {
+        .auto => zigcc.int_param_regs,
         .x86_64_sysv => &SysV.c_abi_int_param_regs,
         .x86_64_win => &Win64.c_abi_int_param_regs,
         else => unreachable,
     };
 }
-pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention) []const Register {
+pub fn getCAbiX87ParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
+    return switch (cc) {
+        .auto => zigcc.x87_param_regs,
+        .x86_64_sysv => &SysV.c_abi_x87_param_regs,
+        .x86_64_win => &Win64.c_abi_x87_param_regs,
+        else => unreachable,
+    };
+}
+pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
     return switch (cc) {
+        .auto => zigcc.sse_param_regs,
         .x86_64_sysv => &SysV.c_abi_sse_param_regs,
         .x86_64_win => &Win64.c_abi_sse_param_regs,
         else => unreachable,
     };
 }
-pub fn getCAbiIntReturnRegs(cc: std.builtin.CallingConvention) []const Register {
+pub fn getCAbiIntReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
     return switch (cc) {
+        .auto => zigcc.int_return_regs,
         .x86_64_sysv => &SysV.c_abi_int_return_regs,
         .x86_64_win => &Win64.c_abi_int_return_regs,
         else => unreachable,
     };
 }
-pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention) []const Register {
+pub fn getCAbiX87ReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
+    return switch (cc) {
+        .auto => zigcc.x87_return_regs,
+        .x86_64_sysv => &SysV.c_abi_x87_return_regs,
+        .x86_64_win => &Win64.c_abi_x87_return_regs,
+        else => unreachable,
+    };
+}
+pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
     return switch (cc) {
+        .auto => zigcc.sse_return_regs,
         .x86_64_sysv => &SysV.c_abi_sse_return_regs,
         .x86_64_win => &Win64.c_abi_sse_return_regs,
         else => unreachable,
     };
 }
+pub fn getCAbiLinkerScratchReg(cc: std.builtin.CallingConvention.Tag) Register {
+    return switch (cc) {
+        .auto => zigcc.int_return_regs[zigcc.int_return_regs.len - 1],
+        .x86_64_sysv => SysV.c_abi_int_return_regs[0],
+        .x86_64_win => Win64.c_abi_int_return_regs[0],
+        else => unreachable,
+    };
+}
 const gp_regs = [_]Register{
-    .rax, .rcx, .rdx, .rbx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15,
+    .rax, .rdx, .rbx, .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15,
 };
 const x87_regs = [_]Register{
     .st0, .st1, .st2, .st3, .st4, .st5, .st6, .st7,

src/arch/x86_64/bits.zig+80-11

...	@@ -150,6 +150,31 @@ pub const Condition = enum(u5) {	...	@@ -150,6 +150,31 @@ pub const Condition = enum(u5) {
150	.nz_or_p => .z_and_np,	150	.nz_or_p => .z_and_np,
151	};	151	};
152	}	152	}
		153
		154	/// Returns the equivalent condition when the operands are swapped.
		155	pub fn commute(cond: Condition) Condition {
		156	return switch (cond) {
		157	else => cond,
		158	.a => .b,
		159	.ae => .be,
		160	.b => .a,
		161	.be => .ae,
		162	.c => .a,
		163	.g => .l,
		164	.ge => .le,
		165	.l => .g,
		166	.le => .ge,
		167	.na => .nb,
		168	.nae => .nbe,
		169	.nb => .na,
		170	.nbe => .nae,
		171	.nc => .na,
		172	.ng => .nl,
		173	.nge => .nle,
		174	.nl => .ng,
		175	.nle => .nge,
		176	};
		177	}
153	};	178	};
154		179
155	pub const Register = enum(u7) {	180	pub const Register = enum(u7) {
...	@@ -454,37 +479,41 @@ pub const RegisterOffset = struct { reg: Register, off: i32 = 0 };	...	@@ -454,37 +479,41 @@ pub const RegisterOffset = struct { reg: Register, off: i32 = 0 };
454	pub const SymbolOffset = struct { sym_index: u32, off: i32 = 0 };	479	pub const SymbolOffset = struct { sym_index: u32, off: i32 = 0 };
455		480
456	pub const Memory = struct {	481	pub const Memory = struct {
457	base: Base,	482	base: Base = .none,
458	mod: Mod,	483	mod: Mod = .{ .rm = .{} },
459		484
460	pub const Base = union(enum(u2)) {	485	pub const Base = union(enum(u3)) {
461	none,	486	none,
462	reg: Register,	487	reg: Register,
463	frame: FrameIndex,	488	frame: FrameIndex,
		489	table,
464	reloc: u32,	490	reloc: u32,
465		491
466	pub const Tag = @typeInfo(Base).@"union".tag_type.?;	492	pub const Tag = @typeInfo(Base).@"union".tag_type.?;
467		493
468	pub fn isExtended(self: Base) bool {	494	pub fn isExtended(self: Base) bool {
469	return switch (self) {	495	return switch (self) {
470	.none, .frame, .reloc => false, // rsp, rbp, and rip are not extended	496	.none, .frame, .table, .reloc => false, // rsp, rbp, and rip are not extended
471	.reg => \|reg\| reg.isExtended(),	497	.reg => \|reg\| reg.isExtended(),
472	};	498	};
473	}	499	}
474	};	500	};
475		501
476	pub const Mod = union(enum(u1)) {	502	pub const Mod = union(enum(u1)) {
477	rm: struct {	503	rm: Rm,
478	size: Size,	504	off: u64,
		505
		506	pub const Rm = struct {
		507	size: Size = .none,
479	index: Register = .none,	508	index: Register = .none,
480	scale: Scale = .@"1",	509	scale: Scale = .@"1",
481	disp: i32 = 0,	510	disp: i32 = 0,
482	},	511	};
483	off: u64,
484	};	512	};
485		513
486	pub const Size = enum(u4) {	514	pub const Size = enum(u4) {
487	none,	515	none,
		516	ptr,
488	byte,	517	byte,
489	word,	518	word,
490	dword,	519	dword,
...	@@ -521,9 +550,10 @@ pub const Memory = struct {	...	@@ -521,9 +550,10 @@ pub const Memory = struct {
521	};	550	};
522	}	551	}
523		552
524	pub fn bitSize(s: Size) u64 {	553	pub fn bitSize(s: Size, target: *const std.Target) u64 {
525	return switch (s) {	554	return switch (s) {
526	.none => 0,	555	.none => 0,
		556	.ptr => target.ptrBitWidth(),
527	.byte => 8,	557	.byte => 8,
528	.word => 16,	558	.word => 16,
529	.dword => 32,	559	.dword => 32,
...	@@ -543,11 +573,50 @@ pub const Memory = struct {	...	@@ -543,11 +573,50 @@ pub const Memory = struct {
543	) @TypeOf(writer).Error!void {	573	) @TypeOf(writer).Error!void {
544	if (s == .none) return;	574	if (s == .none) return;
545	try writer.writeAll(@tagName(s));	575	try writer.writeAll(@tagName(s));
546	try writer.writeAll(" ptr");	576	switch (s) {
		577	.none => unreachable,
		578	.ptr => {},
		579	else => {
		580	try writer.writeByte(' ');
		581	try writer.writeAll("ptr");
		582	},
		583	}
547	}	584	}
548	};	585	};
549		586
550	pub const Scale = enum(u2) { @"1", @"2", @"4", @"8" };	587	pub const Scale = enum(u2) {
		588	@"1",
		589	@"2",
		590	@"4",
		591	@"8",
		592
		593	pub fn fromFactor(factor: u4) Scale {
		594	return switch (factor) {
		595	else => unreachable,
		596	1 => .@"1",
		597	2 => .@"2",
		598	4 => .@"4",
		599	8 => .@"8",
		600	};
		601	}
		602
		603	pub fn toFactor(scale: Scale) u4 {
		604	return switch (scale) {
		605	.@"1" => 1,
		606	.@"2" => 2,
		607	.@"4" => 4,
		608	.@"8" => 8,
		609	};
		610	}
		611
		612	pub fn fromLog2(log2: u2) Scale {
		613	return @enumFromInt(log2);
		614	}
		615
		616	pub fn toLog2(scale: Scale) u2 {
		617	return @intFromEnum(scale);
		618	}
		619	};
551	};	620	};
552		621
553	pub const Immediate = union(enum) {	622	pub const Immediate = union(enum) {

src/arch/x86_64/encoder.zig+94-84

 @@ -138,7 +138,7 @@ pub const Instruction = struct {
                 .moffs => true,
                 .rip => false,
                 .sib => |s| switch (s.base) {
-                    .none, .frame, .reloc => false,
+                    .none, .frame, .table, .reloc => false,
                     .reg => |reg| reg.class() == .segment,
                 },
             };
 @@ -161,17 +161,17 @@ pub const Instruction = struct {
         pub fn disp(mem: Memory) Immediate {
             return switch (mem) {
-                .sib => |s| Immediate.s(s.disp),
+                .sib => |s| .s(s.disp),
-                .rip => |r| Immediate.s(r.disp),
+                .rip => |r| .s(r.disp),
-                .moffs => |m| Immediate.u(m.offset),
+                .moffs => |m| .u(m.offset),
             };
         }
-        pub fn bitSize(mem: Memory) u64 {
+        pub fn bitSize(mem: Memory, target: *const std.Target) u64 {
             return switch (mem) {
-                .rip => |r| r.ptr_size.bitSize(),
+                .rip => |r| r.ptr_size.bitSize(target),
-                .sib => |s| s.ptr_size.bitSize(),
+                .sib => |s| s.ptr_size.bitSize(target),
-                .moffs => 64,
+                .moffs => target.ptrBitWidth(),
             };
         }
     };
 @@ -277,6 +277,7 @@ pub const Instruction = struct {
                             .none => any = false,
                             .reg => |reg| try writer.print("{s}", .{@tagName(reg)}),
                             .frame => |frame_index| try writer.print("{}", .{frame_index}),
+                            .table => try writer.print("Table", .{}),
                             .reloc => |sym_index| try writer.print("Symbol({d})", .{sym_index}),
                         }
                         if (mem.scaleIndex()) |si| {
 @@ -314,28 +315,33 @@ pub const Instruction = struct {
         }
     };
-    pub fn new(prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) !Instruction {
+    pub fn new(
+        prefix: Prefix,
+        mnemonic: Mnemonic,
+        ops: []const Operand,
+        target: *const std.Target,
+    ) !Instruction {
         const encoding: Encoding = switch (prefix) {
-            else => (try Encoding.findByMnemonic(prefix, mnemonic, ops)) orelse {
+            else => (try Encoding.findByMnemonic(prefix, mnemonic, ops, target)) orelse {
                 log.err("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
                     @tagName(prefix),
                     @tagName(mnemonic),
-                    @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0]) else .none),
+                    @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none),
-                    @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1]) else .none),
+                    @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none),
-                    @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2]) else .none),
+                    @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none),
-                    @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3]) else .none),
+                    @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none),
                 });
                 return error.InvalidInstruction;
             },
             .directive => .{
                 .mnemonic = mnemonic,
                 .data = .{
-                    .op_en = .zo,
+                    .op_en = .z,
                     .ops = .{
-                        if (ops.len > 0) Encoding.Op.fromOperand(ops[0]) else .none,
+                        if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none,
-                        if (ops.len > 1) Encoding.Op.fromOperand(ops[1]) else .none,
+                        if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none,
-                        if (ops.len > 2) Encoding.Op.fromOperand(ops[2]) else .none,
+                        if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none,
-                        if (ops.len > 3) Encoding.Op.fromOperand(ops[3]) else .none,
+                        if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none,
                     },
                     .opc_len = 0,
                     .opc = undefined,
 @@ -395,7 +401,7 @@ pub const Instruction = struct {
         }
         switch (data.op_en) {
-            .zo, .o => {},
+            .z, .o, .zo, .oz => {},
             .i, .d => try encodeImm(inst.ops[0].imm, data.ops[0], encoder),
             .zi, .oi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),
             .fd => try encoder.imm64(inst.ops[1].mem.moffs.offset),
 @@ -403,7 +409,7 @@ pub const Instruction = struct {
             else => {
                 const mem_op = switch (data.op_en) {
                     .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
-                    .rm, .rmi, .rm0, .vmi => inst.ops[1],
+                    .rm, .rmi, .rm0, .vmi, .rmv => inst.ops[1],
                     .rvm, .rvmr, .rvmi => inst.ops[2],
                     else => unreachable,
                 };
 @@ -412,7 +418,7 @@ pub const Instruction = struct {
                         const rm = switch (data.op_en) {
                             .m, .mi, .m1, .mc, .vmi => enc.modRmExt(),
                             .mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),
-                            .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0].reg.lowEnc(),
+                            .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0].reg.lowEnc(),
                             .mvr => inst.ops[2].reg.lowEnc(),
                             else => unreachable,
                         };
 @@ -422,7 +428,7 @@ pub const Instruction = struct {
                         const op = switch (data.op_en) {
                             .m, .mi, .m1, .mc, .vmi => .none,
                             .mr, .mri, .mrc => inst.ops[1],
-                            .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0],
+                            .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0],
                             .mvr => inst.ops[2],
                             else => unreachable,
                         };
 @@ -448,7 +454,8 @@ pub const Instruction = struct {
         const final = opcode.len - 1;
         for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte);
         switch (inst.encoding.data.op_en) {
-            .o, .oi => try encoder.opcode_withReg(opcode[final], inst.ops[0].reg.lowEnc()),
+            .o, .oz, .oi => try encoder.opcode_withReg(opcode[final], inst.ops[0].reg.lowEnc()),
+            .zo => try encoder.opcode_withReg(opcode[final], inst.ops[1].reg.lowEnc()),
             else => try encoder.opcode_1byte(opcode[final]),
         }
     }
 @@ -474,7 +481,7 @@ pub const Instruction = struct {
         }
         const segment_override: ?Register = switch (op_en) {
-            .zo, .i, .zi, .o, .oi, .d => null,
+            .z, .i, .zi, .o, .zo, .oz, .oi, .d => null,
             .fd => inst.ops[1].mem.base().reg,
             .td => inst.ops[0].mem.base().reg,
             .rm, .rmi, .rm0 => if (inst.ops[1].isSegmentRegister())
 @@ -493,7 +500,7 @@ pub const Instruction = struct {
                 }
             else
                 null,
-            .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable,
+            .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable,
         };
         if (segment_override) |seg| {
             legacy.setSegmentOverride(seg);
 @@ -510,11 +517,12 @@ pub const Instruction = struct {
         rex.w = inst.encoding.data.mode == .long;
         switch (op_en) {
-            .zo, .i, .zi, .fd, .td, .d => {},
+            .z, .i, .zi, .fd, .td, .d => {},
-            .o, .oi => rex.b = inst.ops[0].reg.isExtended(),
+            .o, .oz, .oi => rex.b = inst.ops[0].reg.isExtended(),
-            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0 => {
+            .zo => rex.b = inst.ops[1].reg.isExtended(),
+            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .rmv => {
                 const r_op = switch (op_en) {
-                    .rm, .rmi, .rm0 => inst.ops[0],
+                    .rm, .rmi, .rm0, .rmv => inst.ops[0],
                     .mr, .mri, .mrc => inst.ops[1],
                     else => .none,
                 };
 @@ -544,11 +552,12 @@ pub const Instruction = struct {
         vex.w = inst.encoding.data.mode.isLong();
         switch (op_en) {
-            .zo, .i, .zi, .fd, .td, .d => {},
+            .z, .i, .zi, .fd, .td, .d => {},
-            .o, .oi => vex.b = inst.ops[0].reg.isExtended(),
+            .o, .oz, .oi => vex.b = inst.ops[0].reg.isExtended(),
-            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr => {
+            .zo => vex.b = inst.ops[1].reg.isExtended(),
+            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => {
                 const r_op = switch (op_en) {
-                    .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0],
+                    .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0],
                     .mr, .mri, .mrc => inst.ops[1],
                     .mvr => inst.ops[2],
                     .m, .mi, .m1, .mc, .vmi => .none,
 @@ -557,7 +566,7 @@ pub const Instruction = struct {
                 vex.r = r_op.isBaseExtended();
                 const b_x_op = switch (op_en) {
-                    .rm, .rmi, .rm0, .vmi => inst.ops[1],
+                    .rm, .rmi, .rm0, .vmi, .rmv => inst.ops[1],
                     .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
                     .rvm, .rvmr, .rvmi => inst.ops[2],
                     else => unreachable,
 @@ -588,6 +597,7 @@ pub const Instruction = struct {
             else => {},
             .vmi => vex.v = inst.ops[0].reg,
             .rvm, .rvmr, .rvmi => vex.v = inst.ops[1].reg,
+            .rmv => vex.v = inst.ops[2].reg,
         }
         try encoder.vex(vex);
 @@ -608,7 +618,7 @@ pub const Instruction = struct {
         switch (mem) {
             .moffs => unreachable,
             .sib => |sib| switch (sib.base) {
-                .none => {
+                .none, .table => {
                     try encoder.modRm_SIBDisp0(operand_enc);
                     if (mem.scaleIndex()) |si| {
                         const scale = math.log2_int(u4, si.scale);
 @@ -676,11 +686,11 @@ pub const Instruction = struct {
                     else => unreachable,
                 },
                 .frame => if (@TypeOf(encoder).options.allow_frame_locs) {
-                    try encoder.modRm_indirectDisp32(operand_enc, undefined);
+                    try encoder.modRm_indirectDisp32(operand_enc, 0);
                     try encoder.disp32(undefined);
                 } else return error.CannotEncode,
                 .reloc => if (@TypeOf(encoder).options.allow_symbols) {
-                    try encoder.modRm_indirectDisp32(operand_enc, undefined);
+                    try encoder.modRm_indirectDisp32(operand_enc, 0);
                     try encoder.disp32(undefined);
                 } else return error.CannotEncode,
             },
 @@ -1185,7 +1195,7 @@ const TestEncode = struct {
     ) !void {
         var stream = std.io.fixedBufferStream(&enc.buffer);
         var count_writer = std.io.countingWriter(stream.writer());
-        const inst = try Instruction.new(.none, mnemonic, ops);
+        const inst: Instruction = try .new(.none, mnemonic, ops);
         try inst.encode(count_writer.writer(), .{});
         enc.index = count_writer.bytes_written;
     }
 @@ -1199,9 +1209,9 @@ test "encode" {
     var buf = std.ArrayList(u8).init(testing.allocator);
     defer buf.deinit();
-    const inst = try Instruction.new(.none, .mov, &.{
+    const inst: Instruction = try .new(.none, .mov, &.{
         .{ .reg = .rbx },
-        .{ .imm = Instruction.Immediate.u(4) },
+        .{ .imm = .u(4) },
     });
     try inst.encode(buf.writer(), .{});
     try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items);
 @@ -1211,47 +1221,47 @@ test "lower I encoding" {
     var enc = TestEncode{};
     try enc.encode(.push, &.{
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10");
     try enc.encode(.push, &.{
-        .{ .imm = Instruction.Immediate.u(0x1000) },
+        .{ .imm = .u(0x1000) },
     });
     try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000");
     try enc.encode(.push, &.{
-        .{ .imm = Instruction.Immediate.u(0x10000000) },
+        .{ .imm = .u(0x10000000) },
     });
     try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000");
     try enc.encode(.adc, &.{
         .{ .reg = .rax },
-        .{ .imm = Instruction.Immediate.u(0x10000000) },
+        .{ .imm = .u(0x10000000) },
     });
     try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000");
     try enc.encode(.add, &.{
         .{ .reg = .al },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10");
     try enc.encode(.add, &.{
         .{ .reg = .rax },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
     try enc.encode(.sbb, &.{
         .{ .reg = .ax },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10");
     try enc.encode(.xor, &.{
         .{ .reg = .al },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10");
 }
 @@ -1261,43 +1271,43 @@ test "lower MI encoding" {
     try enc.encode(.mov, &.{
         .{ .reg = .r12 },
-        .{ .imm = Instruction.Immediate.u(0x1000) },
+        .{ .imm = .u(0x1000) },
     });
     try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
     try enc.encode(.mov, &.{
         .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .r12 } }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
     try enc.encode(.mov, &.{
         .{ .reg = .r12 },
-        .{ .imm = Instruction.Immediate.u(0x1000) },
+        .{ .imm = .u(0x1000) },
     });
     try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
     try enc.encode(.mov, &.{
         .{ .reg = .r12 },
-        .{ .imm = Instruction.Immediate.u(0x1000) },
+        .{ .imm = .u(0x1000) },
     });
     try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
     try enc.encode(.mov, &.{
         .{ .reg = .rax },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
     try enc.encode(.mov, &.{
         .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 } }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
     try enc.encode(.mov, &.{
         .{ .mem = Instruction.Memory.initRip(.qword, 0x10) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings(
         "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
 @@ -1307,19 +1317,19 @@ test "lower MI encoding" {
     try enc.encode(.mov, &.{
         .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -8 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10");
     try enc.encode(.mov, &.{
         .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -2 }) },
-        .{ .imm = Instruction.Immediate.s(-16) },
+        .{ .imm = .s(-16) },
     });
     try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16");
     try enc.encode(.mov, &.{
         .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -1 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10");
 @@ -1329,7 +1339,7 @@ test "lower MI encoding" {
             .disp = 0x10000000,
             .scale_index = .{ .scale = 2, .index = .rcx },
         }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings(
         "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
 @@ -1339,43 +1349,43 @@ test "lower MI encoding" {
     try enc.encode(.adc, &.{
         .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10");
     try enc.encode(.adc, &.{
         .{ .mem = Instruction.Memory.initRip(.qword, 0) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10");
     try enc.encode(.adc, &.{
         .{ .reg = .rax },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10");
     try enc.encode(.add, &.{
         .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .rdx }, .disp = -8 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10");
     try enc.encode(.add, &.{
         .{ .reg = .rax },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
     try enc.encode(.add, &.{
         .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) },
-        .{ .imm = Instruction.Immediate.s(-0x10) },
+        .{ .imm = .s(-0x10) },
     });
     try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10");
     try enc.encode(.@"and", &.{
         .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings(
         "\x83\x24\x25\x00\x00\x00\x10\x10",
 @@ -1385,7 +1395,7 @@ test "lower MI encoding" {
     try enc.encode(.@"and", &.{
         .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .es }, .disp = 0x10000000 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings(
         "\x26\x83\x24\x25\x00\x00\x00\x10\x10",
 @@ -1395,7 +1405,7 @@ test "lower MI encoding" {
     try enc.encode(.@"and", &.{
         .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings(
         "\x41\x83\xA4\x24\x00\x00\x00\x10\x10",
 @@ -1405,7 +1415,7 @@ test "lower MI encoding" {
     try enc.encode(.sub, &.{
         .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings(
         "\x41\x83\xAB\x00\x00\x00\x10\x10",
 @@ -1624,14 +1634,14 @@ test "lower RMI encoding" {
     try enc.encode(.imul, &.{
         .{ .reg = .r11 },
         .{ .reg = .r12 },
-        .{ .imm = Instruction.Immediate.s(-2) },
+        .{ .imm = .s(-2) },
     });
     try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2");
     try enc.encode(.imul, &.{
         .{ .reg = .r11 },
         .{ .mem = Instruction.Memory.initRip(.qword, -16) },
-        .{ .imm = Instruction.Immediate.s(-1024) },
+        .{ .imm = .s(-1024) },
     });
     try expectEqualHexStrings(
         "\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF",
 @@ -1642,7 +1652,7 @@ test "lower RMI encoding" {
     try enc.encode(.imul, &.{
         .{ .reg = .bx },
         .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) },
-        .{ .imm = Instruction.Immediate.s(-1024) },
+        .{ .imm = .s(-1024) },
     });
     try expectEqualHexStrings(
         "\x66\x69\x5D\xF0\x00\xFC",
 @@ -1653,7 +1663,7 @@ test "lower RMI encoding" {
     try enc.encode(.imul, &.{
         .{ .reg = .bx },
         .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) },
-        .{ .imm = Instruction.Immediate.u(1024) },
+        .{ .imm = .u(1024) },
     });
     try expectEqualHexStrings(
         "\x66\x69\x5D\xF0\x00\x04",
 @@ -1769,7 +1779,7 @@ test "lower M encoding" {
     try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
     try enc.encode(.call, &.{
-        .{ .imm = Instruction.Immediate.s(0) },
+        .{ .imm = .s(0) },
     });
     try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
 @@ -1828,7 +1838,7 @@ test "lower OI encoding" {
     try enc.encode(.mov, &.{
         .{ .reg = .rax },
-        .{ .imm = Instruction.Immediate.u(0x1000000000000000) },
+        .{ .imm = .u(0x1000000000000000) },
     });
     try expectEqualHexStrings(
         "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
 @@ -1838,7 +1848,7 @@ test "lower OI encoding" {
     try enc.encode(.mov, &.{
         .{ .reg = .r11 },
-        .{ .imm = Instruction.Immediate.u(0x1000000000000000) },
+        .{ .imm = .u(0x1000000000000000) },
     });
     try expectEqualHexStrings(
         "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
 @@ -1848,19 +1858,19 @@ test "lower OI encoding" {
     try enc.encode(.mov, &.{
         .{ .reg = .r11d },
-        .{ .imm = Instruction.Immediate.u(0x10000000) },
+        .{ .imm = .u(0x10000000) },
     });
     try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000");
     try enc.encode(.mov, &.{
         .{ .reg = .r11w },
-        .{ .imm = Instruction.Immediate.u(0x1000) },
+        .{ .imm = .u(0x1000) },
     });
     try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000");
     try enc.encode(.mov, &.{
         .{ .reg = .r11b },
-        .{ .imm = Instruction.Immediate.u(0x10) },
+        .{ .imm = .u(0x10) },
     });
     try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10");
 }
 @@ -1934,7 +1944,7 @@ test "lower NP encoding" {
 }
 fn invalidInstruction(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void {
-    const err = Instruction.new(.none, mnemonic, ops);
+    const err: Instruction = .new(.none, mnemonic, ops);
     try testing.expectError(error.InvalidInstruction, err);
 }
 @@ -1982,12 +1992,12 @@ test "invalid instruction" {
         .{ .reg = .r12d },
     });
     try invalidInstruction(.push, &.{
-        .{ .imm = Instruction.Immediate.u(0x1000000000000000) },
+        .{ .imm = .u(0x1000000000000000) },
     });
 }
 fn cannotEncode(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void {
-    try testing.expectError(error.CannotEncode, Instruction.new(.none, mnemonic, ops));
+    try testing.expectError(error.CannotEncode, .new(.none, mnemonic, ops));
 }
 test "cannot encode" {
 @@ -2171,7 +2181,7 @@ const Assembler = struct {
     pub fn assemble(as: *Assembler, writer: anytype) !void {
         while (try as.next()) |parsed_inst| {
-            const inst = try Instruction.new(.none, parsed_inst.mnemonic, &parsed_inst.ops);
+            const inst: Instruction = try .new(.none, parsed_inst.mnemonic, &parsed_inst.ops);
             try inst.encode(writer, .{});
         }
     }

src/arch/x86_64/encodings.zig+229-163

...	@@ -124,106 +124,118 @@ pub const table = [_]Entry{	...	@@ -124,106 +124,118 @@ pub const table = [_]Entry{
124	.{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },	124	.{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },
125	.{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none, .none },	125	.{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none, .none },
126		126
127	.{ .cbw, .zo, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },	127	.{ .cbw, .z, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
128	.{ .cwde, .zo, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },	128	.{ .cwde, .z, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
129	.{ .cdqe, .zo, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },	129	.{ .cdqe, .z, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
130		130
131	.{ .cwd, .zo, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },	131	.{ .cwd, .z, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
132	.{ .cdq, .zo, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },	132	.{ .cdq, .z, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
133	.{ .cqo, .zo, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },	133	.{ .cqo, .z, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
		134
		135	.{ .clac, .z, &.{}, &.{ 0x0f, 0x01, 0xca }, 0, .none, .smap },
		136
		137	.{ .clc, .z, &.{}, &.{ 0xf8 }, 0, .none, .none },
		138
		139	.{ .cld, .z, &.{}, &.{ 0xfc }, 0, .none, .none },
134		140
135	.{ .clflush, .m, &.{ .m8 }, &.{ 0x0f, 0xae }, 7, .none, .none },	141	.{ .clflush, .m, &.{ .m8 }, &.{ 0x0f, 0xae }, 7, .none, .none },
136		142
137	.{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },	143	.{ .cli, .z, &.{}, &.{ 0xfa }, 0, .none, .none },
138	.{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },	144
139	.{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },	145	.{ .clts, .z, &.{}, &.{ 0x0f, 0x06 }, 0, .none, .none },
140	.{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },	146
141	.{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },	147	.{ .clui, .z, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xee }, 0, .none, .uintr },
142	.{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },	148
143	.{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },	149	.{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .cmov },
144	.{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },	150	.{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .cmov },
145	.{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },	151	.{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .cmov },
146	.{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },	152	.{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
147	.{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },	153	.{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
148	.{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },	154	.{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
149	.{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },	155	.{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
150	.{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },	156	.{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
151	.{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },	157	.{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
152	.{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },	158	.{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .cmov },
153	.{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },	159	.{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .cmov },
154	.{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },	160	.{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .cmov },
155	.{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },	161	.{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
156	.{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },	162	.{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
157	.{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },	163	.{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
158	.{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },	164	.{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .cmov },
159	.{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },	165	.{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .cmov },
160	.{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },	166	.{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .cmov },
161	.{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },	167	.{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .cmov },
162	.{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },	168	.{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .cmov },
163	.{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },	169	.{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .cmov },
164	.{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },	170	.{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .cmov },
165	.{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },	171	.{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .cmov },
166	.{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },	172	.{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .cmov },
167	.{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },	173	.{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .cmov },
168	.{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },	174	.{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .cmov },
169	.{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },	175	.{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .cmov },
170	.{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },	176	.{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .cmov },
171	.{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },	177	.{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .cmov },
172	.{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },	178	.{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .cmov },
173	.{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },	179	.{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .cmov },
174	.{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },	180	.{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .cmov },
175	.{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },	181	.{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .cmov },
176	.{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },	182	.{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
177	.{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },	183	.{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
178	.{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },	184	.{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
179	.{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },	185	.{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
180	.{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },	186	.{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
181	.{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },	187	.{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
182	.{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },	188	.{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .cmov },
183	.{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },	189	.{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .cmov },
184	.{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },	190	.{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .cmov },
185	.{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },	191	.{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
186	.{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },	192	.{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
187	.{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },	193	.{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
188	.{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },	194	.{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .cmov },
189	.{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },	195	.{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .cmov },
190	.{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },	196	.{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .cmov },
191	.{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },	197	.{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .cmov },
192	.{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },	198	.{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .cmov },
193	.{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },	199	.{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .cmov },
194	.{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },	200	.{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .cmov },
195	.{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },	201	.{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .cmov },
196	.{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },	202	.{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .cmov },
197	.{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .none },	203	.{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .cmov },
198	.{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .none },	204	.{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .cmov },
199	.{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .none },	205	.{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .cmov },
200	.{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },	206	.{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .cmov },
201	.{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },	207	.{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .cmov },
202	.{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },	208	.{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .cmov },
203	.{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .none },	209	.{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .cmov },
204	.{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .none },	210	.{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .cmov },
205	.{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .none },	211	.{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .cmov },
206	.{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },	212	.{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .cmov },
207	.{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },	213	.{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .cmov },
208	.{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },	214	.{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .cmov },
209	.{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .none },	215	.{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .cmov },
210	.{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .none },	216	.{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .cmov },
211	.{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .none },	217	.{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .cmov },
212	.{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },	218	.{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .cmov },
213	.{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },	219	.{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .cmov },
214	.{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },	220	.{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .cmov },
215	.{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },	221	.{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .cmov },
216	.{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },	222	.{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .cmov },
217	.{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },	223	.{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .cmov },
218	.{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },	224	.{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .cmov },
219	.{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },	225	.{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .cmov },
220	.{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },	226	.{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .cmov },
221	.{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .none },	227	.{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .cmov },
222	.{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .none },	228	.{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .cmov },
223	.{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .none },	229	.{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .cmov },
224	.{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },	230	.{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .cmov },
225	.{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },	231	.{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .cmov },
226	.{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },	232	.{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .cmov },
		233	.{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .cmov },
		234	.{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .cmov },
		235	.{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .cmov },
		236	.{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .cmov },
		237	.{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .cmov },
		238	.{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .cmov },
227		239
228	.{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none, .none },	240	.{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none, .none },
229	.{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .short, .none },	241	.{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .short, .none },
...	@@ -248,15 +260,15 @@ pub const table = [_]Entry{	...	@@ -248,15 +260,15 @@ pub const table = [_]Entry{
248	.{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none, .none },	260	.{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none, .none },
249	.{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long, .none },	261	.{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long, .none },
250		262
251	.{ .cmps, .zo, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },	263	.{ .cmps, .z, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },
252	.{ .cmps, .zo, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },	264	.{ .cmps, .z, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },
253	.{ .cmps, .zo, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },	265	.{ .cmps, .z, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },
254	.{ .cmps, .zo, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },	266	.{ .cmps, .z, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },
255		267
256	.{ .cmpsb, .zo, &.{}, &.{ 0xa6 }, 0, .none, .none },	268	.{ .cmpsb, .z, &.{}, &.{ 0xa6 }, 0, .none, .none },
257	.{ .cmpsw, .zo, &.{}, &.{ 0xa7 }, 0, .short, .none },	269	.{ .cmpsw, .z, &.{}, &.{ 0xa7 }, 0, .short, .none },
258	.{ .cmpsd, .zo, &.{}, &.{ 0xa7 }, 0, .none, .none },	270	.{ .cmpsd, .z, &.{}, &.{ 0xa7 }, 0, .none, .none },
259	.{ .cmpsq, .zo, &.{}, &.{ 0xa7 }, 0, .long, .none },	271	.{ .cmpsq, .z, &.{}, &.{ 0xa7 }, 0, .long, .none },
260		272
261	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none, .none },	273	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none, .none },
262	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex, .none },	274	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex, .none },
...	@@ -267,7 +279,7 @@ pub const table = [_]Entry{	...	@@ -267,7 +279,7 @@ pub const table = [_]Entry{
267	.{ .cmpxchg8b, .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none, .none },	279	.{ .cmpxchg8b, .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none, .none },
268	.{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },	280	.{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },
269		281
270	.{ .cpuid, .zo, &.{}, &.{ 0x0f, 0xa2 }, 0, .none, .none },	282	.{ .cpuid, .z, &.{}, &.{ 0x0f, 0xa2 }, 0, .none, .none },
271		283
272	.{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .none, .none },	284	.{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .none, .none },
273	.{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .rex, .none },	285	.{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .rex, .none },
...	@@ -308,7 +320,7 @@ pub const table = [_]Entry{	...	@@ -308,7 +320,7 @@ pub const table = [_]Entry{
308	.{ .inc, .m, &.{ .rm32 }, &.{ 0xff }, 0, .none, .none },	320	.{ .inc, .m, &.{ .rm32 }, &.{ 0xff }, 0, .none, .none },
309	.{ .inc, .m, &.{ .rm64 }, &.{ 0xff }, 0, .long, .none },	321	.{ .inc, .m, &.{ .rm64 }, &.{ 0xff }, 0, .long, .none },
310		322
311	.{ .int3, .zo, &.{}, &.{ 0xcc }, 0, .none, .none },	323	.{ .int3, .z, &.{}, &.{ 0xcc }, 0, .none, .none },
312		324
313	.{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },	325	.{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
314	.{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },	326	.{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
...	@@ -349,23 +361,23 @@ pub const table = [_]Entry{	...	@@ -349,23 +361,23 @@ pub const table = [_]Entry{
349	.{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },	361	.{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },
350	.{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },	362	.{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },
351		363
352	.{ .lfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },	364	.{ .lfence, .z, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
353		365
354	.{ .lods, .zo, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },	366	.{ .lods, .z, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },
355	.{ .lods, .zo, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },	367	.{ .lods, .z, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
356	.{ .lods, .zo, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },	368	.{ .lods, .z, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
357	.{ .lods, .zo, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },	369	.{ .lods, .z, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
358		370
359	.{ .lodsb, .zo, &.{}, &.{ 0xac }, 0, .none, .none },	371	.{ .lodsb, .z, &.{}, &.{ 0xac }, 0, .none, .none },
360	.{ .lodsw, .zo, &.{}, &.{ 0xad }, 0, .short, .none },	372	.{ .lodsw, .z, &.{}, &.{ 0xad }, 0, .short, .none },
361	.{ .lodsd, .zo, &.{}, &.{ 0xad }, 0, .none, .none },	373	.{ .lodsd, .z, &.{}, &.{ 0xad }, 0, .none, .none },
362	.{ .lodsq, .zo, &.{}, &.{ 0xad }, 0, .long, .none },	374	.{ .lodsq, .z, &.{}, &.{ 0xad }, 0, .long, .none },
363		375
364	.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt },	376	.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt },
365	.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt },	377	.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt },
366	.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt },	378	.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt },
367		379
368	.{ .mfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },	380	.{ .mfence, .z, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
369		381
370	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none, .none },	382	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none, .none },
371	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex, .none },	383	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex, .none },
...	@@ -409,15 +421,15 @@ pub const table = [_]Entry{	...	@@ -409,15 +421,15 @@ pub const table = [_]Entry{
409	.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .movbe },	421	.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .movbe },
410	.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .movbe },	422	.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .movbe },
411		423
412	.{ .movs, .zo, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },	424	.{ .movs, .z, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
413	.{ .movs, .zo, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },	425	.{ .movs, .z, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
414	.{ .movs, .zo, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },	426	.{ .movs, .z, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
415	.{ .movs, .zo, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },	427	.{ .movs, .z, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
416		428
417	.{ .movsb, .zo, &.{}, &.{ 0xa4 }, 0, .none, .none },	429	.{ .movsb, .z, &.{}, &.{ 0xa4 }, 0, .none, .none },
418	.{ .movsw, .zo, &.{}, &.{ 0xa5 }, 0, .short, .none },	430	.{ .movsw, .z, &.{}, &.{ 0xa5 }, 0, .short, .none },
419	.{ .movsd, .zo, &.{}, &.{ 0xa5 }, 0, .none, .none },	431	.{ .movsd, .z, &.{}, &.{ 0xa5 }, 0, .none, .none },
420	.{ .movsq, .zo, &.{}, &.{ 0xa5 }, 0, .long, .none },	432	.{ .movsq, .z, &.{}, &.{ 0xa5 }, 0, .long, .none },
421		433
422	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .short, .none },	434	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .short, .none },
423	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex_short, .none },	435	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex_short, .none },
...	@@ -453,7 +465,7 @@ pub const table = [_]Entry{	...	@@ -453,7 +465,7 @@ pub const table = [_]Entry{
453	.{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },	465	.{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },
454	.{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },	466	.{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },
455		467
456	.{ .nop, .zo, &.{}, &.{ 0x90 }, 0, .none, .none },	468	.{ .nop, .z, &.{}, &.{ 0x90 }, 0, .none, .none },
457		469
458	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none, .none },	470	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none, .none },
459	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex, .none },	471	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex, .none },
...	@@ -484,7 +496,7 @@ pub const table = [_]Entry{	...	@@ -484,7 +496,7 @@ pub const table = [_]Entry{
484	.{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none, .none },	496	.{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none, .none },
485	.{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long, .none },	497	.{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long, .none },
486		498
487	.{ .pause, .zo, &.{}, &.{ 0xf3, 0x90 }, 0, .none, .none },	499	.{ .pause, .z, &.{}, &.{ 0xf3, 0x90 }, 0, .none, .none },
488		500
489	.{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .short, .none },	501	.{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .short, .none },
490	.{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none, .none },	502	.{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none, .none },
...	@@ -495,7 +507,7 @@ pub const table = [_]Entry{	...	@@ -495,7 +507,7 @@ pub const table = [_]Entry{
495	.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt },	507	.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt },
496	.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt },	508	.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt },
497		509
498	.{ .popfq, .zo, &.{}, &.{ 0x9d }, 0, .none, .none },	510	.{ .popfq, .z, &.{}, &.{ 0x9d }, 0, .none, .none },
499		511
500	.{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .short, .none },	512	.{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .short, .none },
501	.{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none, .none },	513	.{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none, .none },
...	@@ -505,9 +517,9 @@ pub const table = [_]Entry{	...	@@ -505,9 +517,9 @@ pub const table = [_]Entry{
505	.{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .short, .none },	517	.{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .short, .none },
506	.{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },	518	.{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },
507		519
508	.{ .pushfq, .zo, &.{}, &.{ 0x9c }, 0, .none, .none },	520	.{ .pushfq, .z, &.{}, &.{ 0x9c }, 0, .none, .none },
509		521
510	.{ .ret, .zo, &.{}, &.{ 0xc3 }, 0, .none, .none },	522	.{ .ret, .z, &.{}, &.{ 0xc3 }, 0, .none, .none },
511		523
512	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none, .none },	524	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none, .none },
513	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex, .none },	525	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex, .none },
...	@@ -628,15 +640,15 @@ pub const table = [_]Entry{	...	@@ -628,15 +640,15 @@ pub const table = [_]Entry{
628	.{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none, .none },	640	.{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none, .none },
629	.{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long, .none },	641	.{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long, .none },
630		642
631	.{ .scas, .zo, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },	643	.{ .scas, .z, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },
632	.{ .scas, .zo, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },	644	.{ .scas, .z, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
633	.{ .scas, .zo, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },	645	.{ .scas, .z, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
634	.{ .scas, .zo, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },	646	.{ .scas, .z, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
635		647
636	.{ .scasb, .zo, &.{}, &.{ 0xae }, 0, .none, .none },	648	.{ .scasb, .z, &.{}, &.{ 0xae }, 0, .none, .none },
637	.{ .scasw, .zo, &.{}, &.{ 0xaf }, 0, .short, .none },	649	.{ .scasw, .z, &.{}, &.{ 0xaf }, 0, .short, .none },
638	.{ .scasd, .zo, &.{}, &.{ 0xaf }, 0, .none, .none },	650	.{ .scasd, .z, &.{}, &.{ 0xaf }, 0, .none, .none },
639	.{ .scasq, .zo, &.{}, &.{ 0xaf }, 0, .long, .none },	651	.{ .scasq, .z, &.{}, &.{ 0xaf }, 0, .long, .none },
640		652
641	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },	653	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
642	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },	654	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },
...	@@ -699,7 +711,7 @@ pub const table = [_]Entry{	...	@@ -699,7 +711,7 @@ pub const table = [_]Entry{
699	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },	711	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
700	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },	712	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },
701		713
702	.{ .sfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },	714	.{ .sfence, .z, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
703		715
704	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },	716	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },
705	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },	717	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },
...	@@ -747,15 +759,25 @@ pub const table = [_]Entry{	...	@@ -747,15 +759,25 @@ pub const table = [_]Entry{
747	.{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none, .none },	759	.{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none, .none },
748	.{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long, .none },	760	.{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long, .none },
749		761
750	.{ .stos, .zo, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },	762	.{ .stac, .z, &.{}, &.{ 0x0f, 0x01, 0xcb }, 0, .none, .smap },
751	.{ .stos, .zo, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },	763
752	.{ .stos, .zo, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },	764	.{ .stc, .z, &.{}, &.{ 0xf9 }, 0, .none, .none },
753	.{ .stos, .zo, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },	765
		766	.{ .std, .z, &.{}, &.{ 0xfd }, 0, .none, .none },
		767
		768	.{ .sti, .z, &.{}, &.{ 0xfb }, 0, .none, .none },
754		769
755	.{ .stosb, .zo, &.{}, &.{ 0xaa }, 0, .none, .none },	770	.{ .stui, .z, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xef }, 0, .none, .uintr },
756	.{ .stosw, .zo, &.{}, &.{ 0xab }, 0, .short, .none },	771
757	.{ .stosd, .zo, &.{}, &.{ 0xab }, 0, .none, .none },	772	.{ .stos, .z, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
758	.{ .stosq, .zo, &.{}, &.{ 0xab }, 0, .long, .none },	773	.{ .stos, .z, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
		774	.{ .stos, .z, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
		775	.{ .stos, .z, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
		776
		777	.{ .stosb, .z, &.{}, &.{ 0xaa }, 0, .none, .none },
		778	.{ .stosw, .z, &.{}, &.{ 0xab }, 0, .short, .none },
		779	.{ .stosd, .z, &.{}, &.{ 0xab }, 0, .none, .none },
		780	.{ .stosq, .z, &.{}, &.{ 0xab }, 0, .long, .none },
759		781
760	.{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none, .none },	782	.{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none, .none },
761	.{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .short, .none },	783	.{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .short, .none },
...	@@ -780,7 +802,7 @@ pub const table = [_]Entry{	...	@@ -780,7 +802,7 @@ pub const table = [_]Entry{
780	.{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none, .none },	802	.{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none, .none },
781	.{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long, .none },	803	.{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long, .none },
782		804
783	.{ .syscall, .zo, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },	805	.{ .syscall, .z, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
784		806
785	.{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none, .none },	807	.{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none, .none },
786	.{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .short, .none },	808	.{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .short, .none },
...	@@ -801,7 +823,7 @@ pub const table = [_]Entry{	...	@@ -801,7 +823,7 @@ pub const table = [_]Entry{
801	.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi },	823	.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi },
802	.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi },	824	.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi },
803		825
804	.{ .ud2, .zo, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },	826	.{ .ud2, .z, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
805		827
806	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none, .none },	828	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none, .none },
807	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex, .none },	829	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex, .none },
...	@@ -809,12 +831,12 @@ pub const table = [_]Entry{	...	@@ -809,12 +831,12 @@ pub const table = [_]Entry{
809	.{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },	831	.{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
810	.{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },	832	.{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },
811		833
812	.{ .xchg, .o, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },	834	.{ .xchg, .zo, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },
813	.{ .xchg, .o, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },	835	.{ .xchg, .oz, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },
814	.{ .xchg, .o, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },	836	.{ .xchg, .zo, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },
815	.{ .xchg, .o, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },	837	.{ .xchg, .zo, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },
816	.{ .xchg, .o, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },	838	.{ .xchg, .oz, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },
817	.{ .xchg, .o, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },	839	.{ .xchg, .oz, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },
818	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none, .none },	840	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none, .none },
819	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex, .none },	841	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex, .none },
820	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none, .none },	842	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none, .none },
...	@@ -826,7 +848,7 @@ pub const table = [_]Entry{	...	@@ -826,7 +848,7 @@ pub const table = [_]Entry{
826	.{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none, .none },	848	.{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none, .none },
827	.{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long, .none },	849	.{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long, .none },
828		850
829	.{ .xgetbv, .zo, &.{}, &.{ 0x0f, 0x01, 0xd0 }, 0, .none, .none },	851	.{ .xgetbv, .z, &.{}, &.{ 0x0f, 0x01, 0xd0 }, 0, .none, .none },
830		852
831	.{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none, .none },	853	.{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none, .none },
832	.{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .short, .none },	854	.{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .short, .none },
...	@@ -852,9 +874,9 @@ pub const table = [_]Entry{	...	@@ -852,9 +874,9 @@ pub const table = [_]Entry{
852	.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },	874	.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
853		875
854	// X87	876	// X87
855	.{ .fabs, .zo, &.{}, &.{ 0xd9, 0xe1 }, 0, .none, .x87 },	877	.{ .fabs, .z, &.{}, &.{ 0xd9, 0xe1 }, 0, .none, .x87 },
856		878
857	.{ .fchs, .zo, &.{}, &.{ 0xd9, 0xe0 }, 0, .none, .x87 },	879	.{ .fchs, .z, &.{}, &.{ 0xd9, 0xe0 }, 0, .none, .x87 },
858		880
859	.{ .ffree, .o, &.{ .st }, &.{ 0xdd, 0xc0 }, 0, .none, .x87 },	881	.{ .ffree, .o, &.{ .st }, &.{ 0xdd, 0xc0 }, 0, .none, .x87 },
860		882
...	@@ -927,8 +949,14 @@ pub const table = [_]Entry{	...	@@ -927,8 +949,14 @@ pub const table = [_]Entry{
927		949
928	.{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },	950	.{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
929		951
		952	.{ .movhps, .rm, &.{ .xmm, .m64 }, &.{ 0x0f, 0x16 }, 0, .none, .sse },
		953	.{ .movhps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x17 }, 0, .none, .sse },
		954
930	.{ .movlhps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .none, .sse },	955	.{ .movlhps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .none, .sse },
931		956
		957	.{ .movlps, .rm, &.{ .xmm, .m64 }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
		958	.{ .movlps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x13 }, 0, .none, .sse },
		959
932	.{ .movmskps, .rm, &.{ .r32, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },	960	.{ .movmskps, .rm, &.{ .r32, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },
933	.{ .movmskps, .rm, &.{ .r64, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },	961	.{ .movmskps, .rm, &.{ .r64, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },
934		962
...	@@ -1037,6 +1065,12 @@ pub const table = [_]Entry{	...	@@ -1037,6 +1065,12 @@ pub const table = [_]Entry{
1037	.{ .movdqu, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x6f }, 0, .none, .sse2 },	1065	.{ .movdqu, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x6f }, 0, .none, .sse2 },
1038	.{ .movdqu, .mr, &.{ .xmm_m128, .xmm }, &.{ 0xf3, 0x0f, 0x7f }, 0, .none, .sse2 },	1066	.{ .movdqu, .mr, &.{ .xmm_m128, .xmm }, &.{ 0xf3, 0x0f, 0x7f }, 0, .none, .sse2 },
1039		1067
		1068	.{ .movhpd, .rm, &.{ .xmm, .m64 }, &.{ 0x66, 0x0f, 0x16 }, 0, .none, .sse2 },
		1069	.{ .movhpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x17 }, 0, .none, .sse2 },
		1070
		1071	.{ .movlpd, .rm, &.{ .xmm, .m64 }, &.{ 0x66, 0x0f, 0x12 }, 0, .none, .sse2 },
		1072	.{ .movlpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x13 }, 0, .none, .sse2 },
		1073
1040	.{ .movmskpd, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },	1074	.{ .movmskpd, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },
1041	.{ .movmskpd, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },	1075	.{ .movmskpd, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },
1042		1076
...	@@ -1251,6 +1285,8 @@ pub const table = [_]Entry{	...	@@ -1251,6 +1285,8 @@ pub const table = [_]Entry{
1251		1285
1252	.{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },	1286	.{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
1253		1287
		1288	.{ .ptest, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x17 }, 0, .none, .sse4_1 },
		1289
1254	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },	1290	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
1255		1291
1256	.{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },	1292	.{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
...	@@ -1287,6 +1323,16 @@ pub const table = [_]Entry{	...	@@ -1287,6 +1323,16 @@ pub const table = [_]Entry{
1287	.{ .sha256rnds2, .rm0, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x0f, 0x38, 0xcb }, 0, .none, .sha },	1323	.{ .sha256rnds2, .rm0, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x0f, 0x38, 0xcb }, 0, .none, .sha },
1288		1324
1289	// AVX	1325	// AVX
		1326	.{ .rorx, .rmi, &.{ .r32, .rm32, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w0, .bmi2 },
		1327	.{ .rorx, .rmi, &.{ .r64, .rm64, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w1, .bmi2 },
		1328
		1329	.{ .sarx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 },
		1330	.{ .shlx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 },
		1331	.{ .shrx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 },
		1332	.{ .sarx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 },
		1333	.{ .shlx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 },
		1334	.{ .shrx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 },
		1335
1290	.{ .vaddpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx },	1336	.{ .vaddpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
1291	.{ .vaddpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx },	1337	.{ .vaddpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
1292		1338
...	@@ -1474,8 +1520,20 @@ pub const table = [_]Entry{	...	@@ -1474,8 +1520,20 @@ pub const table = [_]Entry{
1474		1520
1475	.{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },	1521	.{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1476		1522
		1523	.{ .vmovhpd, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x66, 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
		1524	.{ .vmovhpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x17 }, 0, .vex_128_wig, .avx },
		1525
		1526	.{ .vmovhps, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
		1527	.{ .vmovhps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x17 }, 0, .vex_128_wig, .avx },
		1528
1477	.{ .vmovlhps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },	1529	.{ .vmovlhps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
1478		1530
		1531	.{ .vmovlpd, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x66, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
		1532	.{ .vmovlpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x13 }, 0, .vex_128_wig, .avx },
		1533
		1534	.{ .vmovlps, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
		1535	.{ .vmovlps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x13 }, 0, .vex_128_wig, .avx },
		1536
1479	.{ .vmovq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .vex_128_wig, .avx },	1537	.{ .vmovq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .vex_128_wig, .avx },
1480	.{ .vmovq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .vex_128_wig, .avx },	1538	.{ .vmovq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .vex_128_wig, .avx },
1481		1539
...	@@ -1571,14 +1629,14 @@ pub const table = [_]Entry{	...	@@ -1571,14 +1629,14 @@ pub const table = [_]Entry{
1571	.{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },	1629	.{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
1572	.{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },	1630	.{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
1573		1631
1574	.{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx },	1632	.{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .vex_128_w0, .avx },
1575	.{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },	1633	.{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_w0, .avx },
1576		1634
1577	.{ .vpinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },	1635	.{ .vpinsrb, .rvmi, &.{ .xmm, .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
1578	.{ .vpinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },	1636	.{ .vpinsrd, .rvmi, &.{ .xmm, .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
1579	.{ .vpinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },	1637	.{ .vpinsrq, .rvmi, &.{ .xmm, .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
1580		1638
1581	.{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },	1639	.{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_w0, .avx },
1582		1640
1583	.{ .vpmaxsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_128_wig, .avx },	1641	.{ .vpmaxsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_128_wig, .avx },
1584	.{ .vpmaxsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xee }, 0, .vex_128_wig, .avx },	1642	.{ .vpmaxsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xee }, 0, .vex_128_wig, .avx },
...	@@ -1666,6 +1724,9 @@ pub const table = [_]Entry{	...	@@ -1666,6 +1724,9 @@ pub const table = [_]Entry{
1666	.{ .vpsubusb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd8 }, 0, .vex_128_wig, .avx },	1724	.{ .vpsubusb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd8 }, 0, .vex_128_wig, .avx },
1667	.{ .vpsubusw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd9 }, 0, .vex_128_wig, .avx },	1725	.{ .vpsubusw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd9 }, 0, .vex_128_wig, .avx },
1668		1726
		1727	.{ .vptest, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_128_wig, .avx },
		1728	.{ .vptest, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_256_wig, .avx },
		1729
1669	.{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx },	1730	.{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx },
1670	.{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx },	1731	.{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx },
1671	.{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx },	1732	.{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx },
...	@@ -1716,6 +1777,11 @@ pub const table = [_]Entry{	...	@@ -1716,6 +1777,11 @@ pub const table = [_]Entry{
1716		1777
1717	.{ .vsubss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },	1778	.{ .vsubss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
1718		1779
		1780	.{ .vtestps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_128_w0, .avx },
		1781	.{ .vtestps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_256_w0, .avx },
		1782	.{ .vtestpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_128_w0, .avx },
		1783	.{ .vtestpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_256_w0, .avx },
		1784
1719	.{ .vxorpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_128_wig, .avx },	1785	.{ .vxorpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_128_wig, .avx },
1720	.{ .vxorpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_256_wig, .avx },	1786	.{ .vxorpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_256_wig, .avx },
1721		1787

src/codegen/c/Type.zig+5-5

src/codegen/llvm.zig+1-1

...	@@ -581,7 +581,7 @@ const DataLayoutBuilder = struct {	...	@@ -581,7 +581,7 @@ const DataLayoutBuilder = struct {
581	switch (kind) {	581	switch (kind) {
582	.integer => {	582	.integer => {
583	if (self.target.ptrBitWidth() <= 16 and size >= 128) return;	583	if (self.target.ptrBitWidth() <= 16 and size >= 128) return;
584	abi = @min(abi, Type.maxIntAlignment(self.target, true) * 8);	584	abi = @min(abi, Type.maxIntAlignment(self.target) * 8);
585	switch (self.target.cpu.arch) {	585	switch (self.target.cpu.arch) {
586	.aarch64,	586	.aarch64,
587	.aarch64_be,	587	.aarch64_be,

src/dev.zig+1

...	@@ -135,6 +135,7 @@ pub const Env = enum {	...	@@ -135,6 +135,7 @@ pub const Env = enum {
135	else => Env.ast_gen.supports(feature),	135	else => Env.ast_gen.supports(feature),
136	},	136	},
137	.@"x86_64-linux" => switch (feature) {	137	.@"x86_64-linux" => switch (feature) {
		138	.build_command,
138	.stdio_listen,	139	.stdio_listen,
139	.incremental,	140	.incremental,
140	.x86_64_backend,	141	.x86_64_backend,

src/link/C.zig+1-1

...	@@ -396,7 +396,7 @@ fn abiDefines(self: *C, target: std.Target) !std.ArrayList(u8) {	...	@@ -396,7 +396,7 @@ fn abiDefines(self: *C, target: std.Target) !std.ArrayList(u8) {
396	else => {},	396	else => {},
397	}	397	}
398	try writer.print("#define ZIG_TARGET_MAX_INT_ALIGNMENT {d}\n", .{	398	try writer.print("#define ZIG_TARGET_MAX_INT_ALIGNMENT {d}\n", .{
399	Type.maxIntAlignment(target, false),	399	Type.maxIntAlignment(target),
400	});	400	});
401	return defines;	401	return defines;
402	}	402	}

src/link/Elf/Atom.zig+35-32

 @@ -976,6 +976,7 @@ const x86_64 = struct {
         it: *RelocsIterator,
     ) !void {
         dev.check(.x86_64_backend);
+        const t = &elf_file.base.comp.root_mod.resolved_target.result;
         const is_static = elf_file.base.isStatic();
         const is_dyn_lib = elf_file.isEffectivelyDynLib();
 @@ -1046,7 +1047,7 @@ const x86_64 = struct {
             .GOTTPOFF => {
                 const should_relax = blk: {
                     if (is_dyn_lib or symbol.flags.import) break :blk false;
-                    if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..])) break :blk false;
+                    if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..], t)) break :blk false;
                     break :blk true;
                 };
                 if (!should_relax) {
 @@ -1090,6 +1091,7 @@ const x86_64 = struct {
         stream: anytype,
     ) (error{ InvalidInstruction, CannotEncode } || RelocError)!void {
         dev.check(.x86_64_backend);
+        const t = &elf_file.base.comp.root_mod.resolved_target.result;
         const diags = &elf_file.base.comp.link_diags;
         const r_type: elf.R_X86_64 = @enumFromInt(rel.r_type());
         const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow;
 @@ -1120,7 +1122,7 @@ const x86_64 = struct {
             .GOTPCRELX => {
                 if (!target.flags.import and !target.isIFunc(elf_file) and !target.isAbs(elf_file)) blk: {
-                    x86_64.relaxGotpcrelx(code[r_offset - 2 ..]) catch break :blk;
+                    x86_64.relaxGotpcrelx(code[r_offset - 2 ..], t) catch break :blk;
                     try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little);
                     return;
                 }
 @@ -1129,7 +1131,7 @@ const x86_64 = struct {
             .REX_GOTPCRELX => {
                 if (!target.flags.import and !target.isIFunc(elf_file) and !target.isAbs(elf_file)) blk: {
-                    x86_64.relaxRexGotpcrelx(code[r_offset - 3 ..]) catch break :blk;
+                    x86_64.relaxRexGotpcrelx(code[r_offset - 3 ..], t) catch break :blk;
                     try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little);
                     return;
                 }
 @@ -1184,7 +1186,7 @@ const x86_64 = struct {
                     const S_ = target.tlsDescAddress(elf_file);
                     try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
                 } else {
-                    x86_64.relaxGotPcTlsDesc(code[r_offset - 3 ..]) catch {
+                    x86_64.relaxGotPcTlsDesc(code[r_offset - 3 ..], t) catch {
                         var err = try diags.addErrorWithNotes(1);
                         try err.addMsg("could not relax {s}", .{@tagName(r_type)});
                         err.addNote("in {}:{s} at offset 0x{x}", .{
 @@ -1208,7 +1210,7 @@ const x86_64 = struct {
                     const S_ = target.gotTpAddress(elf_file);
                     try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
                 } else {
-                    x86_64.relaxGotTpOff(code[r_offset - 3 ..]);
+                    x86_64.relaxGotTpOff(code[r_offset - 3 ..], t);
                     try cwriter.writeInt(i32, @as(i32, @intCast(S - TP)), .little);
                 }
             },
 @@ -1269,31 +1271,31 @@ const x86_64 = struct {
         }
     }
-    fn relaxGotpcrelx(code: []u8) !void {
+    fn relaxGotpcrelx(code: []u8, t: *const std.Target) !void {
         dev.check(.x86_64_backend);
         const old_inst = disassemble(code) orelse return error.RelaxFailure;
-        const inst = switch (old_inst.encoding.mnemonic) {
+        const inst: Instruction = switch (old_inst.encoding.mnemonic) {
-            .call => try Instruction.new(old_inst.prefix, .call, &.{
+            .call => try .new(old_inst.prefix, .call, &.{
                 // TODO: hack to force imm32s in the assembler
-                .{ .imm = Immediate.s(-129) },
+                .{ .imm = .s(-129) },
-            }),
+            }, t),
-            .jmp => try Instruction.new(old_inst.prefix, .jmp, &.{
+            .jmp => try .new(old_inst.prefix, .jmp, &.{
                 // TODO: hack to force imm32s in the assembler
-                .{ .imm = Immediate.s(-129) },
+                .{ .imm = .s(-129) },
-            }),
+            }, t),
             else => return error.RelaxFailure,
         };
         relocs_log.debug("    relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
-        const nop = try Instruction.new(.none, .nop, &.{});
+        const nop: Instruction = try .new(.none, .nop, &.{}, t);
         try encode(&.{ nop, inst }, code);
     }
-    fn relaxRexGotpcrelx(code: []u8) !void {
+    fn relaxRexGotpcrelx(code: []u8, t: *const std.Target) !void {
         dev.check(.x86_64_backend);
         const old_inst = disassemble(code) orelse return error.RelaxFailure;
         switch (old_inst.encoding.mnemonic) {
             .mov => {
-                const inst = try Instruction.new(old_inst.prefix, .lea, &old_inst.ops);
+                const inst: Instruction = try .new(old_inst.prefix, .lea, &old_inst.ops, t);
                 relocs_log.debug("    relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
                 try encode(&.{inst}, code);
             },
 @@ -1398,23 +1400,24 @@ const x86_64 = struct {
         }
     }
-    fn canRelaxGotTpOff(code: []const u8) bool {
+    fn canRelaxGotTpOff(code: []const u8, t: *const std.Target) bool {
         dev.check(.x86_64_backend);
         const old_inst = disassemble(code) orelse return false;
         switch (old_inst.encoding.mnemonic) {
-            .mov => if (Instruction.new(old_inst.prefix, .mov, &.{
+            .mov => {
-                old_inst.ops[0],
+                const inst = Instruction.new(old_inst.prefix, .mov, &.{
-                // TODO: hack to force imm32s in the assembler
+                    old_inst.ops[0],
-                .{ .imm = Immediate.s(-129) },
+                    // TODO: hack to force imm32s in the assembler
-            })) |inst| {
+                    .{ .imm = .s(-129) },
+                }, t) catch return false;
                 inst.encode(std.io.null_writer, .{}) catch return false;
                 return true;
-            } else |_| return false,
+            },
             else => return false,
         }
     }
-    fn relaxGotTpOff(code: []u8) void {
+    fn relaxGotTpOff(code: []u8, t: *const std.Target) void {
         dev.check(.x86_64_backend);
         const old_inst = disassemble(code) orelse unreachable;
         switch (old_inst.encoding.mnemonic) {
 @@ -1422,8 +1425,8 @@ const x86_64 = struct {
                 const inst = Instruction.new(old_inst.prefix, .mov, &.{
                     old_inst.ops[0],
                     // TODO: hack to force imm32s in the assembler
-                    .{ .imm = Immediate.s(-129) },
+                    .{ .imm = .s(-129) },
-                }) catch unreachable;
+                }, t) catch unreachable;
                 relocs_log.debug("    relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
                 encode(&.{inst}, code) catch unreachable;
             },
 @@ -1431,16 +1434,16 @@ const x86_64 = struct {
         }
     }
-    fn relaxGotPcTlsDesc(code: []u8) !void {
+    fn relaxGotPcTlsDesc(code: []u8, target: *const std.Target) !void {
         dev.check(.x86_64_backend);
         const old_inst = disassemble(code) orelse return error.RelaxFailure;
         switch (old_inst.encoding.mnemonic) {
             .lea => {
-                const inst = try Instruction.new(old_inst.prefix, .mov, &.{
+                const inst: Instruction = try .new(old_inst.prefix, .mov, &.{
                     old_inst.ops[0],
                     // TODO: hack to force imm32s in the assembler
-                    .{ .imm = Immediate.s(-129) },
+                    .{ .imm = .s(-129) },
-                });
+                }, target);
                 relocs_log.debug("    relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
                 try encode(&.{inst}, code);
             },
 @@ -1779,7 +1782,7 @@ const aarch64 = struct {
                     const off: u12 = @truncate(@as(u64, @bitCast(S_ + A)));
                     aarch64_util.writeAddImmInst(off, code);
                 } else {
-                    const old_inst = Instruction{
+                    const old_inst: Instruction = .{
                         .add_subtract_immediate = mem.bytesToValue(std.meta.TagPayload(
                             Instruction,
                             Instruction.add_subtract_immediate,
 @@ -1793,7 +1796,7 @@ const aarch64 = struct {
             },
             .TLSDESC_CALL => if (!target.flags.has_tlsdesc) {
-                const old_inst = Instruction{
+                const old_inst: Instruction = .{
                     .unconditional_branch_register = mem.bytesToValue(std.meta.TagPayload(
                         Instruction,
                         Instruction.unconditional_branch_register,

src/link/MachO.zig+1-1

...	@@ -3548,7 +3548,7 @@ pub fn getTarget(self: MachO) std.Target {	...	@@ -3548,7 +3548,7 @@ pub fn getTarget(self: MachO) std.Target {
3548	pub fn invalidateKernelCache(dir: fs.Dir, sub_path: []const u8) !void {	3548	pub fn invalidateKernelCache(dir: fs.Dir, sub_path: []const u8) !void {
3549	const tracy = trace(@src());	3549	const tracy = trace(@src());
3550	defer tracy.end();	3550	defer tracy.end();
3551	if (comptime builtin.target.isDarwin() and builtin.target.cpu.arch == .aarch64) {	3551	if (builtin.target.isDarwin() and builtin.target.cpu.arch == .aarch64) {
3552	try dir.copyFile(sub_path, dir, sub_path, .{});	3552	try dir.copyFile(sub_path, dir, sub_path, .{});
3553	}	3553	}
3554	}	3554	}

src/link/MachO/Atom.zig+7-5

 @@ -640,7 +640,8 @@ fn resolveRelocInner(
     macho_file: *MachO,
     writer: anytype,
 ) ResolveError!void {
-    const cpu_arch = macho_file.getTarget().cpu.arch;
+    const t = &macho_file.base.comp.root_mod.resolved_target.result;
+    const cpu_arch = t.cpu.arch;
     const rel_offset = math.cast(usize, rel.offset - self.off) orelse return error.Overflow;
     const P = @as(i64, @intCast(self.getAddress(macho_file))) + @as(i64, @intCast(rel_offset));
     const A = rel.addend + rel.getRelocAddend(cpu_arch);
 @@ -747,7 +748,7 @@ fn resolveRelocInner(
                 const S_: i64 = @intCast(sym.getTlvPtrAddress(macho_file));
                 try writer.writeInt(i32, @intCast(S_ + A - P), .little);
             } else {
-                try x86_64.relaxTlv(code[rel_offset - 3 ..]);
+                try x86_64.relaxTlv(code[rel_offset - 3 ..], t);
                 try writer.writeInt(i32, @intCast(S + A - P), .little);
             }
         },
 @@ -893,11 +894,12 @@ fn resolveRelocInner(
 const x86_64 = struct {
     fn relaxGotLoad(self: Atom, code: []u8, rel: Relocation, macho_file: *MachO) ResolveError!void {
         dev.check(.x86_64_backend);
+        const t = &macho_file.base.comp.root_mod.resolved_target.result;
         const diags = &macho_file.base.comp.link_diags;
         const old_inst = disassemble(code) orelse return error.RelaxFail;
         switch (old_inst.encoding.mnemonic) {
             .mov => {
-                const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops) catch return error.RelaxFail;
+                const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t) catch return error.RelaxFail;
                 relocs_log.debug("    relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
                 encode(&.{inst}, code) catch return error.RelaxFail;
             },
 @@ -916,12 +918,12 @@ const x86_64 = struct {
         }
     }
-    fn relaxTlv(code: []u8) error{RelaxFail}!void {
+    fn relaxTlv(code: []u8, t: *const std.Target) error{RelaxFail}!void {
         dev.check(.x86_64_backend);
         const old_inst = disassemble(code) orelse return error.RelaxFail;
         switch (old_inst.encoding.mnemonic) {
             .mov => {
-                const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops) catch return error.RelaxFail;
+                const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t) catch return error.RelaxFail;
                 relocs_log.debug("    relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
                 encode(&.{inst}, code) catch return error.RelaxFail;
             },

src/main.zig+8

...	@@ -39,6 +39,11 @@ test {	...	@@ -39,6 +39,11 @@ test {
39	_ = Package;	39	_ = Package;
40	}	40	}
41		41
		42	const thread_stack_size = switch (builtin.zig_backend) {
		43	else => std.Thread.SpawnConfig.default_stack_size,
		44	.stage2_x86_64 => 32 << 20,
		45	};
		46
42	pub const std_options: std.Options = .{	47	pub const std_options: std.Options = .{
43	.wasiCwd = wasi_cwd,	48	.wasiCwd = wasi_cwd,
44	.logFn = log,	49	.logFn = log,
...	@@ -3320,6 +3325,7 @@ fn buildOutputType(	...	@@ -3320,6 +3325,7 @@ fn buildOutputType(
3320	.allocator = gpa,	3325	.allocator = gpa,
3321	.n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),	3326	.n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
3322	.track_ids = true,	3327	.track_ids = true,
		3328	.stack_size = thread_stack_size,
3323	});	3329	});
3324	defer thread_pool.deinit();	3330	defer thread_pool.deinit();
3325		3331
...	@@ -5024,6 +5030,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {	...	@@ -5024,6 +5030,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5024	.allocator = gpa,	5030	.allocator = gpa,
5025	.n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),	5031	.n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
5026	.track_ids = true,	5032	.track_ids = true,
		5033	.stack_size = thread_stack_size,
5027	});	5034	});
5028	defer thread_pool.deinit();	5035	defer thread_pool.deinit();
5029		5036
...	@@ -5460,6 +5467,7 @@ fn jitCmd(	...	@@ -5460,6 +5467,7 @@ fn jitCmd(
5460	.allocator = gpa,	5467	.allocator = gpa,
5461	.n_jobs = @min(@max(std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),	5468	.n_jobs = @min(@max(std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
5462	.track_ids = true,	5469	.track_ids = true,
		5470	.stack_size = thread_stack_size,
5463	});	5471	});
5464	defer thread_pool.deinit();	5472	defer thread_pool.deinit();
5465		5473

src/print_air.zig+12-12

...	@@ -96,8 +96,8 @@ const Writer = struct {	...	@@ -96,8 +96,8 @@ const Writer = struct {
96	fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {	96	fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
97	const tag = w.air.instructions.items(.tag)[@intFromEnum(inst)];	97	const tag = w.air.instructions.items(.tag)[@intFromEnum(inst)];
98	try s.writeByteNTimes(' ', w.indent);	98	try s.writeByteNTimes(' ', w.indent);
99	try s.print("%{d}{c}= {s}(", .{	99	try s.print("{}{c}= {s}(", .{
100	@intFromEnum(inst),	100	inst,
101	@as(u8, if (if (w.liveness) \|liveness\| liveness.isUnused(inst) else false) '!' else ' '),	101	@as(u8, if (if (w.liveness) \|liveness\| liveness.isUnused(inst) else false) '!' else ' '),
102	@tagName(tag),	102	@tagName(tag),
103	});	103	});
...	@@ -409,7 +409,7 @@ const Writer = struct {	...	@@ -409,7 +409,7 @@ const Writer = struct {
409	try s.writeAll("}");	409	try s.writeAll("}");
410		410
411	for (liveness_block.deaths) \|operand\| {	411	for (liveness_block.deaths) \|operand\| {
412	try s.print(" %{d}!", .{@intFromEnum(operand)});	412	try s.print(" {}!", .{operand});
413	}	413	}
414	}	414	}
415		415
...	@@ -728,7 +728,7 @@ const Writer = struct {	...	@@ -728,7 +728,7 @@ const Writer = struct {
728	try s.writeByteNTimes(' ', w.indent);	728	try s.writeByteNTimes(' ', w.indent);
729	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {	729	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
730	if (i != 0) try s.writeAll(" ");	730	if (i != 0) try s.writeAll(" ");
731	try s.print("%{d}!", .{@intFromEnum(operand)});	731	try s.print("{}!", .{operand});
732	}	732	}
733	try s.writeAll("\n");	733	try s.writeAll("\n");
734	}	734	}
...	@@ -739,7 +739,7 @@ const Writer = struct {	...	@@ -739,7 +739,7 @@ const Writer = struct {
739	try s.writeAll("}");	739	try s.writeAll("}");
740		740
741	for (liveness_condbr.then_deaths) \|operand\| {	741	for (liveness_condbr.then_deaths) \|operand\| {
742	try s.print(" %{d}!", .{@intFromEnum(operand)});	742	try s.print(" {}!", .{operand});
743	}	743	}
744	}	744	}
745		745
...	@@ -765,7 +765,7 @@ const Writer = struct {	...	@@ -765,7 +765,7 @@ const Writer = struct {
765	try s.writeByteNTimes(' ', w.indent);	765	try s.writeByteNTimes(' ', w.indent);
766	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {	766	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
767	if (i != 0) try s.writeAll(" ");	767	if (i != 0) try s.writeAll(" ");
768	try s.print("%{d}!", .{@intFromEnum(operand)});	768	try s.print("{}!", .{operand});
769	}	769	}
770	try s.writeAll("\n");	770	try s.writeAll("\n");
771	}	771	}
...	@@ -776,7 +776,7 @@ const Writer = struct {	...	@@ -776,7 +776,7 @@ const Writer = struct {
776	try s.writeAll("}");	776	try s.writeAll("}");
777		777
778	for (liveness_condbr.then_deaths) \|operand\| {	778	for (liveness_condbr.then_deaths) \|operand\| {
779	try s.print(" %{d}!", .{@intFromEnum(operand)});	779	try s.print(" {}!", .{operand});
780	}	780	}
781	}	781	}
782		782
...	@@ -807,7 +807,7 @@ const Writer = struct {	...	@@ -807,7 +807,7 @@ const Writer = struct {
807	try s.writeByteNTimes(' ', w.indent);	807	try s.writeByteNTimes(' ', w.indent);
808	for (liveness_condbr.then_deaths, 0..) \|operand, i\| {	808	for (liveness_condbr.then_deaths, 0..) \|operand, i\| {
809	if (i != 0) try s.writeAll(" ");	809	if (i != 0) try s.writeAll(" ");
810	try s.print("%{d}!", .{@intFromEnum(operand)});	810	try s.print("{}!", .{operand});
811	}	811	}
812	try s.writeAll("\n");	812	try s.writeAll("\n");
813	}	813	}
...	@@ -827,7 +827,7 @@ const Writer = struct {	...	@@ -827,7 +827,7 @@ const Writer = struct {
827	try s.writeByteNTimes(' ', w.indent);	827	try s.writeByteNTimes(' ', w.indent);
828	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {	828	for (liveness_condbr.else_deaths, 0..) \|operand, i\| {
829	if (i != 0) try s.writeAll(" ");	829	if (i != 0) try s.writeAll(" ");
830	try s.print("%{d}!", .{@intFromEnum(operand)});	830	try s.print("{}!", .{operand});
831	}	831	}
832	try s.writeAll("\n");	832	try s.writeAll("\n");
833	}	833	}
...	@@ -884,7 +884,7 @@ const Writer = struct {	...	@@ -884,7 +884,7 @@ const Writer = struct {
884	try s.writeByteNTimes(' ', w.indent);	884	try s.writeByteNTimes(' ', w.indent);
885	for (deaths, 0..) \|operand, i\| {	885	for (deaths, 0..) \|operand, i\| {
886	if (i != 0) try s.writeAll(" ");	886	if (i != 0) try s.writeAll(" ");
887	try s.print("%{d}!", .{@intFromEnum(operand)});	887	try s.print("{}!", .{operand});
888	}	888	}
889	try s.writeAll("\n");	889	try s.writeAll("\n");
890	}	890	}
...	@@ -910,7 +910,7 @@ const Writer = struct {	...	@@ -910,7 +910,7 @@ const Writer = struct {
910	try s.writeByteNTimes(' ', w.indent);	910	try s.writeByteNTimes(' ', w.indent);
911	for (deaths, 0..) \|operand, i\| {	911	for (deaths, 0..) \|operand, i\| {
912	if (i != 0) try s.writeAll(" ");	912	if (i != 0) try s.writeAll(" ");
913	try s.print("%{d}!", .{@intFromEnum(operand)});	913	try s.print("{}!", .{operand});
914	}	914	}
915	try s.writeAll("\n");	915	try s.writeAll("\n");
916	}	916	}
...	@@ -994,7 +994,7 @@ const Writer = struct {	...	@@ -994,7 +994,7 @@ const Writer = struct {
994	dies: bool,	994	dies: bool,
995	) @TypeOf(s).Error!void {	995	) @TypeOf(s).Error!void {
996	_ = w;	996	_ = w;
997	try s.print("%{d}", .{@intFromEnum(inst)});	997	try s.print("{}", .{inst});
998	if (dies) try s.writeByte('!');	998	if (dies) try s.writeByte('!');
999	}	999	}
1000		1000

src/register_manager.zig+50-71

...	@@ -41,12 +41,12 @@ pub fn RegisterManager(	...	@@ -41,12 +41,12 @@ pub fn RegisterManager(
41	registers: TrackedRegisters = undefined,	41	registers: TrackedRegisters = undefined,
42	/// Tracks which registers are free (in which case the	42	/// Tracks which registers are free (in which case the
43	/// corresponding bit is set to 1)	43	/// corresponding bit is set to 1)
44	free_registers: RegisterBitSet = RegisterBitSet.initFull(),	44	free_registers: RegisterBitSet = .initFull(),
45	/// Tracks all registers allocated in the course of this	45	/// Tracks all registers allocated in the course of this
46	/// function	46	/// function
47	allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(),	47	allocated_registers: RegisterBitSet = .initEmpty(),
48	/// Tracks registers which are locked from being allocated	48	/// Tracks registers which are locked from being allocated
49	locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(),	49	locked_registers: RegisterBitSet = .initEmpty(),
50		50
51	const Self = @This();	51	const Self = @This();
52		52
...	@@ -58,11 +58,6 @@ pub fn RegisterManager(	...	@@ -58,11 +58,6 @@ pub fn RegisterManager(
58	return @alignCast(@fieldParentPtr("register_manager", self));	58	return @alignCast(@fieldParentPtr("register_manager", self));
59	}	59	}
60		60
61	fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool {
62	const index = indexOfRegIntoTracked(reg) orelse return true;
63	return !register_class.isSet(index);
64	}
65
66	fn markRegIndexAllocated(self: *Self, tracked_index: TrackedIndex) void {	61	fn markRegIndexAllocated(self: *Self, tracked_index: TrackedIndex) void {
67	self.allocated_registers.set(tracked_index);	62	self.allocated_registers.set(tracked_index);
68	}	63	}
...	@@ -99,8 +94,7 @@ pub fn RegisterManager(	...	@@ -99,8 +94,7 @@ pub fn RegisterManager(
99	max_id = @max(elem_id, max_id);	94	max_id = @max(elem_id, max_id);
100	}	95	}
101		96
102	const OptionalIndex = std.math.IntFittingRange(0, set.len);	97	comptime var map: [max_id - min_id + 1]std.math.IntFittingRange(0, set.len) = @splat(set.len);
103	comptime var map = [1]OptionalIndex{set.len} ** (max_id - min_id + 1);
104	inline for (set, 0..) \|elem, elem_index\| map[comptime elem.id() - min_id] = elem_index;	98	inline for (set, 0..) \|elem, elem_index\| map[comptime elem.id() - min_id] = elem_index;
105		99
106	const id_index = reg.id() -% min_id;	100	const id_index = reg.id() -% min_id;
...	@@ -112,6 +106,9 @@ pub fn RegisterManager(	...	@@ -112,6 +106,9 @@ pub fn RegisterManager(
112	pub fn indexOfRegIntoTracked(reg: Register) ?TrackedIndex {	106	pub fn indexOfRegIntoTracked(reg: Register) ?TrackedIndex {
113	return indexOfReg(tracked_registers, reg);	107	return indexOfReg(tracked_registers, reg);
114	}	108	}
		109	pub inline fn indexOfKnownRegIntoTracked(comptime reg: Register) ?TrackedIndex {
		110	return comptime indexOfRegIntoTracked(reg);
		111	}
115		112
116	pub fn regAtTrackedIndex(tracked_index: TrackedIndex) Register {	113	pub fn regAtTrackedIndex(tracked_index: TrackedIndex) Register {
117	return tracked_registers[tracked_index];	114	return tracked_registers[tracked_index];
...	@@ -124,6 +121,9 @@ pub fn RegisterManager(	...	@@ -124,6 +121,9 @@ pub fn RegisterManager(
124	pub fn isRegFree(self: Self, reg: Register) bool {	121	pub fn isRegFree(self: Self, reg: Register) bool {
125	return self.isRegIndexFree(indexOfRegIntoTracked(reg) orelse return true);	122	return self.isRegIndexFree(indexOfRegIntoTracked(reg) orelse return true);
126	}	123	}
		124	pub fn isKnownRegFree(self: Self, comptime reg: Register) bool {
		125	return self.isRegIndexFree(indexOfKnownRegIntoTracked(reg) orelse return true);
		126	}
127		127
128	/// Returns whether this register was allocated in the course	128	/// Returns whether this register was allocated in the course
129	/// of this function.	129	/// of this function.
...	@@ -143,6 +143,9 @@ pub fn RegisterManager(	...	@@ -143,6 +143,9 @@ pub fn RegisterManager(
143	pub fn isRegLocked(self: Self, reg: Register) bool {	143	pub fn isRegLocked(self: Self, reg: Register) bool {
144	return self.isRegIndexLocked(indexOfRegIntoTracked(reg) orelse return false);	144	return self.isRegIndexLocked(indexOfRegIntoTracked(reg) orelse return false);
145	}	145	}
		146	pub fn isKnownRegLocked(self: Self, comptime reg: Register) bool {
		147	return self.isRegIndexLocked(indexOfKnownRegIntoTracked(reg) orelse return false);
		148	}
146		149
147	pub const RegisterLock = struct { tracked_index: TrackedIndex };	150	pub const RegisterLock = struct { tracked_index: TrackedIndex };
148		151
...	@@ -176,6 +179,9 @@ pub fn RegisterManager(	...	@@ -176,6 +179,9 @@ pub fn RegisterManager(
176	pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {	179	pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
177	return self.lockRegIndexAssumeUnused(indexOfRegIntoTracked(reg) orelse unreachable);	180	return self.lockRegIndexAssumeUnused(indexOfRegIntoTracked(reg) orelse unreachable);
178	}	181	}
		182	pub fn lockKnownRegAssumeUnused(self: *Self, comptime reg: Register) RegisterLock {
		183	return self.lockRegIndexAssumeUnused(indexOfKnownRegIntoTracked(reg) orelse unreachable);
		184	}
179		185
180	/// Like `lockReg` but locks multiple registers.	186	/// Like `lockReg` but locks multiple registers.
181	pub fn lockRegs(	187	pub fn lockRegs(
...	@@ -223,28 +229,20 @@ pub fn RegisterManager(	...	@@ -223,28 +229,20 @@ pub fn RegisterManager(
223	) ?[count]Register {	229	) ?[count]Register {
224	comptime assert(count > 0 and count <= tracked_registers.len);	230	comptime assert(count > 0 and count <= tracked_registers.len);
225		231
226	var free_and_not_locked_registers = self.free_registers;	232	var free_and_unlocked_registers = self.locked_registers;
227	free_and_not_locked_registers.setIntersection(register_class);	233	free_and_unlocked_registers.toggleAll();
228		234	free_and_unlocked_registers.setIntersection(self.free_registers);
229	var unlocked_registers = self.locked_registers;	235	free_and_unlocked_registers.setIntersection(register_class);
230	unlocked_registers.toggleAll();
231
232	free_and_not_locked_registers.setIntersection(unlocked_registers);
233
234	if (free_and_not_locked_registers.count() < count) return null;
235		236
236	var regs: [count]Register = undefined;	237	var regs: [count]Register = undefined;
237	var i: usize = 0;	238	var i: usize = 0;
238	for (tracked_registers) \|reg\| {	239	var it = free_and_unlocked_registers.iterator(.{});
239	if (i >= count) break;	240	while (it.next()) \|reg_index\| {
240	if (excludeRegister(reg, register_class)) continue;	241	regs[i] = regAtTrackedIndex(@intCast(reg_index));
241	if (self.isRegLocked(reg)) continue;
242	if (!self.isRegFree(reg)) continue;
243
244	regs[i] = reg;
245	i += 1;	242	i += 1;
		243	if (i >= count) break;
246	}	244	}
247	assert(i == count);	245	if (i < count) return null;
248		246
249	for (regs, insts) \|reg, inst\| {	247	for (regs, insts) \|reg, inst\| {
250	log.debug("tryAllocReg {} for inst {?}", .{ reg, inst });	248	log.debug("tryAllocReg {} for inst {?}", .{ reg, inst });
...	@@ -279,46 +277,27 @@ pub fn RegisterManager(	...	@@ -279,46 +277,27 @@ pub fn RegisterManager(
279	) AllocationError![count]Register {	277	) AllocationError![count]Register {
280	comptime assert(count > 0 and count <= tracked_registers.len);	278	comptime assert(count > 0 and count <= tracked_registers.len);
281		279
282	var locked_registers = self.locked_registers;
283	locked_registers.setIntersection(register_class);
284
285	if (count > register_class.count() - locked_registers.count()) return error.OutOfRegisters;
286
287	const result = self.tryAllocRegs(count, insts, register_class) orelse blk: {	280	const result = self.tryAllocRegs(count, insts, register_class) orelse blk: {
		281	var unlocked_registers = self.locked_registers;
		282	unlocked_registers.toggleAll();
		283	unlocked_registers.setIntersection(register_class);
		284
288	// We'll take over the first count registers. Spill	285	// We'll take over the first count registers. Spill
289	// the instructions that were previously there to a	286	// the instructions that were previously there to a
290	// stack allocations.	287	// stack allocations.
291	var regs: [count]Register = undefined;	288	var regs: [count]Register = undefined;
292	var i: usize = 0;	289	var i: usize = 0;
293	for (tracked_registers) \|reg\| {	290	var it = unlocked_registers.iterator(.{});
294	if (i >= count) break;	291	while (it.next()) \|reg_index\| {
295	if (excludeRegister(reg, register_class)) break;	292	const tracked_index: TrackedIndex = @intCast(reg_index);
296	if (self.isRegLocked(reg)) continue;	293	if (!self.isRegIndexFree(tracked_index) and
297		294	self.registers[tracked_index].unwrap() == .target) continue;
298	log.debug("allocReg {} for inst {?}", .{ reg, insts[i] });	295	try self.getRegIndex(tracked_index, insts[i]);
299	regs[i] = reg;	296	regs[i] = regAtTrackedIndex(tracked_index);
300	self.markRegAllocated(reg);
301	const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null
302	if (insts[i]) \|inst\| {
303	// Track the register
304	if (self.isRegFree(reg)) {
305	self.markRegUsed(reg);
306	} else {
307	const spilled_inst = self.registers[index];
308	try self.getFunction().spillInstruction(reg, spilled_inst);
309	}
310	self.registers[index] = inst;
311	} else {
312	// Don't track the register
313	if (!self.isRegFree(reg)) {
314	const spilled_inst = self.registers[index];
315	try self.getFunction().spillInstruction(reg, spilled_inst);
316	self.freeReg(reg);
317	}
318	}
319
320	i += 1;	297	i += 1;
		298	if (i >= count) break;
321	}	299	}
		300	if (i < count) return error.OutOfRegisters;
322		301
323	break :blk regs;	302	break :blk regs;
324	};	303	};
...	@@ -340,7 +319,7 @@ pub fn RegisterManager(	...	@@ -340,7 +319,7 @@ pub fn RegisterManager(
340	/// Spills the register if it is currently allocated. If a	319	/// Spills the register if it is currently allocated. If a
341	/// corresponding instruction is passed, will also track this	320	/// corresponding instruction is passed, will also track this
342	/// register.	321	/// register.
343	fn getRegIndex(	322	pub fn getRegIndex(
344	self: *Self,	323	self: *Self,
345	tracked_index: TrackedIndex,	324	tracked_index: TrackedIndex,
346	inst: ?Air.Inst.Index,	325	inst: ?Air.Inst.Index,
...	@@ -366,13 +345,13 @@ pub fn RegisterManager(	...	@@ -366,13 +345,13 @@ pub fn RegisterManager(
366	comptime reg: Register,	345	comptime reg: Register,
367	inst: ?Air.Inst.Index,	346	inst: ?Air.Inst.Index,
368	) AllocationError!void {	347	) AllocationError!void {
369	return self.getRegIndex((comptime indexOfRegIntoTracked(reg)) orelse return, inst);	348	return self.getRegIndex(indexOfKnownRegIntoTracked(reg) orelse return, inst);
370	}	349	}
371		350
372	/// Allocates the specified register with the specified	351	/// Allocates the specified register with the specified
373	/// instruction. Asserts that the register is free and no	352	/// instruction. Asserts that the register is free and no
374	/// spilling is necessary.	353	/// spilling is necessary.
375	fn getRegIndexAssumeFree(	354	pub fn getRegIndexAssumeFree(
376	self: *Self,	355	self: *Self,
377	tracked_index: TrackedIndex,	356	tracked_index: TrackedIndex,
378	inst: ?Air.Inst.Index,	357	inst: ?Air.Inst.Index,
...	@@ -391,7 +370,7 @@ pub fn RegisterManager(	...	@@ -391,7 +370,7 @@ pub fn RegisterManager(
391	}	370	}
392		371
393	/// Marks the specified register as free	372	/// Marks the specified register as free
394	fn freeRegIndex(self: *Self, tracked_index: TrackedIndex) void {	373	pub fn freeRegIndex(self: *Self, tracked_index: TrackedIndex) void {
395	log.debug("freeing register {}", .{regAtTrackedIndex(tracked_index)});	374	log.debug("freeing register {}", .{regAtTrackedIndex(tracked_index)});
396	self.registers[tracked_index] = undefined;	375	self.registers[tracked_index] = undefined;
397	self.markRegIndexFree(tracked_index);	376	self.markRegIndexFree(tracked_index);
...	@@ -420,8 +399,8 @@ const MockRegister1 = enum(u2) {	...	@@ -420,8 +399,8 @@ const MockRegister1 = enum(u2) {
420	&MockRegister1.allocatable_registers,	399	&MockRegister1.allocatable_registers,
421	);	400	);
422		401
423	const gp: RM.RegisterBitSet = blk: {	402	const gp = blk: {
424	var set = RM.RegisterBitSet.initEmpty();	403	var set: RM.RegisterBitSet = .initEmpty();
425	set.setRangeValue(.{	404	set.setRangeValue(.{
426	.start = 0,	405	.start = 0,
427	.end = allocatable_registers.len,	406	.end = allocatable_registers.len,
...	@@ -448,8 +427,8 @@ const MockRegister2 = enum(u2) {	...	@@ -448,8 +427,8 @@ const MockRegister2 = enum(u2) {
448	&MockRegister2.allocatable_registers,	427	&MockRegister2.allocatable_registers,
449	);	428	);
450		429
451	const gp: RM.RegisterBitSet = blk: {	430	const gp = blk: {
452	var set = RM.RegisterBitSet.initEmpty();	431	var set: RM.RegisterBitSet = .initEmpty();
453	set.setRangeValue(.{	432	set.setRangeValue(.{
454	.start = 0,	433	.start = 0,
455	.end = allocatable_registers.len,	434	.end = allocatable_registers.len,
...	@@ -489,16 +468,16 @@ const MockRegister3 = enum(u3) {	...	@@ -489,16 +468,16 @@ const MockRegister3 = enum(u3) {
489	&MockRegister3.allocatable_registers,	468	&MockRegister3.allocatable_registers,
490	);	469	);
491		470
492	const gp: RM.RegisterBitSet = blk: {	471	const gp = blk: {
493	var set = RM.RegisterBitSet.initEmpty();	472	var set: RM.RegisterBitSet = .initEmpty();
494	set.setRangeValue(.{	473	set.setRangeValue(.{
495	.start = 0,	474	.start = 0,
496	.end = gp_regs.len,	475	.end = gp_regs.len,
497	}, true);	476	}, true);
498	break :blk set;	477	break :blk set;
499	};	478	};
500	const ext: RM.RegisterBitSet = blk: {	479	const ext = blk: {
501	var set = RM.RegisterBitSet.initEmpty();	480	var set: RM.RegisterBitSet = .initEmpty();
502	set.setRangeValue(.{	481	set.setRangeValue(.{
503	.start = gp_regs.len,	482	.start = gp_regs.len,
504	.end = allocatable_registers.len,	483	.end = allocatable_registers.len,

test/behavior.zig+2

...	@@ -110,6 +110,8 @@ test {	...	@@ -110,6 +110,8 @@ test {
110	_ = @import("behavior/widening.zig");	110	_ = @import("behavior/widening.zig");
111	_ = @import("behavior/abs.zig");	111	_ = @import("behavior/abs.zig");
112		112
		113	_ = @import("behavior/x86_64.zig");
		114
113	if (builtin.cpu.arch == .wasm32) {	115	if (builtin.cpu.arch == .wasm32) {
114	_ = @import("behavior/wasm.zig");	116	_ = @import("behavior/wasm.zig");
115	}	117	}

test/behavior/align.zig+19-33

 @@ -144,31 +144,17 @@ test "alignment and size of structs with 128-bit fields" {
             },
         },
-        .x86_64 => switch (builtin.zig_backend) {
+        .x86_64 => .{
-            .stage2_x86_64 => .{
+            .a_align = 16,
-                .a_align = 8,
+            .a_size = 16,
-                .a_size = 16,
-                .b_align = 16,
-                .b_size = 32,
-                .u128_align = 8,
-                .u128_size = 16,
-                .u129_align = 8,
-                .u129_size = 24,
-            },
-            else => .{
-                .a_align = 16,
-                .a_size = 16,
-                .b_align = 16,
+            .b_align = 16,
-                .b_size = 32,
+            .b_size = 32,
-                .u128_align = 16,
+            .u128_align = 16,
-                .u128_size = 16,
+            .u128_size = 16,
-                .u129_align = 16,
+            .u129_align = 16,
-                .u129_size = 32,
+            .u129_size = 32,
-            },
         },
         .x86,
 @@ -291,8 +277,8 @@ test "function alignment" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    // function alignment is a compile error on wasm32/wasm64
+    // function alignment is a compile error on wasm
-    if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
+    if (native_arch.isWasm()) return error.SkipZigTest;
     const S = struct {
         fn alignExpr() align(@sizeOf(usize) * 2) i32 {
 @@ -321,8 +307,8 @@ test "implicitly decreasing fn alignment" {
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
-    // function alignment is a compile error on wasm32/wasm64
+    // function alignment is a compile error on wasm
-    if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
+    if (native_arch.isWasm()) return error.SkipZigTest;
     try testImplicitlyDecreaseFnAlign(alignedSmall, 1234);
     try testImplicitlyDecreaseFnAlign(alignedBig, 5678);
 @@ -345,9 +331,9 @@ test "@alignCast functions" {
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
-    // function alignment is a compile error on wasm32/wasm64
+    // function alignment is a compile error on wasm
-    if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
+    if (native_arch.isWasm()) return error.SkipZigTest;
-    if (native_arch == .thumb or native_arch == .thumbeb) return error.SkipZigTest;
+    if (native_arch.isThumb()) return error.SkipZigTest;
     try expect(fnExpectsOnly1(simple4) == 0x19);
 }
 @@ -510,9 +496,9 @@ test "align(N) on functions" {
         return error.SkipZigTest;
     }
-    // function alignment is a compile error on wasm32/wasm64
+    // function alignment is a compile error on wasm
-    if (native_arch == .wasm32 or native_arch == .wasm64) return error.SkipZigTest;
+    if (native_arch.isWasm()) return error.SkipZigTest;
-    if (native_arch == .thumb or native_arch == .thumbeb) return error.SkipZigTest;
+    if (native_arch.isThumb()) return error.SkipZigTest;
     try expect((@intFromPtr(&overaligned_fn) & (0x1000 - 1)) == 0);
 }

test/behavior/asm.zig+1-1

...	@@ -178,7 +178,7 @@ test "rw constraint (x86_64)" {	...	@@ -178,7 +178,7 @@ test "rw constraint (x86_64)" {
178	}	178	}
179		179
180	test "asm modifiers (AArch64)" {	180	test "asm modifiers (AArch64)" {
181	if (builtin.target.cpu.arch != .aarch64) return error.SkipZigTest;	181	if (!builtin.target.cpu.arch.isAARCH64()) return error.SkipZigTest;
182	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	182	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
183		183
184	if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest; // MSVC doesn't support inline assembly	184	if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest; // MSVC doesn't support inline assembly

test/behavior/basic.zig+1-1

...	@@ -1169,7 +1169,7 @@ test "arrays and vectors with big integers" {	...	@@ -1169,7 +1169,7 @@ test "arrays and vectors with big integers" {
1169	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	1169	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1170	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;	1170	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
1171	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	1171	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
1172	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;	1172	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1173	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;	1173	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
1174	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	1174	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1175	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	1175	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;

test/behavior/call.zig+1

...	@@ -660,6 +660,7 @@ test "arguments pointed to on stack into tailcall" {	...	@@ -660,6 +660,7 @@ test "arguments pointed to on stack into tailcall" {
660		660
661	switch (builtin.cpu.arch) {	661	switch (builtin.cpu.arch) {
662	.wasm32,	662	.wasm32,
		663	.wasm64,
663	.mips,	664	.mips,
664	.mipsel,	665	.mipsel,
665	.mips64,	666	.mips64,

test/behavior/cast.zig+2-3

test/behavior/eval.zig+1-1

...	@@ -522,7 +522,7 @@ test "runtime 128 bit integer division" {	...	@@ -522,7 +522,7 @@ test "runtime 128 bit integer division" {
522	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	522	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
523	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	523	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
524	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	524	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
525	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	525	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
526	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	526	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
527	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	527	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
528		528

test/behavior/math.zig+12-8

test/behavior/maximum_minimum.zig+1-1

...	@@ -122,7 +122,7 @@ test "@min/max for floats" {	...	@@ -122,7 +122,7 @@ test "@min/max for floats" {
122	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	122	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
123	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	123	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
124	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	124	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
125	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	125	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
126	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;	126	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
127	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	127	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
128		128

test/behavior/muladd.zig+4-4

test/behavior/optional.zig+1-1

...	@@ -57,7 +57,7 @@ fn testNullPtrsEql() !void {	...	@@ -57,7 +57,7 @@ fn testNullPtrsEql() !void {
57		57
58	test "optional with zero-bit type" {	58	test "optional with zero-bit type" {
59	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;	59	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
60	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;	60	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
61	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	61	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
62		62
63	const S = struct {	63	const S = struct {

test/behavior/packed-union.zig-1

...	@@ -138,7 +138,6 @@ test "packed union initialized with a runtime value" {	...	@@ -138,7 +138,6 @@ test "packed union initialized with a runtime value" {
138	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	138	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
139	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	139	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
140	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	140	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
141	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
142	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	141	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
143		142
144	const Fields = packed struct {	143	const Fields = packed struct {

test/behavior/saturating_arithmetic.zig+5-5

test/behavior/struct.zig+4-4

test/behavior/union.zig+1-1

 @@ -2246,12 +2246,12 @@ test "matching captures causes union equivalence" {
 }
 test "signed enum tag with negative value" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
     const Enum = enum(i8) {
         a = -1,

test/behavior/var_args.zig+5-5

test/behavior/vector.zig+6-9

test/behavior/wrapping_arithmetic.zig+1-1

...	@@ -83,7 +83,7 @@ test "wrapping multiplication" {	...	@@ -83,7 +83,7 @@ test "wrapping multiplication" {
83	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	83	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
84		84
85	// TODO: once #9660 has been solved, remove this line	85	// TODO: once #9660 has been solved, remove this line
86	if (builtin.cpu.arch == .wasm32) return error.SkipZigTest;	86	if (builtin.cpu.arch.isWasm()) return error.SkipZigTest;
87		87
88	const S = struct {	88	const S = struct {
89	fn doTheTest() !void {	89	fn doTheTest() !void {

test/behavior/x86_64.zig created+9

...	@@ -0,0 +1,9 @@
1	//! CodeGen tests for the x86_64 backend.
2
3	test {
4	const builtin = @import("builtin");
5	if (builtin.zig_backend != .stage2_x86_64) return error.SkipZigTest;
6	if (builtin.object_format == .coff) return error.SkipZigTest;
7	_ = @import("x86_64/math.zig");
8	_ = @import("x86_64/mem.zig");
9	}

test/behavior/x86_64/build.zig created+115

...	@@ -0,0 +1,115 @@
1	const std = @import("std");
2	pub fn build(b: *std.Build) void {
3	const compiler_rt_lib = b.addStaticLibrary(.{
4	.name = "compiler_rt",
5	.use_llvm = false,
6	.use_lld = false,
7	.root_module = b.createModule(.{
8	.root_source_file = b.addWriteFiles().add("compiler_rt.zig", ""),
9	.target = b.resolveTargetQuery(.{ .cpu_arch = .x86_64 }),
10	}),
11	});
12	compiler_rt_lib.bundle_compiler_rt = true;
13
14	for ([_]std.Target.Query{
15	.{
16	.cpu_arch = .x86_64,
17	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
18	.cpu_features_add = std.Target.x86.featureSet(&.{.bsf_bsr_0_clobbers_result}),
19	//.cpu_features_sub = std.Target.x86.featureSet(&.{.sse}),
20	},
21	.{
22	.cpu_arch = .x86_64,
23	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
24	.cpu_features_add = std.Target.x86.featureSet(&.{.bsf_bsr_0_clobbers_result}),
25	.cpu_features_sub = std.Target.x86.featureSet(&.{
26	.cmov,
27	//.sse,
28	}),
29	},
30	//.{
31	// .cpu_arch = .x86_64,
32	// .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
33	// .cpu_features_sub = std.Target.x86.featureSet(&.{.sse}),
34	//},
35	.{
36	.cpu_arch = .x86_64,
37	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
38	.cpu_features_sub = std.Target.x86.featureSet(&.{.sse2}),
39	},
40	.{
41	.cpu_arch = .x86_64,
42	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
43	},
44	.{
45	.cpu_arch = .x86_64,
46	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
47	.cpu_features_add = std.Target.x86.featureSet(&.{.sse3}),
48	},
49	.{
50	.cpu_arch = .x86_64,
51	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
52	.cpu_features_add = std.Target.x86.featureSet(&.{.ssse3}),
53	},
54	.{
55	.cpu_arch = .x86_64,
56	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
57	.cpu_features_add = std.Target.x86.featureSet(&.{.sse4_1}),
58	},
59	.{
60	.cpu_arch = .x86_64,
61	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
62	.cpu_features_add = std.Target.x86.featureSet(&.{.sse4_2}),
63	},
64	.{
65	.cpu_arch = .x86_64,
66	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v2 },
67	},
68	.{
69	.cpu_arch = .x86_64,
70	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v2 },
71	.cpu_features_add = std.Target.x86.featureSet(&.{.avx}),
72	},
73	.{
74	.cpu_arch = .x86_64,
75	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 },
76	.cpu_features_sub = std.Target.x86.featureSet(&.{.avx2}),
77	},
78	.{
79	.cpu_arch = .x86_64,
80	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 },
81	},
82	.{
83	.cpu_arch = .x86_64,
84	.cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v4 },
85	},
86	}) \|query\| {
87	const target = b.resolveTargetQuery(query);
88	const cpu = query.serializeCpuAlloc(b.allocator) catch @panic("OOM");
89	for ([_][]const u8{
90	"math.zig",
91	"mem.zig",
92	}) \|path\| {
93	const test_mod = b.createModule(.{
94	.root_source_file = b.path(path),
95	.target = target,
96	});
97	const test_exe = b.addTest(.{
98	.name = std.fs.path.stem(path),
99	.use_llvm = false,
100	.use_lld = false,
101	.root_module = test_mod,
102	});
103	if (!std.Target.x86.featureSetHas(target.result.cpu.features, .sse2)) {
104	test_exe.bundle_compiler_rt = false;
105	test_mod.linkLibrary(compiler_rt_lib);
106	}
107	const test_run = b.addRunArtifact(test_exe);
108	b.default_step.dependOn(&test_run.step);
109	for ([_]*std.Build.Step{
110	&test_exe.step,
111	&test_run.step,
112	}) \|step\| step.name = b.fmt("{s} {s}", .{ step.name, cpu });
113	}
114	}
115	}

test/behavior/x86_64/math.zig created+1103

...	@@ -0,0 +1,1103 @@
1	fn testUnary(comptime op: anytype) !void {
2	const testType = struct {
3	fn testType(comptime Type: type, comptime imm_arg: Type) !void {
4	const expected = op(Type, imm_arg);
5	try struct {
6	fn testOne(actual: @TypeOf(expected)) !void {
7	if (switch (@typeInfo(@TypeOf(expected))) {
8	else => actual != expected,
9	.vector => @reduce(.Or, actual != expected),
10	}) return error.Unexpected;
11	}
12	noinline fn testOps(mem_arg: Type) !void {
13	var reg_arg = mem_arg;
14	_ = .{&reg_arg};
15	try testOne(op(Type, reg_arg));
16	try testOne(op(Type, mem_arg));
17	try testOne(op(Type, imm_arg));
18	}
19	}.testOps(imm_arg);
20	}
21	}.testType;
22
23	try testType(i0, 0);
24	try testType(u0, 0);
25
26	try testType(i1, -1);
27	try testType(i1, 0);
28	try testType(u1, 0);
29	try testType(u1, 1 << 0);
30
31	try testType(i2, -1 << 1);
32	try testType(i2, -1);
33	try testType(i2, 0);
34	try testType(u2, 0);
35	try testType(u2, 1 << 0);
36	try testType(u2, 1 << 1);
37
38	try testType(i3, -1 << 2);
39	try testType(i3, -1);
40	try testType(i3, 0);
41	try testType(u3, 0);
42	try testType(u3, 1 << 0);
43	try testType(u3, 1 << 1);
44	try testType(u3, 1 << 2);
45
46	try testType(i4, -1 << 3);
47	try testType(i4, -1);
48	try testType(i4, 0);
49	try testType(u4, 0);
50	try testType(u4, 1 << 0);
51	try testType(u4, 1 << 1);
52	try testType(u4, 1 << 2);
53	try testType(u4, 1 << 3);
54
55	try testType(i5, -1 << 4);
56	try testType(i5, -1);
57	try testType(i5, 0);
58	try testType(u5, 0);
59	try testType(u5, 1 << 0);
60	try testType(u5, 1 << 1);
61	try testType(u5, 1 << 3);
62	try testType(u5, 1 << 4);
63
64	try testType(i7, -1 << 6);
65	try testType(i7, -1);
66	try testType(i7, 0);
67	try testType(u7, 0);
68	try testType(u7, 1 << 0);
69	try testType(u7, 1 << 1);
70	try testType(u7, 1 << 5);
71	try testType(u7, 1 << 6);
72
73	try testType(i8, -1 << 7);
74	try testType(i8, -1);
75	try testType(i8, 0);
76	try testType(u8, 0);
77	try testType(u8, 1 << 0);
78	try testType(u8, 1 << 1);
79	try testType(u8, 1 << 6);
80	try testType(u8, 1 << 7);
81
82	try testType(i9, -1 << 8);
83	try testType(i9, -1);
84	try testType(i9, 0);
85	try testType(u9, 0);
86	try testType(u9, 1 << 0);
87	try testType(u9, 1 << 1);
88	try testType(u9, 1 << 7);
89	try testType(u9, 1 << 8);
90
91	try testType(i15, -1 << 14);
92	try testType(i15, -1);
93	try testType(i15, 0);
94	try testType(u15, 0);
95	try testType(u15, 1 << 0);
96	try testType(u15, 1 << 1);
97	try testType(u15, 1 << 13);
98	try testType(u15, 1 << 14);
99
100	try testType(i16, -1 << 15);
101	try testType(i16, -1);
102	try testType(i16, 0);
103	try testType(u16, 0);
104	try testType(u16, 1 << 0);
105	try testType(u16, 1 << 1);
106	try testType(u16, 1 << 14);
107	try testType(u16, 1 << 15);
108
109	try testType(i17, -1 << 16);
110	try testType(i17, -1);
111	try testType(i17, 0);
112	try testType(u17, 0);
113	try testType(u17, 1 << 0);
114	try testType(u17, 1 << 1);
115	try testType(u17, 1 << 15);
116	try testType(u17, 1 << 16);
117
118	try testType(i31, -1 << 30);
119	try testType(i31, -1);
120	try testType(i31, 0);
121	try testType(u31, 0);
122	try testType(u31, 1 << 0);
123	try testType(u31, 1 << 1);
124	try testType(u31, 1 << 29);
125	try testType(u31, 1 << 30);
126
127	try testType(i32, -1 << 31);
128	try testType(i32, -1);
129	try testType(i32, 0);
130	try testType(u32, 0);
131	try testType(u32, 1 << 0);
132	try testType(u32, 1 << 1);
133	try testType(u32, 1 << 30);
134	try testType(u32, 1 << 31);
135
136	try testType(i33, -1 << 32);
137	try testType(i33, -1);
138	try testType(i33, 0);
139	try testType(u33, 0);
140	try testType(u33, 1 << 0);
141	try testType(u33, 1 << 1);
142	try testType(u33, 1 << 31);
143	try testType(u33, 1 << 32);
144
145	try testType(i63, -1 << 62);
146	try testType(i63, -1);
147	try testType(i63, 0);
148	try testType(u63, 0);
149	try testType(u63, 1 << 0);
150	try testType(u63, 1 << 1);
151	try testType(u63, 1 << 61);
152	try testType(u63, 1 << 62);
153
154	try testType(i64, -1 << 63);
155	try testType(i64, -1);
156	try testType(i64, 0);
157	try testType(u64, 0);
158	try testType(u64, 1 << 0);
159	try testType(u64, 1 << 1);
160	try testType(u64, 1 << 62);
161	try testType(u64, 1 << 63);
162
163	try testType(i65, -1 << 64);
164	try testType(i65, -1);
165	try testType(i65, 0);
166	try testType(u65, 0);
167	try testType(u65, 1 << 0);
168	try testType(u65, 1 << 1);
169	try testType(u65, 1 << 63);
170	try testType(u65, 1 << 64);
171
172	try testType(i95, -1 << 94);
173	try testType(i95, -1);
174	try testType(i95, 0);
175	try testType(u95, 0);
176	try testType(u95, 1 << 0);
177	try testType(u95, 1 << 1);
178	try testType(u95, 1 << 93);
179	try testType(u95, 1 << 94);
180
181	try testType(i96, -1 << 95);
182	try testType(i96, -1);
183	try testType(i96, 0);
184	try testType(u96, 0);
185	try testType(u96, 1 << 0);
186	try testType(u96, 1 << 1);
187	try testType(u96, 1 << 94);
188	try testType(u96, 1 << 95);
189
190	try testType(i97, -1 << 96);
191	try testType(i97, -1);
192	try testType(i97, 0);
193	try testType(u97, 0);
194	try testType(u97, 1 << 0);
195	try testType(u97, 1 << 1);
196	try testType(u97, 1 << 95);
197	try testType(u97, 1 << 96);
198
199	try testType(i127, -1 << 126);
200	try testType(i127, -1);
201	try testType(i127, 0);
202	try testType(u127, 0);
203	try testType(u127, 1 << 0);
204	try testType(u127, 1 << 1);
205	try testType(u127, 1 << 125);
206	try testType(u127, 1 << 126);
207
208	try testType(i128, -1 << 127);
209	try testType(i128, -1);
210	try testType(i128, 0);
211	try testType(u128, 0);
212	try testType(u128, 1 << 0);
213	try testType(u128, 1 << 1);
214	try testType(u128, 1 << 126);
215	try testType(u128, 1 << 127);
216
217	try testType(i129, -1 << 128);
218	try testType(i129, -1);
219	try testType(i129, 0);
220	try testType(u129, 0);
221	try testType(u129, 1 << 0);
222	try testType(u129, 1 << 1);
223	try testType(u129, 1 << 127);
224	try testType(u129, 1 << 128);
225
226	try testType(i159, -1 << 158);
227	try testType(i159, -1);
228	try testType(i159, 0);
229	try testType(u159, 0);
230	try testType(u159, 1 << 0);
231	try testType(u159, 1 << 1);
232	try testType(u159, 1 << 157);
233	try testType(u159, 1 << 158);
234
235	try testType(i160, -1 << 159);
236	try testType(i160, -1);
237	try testType(i160, 0);
238	try testType(u160, 0);
239	try testType(u160, 1 << 0);
240	try testType(u160, 1 << 1);
241	try testType(u160, 1 << 158);
242	try testType(u160, 1 << 159);
243
244	try testType(i161, -1 << 160);
245	try testType(i161, -1);
246	try testType(i161, 0);
247	try testType(u161, 0);
248	try testType(u161, 1 << 0);
249	try testType(u161, 1 << 1);
250	try testType(u161, 1 << 159);
251	try testType(u161, 1 << 160);
252
253	try testType(i191, -1 << 190);
254	try testType(i191, -1);
255	try testType(i191, 0);
256	try testType(u191, 0);
257	try testType(u191, 1 << 0);
258	try testType(u191, 1 << 1);
259	try testType(u191, 1 << 189);
260	try testType(u191, 1 << 190);
261
262	try testType(i192, -1 << 191);
263	try testType(i192, -1);
264	try testType(i192, 0);
265	try testType(u192, 0);
266	try testType(u192, 1 << 0);
267	try testType(u192, 1 << 1);
268	try testType(u192, 1 << 190);
269	try testType(u192, 1 << 191);
270
271	try testType(i193, -1 << 192);
272	try testType(i193, -1);
273	try testType(i193, 0);
274	try testType(u193, 0);
275	try testType(u193, 1 << 0);
276	try testType(u193, 1 << 1);
277	try testType(u193, 1 << 191);
278	try testType(u193, 1 << 192);
279
280	try testType(i223, -1 << 222);
281	try testType(i223, -1);
282	try testType(i223, 0);
283	try testType(u223, 0);
284	try testType(u223, 1 << 0);
285	try testType(u223, 1 << 1);
286	try testType(u223, 1 << 221);
287	try testType(u223, 1 << 222);
288
289	try testType(i224, -1 << 223);
290	try testType(i224, -1);
291	try testType(i224, 0);
292	try testType(u224, 0);
293	try testType(u224, 1 << 0);
294	try testType(u224, 1 << 1);
295	try testType(u224, 1 << 222);
296	try testType(u224, 1 << 223);
297
298	try testType(i225, -1 << 224);
299	try testType(i225, -1);
300	try testType(i225, 0);
301	try testType(u225, 0);
302	try testType(u225, 1 << 0);
303	try testType(u225, 1 << 1);
304	try testType(u225, 1 << 223);
305	try testType(u225, 1 << 224);
306
307	try testType(i255, -1 << 254);
308	try testType(i255, -1);
309	try testType(i255, 0);
310	try testType(u255, 0);
311	try testType(u255, 1 << 0);
312	try testType(u255, 1 << 1);
313	try testType(u255, 1 << 253);
314	try testType(u255, 1 << 254);
315
316	try testType(i256, -1 << 255);
317	try testType(i256, -1);
318	try testType(i256, 0);
319	try testType(u256, 0);
320	try testType(u256, 1 << 0);
321	try testType(u256, 1 << 1);
322	try testType(u256, 1 << 254);
323	try testType(u256, 1 << 255);
324
325	try testType(i257, -1 << 256);
326	try testType(i257, -1);
327	try testType(i257, 0);
328	try testType(u257, 0);
329	try testType(u257, 1 << 0);
330	try testType(u257, 1 << 1);
331	try testType(u257, 1 << 255);
332	try testType(u257, 1 << 256);
333
334	try testType(i511, -1 << 510);
335	try testType(i511, -1);
336	try testType(i511, 0);
337	try testType(u511, 0);
338	try testType(u511, 1 << 0);
339	try testType(u511, 1 << 1);
340	try testType(u511, 1 << 509);
341	try testType(u511, 1 << 510);
342
343	try testType(i512, -1 << 511);
344	try testType(i512, -1);
345	try testType(i512, 0);
346	try testType(u512, 0);
347	try testType(u512, 1 << 0);
348	try testType(u512, 1 << 1);
349	try testType(u512, 1 << 510);
350	try testType(u512, 1 << 511);
351
352	try testType(i513, -1 << 512);
353	try testType(i513, -1);
354	try testType(i513, 0);
355	try testType(u513, 0);
356	try testType(u513, 1 << 0);
357	try testType(u513, 1 << 1);
358	try testType(u513, 1 << 511);
359	try testType(u513, 1 << 512);
360
361	try testType(i1023, -1 << 1022);
362	try testType(i1023, -1);
363	try testType(i1023, 0);
364	try testType(u1023, 0);
365	try testType(u1023, 1 << 0);
366	try testType(u1023, 1 << 1);
367	try testType(u1023, 1 << 1021);
368	try testType(u1023, 1 << 1022);
369
370	try testType(i1024, -1 << 1023);
371	try testType(i1024, -1);
372	try testType(i1024, 0);
373	try testType(u1024, 0);
374	try testType(u1024, 1 << 0);
375	try testType(u1024, 1 << 1);
376	try testType(u1024, 1 << 1022);
377	try testType(u1024, 1 << 1023);
378
379	try testType(i1025, -1 << 1024);
380	try testType(i1025, -1);
381	try testType(i1025, 0);
382	try testType(u1025, 0);
383	try testType(u1025, 1 << 0);
384	try testType(u1025, 1 << 1);
385	try testType(u1025, 1 << 1023);
386	try testType(u1025, 1 << 1024);
387
388	try testType(@Vector(3, i0), .{ 0 << 0, 0, 0 });
389	try testType(@Vector(3, u0), .{ 0, 0, 0 << 0 });
390
391	try testType(@Vector(3, i1), .{ -1 << 0, -1, 0 });
392	try testType(@Vector(3, u1), .{ 0, 1, 1 << 0 });
393
394	try testType(@Vector(3, i2), .{ -1 << 1, -1, 0 });
395	try testType(@Vector(3, u2), .{ 0, 1, 1 << 1 });
396
397	try testType(@Vector(3, i3), .{ -1 << 2, -1, 0 });
398	try testType(@Vector(3, u3), .{ 0, 1, 1 << 2 });
399
400	try testType(@Vector(3, i4), .{ -1 << 3, -1, 0 });
401	try testType(@Vector(3, u4), .{ 0, 1, 1 << 3 });
402	try testType(@Vector(1, u4), .{
403	0xb,
404	});
405	try testType(@Vector(2, u4), .{
406	0x3, 0x4,
407	});
408	try testType(@Vector(4, u4), .{
409	0x9, 0x2, 0xf, 0xe,
410	});
411	try testType(@Vector(8, u4), .{
412	0x8, 0x1, 0xb, 0x1, 0xf, 0x5, 0x9, 0x6,
413	});
414	try testType(@Vector(16, u4), .{
415	0xb, 0x6, 0x0, 0x7, 0x8, 0x5, 0x6, 0x9, 0xe, 0xb, 0x3, 0xa, 0xb, 0x5, 0x8, 0xc,
416	});
417	try testType(@Vector(32, u4), .{
418	0xe, 0x6, 0xe, 0xa, 0xb, 0x4, 0xa, 0xb, 0x1, 0x3, 0xb, 0xc, 0x0, 0xb, 0x9, 0x4, 0xd, 0xa, 0xd, 0xd, 0x4, 0x8, 0x8, 0x6, 0xb, 0xe, 0x9, 0x6, 0xc, 0xd, 0x5, 0xd,
419	});
420	try testType(@Vector(64, u4), .{
421	0x1, 0xc, 0xe, 0x9, 0x9, 0xf, 0x3, 0xf, 0x9, 0x9, 0x5, 0x3, 0xb, 0xd, 0xd, 0xf, 0x1, 0x2, 0xf, 0x9, 0x4, 0x4, 0x8, 0x9, 0x2, 0x9, 0x8, 0xe, 0x8, 0xa, 0x4, 0x3,
422	0x4, 0xc, 0xb, 0x6, 0x4, 0x0, 0xa, 0x5, 0x1, 0xa, 0x4, 0xe, 0xa, 0x7, 0xd, 0x0, 0x4, 0xe, 0xe, 0x7, 0x7, 0xa, 0x4, 0x5, 0x6, 0xc, 0x6, 0x2, 0x6, 0xa, 0xe, 0xa,
423	});
424	try testType(@Vector(128, u4), .{
425	0xd, 0x5, 0x6, 0xe, 0x3, 0x3, 0x3, 0xe, 0xd, 0xd, 0x9, 0x0, 0x0, 0xe, 0xa, 0x9, 0x8, 0x7, 0xb, 0x5, 0x7, 0xf, 0xb, 0x8, 0x0, 0xf, 0xb, 0x3, 0xa, 0x2, 0xb, 0xc,
426	0x1, 0x1, 0xc, 0x8, 0x8, 0x6, 0x9, 0x1, 0xb, 0x0, 0x2, 0xb, 0x2, 0x2, 0x7, 0x6, 0x1, 0x1, 0xb, 0x4, 0x6, 0x4, 0x7, 0xc, 0xd, 0xc, 0xa, 0x8, 0x1, 0x7, 0x8, 0xa,
427	0x9, 0xa, 0x1, 0x8, 0x1, 0x7, 0x9, 0x4, 0x5, 0x9, 0xd, 0x0, 0xa, 0xf, 0x3, 0x3, 0x9, 0x2, 0xf, 0x5, 0xb, 0x8, 0x6, 0xb, 0xf, 0x5, 0x8, 0x3, 0x9, 0xf, 0x6, 0x8,
428	0xc, 0x8, 0x3, 0x4, 0xa, 0xe, 0xc, 0x1, 0xe, 0x9, 0x1, 0x8, 0xf, 0x6, 0xc, 0xc, 0x6, 0xf, 0x6, 0xd, 0xb, 0x9, 0xc, 0x3, 0xd, 0xa, 0x6, 0x8, 0x4, 0xa, 0x6, 0x9,
429	});
430	try testType(@Vector(256, u4), .{
431	0x6, 0xc, 0xe, 0x3, 0x8, 0x2, 0xb, 0xd, 0x3, 0xa, 0x3, 0x8, 0xb, 0x8, 0x3, 0x0, 0xb, 0x5, 0x1, 0x3, 0x2, 0x2, 0xf, 0xc, 0x5, 0x1, 0x3, 0xb, 0x1, 0xc, 0x2, 0xd,
432	0xa, 0x8, 0x1, 0xc, 0xb, 0xa, 0x3, 0x1, 0xe, 0x4, 0xf, 0xb, 0xd, 0x8, 0xf, 0xa, 0xc, 0xb, 0xb, 0x0, 0xa, 0xc, 0xf, 0xe, 0x8, 0xd, 0x9, 0x3, 0xa, 0xe, 0x8, 0x7,
433	0x5, 0xa, 0x0, 0xe, 0x0, 0xd, 0x2, 0x2, 0x9, 0x4, 0x8, 0x9, 0x0, 0x4, 0x4, 0x8, 0xe, 0x1, 0xf, 0x1, 0x9, 0x3, 0xf, 0xc, 0xa, 0x0, 0x3, 0x2, 0x4, 0x1, 0x2, 0x3,
434	0xf, 0x2, 0x7, 0xb, 0x5, 0x0, 0xd, 0x3, 0x4, 0xf, 0xa, 0x3, 0xc, 0x2, 0x5, 0xe, 0x7, 0x5, 0xd, 0x7, 0x9, 0x0, 0xd, 0x7, 0x9, 0xd, 0x5, 0x7, 0xf, 0xd, 0xb, 0x4,
435	0x9, 0x6, 0xf, 0xb, 0x1, 0xb, 0x6, 0xb, 0xf, 0x7, 0xf, 0x0, 0x4, 0x7, 0x5, 0xa, 0x8, 0x1, 0xf, 0x9, 0x9, 0x0, 0x6, 0xb, 0x1, 0x2, 0x4, 0x3, 0x2, 0x0, 0x7, 0x0,
436	0x6, 0x7, 0xf, 0x1, 0xe, 0xa, 0x8, 0x2, 0x9, 0xc, 0x1, 0x5, 0x7, 0x1, 0xb, 0x0, 0x1, 0x3, 0xd, 0x3, 0x0, 0x1, 0xa, 0x0, 0x3, 0x7, 0x1, 0x2, 0xb, 0xc, 0x2, 0x9,
437	0x8, 0x8, 0x7, 0x0, 0xd, 0x5, 0x1, 0x5, 0x7, 0x7, 0x2, 0x3, 0x8, 0x7, 0xc, 0x8, 0xf, 0xa, 0xf, 0xf, 0x3, 0x2, 0x0, 0x4, 0x7, 0x5, 0x6, 0xd, 0x6, 0x3, 0xa, 0x4,
438	0x1, 0x1, 0x2, 0xc, 0x3, 0xe, 0x2, 0xc, 0x7, 0x6, 0xe, 0xf, 0xb, 0x8, 0x6, 0x6, 0x9, 0x0, 0x4, 0xb, 0xe, 0x4, 0x2, 0x7, 0xf, 0xc, 0x0, 0x6, 0xd, 0xa, 0xe, 0xc,
439	});
440
441	try testType(@Vector(3, i5), .{ -1 << 4, -1, 0 });
442	try testType(@Vector(3, u5), .{ 0, 1, 1 << 4 });
443
444	try testType(@Vector(3, i7), .{ -1 << 6, -1, 0 });
445	try testType(@Vector(3, u7), .{ 0, 1, 1 << 6 });
446
447	try testType(@Vector(3, i8), .{ -1 << 7, -1, 0 });
448	try testType(@Vector(3, u8), .{ 0, 1, 1 << 7 });
449	try testType(@Vector(1, u8), .{
450	0x33,
451	});
452	try testType(@Vector(2, u8), .{
453	0x66, 0x87,
454	});
455	try testType(@Vector(4, u8), .{
456	0x9d, 0xcb, 0x30, 0x7b,
457	});
458	try testType(@Vector(8, u8), .{
459	0x4b, 0x35, 0x3f, 0x5c, 0xa5, 0x91, 0x23, 0x6d,
460	});
461	try testType(@Vector(16, u8), .{
462	0xb7, 0x57, 0x27, 0x29, 0x58, 0xf8, 0xc9, 0x6c, 0xbe, 0x41, 0xf4, 0xd7, 0x4d, 0x01, 0xf0, 0x37,
463	});
464	try testType(@Vector(32, u8), .{
465	0x5f, 0x61, 0x34, 0xe8, 0x37, 0x12, 0xba, 0x5a, 0x85, 0xf3, 0x3e, 0xa2, 0x0f, 0xd0, 0x65, 0xae,
466	0xed, 0xf5, 0xe8, 0x65, 0x61, 0x28, 0x4a, 0x27, 0x2e, 0x01, 0x40, 0x8c, 0xe3, 0x36, 0x5d, 0xb6,
467	});
468	try testType(@Vector(64, u8), .{
469	0xb0, 0x19, 0x5c, 0xc2, 0x3b, 0x16, 0x70, 0xad, 0x26, 0x45, 0xf2, 0xe1, 0x4f, 0x0f, 0x01, 0x72,
470	0x7f, 0x1f, 0x07, 0x9e, 0xee, 0x9b, 0xb3, 0x38, 0x50, 0xf3, 0x56, 0x73, 0xd0, 0xd1, 0xee, 0xe3,
471	0xeb, 0xf3, 0x1b, 0xe0, 0x77, 0x78, 0x75, 0xc6, 0x19, 0xe4, 0x69, 0xaa, 0x73, 0x08, 0xcd, 0x0c,
472	0xf9, 0xed, 0x94, 0xf8, 0x79, 0x86, 0x63, 0x31, 0xbf, 0xd1, 0xe3, 0x17, 0x2b, 0xb9, 0xa1, 0x72,
473	});
474	try testType(@Vector(128, u8), .{
475	0x2e, 0x93, 0x87, 0x09, 0x4f, 0x68, 0x14, 0xab, 0x3f, 0x04, 0x86, 0xc1, 0x95, 0xe8, 0x74, 0x11,
476	0x57, 0x25, 0xe1, 0x88, 0xc0, 0x96, 0x33, 0x99, 0x15, 0x86, 0x2c, 0x84, 0x2e, 0xd7, 0x57, 0x21,
477	0xd3, 0x18, 0xd5, 0x0e, 0xb4, 0x60, 0xe2, 0x08, 0xce, 0xbc, 0xd5, 0x4d, 0x8f, 0x59, 0x01, 0x67,
478	0x71, 0x0a, 0x74, 0x48, 0xef, 0x39, 0x49, 0x7e, 0xa8, 0x39, 0x34, 0x75, 0x95, 0x3b, 0x38, 0xea,
479	0x60, 0xd7, 0xed, 0x8f, 0xbb, 0xc0, 0x7d, 0xc2, 0x79, 0x2d, 0xbf, 0xa5, 0x64, 0xf4, 0x09, 0x86,
480	0xfb, 0x29, 0xfe, 0xc7, 0xff, 0x62, 0x1a, 0x6f, 0xf8, 0xbd, 0xfe, 0xa4, 0xac, 0x24, 0xcf, 0x56,
481	0x82, 0x69, 0x81, 0x0d, 0xc1, 0x51, 0x8d, 0x85, 0xf4, 0x00, 0xe7, 0x25, 0xab, 0xa5, 0x33, 0x45,
482	0x66, 0x2e, 0x33, 0xc8, 0xf3, 0x35, 0x16, 0x7d, 0x1f, 0xc9, 0xf7, 0x44, 0xab, 0x66, 0x28, 0x0d,
483	});
484
485	try testType(@Vector(3, i9), .{ -1 << 8, -1, 0 });
486	try testType(@Vector(3, u9), .{ 0, 1, 1 << 8 });
487
488	try testType(@Vector(3, i15), .{ -1 << 14, -1, 0 });
489	try testType(@Vector(3, u15), .{ 0, 1, 1 << 14 });
490
491	try testType(@Vector(3, i16), .{ -1 << 15, -1, 0 });
492	try testType(@Vector(3, u16), .{ 0, 1, 1 << 15 });
493	try testType(@Vector(1, u16), .{
494	0x4da6,
495	});
496	try testType(@Vector(2, u16), .{
497	0x04d7, 0x50c6,
498	});
499	try testType(@Vector(4, u16), .{
500	0x4c06, 0xd71f, 0x4d8f, 0xe0a4,
501	});
502	try testType(@Vector(8, u16), .{
503	0xee9a, 0x881d, 0x31fb, 0xd3f7, 0x2c74, 0x6949, 0x4e04, 0x53d7,
504	});
505	try testType(@Vector(16, u16), .{
506	0xeafe, 0x9a7b, 0x0d6f, 0x18cb, 0xaf8f, 0x8ee4, 0xa47e, 0xd39a,
507	0x6572, 0x9c53, 0xf36e, 0x982e, 0x41c1, 0x8682, 0xf5dc, 0x7e01,
508	});
509	try testType(@Vector(32, u16), .{
510	0xdfb3, 0x7de6, 0xd9ed, 0xb42e, 0x95ac, 0x9b5b, 0x0422, 0xdfcd,
511	0x6196, 0x4dbe, 0x1818, 0x8816, 0x75e7, 0xc9b0, 0x92f7, 0x1f71,
512	0xe584, 0x576c, 0x043a, 0x0f31, 0xfc4c, 0x2c87, 0x6b02, 0x0229,
513	0x25b7, 0x53cd, 0x9bab, 0x866b, 0x9008, 0xf0f3, 0xeb21, 0x88e2,
514	});
515	try testType(@Vector(64, u16), .{
516	0x084c, 0x445f, 0xce89, 0xd3ee, 0xb399, 0x315d, 0x8ef8, 0x4f6f,
517	0xf9af, 0xcbc4, 0x0332, 0xcd55, 0xa4dc, 0xbc38, 0x6e33, 0x8ead,
518	0xd15a, 0x5057, 0x58ef, 0x657a, 0xe9f0, 0x1418, 0x2b62, 0x3387,
519	0x1c15, 0x04e1, 0x0276, 0x3783, 0xad9c, 0xea9a, 0x0e5e, 0xe803,
520	0x2ee7, 0x0cf1, 0x30f1, 0xb12a, 0x381b, 0x353d, 0xf637, 0xf853,
521	0x2ac1, 0x7ce8, 0x6a50, 0xcbb8, 0xc9b8, 0x9b25, 0xd1e9, 0xeff0,
522	0xc0a2, 0x8e51, 0xde7a, 0x4e58, 0x5685, 0xeb3f, 0xd29b, 0x66ed,
523	0x3dd5, 0xcb59, 0x6003, 0xf710, 0x943a, 0x7276, 0xe547, 0xe48f,
524	});
525
526	try testType(@Vector(3, i17), .{ -1 << 16, -1, 0 });
527	try testType(@Vector(3, u17), .{ 0, 1, 1 << 16 });
528
529	try testType(@Vector(3, i31), .{ -1 << 30, -1, 0 });
530	try testType(@Vector(3, u31), .{ 0, 1, 1 << 30 });
531
532	try testType(@Vector(3, i32), .{ -1 << 31, -1, 0 });
533	try testType(@Vector(3, u32), .{ 0, 1, 1 << 31 });
534	try testType(@Vector(1, u32), .{
535	0x17e2805c,
536	});
537	try testType(@Vector(2, u32), .{
538	0xdb6aadc5, 0xb1ff3754,
539	});
540	try testType(@Vector(4, u32), .{
541	0xf7897b31, 0x342e1af9, 0x190fd76b, 0x283b5374,
542	});
543	try testType(@Vector(8, u32), .{
544	0x81a0bd16, 0xc55da94e, 0x910f7e7c, 0x078d5ef7,
545	0x0bdb1e4a, 0xf1a96e99, 0xcdd729b5, 0xe6966a1c,
546	});
547	try testType(@Vector(16, u32), .{
548	0xfee812db, 0x29eacbed, 0xaed48136, 0x3053de13,
549	0xbbda20df, 0x6faa274a, 0xe0b5ec3a, 0x1878b0dc,
550	0x98204475, 0x810d8d05, 0x1e6996b6, 0xc543826a,
551	0x53b47d8c, 0xc72c3142, 0x12f7e1f9, 0xf6782e54,
552	});
553	try testType(@Vector(32, u32), .{
554	0xf0cf30d3, 0xe3c587b8, 0xcee44739, 0xe4a0bd72,
555	0x41d44cce, 0x6d7c4259, 0xd85580a5, 0xec4b02d7,
556	0xa366483d, 0x2d7b59d4, 0xe9c0ace4, 0x82cb441c,
557	0xa23958ba, 0x04a70148, 0x3f0d20a3, 0xf9e21e37,
558	0x009fce8b, 0x4a34a229, 0xf09c35cf, 0xc0977d4d,
559	0xcc4d4647, 0xa30f1363, 0x27a65b14, 0xe572c785,
560	0x8f42e320, 0x2b2cdeca, 0x11205bd4, 0x739d26aa,
561	0xcbcc2df0, 0x5f7a3649, 0xbde1b7aa, 0x180a169f,
562	});
563
564	try testType(@Vector(3, i33), .{ -1 << 32, -1, 0 });
565	try testType(@Vector(3, u33), .{ 0, 1, 1 << 32 });
566
567	try testType(@Vector(3, i63), .{ -1 << 62, -1, 0 });
568	try testType(@Vector(3, u63), .{ 0, 1, 1 << 62 });
569
570	try testType(@Vector(3, i64), .{ -1 << 63, -1, 0 });
571	try testType(@Vector(3, u64), .{ 0, 1, 1 << 63 });
572	try testType(@Vector(1, u64), .{
573	0x7d2e439abb0edba7,
574	});
575	try testType(@Vector(2, u64), .{
576	0x3749ee5a2d237b9f, 0x6d8f4c3e1378f389,
577	});
578	try testType(@Vector(4, u64), .{
579	0x03c127040e10d52b, 0xa86fe019072e27eb,
580	0x0a554a47b709cdba, 0xf4342cc597e196c3,
581	});
582	try testType(@Vector(8, u64), .{
583	0xea455c104375a055, 0x5c35d9d945edb2fa,
584	0xc11b73d9d9d546fc, 0x2a9d63aae838dd5b,
585	0xed6603f1f5d574b3, 0x2f37b354c81c1e56,
586	0xbe7f5e2476bc76bd, 0xb0c88eacfffa9a8f,
587	});
588	try testType(@Vector(16, u64), .{
589	0x2258fc04b31f8dbe, 0x3a2e5483003a10d8,
590	0xebf24b31c0460510, 0x15d5b4c09b53ffa5,
591	0x05abf6e744b17cc6, 0x9747b483f2d159fe,
592	0x4616d8b2c8673125, 0x8ae3f91d422447eb,
593	0x18da2f101a9e9776, 0x77a1197fb0441007,
594	0x4ba480c8ec2dd10b, 0xeb99b9c0a1725278,
595	0xd9d0acc5084ecdf0, 0xa0a23317fff4f515,
596	0x0901c59a9a6a408b, 0x7c77ca72e25df033,
597	});
598
599	try testType(@Vector(3, i65), .{ -1 << 64, -1, 0 });
600	try testType(@Vector(3, u65), .{ 0, 1, 1 << 64 });
601
602	try testType(@Vector(3, i127), .{ -1 << 126, -1, 0 });
603	try testType(@Vector(3, u127), .{ 0, 1, 1 << 126 });
604
605	try testType(@Vector(3, i128), .{ -1 << 127, -1, 0 });
606	try testType(@Vector(3, u128), .{ 0, 1, 1 << 127 });
607	try testType(@Vector(1, u128), .{
608	0x809f29e7fbafadc01145e1732590e7d9,
609	});
610	try testType(@Vector(2, u128), .{
611	0x5150ac3438aacd0d51132cc2723b2995,
612	0x151be9c47ad29cf719cf8358dd40165c,
613	});
614	try testType(@Vector(4, u128), .{
615	0x4bae22df929f2f7cb9bd84deaad3e7a8,
616	0x1ed46b2d6e1f3569f56b2ac33d8bc1cb,
617	0xae93ea459d2ccfd5fb794e6d5c31aabb,
618	0xb1177136acf099f550b70949ac202ec4,
619	});
620	try testType(@Vector(8, u128), .{
621	0x7cd78db6baed6bfdf8c5265136c4e0fd,
622	0xa41b8984c6bbde84640068194b7eba98,
623	0xd33102778f2ae1a48d1e9bf8801bbbf0,
624	0x0d59f6de003513a60055c86cbce2c200,
625	0x825579d90012afddfbf04851c0748561,
626	0xc2647c885e9d6f0ee1f5fac5da8ef7f5,
627	0xcb4bbc1f81aa8ee68aa4dc140745687b,
628	0x4ff10f914f74b46c694407f5bf7c7836,
629	});
630
631	try testType(@Vector(3, i129), .{ -1 << 128, -1, 0 });
632	try testType(@Vector(3, u129), .{ 0, 1, 1 << 128 });
633
634	try testType(@Vector(3, i191), .{ -1 << 190, -1, 0 });
635	try testType(@Vector(3, u191), .{ 0, 1, 1 << 190 });
636
637	try testType(@Vector(3, i192), .{ -1 << 191, -1, 0 });
638	try testType(@Vector(3, u192), .{ 0, 1, 1 << 191 });
639	try testType(@Vector(1, u192), .{
640	0xe7baafcb9781626a77571b0539b9471a60c97d6c02106c8b,
641	});
642	try testType(@Vector(2, u192), .{
643	0xbc9510913ed09e2c2aa50ffab9f1bc7b303a87f36e232a83,
644	0x1f37bee446d7712d1ad457c47a66812cb926198d052aee65,
645	});
646	try testType(@Vector(4, u192), .{
647	0xdca6a7cfc19c69efc34022062a8ca36f2569ab3dce001202,
648	0xd25a4529e621c9084181fdb6917c6a32eccc58b63601b35d,
649	0x0a258afd6debbaf8c158f1caa61fed63b31871d13f51b43d,
650	0x6b40a178674fcb82c623ac322f851623d5e993dac97a219a,
651	});
652
653	try testType(@Vector(3, i193), .{ -1 << 192, -1, 0 });
654	try testType(@Vector(3, u193), .{ 0, 1, 1 << 192 });
655
656	try testType(@Vector(3, i255), .{ -1 << 254, -1, 0 });
657	try testType(@Vector(3, u255), .{ 0, 1, 1 << 254 });
658
659	try testType(@Vector(3, i256), .{ -1 << 255, -1, 0 });
660	try testType(@Vector(3, u256), .{ 0, 1, 1 << 255 });
661	try testType(@Vector(1, u256), .{
662	0x230413bb481fa3a997796acf282010c560d1942e7339fd584a0f15a90c83fbda,
663	});
664	try testType(@Vector(2, u256), .{
665	0x3ad569f8d91fdbc9da8ec0e933565919f2feb90b996c90c352b461aa0908e62d,
666	0x0f109696d64647983f1f757042515510729ad1350e862cbf38cb73b5cf99f0f7,
667	});
668	try testType(@Vector(4, u256), .{
669	0x1717c6ded4ac6de282d59f75f068da47d5a47a30f2c5053d2d59e715f9d28b97,
670	0x3087189ce7540e2e0028b80af571ebc6353a00b2917f243a869ed29ecca0adaa,
671	0x1507c6a9d104684bf503cdb08841cf91adab4644306bd67aafff5326604833ce,
672	0x857e134ff9179733c871295b25f824bd3eb562977bad30890964fa0cdc15bb07,
673	});
674
675	try testType(@Vector(3, i257), .{ -1 << 256, -1, 0 });
676	try testType(@Vector(3, u257), .{ 0, 1, 1 << 256 });
677
678	try testType(@Vector(3, i511), .{ -1 << 510, -1, 0 });
679	try testType(@Vector(3, u511), .{ 0, 1, 1 << 510 });
680
681	try testType(@Vector(3, i512), .{ -1 << 511, -1, 0 });
682	try testType(@Vector(3, u512), .{ 0, 1, 1 << 511 });
683	try testType(@Vector(1, u512), .{
684	0xa3ff51a609f1370e5eeb96b05169bf7469e465cf76ac5b4ea8ffd166c1ba3cd94f2dedf0d647a1fe424f3a06e6d7940f03e257f28100970b00bd5528c52b9ae6,
685	});
686	try testType(@Vector(2, u512), .{
687	0xc6d43cd46ae31ab71f9468a895c83bf17516c6b2f1c9b04b9aa113bf7fe1b789eb7d95fcf951f12a9a6f2124589551efdd8c00f528b366a7bfb852faf8f3da53,
688	0xc9099d2bdf8d1a0d30485ec6db4a24cbc0d89a863de30e18313ee1d66f71dd2d26235caaa703286cf4a2b51e1a12ef96d2d944c66c0bd3f0d72dd4cf0fc8100e,
689	});
690
691	try testType(@Vector(3, i513), .{ -1 << 512, -1, 0 });
692	try testType(@Vector(3, u513), .{ 0, 1, 1 << 512 });
693
694	try testType(@Vector(3, i1023), .{ -1 << 1022, -1, 0 });
695	try testType(@Vector(3, u1023), .{ 0, 1, 1 << 1022 });
696
697	try testType(@Vector(3, i1024), .{ -1 << 1023, -1, 0 });
698	try testType(@Vector(3, u1024), .{ 0, 1, 1 << 1023 });
699	try testType(@Vector(1, u1024), .{
700	0xc6cfaa6571139552e1f067402dfc131d9b9a58aafda97198a78764b05138fb68cf26f085b7652f3d5ae0e56aa21732f296a581bb411d4a73795c213de793489fa49b173b9f5c089aa6295ff1fcdc14d491a05035b45d08fc35cd67a83d887a02b8db512f07518132e0ba56533c7d6fbe958255eddf5649bd8aba288c0dd84a25,
701	});
702
703	try testType(@Vector(3, i1025), .{ -1 << 1024, -1, 0 });
704	try testType(@Vector(3, u1025), .{ 0, 1, 1 << 1024 });
705	}
706
707	fn testBinary(comptime op: anytype) !void {
708	const testType = struct {
709	fn testType(comptime Type: type, comptime imm_lhs: Type, comptime imm_rhs: Type) !void {
710	const expected = op(Type, imm_lhs, imm_rhs);
711	try struct {
712	fn testOne(actual: @TypeOf(expected)) !void {
713	if (switch (@typeInfo(@TypeOf(expected))) {
714	else => actual != expected,
715	.vector => @reduce(.Or, actual != expected),
716	}) return error.Unexpected;
717	}
718	noinline fn testOps(mem_lhs: Type, mem_rhs: Type) !void {
719	var reg_lhs = mem_lhs;
720	var reg_rhs = mem_rhs;
721	_ = .{ &reg_lhs, &reg_rhs };
722	try testOne(op(Type, reg_lhs, reg_rhs));
723	try testOne(op(Type, reg_lhs, mem_rhs));
724	try testOne(op(Type, reg_lhs, imm_rhs));
725	try testOne(op(Type, mem_lhs, reg_rhs));
726	try testOne(op(Type, mem_lhs, mem_rhs));
727	try testOne(op(Type, mem_lhs, imm_rhs));
728	try testOne(op(Type, imm_lhs, reg_rhs));
729	try testOne(op(Type, imm_lhs, mem_rhs));
730	}
731	}.testOps(imm_lhs, imm_rhs);
732	}
733	}.testType;
734
735	try testType(u8, 0xbb, 0x43);
736	try testType(u16, 0xb8bf, 0x626d);
737	try testType(u32, 0x80d7a2c6, 0xbff6a402);
738	try testType(u64, 0x71138bc6b4a38898, 0x1bc4043de9438c7b);
739	try testType(u128, 0xe05fc132ef2cd8affee00a907f0a851f, 0x29f912a72cfc6a7c6973426a9636da9a);
740	try testType(
741	u256,
742	0xb7935f5c2f3b1ae7a422c0a7c446884294b7d5370bada307d2fe5a4c4284a999,
743	0x310e6e196ba4f143b8d285ca6addf7f3bb3344224aff221b27607a31e148be08,
744	);
745	try testType(
746	u258,
747	0x186d5ddaab8cb8cb04e5b41e36f812e039d008baf49f12894c39e29a07796d800,
748	0x2072daba6ffad168826163eb136f6d28ca4360c8e7e5e41e29755e19e4753a4f5,
749	);
750	try testType(
751	u495,
752	0x6eaf4e252b3bf74b75bac59e0b43ca5326bad2a25b3fdb74a67ef132ac5e47d72eebc3316fb2351ee66c50dc5afb92a75cea9b0e35160652c7db39eeb158,
753	0x49fbed744a92b549d8c05bb3512c617d24dd824f3f69bdf3923bc326a75674b85f5b828d2566fab9c86f571d12c2a63c9164feb0d191d27905533d09622a,
754	);
755	try testType(
756	u512,
757	0xe5b1fedca3c77db765e517aabd05ffc524a3a8aff1784bbf67c45b894447ede32b65b9940e78173c591e56e078932d465f235aece7ad47b7f229df7ba8f12295,
758	0x8b4bb7c2969e3b121cc1082c442f8b4330f0a50058438fed56447175bb10178607ecfe425cb54dacc25ef26810f3e04681de1844f1aa8d029aca75d658634806,
759	);
760
761	try testType(@Vector(1, u8), .{
762	0x1f,
763	}, .{
764	0x06,
765	});
766	try testType(@Vector(2, u8), .{
767	0x80, 0x63,
768	}, .{
769	0xe4, 0x28,
770	});
771	try testType(@Vector(4, u8), .{
772	0x83, 0x9e, 0x1e, 0xc1,
773	}, .{
774	0xf0, 0x5c, 0x46, 0x85,
775	});
776	try testType(@Vector(8, u8), .{
777	0x1e, 0x4d, 0x9d, 0x2a, 0x4c, 0x74, 0x0a, 0x83,
778	}, .{
779	0x28, 0x60, 0xa9, 0xb5, 0xd9, 0xa6, 0xf1, 0xb6,
780	});
781	try testType(@Vector(16, u8), .{
782	0xea, 0x80, 0xbb, 0xe8, 0x74, 0x81, 0xc8, 0x66, 0x7b, 0x41, 0x90, 0xcb, 0x30, 0x70, 0x4b, 0x0f,
783	}, .{
784	0x61, 0x26, 0xbe, 0x47, 0x00, 0x9c, 0x55, 0xa5, 0x59, 0xf0, 0xb2, 0x20, 0x30, 0xaf, 0x82, 0x3e,
785	});
786	try testType(@Vector(32, u8), .{
787	0xa1, 0x88, 0xc4, 0xf4, 0x77, 0x0b, 0xf5, 0xbb, 0x09, 0x03, 0xbf, 0xf5, 0xcc, 0x7f, 0x6b, 0x2a,
788	0x4c, 0x05, 0x37, 0xc9, 0x8a, 0xcb, 0x91, 0x23, 0x09, 0x5f, 0xb8, 0x99, 0x4a, 0x75, 0x26, 0xe4,
789	}, .{
790	0xff, 0x0f, 0x99, 0x49, 0xa6, 0x25, 0xa7, 0xd4, 0xc9, 0x2f, 0x97, 0x6a, 0x01, 0xd6, 0x6e, 0x41,
791	0xa4, 0xb5, 0x3c, 0x03, 0xea, 0x82, 0x9c, 0x5f, 0xac, 0x07, 0x16, 0x15, 0x1c, 0x64, 0x25, 0x2f,
792	});
793	try testType(@Vector(64, u8), .{
794	0xaa, 0x08, 0xeb, 0xb2, 0xd7, 0x89, 0x0f, 0x98, 0xda, 0x9f, 0xa6, 0x4e, 0x3c, 0xce, 0x1b, 0x1b,
795	0x9e, 0x5f, 0x2b, 0xd6, 0x59, 0x26, 0x47, 0x05, 0x2a, 0xb7, 0xd1, 0x10, 0xde, 0xd9, 0x84, 0x00,
796	0x07, 0xc0, 0xaa, 0x6e, 0xfa, 0x3b, 0x97, 0x85, 0xa8, 0x42, 0xd7, 0xa5, 0x90, 0xe6, 0x10, 0x1a,
797	0x47, 0x84, 0xe1, 0x3e, 0xb0, 0x70, 0x26, 0x3f, 0xea, 0x24, 0xb8, 0x5f, 0xe3, 0xe3, 0x4c, 0xed,
798	}, .{
799	0x3b, 0xc5, 0xe0, 0x3d, 0x4f, 0x2e, 0x1d, 0xa9, 0xf7, 0x7b, 0xc7, 0xc1, 0x48, 0xc6, 0xe5, 0x9e,
800	0x4d, 0xa8, 0x21, 0x37, 0xa1, 0x1a, 0x95, 0x69, 0x89, 0x2f, 0x15, 0x07, 0x3d, 0x7b, 0x69, 0x89,
801	0xea, 0x87, 0xf0, 0x94, 0x67, 0xf2, 0x3d, 0x04, 0x96, 0x8a, 0xd6, 0x70, 0x7c, 0x16, 0xe7, 0x62,
802	0xf0, 0x8d, 0x96, 0x65, 0xd1, 0x4a, 0x35, 0x3e, 0x7a, 0x67, 0xa6, 0x1f, 0x37, 0x66, 0xe3, 0x45,
803	});
804	try testType(@Vector(128, u8), .{
805	0xa1, 0xd0, 0x7b, 0xf9, 0x7b, 0x77, 0x7b, 0x3d, 0x2d, 0x68, 0xc2, 0x7b, 0xb0, 0xb8, 0xd4, 0x7c,
806	0x1a, 0x1f, 0xd2, 0x92, 0x3e, 0xcb, 0xc1, 0x6b, 0xb9, 0x4d, 0xf1, 0x67, 0x58, 0x8e, 0x77, 0xa6,
807	0xb9, 0xdf, 0x10, 0x6f, 0xbe, 0xe3, 0x33, 0xb6, 0x93, 0x77, 0x80, 0xef, 0x09, 0x9d, 0x61, 0x40,
808	0xa2, 0xf4, 0x52, 0x18, 0x9d, 0xe4, 0xb0, 0xaf, 0x0a, 0xa7, 0x0b, 0x09, 0x67, 0x38, 0x71, 0x04,
809	0x72, 0xa1, 0xd2, 0xfd, 0xf8, 0xf0, 0xa7, 0x23, 0x24, 0x5b, 0x7d, 0xfb, 0x43, 0xba, 0x6c, 0xc4,
810	0x83, 0x46, 0x0e, 0x4d, 0x6c, 0x92, 0xab, 0x4f, 0xd2, 0x70, 0x9d, 0xfe, 0xce, 0xf8, 0x05, 0x9f,
811	0x98, 0x36, 0x9c, 0x90, 0x9a, 0xd0, 0xb5, 0x76, 0x16, 0xe8, 0x25, 0xc2, 0xbd, 0x91, 0xab, 0xf9,
812	0x6f, 0x6c, 0xc5, 0x60, 0xe5, 0x30, 0xf2, 0xb7, 0x59, 0xc4, 0x9c, 0xdd, 0xdf, 0x04, 0x65, 0xd9,
813	}, .{
814	0xed, 0xe1, 0x8a, 0xf6, 0xf3, 0x8b, 0xfd, 0x1d, 0x3c, 0x87, 0xbf, 0xfe, 0x04, 0x52, 0x15, 0x82,
815	0x0b, 0xb0, 0xcf, 0xcf, 0xf8, 0x03, 0x9c, 0xef, 0xc1, 0x76, 0x7e, 0xe3, 0xe9, 0xa8, 0x18, 0x90,
816	0xd4, 0xc4, 0x91, 0x15, 0x68, 0x7f, 0x65, 0xd8, 0xe1, 0xb3, 0x23, 0xc2, 0x7d, 0x84, 0x3b, 0xaf,
817	0x74, 0x69, 0x07, 0x2a, 0x1b, 0x5f, 0x0e, 0x44, 0x0d, 0x2b, 0x9c, 0x82, 0x41, 0xf9, 0x7f, 0xb5,
818	0xc4, 0xd9, 0xcb, 0xd3, 0xc5, 0x31, 0x8b, 0x5f, 0xda, 0x09, 0x9b, 0x29, 0xa3, 0xb7, 0x13, 0x0d,
819	0x55, 0x9b, 0x59, 0x33, 0x2a, 0x59, 0x3a, 0x44, 0x1f, 0xd3, 0x40, 0x4e, 0xde, 0x2c, 0xe4, 0x16,
820	0xfd, 0xc3, 0x02, 0x74, 0xaa, 0x65, 0xfd, 0xc8, 0x2a, 0x8a, 0xdb, 0xae, 0x44, 0x28, 0x62, 0xa4,
821	0x56, 0x4f, 0xf1, 0xaa, 0x0a, 0x0f, 0xdb, 0x1b, 0xc8, 0x45, 0x9b, 0x12, 0xb4, 0x1a, 0xe4, 0xa3,
822	});
823
824	try testType(@Vector(1, u16), .{
825	0x9d6f,
826	}, .{
827	0x44b1,
828	});
829	try testType(@Vector(2, u16), .{
830	0xa0fa, 0xc365,
831	}, .{
832	0xe736, 0xc394,
833	});
834	try testType(@Vector(4, u16), .{
835	0x9608, 0xa558, 0x161b, 0x206f,
836	}, .{
837	0x3088, 0xf25c, 0x7837, 0x9b3f,
838	});
839	try testType(@Vector(8, u16), .{
840	0xcf61, 0xb121, 0x3cf1, 0x3e9f, 0x43a7, 0x8d69, 0x96f5, 0xc11e,
841	}, .{
842	0xee30, 0x82f0, 0x270b, 0x1498, 0x4c60, 0x6e72, 0x0b64, 0x02d4,
843	});
844	try testType(@Vector(16, u16), .{
845	0x9191, 0xd23e, 0xf844, 0xd84a, 0xe907, 0xf1e8, 0x712d, 0x90af,
846	0x6541, 0x3fa6, 0x92eb, 0xe35a, 0xc0c9, 0xcb47, 0xb790, 0x4453,
847	}, .{
848	0x21c3, 0x4039, 0x9b71, 0x60bd, 0xcd7f, 0x2ec8, 0x50ba, 0xe810,
849	0xebd4, 0x06e5, 0xed18, 0x2f66, 0x7e31, 0xe282, 0xad63, 0xb25e,
850	});
851	try testType(@Vector(32, u16), .{
852	0x6b6a, 0x30a9, 0xc267, 0x2231, 0xbf4c, 0x00bc, 0x9c2c, 0x2928,
853	0xecad, 0x82df, 0xcfb0, 0xa4e5, 0x909b, 0x1b05, 0xaf40, 0x1fd9,
854	0xcec6, 0xd8dc, 0xd4b5, 0x6d59, 0x8e3f, 0x4d8a, 0xb83a, 0x808e,
855	0x47e2, 0x5782, 0x59bf, 0xcefc, 0x5179, 0x3f48, 0x93dc, 0x66d2,
856	}, .{
857	0x1be8, 0xe98c, 0xf9b3, 0xb008, 0x2f8d, 0xf087, 0xc9b9, 0x75aa,
858	0xbd16, 0x9540, 0xc5bd, 0x2b2c, 0xd43f, 0x9394, 0x3e1d, 0xf695,
859	0x167d, 0xff7a, 0xf09d, 0xdff8, 0xdfa2, 0xc779, 0x70b7, 0x01bd,
860	0x46b3, 0x995a, 0xb7bc, 0xa79d, 0x5542, 0x961e, 0x37cd, 0x9c2a,
861	});
862	try testType(@Vector(64, u16), .{
863	0x6b87, 0xfd84, 0x436b, 0xe345, 0xfb82, 0x81fc, 0x0992, 0x45f9,
864	0x5527, 0x1f6d, 0xda46, 0x6a16, 0xf6e1, 0x8fb7, 0x3619, 0xdfe3,
865	0x64ce, 0x8ac6, 0x3ae8, 0x30e3, 0xec3b, 0x4ba7, 0x02a4, 0xa694,
866	0x8e68, 0x8f0c, 0x5e30, 0x0e55, 0x6538, 0x9852, 0xea35, 0x7be2,
867	0xdabd, 0x57e6, 0x5b38, 0x0fb2, 0x2604, 0x85e7, 0x6595, 0x8de9,
868	0x49b1, 0xe9a2, 0x3758, 0xa4d9, 0x505b, 0xc9d3, 0xddc5, 0x9a43,
869	0xfd44, 0x50f5, 0x379e, 0x03b6, 0x6375, 0x692f, 0x5586, 0xc717,
870	0x94dd, 0xee06, 0xb32d, 0x0bb9, 0x0e35, 0x5f8f, 0x0ba4, 0x19a8,
871	}, .{
872	0xbeeb, 0x3e54, 0x6486, 0x5167, 0xe432, 0x57cf, 0x9cac, 0x922e,
873	0xd2f8, 0x5614, 0x2e7f, 0x19cf, 0x9a07, 0x0524, 0x168f, 0x4464,
874	0x4def, 0x83ce, 0x97b4, 0xf269, 0xda5f, 0x28c1, 0x9cc3, 0xfa7c,
875	0x25a0, 0x912d, 0x25b2, 0xd60d, 0xcd82, 0x0e03, 0x40cc, 0xc9dc,
876	0x18eb, 0xc609, 0xb06d, 0x29e0, 0xf3c7, 0x997b, 0x8ca2, 0xa750,
877	0xc9bc, 0x8f0e, 0x3916, 0xd905, 0x94f8, 0x397f, 0x98b5, 0xc61d,
878	0x05db, 0x3e7a, 0xf750, 0xe8de, 0x3225, 0x81d9, 0x612e, 0x0a7e,
879	0x2c02, 0xff5b, 0x19ca, 0xbbf5, 0x870e, 0xc9ca, 0x47bb, 0xcfcc,
880	});
881
882	try testType(@Vector(1, u32), .{
883	0x1d0d9cc4,
884	}, .{
885	0xce2d0ab6,
886	});
887	try testType(@Vector(2, u32), .{
888	0x5ab78c03, 0xd21bb513,
889	}, .{
890	0x8a6664eb, 0x79eac37d,
891	});
892	try testType(@Vector(4, u32), .{
893	0x234d576e, 0x4151cc9c, 0x39f558e4, 0xba935a32,
894	}, .{
895	0x398f2a9d, 0x4540f093, 0x9225551c, 0x3bac865b,
896	});
897	try testType(@Vector(8, u32), .{
898	0xb8336635, 0x2fc3182c, 0x27a00123, 0x71587fbe,
899	0x9cbc65d2, 0x6f4bb0e6, 0x362594ce, 0x9971df38,
900	}, .{
901	0x5727e734, 0x972b0313, 0xff25f5dc, 0x924f8e55,
902	0x04920a61, 0xa1c3b334, 0xf52df4b6, 0x5ef72ecc,
903	});
904	try testType(@Vector(16, u32), .{
905	0xfb566f9e, 0x9ad4691a, 0x5b5f9ec0, 0x5a572d2a,
906	0x8f2f226b, 0x2dfc7e33, 0x9fb07e32, 0x9d672a2e,
907	0xbedc3cee, 0x6872428d, 0xbc73a9fd, 0xd4d5f055,
908	0x69c1e9ee, 0x65038deb, 0x1449061a, 0x48412ec2,
909	}, .{
910	0x96cbe946, 0x3f24f60b, 0xaeacdc53, 0x7611a8b4,
911	0x031a67a8, 0x52a26828, 0x75646f4b, 0xb75902c3,
912	0x1f881f08, 0x834e02a4, 0x5e5b40eb, 0xc75c264d,
913	0xa8251e09, 0x28e46bbd, 0x12cb1f31, 0x9a2af615,
914	});
915	try testType(@Vector(32, u32), .{
916	0x131bbb7b, 0xa7311026, 0x9d5e59a0, 0x99b090d6,
917	0xfe969e2e, 0x04547697, 0x357d3250, 0x43be6d7a,
918	0x16ecf5c5, 0xf60febcc, 0x1d1e2602, 0x138a96d2,
919	0x9117ba72, 0x9f185b32, 0xc10e23fd, 0x3e6b7fd8,
920	0x4dc9be70, 0x2ee30047, 0xaffeab60, 0x7172d362,
921	0x6154bfcf, 0x5388dc3e, 0xd6e5a76e, 0x8b782f2d,
922	0xacbef4a2, 0x843aca71, 0x25d8ab5c, 0xe1a63a39,
923	0xc26212e5, 0x0847b84b, 0xb53541e5, 0x0c8e44db,
924	}, .{
925	0x4ad92822, 0x715b623f, 0xa5bed8a7, 0x937447a9,
926	0x7ecb38eb, 0x0a2f3dfc, 0x96f467a2, 0xec882793,
927	0x41a8707f, 0xf7310656, 0x76217b80, 0x2058e5fc,
928	0x26682154, 0x87313e31, 0x4bdc480a, 0x193572ff,
929	0x60b03c75, 0x0fe45908, 0x56c73703, 0xdb86554c,
930	0xdda2dd7d, 0x34371b27, 0xe4e6ad50, 0x422d1828,
931	0x1de3801b, 0xdce268d3, 0x20af9ec8, 0x188a591f,
932	0xf080e943, 0xc8718d14, 0x3f920382, 0x18d101b5,
933	});
934
935	try testType(@Vector(1, u64), .{
936	0x333f593bf9d08546,
937	}, .{
938	0x6918bd767e730778,
939	});
940	try testType(@Vector(2, u64), .{
941	0x4cd89a317b03d430, 0x28998f61842f63a9,
942	}, .{
943	0x6c34db64af0e217e, 0x57aa5d02cd45dceb,
944	});
945	try testType(@Vector(4, u64), .{
946	0x946cf7e7484691c9, 0xf4fc5be2a762fcbf,
947	0x71cc83bc25abaf14, 0xc69cef44c6f833a1,
948	}, .{
949	0x9f90cbd6c3ce1d4e, 0x182f65295dff4e84,
950	0x4dfe62c59fed0040, 0x18402347c1db1999,
951	});
952	try testType(@Vector(8, u64), .{
953	0x92c6281333943e2c, 0xa97750504668efb5,
954	0x234be51057c0181f, 0xefbc1f407f3df4fb,
955	0x8da6cc7c39cebb94, 0xb408f7e56feee497,
956	0x2363f1f8821592ed, 0x01716e800c0619e1,
957	}, .{
958	0xa617426684147e7e, 0x7542da7ebe093a7b,
959	0x3f21d99ac57606b7, 0x65cd36d697d22de4,
960	0xed23d6bdf176c844, 0x2d4573f100ff7b58,
961	0x4968f4d21b49f8ab, 0xf5d9a205d453e933,
962	});
963	try testType(@Vector(16, u64), .{
964	0x2f61a4ee66177b4a, 0xf13b286b279f6a93,
965	0x36b46beb63665318, 0x74294dbde0da98d2,
966	0x3aa872ba60b936eb, 0xe8f698b36e62600b,
967	0x9e8930c21a6a1a76, 0x876998b09b8eb03c,
968	0xa0244771a2ec0adb, 0xb4c72bff3d3ac1a2,
969	0xd70677210830eced, 0x6622abc1734dd72d,
970	0x157e2bb0d57d6596, 0x2aac8192fb7ef973,
971	0xc4a0ca92f34d7b13, 0x04300f8ad1845246,
972	}, .{
973	0xeaf71dcf0eb76f5d, 0x0e84b1b63dc97139,
974	0x0f64cc38d23c94a1, 0x12775cf0816349b7,
975	0xfdcf13387ba48d54, 0xf8d3c672cacd8779,
976	0xe728c1f5eb56ab1e, 0x05931a34877f7a69,
977	0x1861a763c8dafd1f, 0x4ac97573ecd5739f,
978	0x3384414c9bf77b8c, 0x32c15bbd04a5ddc4,
979	0xbfd88aee1d82ed32, 0x20e91c15b701059a,
980	0xed533d18f8657f3f, 0x1ddd7cd7f6bab957,
981	});
982
983	try testType(@Vector(1, u128), .{
984	0x5f11e16b0ca3392f907a857881455d2e,
985	}, .{
986	0xf9142d73b408fd6955922f9fc147f7d7,
987	});
988	try testType(@Vector(2, u128), .{
989	0xee0fb41fabd805923fb21b5c658e3a87,
990	0x2352e74aad6c58b3255ff0bba5aa6552,
991	}, .{
992	0x8d822f9fdd9cb9a5b43513b14419b224,
993	0x1aef2a02704379e38ead4d53d69e4cc4,
994	});
995	try testType(@Vector(4, u128), .{
996	0xc74437a4ea3bbbb193dbf0ea2f0c5281,
997	0x039e4b1640868248780db1834a0027eb,
998	0xb9e8bb34155b2b238da20331d08ff85b,
999	0x863802d34a54c2e6aa71dd0f067c4904,
1000	}, .{
1001	0x7471bae24ff7b84ab107f86ba2b7d1e7,
1002	0x8f34c449d0576e682c20bda74aa6b6c9,
1003	0x1f34c3efa167b61c48c9d5ec01a1a93f,
1004	0x71c8318fcf3ddc7be058c73a52dce9e3,
1005	});
1006	try testType(@Vector(8, u128), .{
1007	0xbf2db71463037f55ee338431f902a906,
1008	0xb7ad317626655f38ab25ae30d8a1aa67,
1009	0x7d3c5a3ffaa607b5560d69ae3fcf7863,
1010	0x009a39a8badf8b628c686dc176aa1273,
1011	0x49dba3744c91304cc7bbbdab61b6c969,
1012	0x6ec664b624f7acf79ce69d80ed7bc85c,
1013	0xe02d7a303c0f00c39010f3b815547f1c,
1014	0xb13e1ee914616f58cffe6acd33d9b5c8,
1015	}, .{
1016	0x2f2d355a071942a7384f82ba72a945b8,
1017	0x61f151b3afec8cb7664f813cecf581d1,
1018	0x5bfbf5484f3a07f0eacc4739ff48af80,
1019	0x59c0abbf8d829cf525a87d5c9c41a38a,
1020	0xdad8b18eb680f0520ca49ebfb5842e22,
1021	0xa05adcaedd9057480b3ba0413d003cec,
1022	0x8b0b4a27fc94a0e90652d19bc755b63d,
1023	0xa858bce5ad0e48c13588a4e170e8667c,
1024	});
1025
1026	try testType(@Vector(1, u256), .{
1027	0x28df37e1f57a56133ba3f5b5b2164ce24eb6c29a8973a597fd91fbee8ab4bafb,
1028	}, .{
1029	0x63f725028cab082b5b1e6cb474428c8c3655cf438f3bb05c7a87f8270198f357,
1030	});
1031	try testType(@Vector(2, u256), .{
1032	0xcc79740b85597ef411e6d7e92049dfaa2328781ea4911540a3dcb512b71c7f3c,
1033	0x51ae46d2f93cbecff1578481f6ddc633dacee94ecaf81597c752c5c5db0ae766,
1034	}, .{
1035	0x257f0107305cb71cef582a9a58612a019f335e390d7998f51f5898f245874a6e,
1036	0x0a95a17323a4d16a715720f122b752785e9877e3dd3d3f9b72cdac3d1139a81f,
1037	});
1038	try testType(@Vector(4, u256), .{
1039	0x19667a6e269342cba437a8904c7ba42a762358d32723723ae2637b01124e63c5,
1040	0x14f7d3599a7edc7bcc46874f68d4291793e6ef72bd1f3763bc5e923f54f2f781,
1041	0x1c939de0ae980b80de773a04088ba45813441336cdfdc281ee356c98d71f653b,
1042	0x39f5d755965382fe13d1b1d6690b8e3827f153f8166768c4ad8a28a963b781f2,
1043	}, .{
1044	0xbe03de37cdcb8126083b4e86cd8a9803121d31b186fd5ce555ad77ce624dd6c7,
1045	0xa0c0730f0d7f141cc959849d09730b049f00693361539f1bc4758270554a60c1,
1046	0x2664bdba8de4eaa36ecee72f6bfec5b4daa6b4e00272d8116f2cc532c29490cc,
1047	0xe47a122bd45d5e7d69722d864a6b795ddee965a0993094f8791dd309d692de8b,
1048	});
1049
1050	try testType(@Vector(1, u512), .{
1051	0x651058c1d89a8f34cfc5e66b6d25294eecfcc4a7e1e4a356eb51ee7d7b2db25378e4afee51b7d18d16e520772a60c50a02d7966f40ced1870b32c658e5821397,
1052	}, .{
1053	0xd726e265ec80cb99510ba4f480ca64e959de5c528a7f54c386ecad22eeeefa845f0fd44b1bd64258a5f868197ee2d8fed59df9c9f0b72e74051a7ff20230880e,
1054	});
1055	try testType(@Vector(2, u512), .{
1056	0x22c8183c95cca8b09fdf541e431b73e9e4a1a5a00dff12381937fab52681d09d38ea25727d7025a2be08942cfa01535759e1644792e347c7901ec94b343c6337,
1057	0x292fdf644e75927e1aea9465ae2f60fb27550cd095f1afdea2cf7855286d26fbeed1c0b9c0474b73cb6b75621f7eadaa2f94ec358179ce2aaa0766df20da1ef3,
1058	}, .{
1059	0xe1cd8c0ca244c6626d4415e10b4ac43fa69e454c529c24fec4b13e6b945684d4ea833709c16c636ca78cffa5c5bf0fe945cd714a9ad695184a6bdad31dec9e31,
1060	0x8fa3d86099e9e2789d72f8e792290356d659ab20ac0414ff94745984c6ae7d986082197bb849889f912e896670aa2c1a11bd7e66e3f650710b0f0a18a1533f90,
1061	});
1062
1063	try testType(@Vector(1, u1024), .{
1064	0x0ca1a0dfaf8bb1da714b457d23c71aef948e66c7cd45c0aa941498a796fb18502ec32f34e885d0a107d44ae81595f8b52c2f0fb38e584b7139903a0e8a823ae20d01ca0662722dd474e7efc40f32d74cc065d97d8a09d0447f1ab6107fa0a57f3f8c866ae872506627ce82f18add79cee8dc69837f4ead3ca770c4d622d7e544,
1065	}, .{
1066	0xf1e3bbe031d59351770a7a501b6e969b2c00d144f17648db3f944b69dfeb7be72e5ff933a061eba4eaa422f8ca09e5a97d0b0dd740fd4076eba8c72d7a278523f399202dc2d043c4e0eb58a2bcd4066e2146e321810b1ee4d3afdddb4f026bcc7905ce17e033a7727b4e08f33b53c63d8c9f763fc6c31d0523eb38c30d5e40bc,
1067	});
1068	}
1069
1070	inline fn bitNot(comptime Type: type, rhs: Type) @TypeOf(~rhs) {
1071	return ~rhs;
1072	}
1073	test bitNot {
1074	try testUnary(bitNot);
1075	}
1076
1077	inline fn clz(comptime Type: type, rhs: Type) @TypeOf(@clz(rhs)) {
1078	return @clz(rhs);
1079	}
1080	test clz {
1081	try testUnary(clz);
1082	}
1083
1084	inline fn bitAnd(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs & rhs) {
1085	return lhs & rhs;
1086	}
1087	test bitAnd {
1088	try testBinary(bitAnd);
1089	}
1090
1091	inline fn bitOr(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs \| rhs) {
1092	return lhs \| rhs;
1093	}
1094	test bitOr {
1095	try testBinary(bitOr);
1096	}
1097
1098	inline fn bitXor(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs ^ rhs) {
1099	return lhs ^ rhs;
1100	}
1101	test bitXor {
1102	try testBinary(bitXor);
1103	}

test/behavior/x86_64/mem.zig created+37

...	@@ -0,0 +1,37 @@
1	fn access(comptime array: anytype) !void {
2	var slice: []const @typeInfo(@TypeOf(array)).array.child = undefined;
3	slice = &array;
4	inline for (0.., &array) \|ct_index, *elem\| {
5	var rt_index: usize = undefined;
6	rt_index = ct_index;
7	if (&(slice.ptr + ct_index)[0] != elem) return error.Unexpected;
8	if (&(slice.ptr + rt_index)[0] != elem) return error.Unexpected;
9	if (&slice.ptr[ct_index..][0] != elem) return error.Unexpected;
10	if (&slice.ptr[rt_index..][0] != elem) return error.Unexpected;
11	if (&slice.ptr[ct_index] != elem) return error.Unexpected;
12	if (&slice.ptr[rt_index] != elem) return error.Unexpected;
13	if (&slice[ct_index..].ptr[0] != elem) return error.Unexpected;
14	if (&slice[rt_index..].ptr[0] != elem) return error.Unexpected;
15	if (&slice[ct_index] != elem) return error.Unexpected;
16	if (&slice[rt_index] != elem) return error.Unexpected;
17	if (slice.ptr[ct_index] != elem.*) return error.Unexpected;
18	if (slice.ptr[rt_index] != elem.*) return error.Unexpected;
19	if (slice[ct_index] != elem.*) return error.Unexpected;
20	if (slice[rt_index] != elem.*) return error.Unexpected;
21	}
22	}
23	test access {
24	try access([3]u8{ 0xdb, 0xef, 0xbd });
25	try access([3]u16{ 0x340e, 0x3654, 0x88d7 });
26	try access([3]u32{ 0xd424c2c0, 0x2d6ac466, 0x5a0cfaba });
27	try access([3]u64{
28	0x9327a4f5221666a6,
29	0x5c34d3ddd84a8b12,
30	0xbae087f39f649260,
31	});
32	try access([3]u128{
33	0x601cf010065444d4d42d5536dd9b95db,
34	0xa03f592fcaa22d40af23a0c735531e3c,
35	0x5da44907b31602b95c2d93f0b582ceab,
36	});
37	}

tools/lldb_pretty_printers.py+1-1

...	@@ -383,7 +383,7 @@ def InstRef_SummaryProvider(value, _=None):	...	@@ -383,7 +383,7 @@ def InstRef_SummaryProvider(value, _=None):
383	'InternPool.Index(%d)' % value.unsigned if value.unsigned < 0x80000000 else 'instructions[%d]' % (value.unsigned - 0x80000000))	383	'InternPool.Index(%d)' % value.unsigned if value.unsigned < 0x80000000 else 'instructions[%d]' % (value.unsigned - 0x80000000))
384		384
385	def InstIndex_SummaryProvider(value, _=None):	385	def InstIndex_SummaryProvider(value, _=None):
386	return 'instructions[%d]' % value.unsigned	386	return 'instructions[%d]' % value.unsigned if value.unsigned < 0x80000000 else 'temps[%d]' % (value.unsigned - 0x80000000)
387		387
388	class zig_DeclIndex_SynthProvider:	388	class zig_DeclIndex_SynthProvider:
389	def __init__(self, value, _=None): self.value = value	389	def __init__(self, value, _=None): self.value = value

tools/update_cpu_features.zig+135-3

...	@@ -902,8 +902,8 @@ const llvm_targets = [_]LlvmTarget{	...	@@ -902,8 +902,8 @@ const llvm_targets = [_]LlvmTarget{
902	.features = &.{ "v8a", "exynos" },	902	.features = &.{ "v8a", "exynos" },
903	},	903	},
904	},	904	},
905	// LLVM removed support for v2 and v3 but zig wants to support targeting old hardware
906	.extra_features = &.{	905	.extra_features = &.{
		906	// LLVM removed support for v2 and v3 but zig wants to support targeting old hardware
907	.{	907	.{
908	.zig_name = "v2",	908	.zig_name = "v2",
909	.desc = "ARMv2 architecture",	909	.desc = "ARMv2 architecture",
...	@@ -1043,10 +1043,22 @@ const llvm_targets = [_]LlvmTarget{	...	@@ -1043,10 +1043,22 @@ const llvm_targets = [_]LlvmTarget{
1043	.llvm_name = "64bit-mode",	1043	.llvm_name = "64bit-mode",
1044	.omit = true,	1044	.omit = true,
1045	},	1045	},
		1046	.{
		1047	.llvm_name = "alderlake",
		1048	.extra_deps = &.{ "smap", "smep" },
		1049	},
1046	.{	1050	.{
1047	.llvm_name = "amdfam10",	1051	.llvm_name = "amdfam10",
1048	.extra_deps = &.{"3dnowa"},	1052	.extra_deps = &.{"3dnowa"},
1049	},	1053	},
		1054	.{
		1055	.llvm_name = "arrowlake",
		1056	.extra_deps = &.{ "smap", "smep" },
		1057	},
		1058	.{
		1059	.llvm_name = "arrowlake-s",
		1060	.extra_deps = &.{ "smap", "smep" },
		1061	},
1050	.{	1062	.{
1051	.llvm_name = "athlon",	1063	.llvm_name = "athlon",
1052	.extra_deps = &.{"3dnowa"},	1064	.extra_deps = &.{"3dnowa"},
...	@@ -1081,16 +1093,64 @@ const llvm_targets = [_]LlvmTarget{	...	@@ -1081,16 +1093,64 @@ const llvm_targets = [_]LlvmTarget{
1081	},	1093	},
1082	.{	1094	.{
1083	.llvm_name = "barcelona",	1095	.llvm_name = "barcelona",
1084	.extra_deps = &.{"3dnowa"},	1096	.extra_deps = &.{ "3dnowa", "smap", "smep" },
		1097	},
		1098	.{
		1099	.llvm_name = "broadwell",
		1100	.extra_deps = &.{ "smap", "smep" },
1085	},	1101	},
1086	.{	1102	.{
1087	.llvm_name = "c3",	1103	.llvm_name = "c3",
1088	.extra_deps = &.{"3dnow"},	1104	.extra_deps = &.{"3dnow"},
1089	},	1105	},
		1106	.{
		1107	.llvm_name = "cannonlake",
		1108	.extra_deps = &.{ "smap", "smep" },
		1109	},
		1110	.{
		1111	.llvm_name = "cascadelake",
		1112	.extra_deps = &.{ "smap", "smep" },
		1113	},
		1114	.{
		1115	.llvm_name = "emeraldrapids",
		1116	.extra_deps = &.{ "smap", "smep" },
		1117	},
1090	.{	1118	.{
1091	.llvm_name = "geode",	1119	.llvm_name = "geode",
1092	.extra_deps = &.{"3dnowa"},	1120	.extra_deps = &.{"3dnowa"},
1093	},	1121	},
		1122	.{
		1123	.llvm_name = "goldmont",
		1124	.extra_deps = &.{ "smap", "smep" },
		1125	},
		1126	.{
		1127	.llvm_name = "goldmont_plus",
		1128	.extra_deps = &.{ "smap", "smep" },
		1129	},
		1130	.{
		1131	.llvm_name = "haswell",
		1132	.extra_deps = &.{"smep"},
		1133	},
		1134	.{
		1135	.llvm_name = "i386",
		1136	.extra_deps = &.{"bsf_bsr_0_clobbers_result"},
		1137	},
		1138	.{
		1139	.llvm_name = "i486",
		1140	.extra_deps = &.{"bsf_bsr_0_clobbers_result"},
		1141	},
		1142	.{
		1143	.llvm_name = "icelake_client",
		1144	.extra_deps = &.{ "smap", "smep" },
		1145	},
		1146	.{
		1147	.llvm_name = "icelake_server",
		1148	.extra_deps = &.{ "smap", "smep" },
		1149	},
		1150	.{
		1151	.llvm_name = "ivybridge",
		1152	.extra_deps = &.{"smep"},
		1153	},
1094	.{	1154	.{
1095	.llvm_name = "k6-2",	1155	.llvm_name = "k6-2",
1096	.extra_deps = &.{"3dnow"},	1156	.extra_deps = &.{"3dnow"},
...	@@ -1127,6 +1187,10 @@ const llvm_targets = [_]LlvmTarget{	...	@@ -1127,6 +1187,10 @@ const llvm_targets = [_]LlvmTarget{
1127	.llvm_name = "lakemont",	1187	.llvm_name = "lakemont",
1128	.extra_deps = &.{"soft_float"},	1188	.extra_deps = &.{"soft_float"},
1129	},	1189	},
		1190	.{
		1191	.llvm_name = "meteorlake",
		1192	.extra_deps = &.{ "smap", "smep" },
		1193	},
1130	.{	1194	.{
1131	.llvm_name = "opteron",	1195	.llvm_name = "opteron",
1132	.extra_deps = &.{"3dnowa"},	1196	.extra_deps = &.{"3dnowa"},
...	@@ -1135,6 +1199,38 @@ const llvm_targets = [_]LlvmTarget{	...	@@ -1135,6 +1199,38 @@ const llvm_targets = [_]LlvmTarget{
1135	.llvm_name = "opteron-sse3",	1199	.llvm_name = "opteron-sse3",
1136	.extra_deps = &.{"3dnowa"},	1200	.extra_deps = &.{"3dnowa"},
1137	},	1201	},
		1202	.{
		1203	.llvm_name = "raptorlake",
		1204	.extra_deps = &.{ "smap", "smep" },
		1205	},
		1206	.{
		1207	.llvm_name = "rocketlake",
		1208	.extra_deps = &.{ "smap", "smep" },
		1209	},
		1210	.{
		1211	.llvm_name = "sapphirerapids",
		1212	.extra_deps = &.{ "smap", "smep" },
		1213	},
		1214	.{
		1215	.llvm_name = "silvermont",
		1216	.extra_deps = &.{"smep"},
		1217	},
		1218	.{
		1219	.llvm_name = "skx",
		1220	.extra_deps = &.{ "smap", "smep" },
		1221	},
		1222	.{
		1223	.llvm_name = "skylake",
		1224	.extra_deps = &.{ "smap", "smep" },
		1225	},
		1226	.{
		1227	.llvm_name = "skylake_avx512",
		1228	.extra_deps = &.{ "smap", "smep" },
		1229	},
		1230	.{
		1231	.llvm_name = "tigerlake",
		1232	.extra_deps = &.{ "smap", "smep" },
		1233	},
1138	.{	1234	.{
1139	.llvm_name = "winchip2",	1235	.llvm_name = "winchip2",
1140	.extra_deps = &.{"3dnow"},	1236	.extra_deps = &.{"3dnow"},
...	@@ -1143,9 +1239,29 @@ const llvm_targets = [_]LlvmTarget{	...	@@ -1143,9 +1239,29 @@ const llvm_targets = [_]LlvmTarget{
1143	.llvm_name = "sse4.2",	1239	.llvm_name = "sse4.2",
1144	.extra_deps = &.{"crc32"},	1240	.extra_deps = &.{"crc32"},
1145	},	1241	},
		1242	.{
		1243	.llvm_name = "znver1",
		1244	.extra_deps = &.{ "smap", "smep" },
		1245	},
		1246	.{
		1247	.llvm_name = "znver2",
		1248	.extra_deps = &.{ "smap", "smep" },
		1249	},
		1250	.{
		1251	.llvm_name = "znver3",
		1252	.extra_deps = &.{ "smap", "smep" },
		1253	},
		1254	.{
		1255	.llvm_name = "znver4",
		1256	.extra_deps = &.{ "smap", "smep" },
		1257	},
		1258	.{
		1259	.llvm_name = "znver5",
		1260	.extra_deps = &.{ "smap", "smep" },
		1261	},
1146	},	1262	},
1147	// Features removed from LLVM
1148	.extra_features = &.{	1263	.extra_features = &.{
		1264	// Features removed from LLVM
1149	.{	1265	.{
1150	.zig_name = "3dnow",	1266	.zig_name = "3dnow",
1151	.desc = "Enable 3DNow! instructions",	1267	.desc = "Enable 3DNow! instructions",
...	@@ -1171,6 +1287,22 @@ const llvm_targets = [_]LlvmTarget{	...	@@ -1171,6 +1287,22 @@ const llvm_targets = [_]LlvmTarget{
1171	.desc = "Prefetch with Intent to Write and T1 Hint",	1287	.desc = "Prefetch with Intent to Write and T1 Hint",
1172	.deps = &.{},	1288	.deps = &.{},
1173	},	1289	},
		1290	// Custom Zig features
		1291	.{
		1292	.zig_name = "bsf_bsr_0_clobbers_result",
		1293	.desc = "BSF/BSR may clobber the lower 32-bits of the result register when the source is zero",
		1294	.deps = &.{},
		1295	},
		1296	.{
		1297	.zig_name = "smap",
		1298	.desc = "Enable Supervisor Mode Access Prevention",
		1299	.deps = &.{},
		1300	},
		1301	.{
		1302	.zig_name = "smep",
		1303	.desc = "Enable Supervisor Mode Execution Prevention",
		1304	.deps = &.{},
		1305	},
1174	},	1306	},
1175	.omit_cpus = &.{	1307	.omit_cpus = &.{
1176	// LLVM defines a bunch of dumb aliases with foreach loops in X86.td.	1308	// LLVM defines a bunch of dumb aliases with foreach loops in X86.td.

...	@@ -6,13 +6,13 @@	...	@@ -6,13 +6,13 @@
6	/// `null` means native.	6	/// `null` means native.
7	cpu_arch: ?Target.Cpu.Arch = null,	7	cpu_arch: ?Target.Cpu.Arch = null,
8		8
9	cpu_model: CpuModel = CpuModel.determined_by_arch_os,	9	cpu_model: CpuModel = .determined_by_arch_os,
10		10
11	/// Sparse set of CPU features to add to the set from `cpu_model`.	11	/// Sparse set of CPU features to add to the set from `cpu_model`.
12	cpu_features_add: Target.Cpu.Feature.Set = Target.Cpu.Feature.Set.empty,	12	cpu_features_add: Target.Cpu.Feature.Set = .empty,
13		13
14	/// Sparse set of CPU features to remove from the set from `cpu_model`.	14	/// Sparse set of CPU features to remove from the set from `cpu_model`.
15	cpu_features_sub: Target.Cpu.Feature.Set = Target.Cpu.Feature.Set.empty,	15	cpu_features_sub: Target.Cpu.Feature.Set = .empty,
16		16
17	/// `null` means native.	17	/// `null` means native.
18	os_tag: ?Target.Os.Tag = null,	18	os_tag: ?Target.Os.Tag = null,
...	@@ -38,7 +38,7 @@ abi: ?Target.Abi = null,	...	@@ -38,7 +38,7 @@ abi: ?Target.Abi = null,
38		38
39	/// When `os_tag` is `null`, then `null` means native. Otherwise it means the standard path	39	/// When `os_tag` is `null`, then `null` means native. Otherwise it means the standard path
40	/// based on the `os_tag`.	40	/// based on the `os_tag`.
41	dynamic_linker: Target.DynamicLinker = Target.DynamicLinker.none,	41	dynamic_linker: Target.DynamicLinker = .none,
42		42
43	/// `null` means default for the cpu/arch/os combo.	43	/// `null` means default for the cpu/arch/os combo.
44	ofmt: ?Target.ObjectFormat = null,	44	ofmt: ?Target.ObjectFormat = null,

...	@@ -23,6 +23,7 @@ pub const Coverage = @import("debug/Coverage.zig");	...	@@ -23,6 +23,7 @@ pub const Coverage = @import("debug/Coverage.zig");
23		23
24	pub const FormattedPanic = @import("debug/FormattedPanic.zig");	24	pub const FormattedPanic = @import("debug/FormattedPanic.zig");
25	pub const SimplePanic = @import("debug/SimplePanic.zig");	25	pub const SimplePanic = @import("debug/SimplePanic.zig");
		26	pub const NoPanic = @import("debug/NoPanic.zig");
26		27
27	/// Unresolved source locations can be represented with a single `usize` that	28	/// Unresolved source locations can be represented with a single `usize` that
28	/// corresponds to a virtual memory address of the program counter. Combined	29	/// corresponds to a virtual memory address of the program counter. Combined
...	@@ -179,7 +180,7 @@ pub fn dumpHexFallible(bytes: []const u8) !void {	...	@@ -179,7 +180,7 @@ pub fn dumpHexFallible(bytes: []const u8) !void {
179	/// TODO multithreaded awareness	180	/// TODO multithreaded awareness
180	pub fn dumpCurrentStackTrace(start_addr: ?usize) void {	181	pub fn dumpCurrentStackTrace(start_addr: ?usize) void {
181	nosuspend {	182	nosuspend {
182	if (comptime builtin.target.isWasm()) {	183	if (builtin.target.isWasm()) {
183	if (native_os == .wasi) {	184	if (native_os == .wasi) {
184	const stderr = io.getStdErr().writer();	185	const stderr = io.getStdErr().writer();
185	stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;	186	stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;
...	@@ -267,7 +268,7 @@ pub inline fn getContext(context: *ThreadContext) bool {	...	@@ -267,7 +268,7 @@ pub inline fn getContext(context: *ThreadContext) bool {
267	/// TODO multithreaded awareness	268	/// TODO multithreaded awareness
268	pub fn dumpStackTraceFromBase(context: *ThreadContext) void {	269	pub fn dumpStackTraceFromBase(context: *ThreadContext) void {
269	nosuspend {	270	nosuspend {
270	if (comptime builtin.target.isWasm()) {	271	if (builtin.target.isWasm()) {
271	if (native_os == .wasi) {	272	if (native_os == .wasi) {
272	const stderr = io.getStdErr().writer();	273	const stderr = io.getStdErr().writer();
273	stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;	274	stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;
...	@@ -365,7 +366,7 @@ pub fn captureStackTrace(first_address: ?usize, stack_trace: *std.builtin.StackT	...	@@ -365,7 +366,7 @@ pub fn captureStackTrace(first_address: ?usize, stack_trace: *std.builtin.StackT
365	/// TODO multithreaded awareness	366	/// TODO multithreaded awareness
366	pub fn dumpStackTrace(stack_trace: std.builtin.StackTrace) void {	367	pub fn dumpStackTrace(stack_trace: std.builtin.StackTrace) void {
367	nosuspend {	368	nosuspend {
368	if (comptime builtin.target.isWasm()) {	369	if (builtin.target.isWasm()) {
369	if (native_os == .wasi) {	370	if (native_os == .wasi) {
370	const stderr = io.getStdErr().writer();	371	const stderr = io.getStdErr().writer();
371	stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;	372	stderr.print("Unable to dump stack trace: not implemented for Wasm\n", .{}) catch return;

...	@@ -121,13 +121,13 @@ pub fn deinit(self: *SelfInfo) void {	...	@@ -121,13 +121,13 @@ pub fn deinit(self: *SelfInfo) void {
121	}	121	}
122		122
123	pub fn getModuleForAddress(self: SelfInfo, address: usize) !Module {	123	pub fn getModuleForAddress(self: SelfInfo, address: usize) !Module {
124	if (comptime builtin.target.isDarwin()) {	124	if (builtin.target.isDarwin()) {
125	return self.lookupModuleDyld(address);	125	return self.lookupModuleDyld(address);
126	} else if (native_os == .windows) {	126	} else if (native_os == .windows) {
127	return self.lookupModuleWin32(address);	127	return self.lookupModuleWin32(address);
128	} else if (native_os == .haiku) {	128	} else if (native_os == .haiku) {
129	return self.lookupModuleHaiku(address);	129	return self.lookupModuleHaiku(address);
130	} else if (comptime builtin.target.isWasm()) {	130	} else if (builtin.target.isWasm()) {
131	return self.lookupModuleWasm(address);	131	return self.lookupModuleWasm(address);
132	} else {	132	} else {
133	return self.lookupModuleDl(address);	133	return self.lookupModuleDl(address);
...	@@ -138,13 +138,13 @@ pub fn getModuleForAddress(self: SelfInfo, address: usize) !Module {	...	@@ -138,13 +138,13 @@ pub fn getModuleForAddress(self: SelfInfo, address: usize) !Module {
138	// This can be called when getModuleForAddress fails, so implementations should provide	138	// This can be called when getModuleForAddress fails, so implementations should provide
139	// a path that doesn't rely on any side-effects of a prior successful module lookup.	139	// a path that doesn't rely on any side-effects of a prior successful module lookup.
140	pub fn getModuleNameForAddress(self: *SelfInfo, address: usize) ?[]const u8 {	140	pub fn getModuleNameForAddress(self: *SelfInfo, address: usize) ?[]const u8 {
141	if (comptime builtin.target.isDarwin()) {	141	if (builtin.target.isDarwin()) {
142	return self.lookupModuleNameDyld(address);	142	return self.lookupModuleNameDyld(address);
143	} else if (native_os == .windows) {	143	} else if (native_os == .windows) {
144	return self.lookupModuleNameWin32(address);	144	return self.lookupModuleNameWin32(address);
145	} else if (native_os == .haiku) {	145	} else if (native_os == .haiku) {
146	return null;	146	return null;
147	} else if (comptime builtin.target.isWasm()) {	147	} else if (builtin.target.isWasm()) {
148	return null;	148	return null;
149	} else {	149	} else {
150	return self.lookupModuleNameDl(address);	150	return self.lookupModuleNameDl(address);

...	@@ -163,7 +163,7 @@ pub fn interlace(vecs: anytype) @Vector(vectorLength(@TypeOf(vecs[0])) * vecs.le	...	@@ -163,7 +163,7 @@ pub fn interlace(vecs: anytype) @Vector(vectorLength(@TypeOf(vecs[0])) * vecs.le
163	// The indices are correct. The problem seems to be with the @shuffle builtin.	163	// The indices are correct. The problem seems to be with the @shuffle builtin.
164	// On MIPS, the test that interlaces small_base gives { 0, 2, 0, 0, 64, 255, 248, 200, 0, 0 }.	164	// On MIPS, the test that interlaces small_base gives { 0, 2, 0, 0, 64, 255, 248, 200, 0, 0 }.
165	// Calling this with two inputs seems to work fine, but I'll let the compile error trigger for all inputs, just to be safe.	165	// Calling this with two inputs seems to work fine, but I'll let the compile error trigger for all inputs, just to be safe.
166	comptime if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why interlace() doesn't work on MIPS");	166	if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why interlace() doesn't work on MIPS");
167		167
168	const VecType = @TypeOf(vecs[0]);	168	const VecType = @TypeOf(vecs[0]);
169	const vecs_arr = @as([vecs.len]VecType, vecs);	169	const vecs_arr = @as([vecs.len]VecType, vecs);
...	@@ -248,7 +248,7 @@ test "vector patterns" {	...	@@ -248,7 +248,7 @@ test "vector patterns" {
248	try std.testing.expectEqual([8]u32{ 10, 20, 30, 40, 55, 66, 77, 88 }, join(base, other_base));	248	try std.testing.expectEqual([8]u32{ 10, 20, 30, 40, 55, 66, 77, 88 }, join(base, other_base));
249	try std.testing.expectEqual([2]u32{ 20, 30 }, extract(base, 1, 2));	249	try std.testing.expectEqual([2]u32{ 20, 30 }, extract(base, 1, 2));
250		250
251	if (comptime !builtin.cpu.arch.isMIPS()) {	251	if (!builtin.cpu.arch.isMIPS()) {
252	try std.testing.expectEqual([8]u32{ 10, 55, 20, 66, 30, 77, 40, 88 }, interlace(.{ base, other_base }));	252	try std.testing.expectEqual([8]u32{ 10, 55, 20, 66, 30, 77, 40, 88 }, interlace(.{ base, other_base }));
253		253
254	const small_braid = interlace(small_bases);	254	const small_braid = interlace(small_bases);
...	@@ -390,7 +390,7 @@ pub fn prefixScanWithFunc(	...	@@ -390,7 +390,7 @@ pub fn prefixScanWithFunc(
390	comptime identity: std.meta.Child(@TypeOf(vec)),	390	comptime identity: std.meta.Child(@TypeOf(vec)),
391	) if (ErrorType == void) @TypeOf(vec) else ErrorType!@TypeOf(vec) {	391	) if (ErrorType == void) @TypeOf(vec) else ErrorType!@TypeOf(vec) {
392	// I haven't debugged this, but it might be a cousin of sorts to what's going on with interlace.	392	// I haven't debugged this, but it might be a cousin of sorts to what's going on with interlace.
393	comptime if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why prefixScan doesn't work on MIPS");	393	if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why prefixScan doesn't work on MIPS");
394		394
395	const len = vectorLength(@TypeOf(vec));	395	const len = vectorLength(@TypeOf(vec));
396		396
...	@@ -465,9 +465,7 @@ test "vector prefix scan" {	...	@@ -465,9 +465,7 @@ test "vector prefix scan" {
465	if ((builtin.cpu.arch == .armeb or builtin.cpu.arch == .thumbeb) and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22060	465	if ((builtin.cpu.arch == .armeb or builtin.cpu.arch == .thumbeb) and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22060
466	if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21893	466	if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21893
467		467
468	if (comptime builtin.cpu.arch.isMIPS()) {	468	if (builtin.cpu.arch.isMIPS()) return error.SkipZigTest;
469	return error.SkipZigTest;
470	}
471		469
472	const int_base = @Vector(4, i32){ 11, 23, 9, -21 };	470	const int_base = @Vector(4, i32){ 11, 23, 9, -21 };
473	const float_base = @Vector(4, f32){ 2, 0.5, -10, 6.54321 };	471	const float_base = @Vector(4, f32){ 2, 0.5, -10, 6.54321 };

...	@@ -369,6 +369,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {	...	@@ -369,6 +369,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
369	setFeature(cpu, .bmi, bit(leaf.ebx, 3));	369	setFeature(cpu, .bmi, bit(leaf.ebx, 3));
370	// AVX2 is only supported if we have the OS save support from AVX.	370	// AVX2 is only supported if we have the OS save support from AVX.
371	setFeature(cpu, .avx2, bit(leaf.ebx, 5) and has_avx_save);	371	setFeature(cpu, .avx2, bit(leaf.ebx, 5) and has_avx_save);
		372	setFeature(cpu, .smep, bit(leaf.ebx, 7));
372	setFeature(cpu, .bmi2, bit(leaf.ebx, 8));	373	setFeature(cpu, .bmi2, bit(leaf.ebx, 8));
373	setFeature(cpu, .invpcid, bit(leaf.ebx, 10));	374	setFeature(cpu, .invpcid, bit(leaf.ebx, 10));
374	setFeature(cpu, .rtm, bit(leaf.ebx, 11));	375	setFeature(cpu, .rtm, bit(leaf.ebx, 11));
...	@@ -377,6 +378,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {	...	@@ -377,6 +378,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
377	setFeature(cpu, .avx512dq, bit(leaf.ebx, 17) and has_avx512_save);	378	setFeature(cpu, .avx512dq, bit(leaf.ebx, 17) and has_avx512_save);
378	setFeature(cpu, .rdseed, bit(leaf.ebx, 18));	379	setFeature(cpu, .rdseed, bit(leaf.ebx, 18));
379	setFeature(cpu, .adx, bit(leaf.ebx, 19));	380	setFeature(cpu, .adx, bit(leaf.ebx, 19));
		381	setFeature(cpu, .smap, bit(leaf.ebx, 20));
380	setFeature(cpu, .avx512ifma, bit(leaf.ebx, 21) and has_avx512_save);	382	setFeature(cpu, .avx512ifma, bit(leaf.ebx, 21) and has_avx512_save);
381	setFeature(cpu, .clflushopt, bit(leaf.ebx, 23));	383	setFeature(cpu, .clflushopt, bit(leaf.ebx, 23));
382	setFeature(cpu, .clwb, bit(leaf.ebx, 24));	384	setFeature(cpu, .clwb, bit(leaf.ebx, 24));

...	@@ -1312,10 +1312,10 @@ pub const Pool = struct {	...	@@ -1312,10 +1312,10 @@ pub const Pool = struct {
1312	},	1312	},
1313	else => {	1313	else => {
1314	const target = &mod.resolved_target.result;	1314	const target = &mod.resolved_target.result;
1315	const abi_align = Type.intAbiAlignment(int_info.bits, target.*, false);	1315	const abi_align = Type.intAbiAlignment(int_info.bits, target.*);
1316	const abi_align_bytes = abi_align.toByteUnits().?;	1316	const abi_align_bytes = abi_align.toByteUnits().?;
1317	const array_ctype = try pool.getArray(allocator, .{	1317	const array_ctype = try pool.getArray(allocator, .{
1318	.len = @divExact(Type.intAbiSize(int_info.bits, target.*, false), abi_align_bytes),	1318	.len = @divExact(Type.intAbiSize(int_info.bits, target.*), abi_align_bytes),
1319	.elem_ctype = try pool.fromIntInfo(allocator, .{	1319	.elem_ctype = try pool.fromIntInfo(allocator, .{
1320	.signedness = .unsigned,	1320	.signedness = .unsigned,
1321	.bits = @intCast(abi_align_bytes * 8),	1321	.bits = @intCast(abi_align_bytes * 8),
...	@@ -1429,7 +1429,7 @@ pub const Pool = struct {	...	@@ -1429,7 +1429,7 @@ pub const Pool = struct {
1429	.name = .{ .index = .len },	1429	.name = .{ .index = .len },
1430	.ctype = CType.usize,	1430	.ctype = CType.usize,
1431	.alignas = AlignAs.fromAbiAlignment(	1431	.alignas = AlignAs.fromAbiAlignment(
1432	Type.intAbiAlignment(target.ptrBitWidth(), target.*, false),	1432	Type.intAbiAlignment(target.ptrBitWidth(), target.*),
1433	),	1433	),
1434	},	1434	},
1435	};	1435	};
...	@@ -1524,7 +1524,7 @@ pub const Pool = struct {	...	@@ -1524,7 +1524,7 @@ pub const Pool = struct {
1524	.name = .{ .index = .len },	1524	.name = .{ .index = .len },
1525	.ctype = CType.usize,	1525	.ctype = CType.usize,
1526	.alignas = AlignAs.fromAbiAlignment(	1526	.alignas = AlignAs.fromAbiAlignment(
1527	Type.intAbiAlignment(target.ptrBitWidth(), target.*, false),	1527	Type.intAbiAlignment(target.ptrBitWidth(), target.*),
1528	),	1528	),
1529	},	1529	},
1530	};	1530	};
...	@@ -1644,7 +1644,7 @@ pub const Pool = struct {	...	@@ -1644,7 +1644,7 @@ pub const Pool = struct {
1644	.name = .{ .index = .@"error" },	1644	.name = .{ .index = .@"error" },
1645	.ctype = error_set_ctype,	1645	.ctype = error_set_ctype,
1646	.alignas = AlignAs.fromAbiAlignment(	1646	.alignas = AlignAs.fromAbiAlignment(
1647	Type.intAbiAlignment(error_set_bits, target.*, false),	1647	Type.intAbiAlignment(error_set_bits, target.*),
1648	),	1648	),
1649	},	1649	},
1650	.{	1650	.{

...	@@ -124,7 +124,7 @@ test "@floatFromInt(f80)" {	...	@@ -124,7 +124,7 @@ test "@floatFromInt(f80)" {
124	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	124	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
125	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	125	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
126	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	126	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
127	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	127	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
128	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	128	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
129	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	129	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
130		130
...	@@ -1362,7 +1362,7 @@ test "cast f16 to wider types" {	...	@@ -1362,7 +1362,7 @@ test "cast f16 to wider types" {
1362	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	1362	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1363	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1363	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1364	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	1364	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1365	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	1365	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
1366	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	1366	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1367	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	1367	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1368		1368
...	@@ -2539,7 +2539,6 @@ test "@intFromBool on vector" {	...	@@ -2539,7 +2539,6 @@ test "@intFromBool on vector" {
2539	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	2539	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
2540	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	2540	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
2541	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	2541	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
2542	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2543	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	2542	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
2544		2543
2545	const S = struct {	2544	const S = struct {

...	@@ -65,6 +65,8 @@ test "@clz" {	...	@@ -65,6 +65,8 @@ test "@clz" {
65	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	65	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
66	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	66	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
67	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	67	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
		68	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
		69	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
68		70
69	try testClz();	71	try testClz();
70	try comptime testClz();	72	try comptime testClz();
...	@@ -75,6 +77,7 @@ fn testClz() !void {	...	@@ -75,6 +77,7 @@ fn testClz() !void {
75	try expect(testOneClz(u8, 0b00001010) == 4);	77	try expect(testOneClz(u8, 0b00001010) == 4);
76	try expect(testOneClz(u8, 0b00011010) == 3);	78	try expect(testOneClz(u8, 0b00011010) == 3);
77	try expect(testOneClz(u8, 0b00000000) == 8);	79	try expect(testOneClz(u8, 0b00000000) == 8);
		80	try expect(testOneClz(i8, -1) == 0);
78	}	81	}
79		82
80	test "@clz big ints" {	83	test "@clz big ints" {
...	@@ -100,7 +103,7 @@ fn testOneClz(comptime T: type, x: T) u32 {	...	@@ -100,7 +103,7 @@ fn testOneClz(comptime T: type, x: T) u32 {
100		103
101	test "@clz vectors" {	104	test "@clz vectors" {
102	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	105	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
103	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	106	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
104	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	107	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
105	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	108	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
106	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	109	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -159,6 +162,8 @@ fn testCtz() !void {	...	@@ -159,6 +162,8 @@ fn testCtz() !void {
159	try expect(testOneCtz(u8, 0b10100000) == 5);	162	try expect(testOneCtz(u8, 0b10100000) == 5);
160	try expect(testOneCtz(u8, 0b10001010) == 1);	163	try expect(testOneCtz(u8, 0b10001010) == 1);
161	try expect(testOneCtz(u8, 0b00000000) == 8);	164	try expect(testOneCtz(u8, 0b00000000) == 8);
		165	try expect(testOneCtz(i8, -1) == 0);
		166	try expect(testOneCtz(i8, -2) == 1);
162	try expect(testOneCtz(u16, 0b00000000) == 16);	167	try expect(testOneCtz(u16, 0b00000000) == 16);
163	}	168	}
164		169
...	@@ -467,7 +472,6 @@ test "division" {	...	@@ -467,7 +472,6 @@ test "division" {
467	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	472	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
468	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	473	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
469	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	474	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
470	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
471	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	475	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
472	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	476	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
473		477
...	@@ -583,7 +587,7 @@ fn testFloatDivision() !void {	...	@@ -583,7 +587,7 @@ fn testFloatDivision() !void {
583	}	587	}
584		588
585	test "large integer division" {	589	test "large integer division" {
586	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;	590	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
587	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	591	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
588	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	592	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
589	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;	593	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
...	@@ -780,7 +784,7 @@ test "128-bit multiplication" {	...	@@ -780,7 +784,7 @@ test "128-bit multiplication" {
780	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	784	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
781	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	785	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
782	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	786	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
783	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	787	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
784	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	788	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
785		789
786	{	790	{
...	@@ -1369,7 +1373,7 @@ test "remainder division" {	...	@@ -1369,7 +1373,7 @@ test "remainder division" {
1369	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1373	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1370	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	1374	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1371	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	1375	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1372	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	1376	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
1373	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	1377	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1374		1378
1375	if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {	1379	if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {
...	@@ -1522,7 +1526,7 @@ test "@round f80" {	...	@@ -1522,7 +1526,7 @@ test "@round f80" {
1522	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	1526	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1523	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1527	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1524	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	1528	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1525	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	1529	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
1526	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	1530	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1527	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	1531	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1528		1532
...	@@ -1535,7 +1539,7 @@ test "@round f128" {	...	@@ -1535,7 +1539,7 @@ test "@round f128" {
1535	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	1539	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1536	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1540	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1537	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	1541	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1538	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	1542	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
1539	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	1543	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1540	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	1544	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1541		1545
...	@@ -1712,7 +1716,7 @@ test "mod lazy values" {	...	@@ -1712,7 +1716,7 @@ test "mod lazy values" {
1712		1716
1713	test "@clz works on both vector and scalar inputs" {	1717	test "@clz works on both vector and scalar inputs" {
1714	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	1718	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1715	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	1719	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1716	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	1720	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1717	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	1721	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1718	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1722	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

...	@@ -58,7 +58,7 @@ test "@mulAdd f80" {	...	@@ -58,7 +58,7 @@ test "@mulAdd f80" {
58	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	58	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
59	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	59	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
60	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	60	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
61	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	61	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
62	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	62	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
63	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	63	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
64		64
...	@@ -79,7 +79,7 @@ test "@mulAdd f128" {	...	@@ -79,7 +79,7 @@ test "@mulAdd f128" {
79	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	79	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
80	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	80	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
81	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	81	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
82	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	82	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
83	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	83	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
84	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	84	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
85		85
...	@@ -189,7 +189,7 @@ test "vector f80" {	...	@@ -189,7 +189,7 @@ test "vector f80" {
189	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	189	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
190	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	190	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
191	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	191	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
192	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	192	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
193	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	193	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
194		194
195	try comptime vector80();	195	try comptime vector80();
...	@@ -216,7 +216,7 @@ test "vector f128" {	...	@@ -216,7 +216,7 @@ test "vector f128" {
216	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	216	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
217	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	217	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
218	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	218	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
219	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	219	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
220	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	220	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
221		221
222	try comptime vector128();	222	try comptime vector128();

...	@@ -164,10 +164,10 @@ test "saturating multiplication <= 32 bits" {	...	@@ -164,10 +164,10 @@ test "saturating multiplication <= 32 bits" {
164	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	164	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
165	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	165	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
166	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	166	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
167	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	167	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
168	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	168	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
169		169
170	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .wasm32) {	170	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isWasm()) {
171	// https://github.com/ziglang/zig/issues/9660	171	// https://github.com/ziglang/zig/issues/9660
172	return error.SkipZigTest;	172	return error.SkipZigTest;
173	}	173	}
...	@@ -264,10 +264,10 @@ test "saturating multiplication" {	...	@@ -264,10 +264,10 @@ test "saturating multiplication" {
264	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	264	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
265	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	265	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
266	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	266	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
267	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	267	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
268	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	268	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
269		269
270	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .wasm32) {	270	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isWasm()) {
271	// https://github.com/ziglang/zig/issues/9660	271	// https://github.com/ziglang/zig/issues/9660
272	return error.SkipZigTest;	272	return error.SkipZigTest;
273	}	273	}
...	@@ -311,7 +311,7 @@ test "saturating shift-left" {	...	@@ -311,7 +311,7 @@ test "saturating shift-left" {
311	try testSatShl(i8, 127, 1, 127);	311	try testSatShl(i8, 127, 1, 127);
312	try testSatShl(i8, -128, 1, -128);	312	try testSatShl(i8, -128, 1, -128);
313	// TODO: remove this check once #9668 is completed	313	// TODO: remove this check once #9668 is completed
314	if (builtin.cpu.arch != .wasm32) {	314	if (!builtin.cpu.arch.isWasm()) {
315	// skip testing ints > 64 bits on wasm due to miscompilation / wasmtime ci error	315	// skip testing ints > 64 bits on wasm due to miscompilation / wasmtime ci error
316	try testSatShl(i128, maxInt(i128), 64, maxInt(i128));	316	try testSatShl(i128, maxInt(i128), 64, maxInt(i128));
317	try testSatShl(u128, maxInt(u128), 64, maxInt(u128));	317	try testSatShl(u128, maxInt(u128), 64, maxInt(u128));

...	@@ -418,8 +418,8 @@ test "packed struct 24bits" {	...	@@ -418,8 +418,8 @@ test "packed struct 24bits" {
418	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	418	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
419	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	419	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
420	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	420	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
421	if (builtin.cpu.arch == .wasm32) return error.SkipZigTest; // TODO	421	if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; // TODO
422	if (comptime builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO	422	if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
423	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	423	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
424	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	424	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
425		425
...	@@ -818,7 +818,7 @@ test "non-packed struct with u128 entry in union" {	...	@@ -818,7 +818,7 @@ test "non-packed struct with u128 entry in union" {
818	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	818	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
819	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	819	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
820	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	820	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
821	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	821	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
822	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	822	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
823		823
824	const U = union(enum) {	824	const U = union(enum) {
...	@@ -941,7 +941,7 @@ test "tuple assigned to variable" {	...	@@ -941,7 +941,7 @@ test "tuple assigned to variable" {
941		941
942	test "comptime struct field" {	942	test "comptime struct field" {
943	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	943	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
944	if (comptime builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO	944	if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
945		945
946	const T = struct {	946	const T = struct {
947	a: i32,	947	a: i32,

...	@@ -100,7 +100,7 @@ test "simple variadic function" {	...	@@ -100,7 +100,7 @@ test "simple variadic function" {
100	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	100	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
101	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	101	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
102	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	102	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
103	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {	103	if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
104	// https://github.com/ziglang/zig/issues/14096	104	// https://github.com/ziglang/zig/issues/14096
105	return error.SkipZigTest;	105	return error.SkipZigTest;
106	}	106	}
...	@@ -161,7 +161,7 @@ test "coerce reference to var arg" {	...	@@ -161,7 +161,7 @@ test "coerce reference to var arg" {
161	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	161	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
162	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	162	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
163	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	163	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
164	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {	164	if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
165	// https://github.com/ziglang/zig/issues/14096	165	// https://github.com/ziglang/zig/issues/14096
166	return error.SkipZigTest;	166	return error.SkipZigTest;
167	}	167	}
...	@@ -194,7 +194,7 @@ test "variadic functions" {	...	@@ -194,7 +194,7 @@ test "variadic functions" {
194	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	194	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
195	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	195	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
196	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	196	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
197	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {	197	if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
198	// https://github.com/ziglang/zig/issues/14096	198	// https://github.com/ziglang/zig/issues/14096
199	return error.SkipZigTest;	199	return error.SkipZigTest;
200	}	200	}
...	@@ -239,7 +239,7 @@ test "copy VaList" {	...	@@ -239,7 +239,7 @@ test "copy VaList" {
239	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	239	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
240	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	240	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
241	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	241	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
242	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {	242	if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
243	// https://github.com/ziglang/zig/issues/14096	243	// https://github.com/ziglang/zig/issues/14096
244	return error.SkipZigTest;	244	return error.SkipZigTest;
245	}	245	}
...	@@ -273,7 +273,7 @@ test "unused VaList arg" {	...	@@ -273,7 +273,7 @@ test "unused VaList arg" {
273	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	273	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
274	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	274	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
275	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	275	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
276	if (builtin.os.tag != .macos and comptime builtin.cpu.arch.isAARCH64()) {	276	if (builtin.os.tag != .macos and builtin.cpu.arch.isAARCH64()) {
277	// https://github.com/ziglang/zig/issues/14096	277	// https://github.com/ziglang/zig/issues/14096
278	return error.SkipZigTest;	278	return error.SkipZigTest;
279	}	279	}

...	@@ -101,7 +101,7 @@ test "vector float operators" {	...	@@ -101,7 +101,7 @@ test "vector float operators" {
101	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	101	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
102	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	102	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
103	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	103	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
104	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	104	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
105	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	105	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
106		106
107	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {	107	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
...	@@ -205,7 +205,6 @@ test "array vector coercion - odd sizes" {	...	@@ -205,7 +205,6 @@ test "array vector coercion - odd sizes" {
205	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	205	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
206	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	206	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
207	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	207	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
208	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
209	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;	208	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
210	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	209	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
211		210
...	@@ -308,7 +307,6 @@ test "tuple to vector" {	...	@@ -308,7 +307,6 @@ test "tuple to vector" {
308		307
309	test "vector casts of sizes not divisible by 8" {	308	test "vector casts of sizes not divisible by 8" {
310	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	309	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
311	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
312	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	310	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
313	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	311	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
314	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	312	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -646,7 +644,7 @@ test "vector division operators" {	...	@@ -646,7 +644,7 @@ test "vector division operators" {
646		644
647	test "vector bitwise not operator" {	645	test "vector bitwise not operator" {
648	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	646	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
649	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	647	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
650	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	648	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
651	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	649	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
652	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	650	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -754,7 +752,7 @@ test "vector reduce operation" {	...	@@ -754,7 +752,7 @@ test "vector reduce operation" {
754	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	752	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
755	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	753	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
756	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	754	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
757	if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArm()) return error.SkipZigTest;	755	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
758	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	756	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
759	if (builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21091	757	if (builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21091
760		758
...	@@ -989,7 +987,7 @@ test "saturating multiplication" {	...	@@ -989,7 +987,7 @@ test "saturating multiplication" {
989	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	987	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
990		988
991	// TODO: once #9660 has been solved, remove this line	989	// TODO: once #9660 has been solved, remove this line
992	if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest;	990	if (builtin.target.cpu.arch.isWasm()) return error.SkipZigTest;
993		991
994	const S = struct {	992	const S = struct {
995	fn doTheTest() !void {	993	fn doTheTest() !void {
...	@@ -1256,7 +1254,7 @@ test "byte vector initialized in inline function" {	...	@@ -1256,7 +1254,7 @@ test "byte vector initialized in inline function" {
1256	if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;	1254	if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1257		1255
1258	if (comptime builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .x86_64 and	1256	if (comptime builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .x86_64 and
1259	builtin.cpu.features.isEnabled(@intFromEnum(std.Target.x86.Feature.avx512f)))	1257	std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f))
1260	{	1258	{
1261	// TODO https://github.com/ziglang/zig/issues/13279	1259	// TODO https://github.com/ziglang/zig/issues/13279
1262	return error.SkipZigTest;	1260	return error.SkipZigTest;
...	@@ -1363,7 +1361,7 @@ test "load packed vector element" {	...	@@ -1363,7 +1361,7 @@ test "load packed vector element" {
1363	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	1361	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1364	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1362	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1365	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	1363	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1366	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	1364	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1367	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;	1365	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1368		1366
1369	var x: @Vector(2, u15) = .{ 1, 4 };	1367	var x: @Vector(2, u15) = .{ 1, 4 };
...	@@ -1411,7 +1409,6 @@ test "store vector with memset" {	...	@@ -1411,7 +1409,6 @@ test "store vector with memset" {
1411	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	1409	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1412	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	1410	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1413	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	1411	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1414	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1415	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1412	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1416	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // TODO	1413	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // TODO
1417	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO	1414	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO