Commit 9e5c8cb008f75c2e570a0e48d5d014e936c103ca | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2022-05-16 19:45:30+02:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2022-05-19 19:37:29+02:00
log	9e5c8cb008f75c2e570a0e48d5d014e936c103ca
tree	af9bf9c8457a4137b1fc534e7eac8a0a4400b8ec
parent	2aee2302515ba444999b82c2e40cbc35dee08baf

x64: merge general purpose with simd register into one bitset

This way, we do not have to tweak the `RegisterManager` to handle
multiple register types - we have one linear space instead. Additionally
we can use the bitset itself to separate the registers into overlapping
(the ones that are aliases of differing bitwidths) and nonoverlapping
classes (for example, AVX registers do not overlap general purpose
registers, thus they can be allocated simultaneously).

Another huge benefit of this simple approach is the fact that we can
still refer to *all* registers regardless of their class via enum
literals which makes the code so much more readable.

Finally, `RegisterLock` is universal across different register classes.

7 files changed, 1158 insertions(+), 1397 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig
    abi.zig
    bits.zig
- codegen.zig
- register_manager.zig

src/arch/x86_64/CodeGen.zig+427-633

...	...	@@ -35,14 +35,10 @@ const caller_preserved_regs = abi.caller_preserved_regs;
35	35	const allocatable_registers = abi.allocatable_registers;
36	36	const c_abi_int_param_regs = abi.c_abi_int_param_regs;
37	37	const c_abi_int_return_regs = abi.c_abi_int_return_regs;
38		const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers, spillInstruction);
	38	const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
39	39	const RegisterLock = RegisterManager.RegisterLock;
40	40	const Register = bits.Register;
41	41
42		const AvxRegisterManager = RegisterManagerFn(Self, AvxRegister, &abi.avx_regs, spillInstructionAvx);
43		const AvxRegisterLock = AvxRegisterManager.RegisterLock;
44		const AvxRegister = bits.AvxRegister;
45
46	42	const InnerError = error{
47	43	OutOfMemory,
48	44	CodegenFail,
...	...	@@ -92,8 +88,7 @@ branch_stack: *std.ArrayList(Branch),
92	88	// Key is the block instruction
93	89	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
94	90
95		register_manager: RegisterManager,
96		avx_register_manager: AvxRegisterManager,
	91	register_manager: RegisterManager = .{},
97	92	/// Maps offset to what is stored there.
98	93	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
99	94
...	...	@@ -133,8 +128,6 @@ pub const MCValue = union(enum) {
133	128	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register,
134	129	/// and the operation is a signed operation.
135	130	register_overflow_signed: Register,
136		/// The value is in an AVX register.
137		avx_register: AvxRegister,
138	131	/// The value is in memory at a hard-coded address.
139	132	/// If the type is a pointer, it means the pointer address is at this memory location.
140	133	memory: u64,
...	...	@@ -202,7 +195,6 @@ pub const MCValue = union(enum) {
202	195	fn isRegister(mcv: MCValue) bool {
203	196	return switch (mcv) {
204	197	.register => true,
205		.avx_register => true,
206	198	else => false,
207	199	};
208	200	}
...	...	@@ -304,11 +296,7 @@ pub fn generate(
304	296	.mir_to_air_map = if (builtin.mode == .Debug)
305	297	std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator)
306	298	else {},
307		.register_manager = undefined,
308		.avx_register_manager = undefined,
309	299	};
310		function.register_manager = .{ .function = &function };
311		function.avx_register_manager = .{ .function = &function };
312	300	defer function.stack.deinit(bin_file.allocator);
313	301	defer function.blocks.deinit(bin_file.allocator);
314	302	defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
...	...	@@ -400,17 +388,15 @@ fn gen(self: *Self) InnerError!void {
400	388	if (cc != .Naked) {
401	389	_ = try self.addInst(.{
402	390	.tag = .push,
403		.ops = (Mir.Ops(Register, Register){
404		.reg1 = .rbp,
405		}).encode(),
	391	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
406	392	.data = undefined, // unused for push reg,
407	393	});
408	394	_ = try self.addInst(.{
409	395	.tag = .mov,
410		.ops = (Mir.Ops(Register, Register){
	396	.ops = Mir.Inst.Ops.encode(.{
411	397	.reg1 = .rbp,
412	398	.reg2 = .rsp,
413		}).encode(),
	399	}),
414	400	.data = undefined,
415	401	});
416	402	// We want to subtract the aligned stack frame size from rsp here, but we don't
...	...	@@ -447,9 +433,7 @@ fn gen(self: *Self) InnerError!void {
447	433	// push the callee_preserved_regs that were used
448	434	const backpatch_push_callee_preserved_regs_i = try self.addInst(.{
449	435	.tag = .push_regs_from_callee_preserved_regs,
450		.ops = (Mir.Ops(Register, Register){
451		.reg1 = .rbp,
452		}).encode(),
	436	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
453	437	.data = .{ .payload = undefined }, // to be backpatched
454	438	});
455	439
...	...	@@ -489,9 +473,7 @@ fn gen(self: *Self) InnerError!void {
489	473	// pop the callee_preserved_regs
490	474	_ = try self.addInst(.{
491	475	.tag = .pop_regs_from_callee_preserved_regs,
492		.ops = (Mir.Ops(Register, Register){
493		.reg1 = .rbp,
494		}).encode(),
	476	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
495	477	.data = .{ .payload = callee_preserved_regs_payload },
496	478	});
497	479
...	...	@@ -510,17 +492,13 @@ fn gen(self: *Self) InnerError!void {
510	492
511	493	_ = try self.addInst(.{
512	494	.tag = .pop,
513		.ops = (Mir.Ops(Register, Register){
514		.reg1 = .rbp,
515		}).encode(),
	495	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
516	496	.data = undefined,
517	497	});
518	498
519	499	_ = try self.addInst(.{
520	500	.tag = .ret,
521		.ops = (Mir.Ops(Register, Register){
522		.flags = 0b11,
523		}).encode(),
	501	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
524	502	.data = undefined,
525	503	});
526	504
...	...	@@ -534,16 +512,12 @@ fn gen(self: *Self) InnerError!void {
534	512	if (aligned_stack_end > 0) {
535	513	self.mir_instructions.set(backpatch_stack_sub, .{
536	514	.tag = .sub,
537		.ops = (Mir.Ops(Register, Register){
538		.reg1 = .rsp,
539		}).encode(),
	515	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
540	516	.data = .{ .imm = aligned_stack_end },
541	517	});
542	518	self.mir_instructions.set(backpatch_stack_add, .{
543	519	.tag = .add,
544		.ops = (Mir.Ops(Register, Register){
545		.reg1 = .rsp,
546		}).encode(),
	520	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
547	521	.data = .{ .imm = aligned_stack_end },
548	522	});
549	523	}
...	...	@@ -908,8 +882,8 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
908	882	// TODO check if AVX available
909	883	const ptr_bytes: u64 = 32;
910	884	if (abi_size <= ptr_bytes) {
911		if (self.avx_register_manager.tryAllocReg(inst)) \|reg\| {
912		return MCValue{ .avx_register = avxRegisterAlias(reg, abi_size) };
	885	if (self.register_manager.tryAllocReg(inst)) \|reg\| {
	886	return MCValue{ .register = registerAlias(reg, abi_size) };
913	887	}
914	888	}
915	889	},
...	...	@@ -947,21 +921,6 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
947	921	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
948	922	}
949	923
950		pub fn spillInstructionAvx(self: *Self, reg: AvxRegister, inst: Air.Inst.Index) !void {
951		const stack_mcv = try self.allocRegOrMem(inst, false);
952		log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
953		const reg_mcv = self.getResolvedInstValue(inst);
954		switch (reg_mcv) {
955		.avx_register => \|other\| {
956		assert(reg.to256() == other.to256());
957		},
958		else => {},
959		}
960		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
961		try branch.inst_table.put(self.gpa, inst, stack_mcv);
962		try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
963		}
964
965	924	pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
966	925	if (self.compare_flags_inst) \|inst_to_save\| {
967	926	const mcv = self.getResolvedInstValue(inst_to_save);
...	...	@@ -972,7 +931,6 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
972	931	.compare_flags_signed,
973	932	.compare_flags_unsigned,
974	933	=> try self.allocRegOrMem(inst_to_save, true),
975		.avx_register => try self.allocRegOrMem(inst_to_save, false),
976	934	else => unreachable,
977	935	};
978	936
...	...	@@ -990,7 +948,6 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
990	948	.register_overflow_signed,
991	949	.register_overflow_unsigned,
992	950	=> \|reg\| self.register_manager.freeReg(reg),
993		.avx_register => \|reg\| self.avx_register_manager.freeReg(reg),
994	951	else => {},
995	952	}
996	953	}
...	...	@@ -1236,10 +1193,10 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1236	1193	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv);
1237	1194	_ = try self.addInst(.{
1238	1195	.tag = if (signedness == .signed) .cond_mov_lt else .cond_mov_below,
1239		.ops = (Mir.Ops(Register, Register){
	1196	.ops = Mir.Inst.Ops.encode(.{
1240	1197	.reg1 = dst_mcv.register,
1241	1198	.reg2 = lhs_reg,
1242		}).encode(),
	1199	}),
1243	1200	.data = undefined,
1244	1201	});
1245	1202
...	...	@@ -1440,10 +1397,10 @@ fn genSetStackTruncatedOverflowCompare(
1440	1397	};
1441	1398	_ = try self.addInst(.{
1442	1399	.tag = .cond_set_byte_overflow,
1443		.ops = (Mir.Ops(Register, Register){
	1400	.ops = Mir.Inst.Ops.encode(.{
1444	1401	.reg1 = overflow_reg.to8(),
1445	1402	.flags = flags,
1446		}).encode(),
	1403	}),
1447	1404	.data = undefined,
1448	1405	});
1449	1406
...	...	@@ -1460,10 +1417,7 @@ fn genSetStackTruncatedOverflowCompare(
1460	1417	const eq_reg = temp_regs[2];
1461	1418	_ = try self.addInst(.{
1462	1419	.tag = .cond_set_byte_eq_ne,
1463		.ops = (Mir.Ops(Register, Register){
1464		.reg1 = eq_reg.to8(),
1465		.flags = 0b00,
1466		}).encode(),
	1420	.ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }),
1467	1421	.data = undefined,
1468	1422	});
1469	1423
...	...	@@ -1609,19 +1563,17 @@ fn genIntMulDivOpMir(
1609	1563	.signed => {
1610	1564	_ = try self.addInst(.{
1611	1565	.tag = .cwd,
1612		.ops = (Mir.Ops(Register, Register){
1613		.flags = 0b11,
1614		}).encode(),
	1566	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
1615	1567	.data = undefined,
1616	1568	});
1617	1569	},
1618	1570	.unsigned => {
1619	1571	_ = try self.addInst(.{
1620	1572	.tag = .xor,
1621		.ops = (Mir.Ops(Register, Register){
	1573	.ops = Mir.Inst.Ops.encode(.{
1622	1574	.reg1 = .rdx,
1623	1575	.reg2 = .rdx,
1624		}).encode(),
	1576	}),
1625	1577	.data = undefined,
1626	1578	});
1627	1579	},
...	...	@@ -1640,16 +1592,14 @@ fn genIntMulDivOpMir(
1640	1592	.register => \|reg\| {
1641	1593	_ = try self.addInst(.{
1642	1594	.tag = tag,
1643		.ops = (Mir.Ops(Register, Register){
1644		.reg1 = reg,
1645		}).encode(),
	1595	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
1646	1596	.data = undefined,
1647	1597	});
1648	1598	},
1649	1599	.stack_offset => \|off\| {
1650	1600	_ = try self.addInst(.{
1651	1601	.tag = tag,
1652		.ops = (Mir.Ops(Register, Register){
	1602	.ops = Mir.Inst.Ops.encode(.{
1653	1603	.reg2 = .rbp,
1654	1604	.flags = switch (abi_size) {
1655	1605	1 => 0b00,
...	...	@@ -1658,7 +1608,7 @@ fn genIntMulDivOpMir(
1658	1608	8 => 0b11,
1659	1609	else => unreachable,
1660	1610	},
1661		}).encode(),
	1611	}),
1662	1612	.data = .{ .imm = @bitCast(u32, -off) },
1663	1613	});
1664	1614	},
...	...	@@ -1691,34 +1641,34 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
1691	1641
1692	1642	_ = try self.addInst(.{
1693	1643	.tag = .xor,
1694		.ops = (Mir.Ops(Register, Register){
	1644	.ops = Mir.Inst.Ops.encode(.{
1695	1645	.reg1 = divisor.to64(),
1696	1646	.reg2 = dividend.to64(),
1697		}).encode(),
	1647	}),
1698	1648	.data = undefined,
1699	1649	});
1700	1650	_ = try self.addInst(.{
1701	1651	.tag = .sar,
1702		.ops = (Mir.Ops(Register, Register){
	1652	.ops = Mir.Inst.Ops.encode(.{
1703	1653	.reg1 = divisor.to64(),
1704	1654	.flags = 0b10,
1705		}).encode(),
	1655	}),
1706	1656	.data = .{ .imm = 63 },
1707	1657	});
1708	1658	_ = try self.addInst(.{
1709	1659	.tag = .@"test",
1710		.ops = (Mir.Ops(Register, Register){
	1660	.ops = Mir.Inst.Ops.encode(.{
1711	1661	.reg1 = .rdx,
1712	1662	.reg2 = .rdx,
1713		}).encode(),
	1663	}),
1714	1664	.data = undefined,
1715	1665	});
1716	1666	_ = try self.addInst(.{
1717	1667	.tag = .cond_mov_eq,
1718		.ops = (Mir.Ops(Register, Register){
	1668	.ops = Mir.Inst.Ops.encode(.{
1719	1669	.reg1 = divisor.to64(),
1720	1670	.reg2 = .rdx,
1721		}).encode(),
	1671	}),
1722	1672	.data = undefined,
1723	1673	});
1724	1674	try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
...	...	@@ -2102,11 +2052,11 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2102	2052	// mov reg, [rbp - 8]
2103	2053	_ = try self.addInst(.{
2104	2054	.tag = .mov,
2105		.ops = (Mir.Ops(Register, Register){
	2055	.ops = Mir.Inst.Ops.encode(.{
2106	2056	.reg1 = addr_reg.to64(),
2107	2057	.reg2 = .rbp,
2108	2058	.flags = 0b01,
2109		}).encode(),
	2059	}),
2110	2060	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
2111	2061	});
2112	2062	},
...	...	@@ -2187,10 +2137,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2187	2137	// lea reg, [rbp]
2188	2138	_ = try self.addInst(.{
2189	2139	.tag = .lea,
2190		.ops = (Mir.Ops(Register, Register){
	2140	.ops = Mir.Inst.Ops.encode(.{
2191	2141	.reg1 = addr_reg.to64(),
2192	2142	.reg2 = .rbp,
2193		}).encode(),
	2143	}),
2194	2144	.data = .{ .imm = @bitCast(u32, -off) },
2195	2145	});
2196	2146	},
...	...	@@ -2198,10 +2148,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2198	2148	// lea reg, [rbp]
2199	2149	_ = try self.addInst(.{
2200	2150	.tag = .lea,
2201		.ops = (Mir.Ops(Register, Register){
	2151	.ops = Mir.Inst.Ops.encode(.{
2202	2152	.reg1 = addr_reg.to64(),
2203	2153	.reg2 = .rbp,
2204		}).encode(),
	2154	}),
2205	2155	.data = .{ .imm = @bitCast(u32, -off) },
2206	2156	});
2207	2157	},
...	...	@@ -2266,11 +2216,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2266	2216	// mov dst_mcv, [dst_mcv]
2267	2217	_ = try self.addInst(.{
2268	2218	.tag = .mov,
2269		.ops = (Mir.Ops(Register, Register){
2270		.flags = 0b01,
	2219	.ops = Mir.Inst.Ops.encode(.{
2271	2220	.reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),
2272	2221	.reg2 = dst_mcv.register,
2273		}).encode(),
	2222	.flags = 0b01,
	2223	}),
2274	2224	.data = .{ .imm = 0 },
2275	2225	});
2276	2226	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
...	...	@@ -2532,11 +2482,11 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2532	2482	// mov dst_reg, [reg]
2533	2483	_ = try self.addInst(.{
2534	2484	.tag = .mov,
2535		.ops = (Mir.Ops(Register, Register){
	2485	.ops = Mir.Inst.Ops.encode(.{
2536	2486	.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
2537	2487	.reg2 = reg,
2538	2488	.flags = 0b01,
2539		}).encode(),
	2489	}),
2540	2490	.data = .{ .imm = 0 },
2541	2491	});
2542	2492	},
...	...	@@ -2552,9 +2502,6 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2552	2502	else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
2553	2503	}
2554	2504	},
2555		.avx_register => {
2556		return self.fail("TODO load for AVX register", .{});
2557		},
2558	2505	.memory,
2559	2506	.got_load,
2560	2507	.direct_load,
...	...	@@ -2606,10 +2553,10 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
2606	2553	const fn_owner_decl = mod.declPtr(self.mod_fn.owner_decl);
2607	2554	_ = try self.addInst(.{
2608	2555	.tag = .lea_pie,
2609		.ops = (Mir.Ops(Register, Register){
	2556	.ops = Mir.Inst.Ops.encode(.{
2610	2557	.reg1 = registerAlias(reg, abi_size),
2611	2558	.flags = flags,
2612		}).encode(),
	2559	}),
2613	2560	.data = .{
2614	2561	.load_reloc = .{
2615	2562	.atom_index = fn_owner_decl.link.macho.local_sym_index,
...	...	@@ -2670,7 +2617,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2670	2617	});
2671	2618	_ = try self.addInst(.{
2672	2619	.tag = .mov_mem_imm,
2673		.ops = (Mir.Ops(Register, Register){
	2620	.ops = Mir.Inst.Ops.encode(.{
2674	2621	.reg1 = reg.to64(),
2675	2622	.flags = switch (abi_size) {
2676	2623	1 => 0b00,
...	...	@@ -2678,7 +2625,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2678	2625	4 => 0b10,
2679	2626	else => unreachable,
2680	2627	},
2681		}).encode(),
	2628	}),
2682	2629	.data = .{ .payload = payload },
2683	2630	});
2684	2631	},
...	...	@@ -2692,11 +2639,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2692	2639	const tmp_reg = try self.copyToTmpRegister(value_ty, value);
2693	2640	_ = try self.addInst(.{
2694	2641	.tag = .mov,
2695		.ops = (Mir.Ops(Register, Register){
	2642	.ops = Mir.Inst.Ops.encode(.{
2696	2643	.reg1 = reg.to64(),
2697	2644	.reg2 = tmp_reg.to64(),
2698	2645	.flags = 0b10,
2699		}).encode(),
	2646	}),
2700	2647	.data = .{ .imm = 0 },
2701	2648	});
2702	2649	},
...	...	@@ -2708,11 +2655,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2708	2655	.register => \|src_reg\| {
2709	2656	_ = try self.addInst(.{
2710	2657	.tag = .mov,
2711		.ops = (Mir.Ops(Register, Register){
	2658	.ops = Mir.Inst.Ops.encode(.{
2712	2659	.reg1 = reg.to64(),
2713	2660	.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
2714	2661	.flags = 0b10,
2715		}).encode(),
	2662	}),
2716	2663	.data = .{ .imm = 0 },
2717	2664	});
2718	2665	},
...	...	@@ -2736,9 +2683,6 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2736	2683	},
2737	2684	}
2738	2685	},
2739		.avx_register => {
2740		return self.fail("TODO store for AVX register", .{});
2741		},
2742	2686	.got_load,
2743	2687	.direct_load,
2744	2688	.memory,
...	...	@@ -2759,11 +2703,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2759	2703	// mov reg, [reg]
2760	2704	_ = try self.addInst(.{
2761	2705	.tag = .mov,
2762		.ops = (Mir.Ops(Register, Register){
	2706	.ops = Mir.Inst.Ops.encode(.{
2763	2707	.reg1 = addr_reg.to64(),
2764	2708	.reg2 = addr_reg.to64(),
2765	2709	.flags = 0b01,
2766		}).encode(),
	2710	}),
2767	2711	.data = .{ .imm = 0 },
2768	2712	});
2769	2713
...	...	@@ -2798,21 +2742,21 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2798	2742	}
2799	2743	_ = try self.addInst(.{
2800	2744	.tag = .mov_mem_imm,
2801		.ops = (Mir.Ops(Register, Register){
	2745	.ops = Mir.Inst.Ops.encode(.{
2802	2746	.reg1 = addr_reg.to64(),
2803	2747	.flags = flags,
2804		}).encode(),
	2748	}),
2805	2749	.data = .{ .payload = payload },
2806	2750	});
2807	2751	},
2808	2752	.register => \|reg\| {
2809	2753	_ = try self.addInst(.{
2810	2754	.tag = .mov,
2811		.ops = (Mir.Ops(Register, Register){
	2755	.ops = Mir.Inst.Ops.encode(.{
2812	2756	.reg1 = addr_reg.to64(),
2813	2757	.reg2 = reg,
2814	2758	.flags = 0b10,
2815		}).encode(),
	2759	}),
2816	2760	.data = .{ .imm = 0 },
2817	2761	});
2818	2762	},
...	...	@@ -2829,20 +2773,20 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2829	2773
2830	2774	_ = try self.addInst(.{
2831	2775	.tag = .mov,
2832		.ops = (Mir.Ops(Register, Register){
	2776	.ops = Mir.Inst.Ops.encode(.{
2833	2777	.reg1 = tmp_reg,
2834	2778	.reg2 = tmp_reg,
2835	2779	.flags = 0b01,
2836		}).encode(),
	2780	}),
2837	2781	.data = .{ .imm = 0 },
2838	2782	});
2839	2783	_ = try self.addInst(.{
2840	2784	.tag = .mov,
2841		.ops = (Mir.Ops(Register, Register){
	2785	.ops = Mir.Inst.Ops.encode(.{
2842	2786	.reg1 = addr_reg.to64(),
2843	2787	.reg2 = tmp_reg,
2844	2788	.flags = 0b10,
2845		}).encode(),
	2789	}),
2846	2790	.data = .{ .imm = 0 },
2847	2791	});
2848	2792	return;
...	...	@@ -2856,11 +2800,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2856	2800	const tmp_reg = try self.copyToTmpRegister(value_ty, value);
2857	2801	_ = try self.addInst(.{
2858	2802	.tag = .mov,
2859		.ops = (Mir.Ops(Register, Register){
	2803	.ops = Mir.Inst.Ops.encode(.{
2860	2804	.reg1 = addr_reg.to64(),
2861	2805	.reg2 = tmp_reg,
2862	2806	.flags = 0b10,
2863		}).encode(),
	2807	}),
2864	2808	.data = .{ .imm = 0 },
2865	2809	});
2866	2810	return;
...	...	@@ -3017,10 +2961,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3017	2961	if (signedness == .signed and field_size < 8) {
3018	2962	_ = try self.addInst(.{
3019	2963	.tag = .mov_sign_extend,
3020		.ops = (Mir.Ops(Register, Register){
	2964	.ops = Mir.Inst.Ops.encode(.{
3021	2965	.reg1 = dst_mcv.register,
3022	2966	.reg2 = registerAlias(dst_mcv.register, field_size),
3023		}).encode(),
	2967	}),
3024	2968	.data = undefined,
3025	2969	});
3026	2970	}
...	...	@@ -3048,10 +2992,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3048	2992	};
3049	2993	_ = try self.addInst(.{
3050	2994	.tag = .cond_set_byte_overflow,
3051		.ops = (Mir.Ops(Register, Register){
	2995	.ops = Mir.Inst.Ops.encode(.{
3052	2996	.reg1 = dst_reg.to8(),
3053	2997	.flags = flags,
3054		}).encode(),
	2998	}),
3055	2999	.data = undefined,
3056	3000	});
3057	3001	break :result MCValue{ .register = dst_reg.to8() };
...	...	@@ -3092,10 +3036,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3092	3036	1 => {
3093	3037	_ = try self.addInst(.{
3094	3038	.tag = tag,
3095		.ops = (Mir.Ops(Register, Register){
3096		.reg1 = registerAlias(reg, abi_size),
3097		.flags = 0b00,
3098		}).encode(),
	3039	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
3099	3040	.data = undefined,
3100	3041	});
3101	3042	return;
...	...	@@ -3103,10 +3044,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3103	3044	else => {
3104	3045	_ = try self.addInst(.{
3105	3046	.tag = tag,
3106		.ops = (Mir.Ops(Register, Register){
	3047	.ops = Mir.Inst.Ops.encode(.{
3107	3048	.reg1 = registerAlias(reg, abi_size),
3108	3049	.flags = 0b10,
3109		}).encode(),
	3050	}),
3110	3051	.data = .{ .imm = @intCast(u8, imm) },
3111	3052	});
3112	3053	return;
...	...	@@ -3124,10 +3065,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3124	3065
3125	3066	_ = try self.addInst(.{
3126	3067	.tag = tag,
3127		.ops = (Mir.Ops(Register, Register){
	3068	.ops = Mir.Inst.Ops.encode(.{
3128	3069	.reg1 = registerAlias(reg, abi_size),
3129	3070	.flags = 0b01,
3130		}).encode(),
	3071	}),
3131	3072	.data = undefined,
3132	3073	});
3133	3074	}
...	...	@@ -3383,32 +3324,6 @@ fn genBinOp(
3383	3324	return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
3384	3325	}
3385	3326
3386		if (lhs_ty.zigTypeTag() == .Float) {
3387		switch (tag) {
3388		.add => {
3389		const dst_reg: AvxRegister = blk: {
3390		const reg = try self.avx_register_manager.allocReg(null);
3391		try self.genSetAvxReg(lhs_ty, reg, lhs);
3392		break :blk reg.to128();
3393		};
3394		const dst_lock = self.avx_register_manager.lockRegAssumeUnused(dst_reg);
3395		defer self.avx_register_manager.unlockReg(dst_lock);
3396
3397		const src_reg: AvxRegister = blk: {
3398		const reg = try self.avx_register_manager.allocReg(null);
3399		try self.genSetAvxReg(lhs_ty, reg, rhs);
3400		break :blk reg.to128();
3401		};
3402		const src_lock = self.avx_register_manager.lockRegAssumeUnused(src_reg);
3403		defer self.avx_register_manager.unlockReg(src_lock);
3404
3405		try self.genBinOpMir(.add_f64, lhs_ty, .{ .avx_register = dst_reg }, .{ .avx_register = src_reg });
3406		return MCValue{ .avx_register = dst_reg };
3407		},
3408		else => unreachable,
3409		}
3410		}
3411
3412	3327	const is_commutative: bool = switch (tag) {
3413	3328	.add,
3414	3329	.addwrap,
...	...	@@ -3526,25 +3441,39 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3526	3441	const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
3527	3442	return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
3528	3443	},
3529		.register => \|src_reg\| {
3530		_ = try self.addInst(.{
3531		.tag = mir_tag,
3532		.ops = (Mir.Ops(Register, Register){
3533		.reg1 = registerAlias(dst_reg, abi_size),
3534		.reg2 = registerAlias(src_reg, abi_size),
3535		}).encode(),
3536		.data = undefined,
3537		});
3538		},
3539		.avx_register => {
3540		return self.fail("TODO genBinOp for AVX register", .{});
	3444	.register => \|src_reg\| switch (dst_ty.zigTypeTag()) {
	3445	.Float => switch (dst_ty.tag()) {
	3446	.f64 => {
	3447	_ = try self.addInst(.{
	3448	.tag = switch (mir_tag) {
	3449	.add => .add_f64,
	3450	.cmp => .cmp_f64,
	3451	else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
	3452	},
	3453	.ops = Mir.Inst.Ops.encode(.{
	3454	.reg1 = dst_reg.to128(),
	3455	.reg2 = src_reg.to128(),
	3456	}),
	3457	.data = undefined,
	3458	});
	3459	},
	3460	else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
	3461	},
	3462	else => {
	3463	_ = try self.addInst(.{
	3464	.tag = mir_tag,
	3465	.ops = Mir.Inst.Ops.encode(.{
	3466	.reg1 = registerAlias(dst_reg, abi_size),
	3467	.reg2 = registerAlias(src_reg, abi_size),
	3468	}),
	3469	.data = undefined,
	3470	});
	3471	},
3541	3472	},
3542	3473	.immediate => \|imm\| {
3543	3474	_ = try self.addInst(.{
3544	3475	.tag = mir_tag,
3545		.ops = (Mir.Ops(Register, Register){
3546		.reg1 = registerAlias(dst_reg, abi_size),
3547		}).encode(),
	3476	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }),
3548	3477	.data = .{ .imm = @truncate(u32, imm) },
3549	3478	});
3550	3479	},
...	...	@@ -3567,39 +3496,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3567	3496	}
3568	3497	_ = try self.addInst(.{
3569	3498	.tag = mir_tag,
3570		.ops = (Mir.Ops(Register, Register){
	3499	.ops = Mir.Inst.Ops.encode(.{
3571	3500	.reg1 = registerAlias(dst_reg, abi_size),
3572	3501	.reg2 = .rbp,
3573	3502	.flags = 0b01,
3574		}).encode(),
	3503	}),
3575	3504	.data = .{ .imm = @bitCast(u32, -off) },
3576	3505	});
3577	3506	},
3578	3507	}
3579	3508	},
3580		.avx_register => \|dst_reg\| {
3581		switch (src_mcv) {
3582		.avx_register => \|src_reg\| {
3583		switch (dst_ty.zigTypeTag()) {
3584		.Float => switch (dst_ty.tag()) {
3585		.f64 => {
3586		_ = try self.addInst(.{
3587		.tag = mir_tag,
3588		.ops = (Mir.Ops(AvxRegister, AvxRegister){
3589		.reg1 = dst_reg.to128(),
3590		.reg2 = src_reg.to128(),
3591		}).encode(),
3592		.data = undefined,
3593		});
3594		},
3595		else => return self.fail("TODO genBinOp for AVX register and type {}", .{dst_ty.fmtDebug()}),
3596		},
3597		else => return self.fail("TODO genBinOp for AVX register and type {}", .{dst_ty.fmtDebug()}),
3598		}
3599		},
3600		else => return self.fail("TODO genBinOp for AVX register", .{}),
3601		}
3602		},
3603	3509	.ptr_stack_offset, .stack_offset => \|off\| {
3604	3510	if (off > math.maxInt(i32)) {
3605	3511	return self.fail("stack offset too large", .{});
...	...	@@ -3617,17 +3523,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3617	3523	.register => \|src_reg\| {
3618	3524	_ = try self.addInst(.{
3619	3525	.tag = mir_tag,
3620		.ops = (Mir.Ops(Register, Register){
	3526	.ops = Mir.Inst.Ops.encode(.{
3621	3527	.reg1 = .rbp,
3622	3528	.reg2 = registerAlias(src_reg, abi_size),
3623	3529	.flags = 0b10,
3624		}).encode(),
	3530	}),
3625	3531	.data = .{ .imm = @bitCast(u32, -off) },
3626	3532	});
3627	3533	},
3628		.avx_register => {
3629		return self.fail("TODO genBinOp for AVX register", .{});
3630		},
3631	3534	.immediate => \|imm\| {
3632	3535	const tag: Mir.Inst.Tag = switch (mir_tag) {
3633	3536	.add => .add_mem_imm,
...	...	@@ -3651,10 +3554,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3651	3554	});
3652	3555	_ = try self.addInst(.{
3653	3556	.tag = tag,
3654		.ops = (Mir.Ops(Register, Register){
	3557	.ops = Mir.Inst.Ops.encode(.{
3655	3558	.reg1 = .rbp,
3656	3559	.flags = flags,
3657		}).encode(),
	3560	}),
3658	3561	.data = .{ .payload = payload },
3659	3562	});
3660	3563	},
...	...	@@ -3697,7 +3600,6 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3697	3600	.ptr_stack_offset => unreachable,
3698	3601	.register_overflow_unsigned => unreachable,
3699	3602	.register_overflow_signed => unreachable,
3700		.avx_register => unreachable,
3701	3603	.register => \|dst_reg\| {
3702	3604	switch (src_mcv) {
3703	3605	.none => unreachable,
...	...	@@ -3706,15 +3608,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3706	3608	.ptr_stack_offset => unreachable,
3707	3609	.register_overflow_unsigned => unreachable,
3708	3610	.register_overflow_signed => unreachable,
3709		.avx_register => unreachable,
3710	3611	.register => \|src_reg\| {
3711	3612	// register, register
3712	3613	_ = try self.addInst(.{
3713	3614	.tag = .imul_complex,
3714		.ops = (Mir.Ops(Register, Register){
	3615	.ops = Mir.Inst.Ops.encode(.{
3715	3616	.reg1 = registerAlias(dst_reg, abi_size),
3716	3617	.reg2 = registerAlias(src_reg, abi_size),
3717		}).encode(),
	3618	}),
3718	3619	.data = undefined,
3719	3620	});
3720	3621	},
...	...	@@ -3724,11 +3625,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3724	3625	if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
3725	3626	_ = try self.addInst(.{
3726	3627	.tag = .imul_complex,
3727		.ops = (Mir.Ops(Register, Register){
	3628	.ops = Mir.Inst.Ops.encode(.{
3728	3629	.reg1 = dst_reg.to32(),
3729	3630	.reg2 = dst_reg.to32(),
3730	3631	.flags = 0b10,
3731		}).encode(),
	3632	}),
3732	3633	.data = .{ .imm = @truncate(u32, imm) },
3733	3634	});
3734	3635	} else {
...	...	@@ -3740,11 +3641,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3740	3641	.stack_offset => \|off\| {
3741	3642	_ = try self.addInst(.{
3742	3643	.tag = .imul_complex,
3743		.ops = (Mir.Ops(Register, Register){
	3644	.ops = Mir.Inst.Ops.encode(.{
3744	3645	.reg1 = registerAlias(dst_reg, abi_size),
3745	3646	.reg2 = .rbp,
3746	3647	.flags = 0b01,
3747		}).encode(),
	3648	}),
3748	3649	.data = .{ .imm = @bitCast(u32, -off) },
3749	3650	});
3750	3651	},
...	...	@@ -3770,7 +3671,6 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3770	3671	.ptr_stack_offset => unreachable,
3771	3672	.register_overflow_unsigned => unreachable,
3772	3673	.register_overflow_signed => unreachable,
3773		.avx_register => unreachable,
3774	3674	.register => \|src_reg\| {
3775	3675	// copy dst to a register
3776	3676	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
...	...	@@ -3778,10 +3678,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3778	3678	// register, register
3779	3679	_ = try self.addInst(.{
3780	3680	.tag = .imul_complex,
3781		.ops = (Mir.Ops(Register, Register){
	3681	.ops = Mir.Inst.Ops.encode(.{
3782	3682	.reg1 = registerAlias(dst_reg, abi_size),
3783	3683	.reg2 = registerAlias(src_reg, abi_size),
3784		}).encode(),
	3684	}),
3785	3685	.data = undefined,
3786	3686	});
3787	3687	// copy dst_reg back out
...	...	@@ -3888,9 +3788,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
3888	3788	fn airBreakpoint(self: *Self) !void {
3889	3789	_ = try self.addInst(.{
3890	3790	.tag = .interrupt,
3891		.ops = (Mir.Ops{
3892		.flags = 0b00,
3893		}).encode(),
	3791	.ops = Mir.Inst.Ops.encode(.{}),
3894	3792	.data = undefined,
3895	3793	});
3896	3794	return self.finishAirBookkeeping();
...	...	@@ -3973,9 +3871,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3973	3871	.ptr_stack_offset => {
3974	3872	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
3975	3873	},
3976		.avx_register => {
3977		return self.fail("TODO implement calling with MCValue.avx_register arg", .{});
3978		},
3979	3874	.undef => unreachable,
3980	3875	.immediate => unreachable,
3981	3876	.unreach => unreachable,
...	...	@@ -3994,9 +3889,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3994	3889	// Adjust the stack
3995	3890	_ = try self.addInst(.{
3996	3891	.tag = .sub,
3997		.ops = (Mir.Ops(Register, Register){
3998		.reg1 = .rsp,
3999		}).encode(),
	3892	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4000	3893	.data = .{ .imm = info.stack_byte_count },
4001	3894	});
4002	3895	}
...	...	@@ -4020,9 +3913,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4020	3913	unreachable;
4021	3914	_ = try self.addInst(.{
4022	3915	.tag = .call,
4023		.ops = (Mir.Ops(Register, Register){
4024		.flags = 0b01,
4025		}).encode(),
	3916	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4026	3917	.data = .{ .imm = @truncate(u32, got_addr) },
4027	3918	});
4028	3919	} else if (func_value.castTag(.extern_fn)) \|_\| {
...	...	@@ -4036,10 +3927,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4036	3927	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
4037	3928	_ = try self.addInst(.{
4038	3929	.tag = .call,
4039		.ops = (Mir.Ops(Register, Register){
	3930	.ops = Mir.Inst.Ops.encode(.{
4040	3931	.reg1 = .rax,
4041	3932	.flags = 0b01,
4042		}).encode(),
	3933	}),
4043	3934	.data = undefined,
4044	3935	});
4045	3936	}
...	...	@@ -4054,10 +3945,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4054	3945	// callq *%rax
4055	3946	_ = try self.addInst(.{
4056	3947	.tag = .call,
4057		.ops = (Mir.Ops(Register, Register){
	3948	.ops = Mir.Inst.Ops.encode(.{
4058	3949	.reg1 = .rax,
4059	3950	.flags = 0b01,
4060		}).encode(),
	3951	}),
4061	3952	.data = undefined,
4062	3953	});
4063	3954	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
...	...	@@ -4089,10 +3980,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4089	3980	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
4090	3981	_ = try self.addInst(.{
4091	3982	.tag = .call,
4092		.ops = (Mir.Ops(Register, Register){
	3983	.ops = Mir.Inst.Ops.encode(.{
4093	3984	.reg1 = .rax,
4094	3985	.flags = 0b01,
4095		}).encode(),
	3986	}),
4096	3987	.data = undefined,
4097	3988	});
4098	3989	}
...	...	@@ -4107,9 +3998,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4107	3998	const fn_got_addr = got_addr + got_index * ptr_bytes;
4108	3999	_ = try self.addInst(.{
4109	4000	.tag = .call,
4110		.ops = (Mir.Ops(Register, Register){
4111		.flags = 0b01,
4112		}).encode(),
	4001	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4113	4002	.data = .{ .imm = @intCast(u32, fn_got_addr) },
4114	4003	});
4115	4004	} else return self.fail("TODO implement calling extern fn on plan9", .{});
...	...	@@ -4119,10 +4008,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4119	4008	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
4120	4009	_ = try self.addInst(.{
4121	4010	.tag = .call,
4122		.ops = (Mir.Ops(Register, Register){
	4011	.ops = Mir.Inst.Ops.encode(.{
4123	4012	.reg1 = .rax,
4124	4013	.flags = 0b01,
4125		}).encode(),
	4014	}),
4126	4015	.data = undefined,
4127	4016	});
4128	4017	}
...	...	@@ -4132,9 +4021,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4132	4021	// Readjust the stack
4133	4022	_ = try self.addInst(.{
4134	4023	.tag = .add,
4135		.ops = (Mir.Ops(Register, Register){
4136		.reg1 = .rsp,
4137		}).encode(),
	4024	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4138	4025	.data = .{ .imm = info.stack_byte_count },
4139	4026	});
4140	4027	}
...	...	@@ -4192,9 +4079,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
4192	4079	// which is available if the jump is 127 bytes or less forward.
4193	4080	const jmp_reloc = try self.addInst(.{
4194	4081	.tag = .jmp,
4195		.ops = (Mir.Ops(Register, Register){
4196		.flags = 0b00,
4197		}).encode(),
	4082	.ops = Mir.Inst.Ops.encode(.{}),
4198	4083	.data = .{ .inst = undefined },
4199	4084	});
4200	4085	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
...	...	@@ -4227,9 +4112,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
4227	4112	// which is available if the jump is 127 bytes or less forward.
4228	4113	const jmp_reloc = try self.addInst(.{
4229	4114	.tag = .jmp,
4230		.ops = (Mir.Ops(Register, Register){
4231		.flags = 0b00,
4232		}).encode(),
	4115	.ops = Mir.Inst.Ops.encode(.{}),
4233	4116	.data = .{ .inst = undefined },
4234	4117	});
4235	4118	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
...	...	@@ -4256,37 +4139,6 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
4256	4139	self.compare_flags_inst = inst;
4257	4140
4258	4141	const result: MCValue = result: {
4259		if (ty.zigTypeTag() == .Float) {
4260		const lhs = try self.resolveInst(bin_op.lhs);
4261		const rhs = try self.resolveInst(bin_op.rhs);
4262
4263		const dst_reg: AvxRegister = blk: {
4264		const reg = try self.avx_register_manager.allocReg(null);
4265		try self.genSetAvxReg(ty, reg, lhs);
4266		break :blk reg.to128();
4267		};
4268		const dst_lock = self.avx_register_manager.lockRegAssumeUnused(dst_reg);
4269		defer self.avx_register_manager.unlockReg(dst_lock);
4270
4271		const src_reg: AvxRegister = blk: {
4272		const reg = try self.avx_register_manager.allocReg(null);
4273		try self.genSetAvxReg(ty, reg, rhs);
4274		break :blk reg.to128();
4275		};
4276		const src_lock = self.avx_register_manager.lockRegAssumeUnused(src_reg);
4277		defer self.avx_register_manager.unlockReg(src_lock);
4278
4279		_ = try self.addInst(.{
4280		.tag = .cmp_f64,
4281		.ops = (Mir.Ops(AvxRegister, AvxRegister){
4282		.reg1 = dst_reg,
4283		.reg2 = src_reg,
4284		}).encode(),
4285		.data = undefined,
4286		});
4287
4288		break :result MCValue{ .compare_flags_unsigned = op };
4289		}
4290	4142	// There are 2 operands, destination and source.
4291	4143	// Either one, but not both, can be a memory operand.
4292	4144	// Source operand can be an immediate, 8 bits or 32 bits.
...	...	@@ -4304,8 +4156,28 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
4304	4156
4305	4157	const dst_mcv = MCValue{ .register = dst_reg };
4306	4158
	4159	const rhs_ty = self.air.typeOf(bin_op.rhs);
4307	4160	// This instruction supports only signed 32-bit immediates at most.
4308		const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
	4161	const src_mcv: MCValue = blk: {
	4162	switch (rhs_ty.zigTypeTag()) {
	4163	.Float => {
	4164	const rhs = try self.resolveInst(bin_op.rhs);
	4165	const rhs_lock: ?RegisterLock = switch (rhs) {
	4166	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	4167	else => null,
	4168	};
	4169	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	4170	const src_reg = try self.copyToTmpRegister(rhs_ty, rhs);
	4171	break :blk MCValue{ .register = src_reg };
	4172	},
	4173	else => break :blk try self.limitImmediateType(bin_op.rhs, i32),
	4174	}
	4175	};
	4176	const src_lock: ?RegisterLock = switch (src_mcv) {
	4177	.register => \|reg\| self.register_manager.lockReg(reg),
	4178	else => null,
	4179	};
	4180	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
4309	4181
4310	4182	try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
4311	4183	break :result switch (signedness) {
...	...	@@ -4504,9 +4376,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4504	4376	Mir.Inst.Tag.cond_jmp_greater_less;
4505	4377	return self.addInst(.{
4506	4378	.tag = tag,
4507		.ops = (Mir.Ops(Register, Register){
4508		.flags = flags,
4509		}).encode(),
	4379	.ops = Mir.Inst.Ops.encode(.{ .flags = flags }),
4510	4380	.data = .{ .inst = undefined },
4511	4381	});
4512	4382	},
...	...	@@ -4514,17 +4384,12 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4514	4384	try self.spillCompareFlagsIfOccupied();
4515	4385	_ = try self.addInst(.{
4516	4386	.tag = .@"test",
4517		.ops = (Mir.Ops(Register, Register){
4518		.reg1 = reg,
4519		.flags = 0b00,
4520		}).encode(),
	4387	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
4521	4388	.data = .{ .imm = 1 },
4522	4389	});
4523	4390	return self.addInst(.{
4524	4391	.tag = .cond_jmp_eq_ne,
4525		.ops = (Mir.Ops(Register, Register){
4526		.flags = 0b01,
4527		}).encode(),
	4392	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4528	4393	.data = .{ .inst = undefined },
4529	4394	});
4530	4395	},
...	...	@@ -4918,9 +4783,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
4918	4783	try self.genBody(body);
4919	4784	_ = try self.addInst(.{
4920	4785	.tag = .jmp,
4921		.ops = (Mir.Ops(Register, Register){
4922		.flags = 0b00,
4923		}).encode(),
	4786	.ops = Mir.Inst.Ops.encode(.{}),
4924	4787	.data = .{ .inst = jmp_target },
4925	4788	});
4926	4789	return self.finishAirBookkeeping();
...	...	@@ -4971,19 +4834,17 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4971	4834	.immediate => \|imm\| {
4972	4835	_ = try self.addInst(.{
4973	4836	.tag = .xor,
4974		.ops = (Mir.Ops(Register, Register){
4975		.reg1 = registerAlias(cond_reg, abi_size),
4976		}).encode(),
	4837	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }),
4977	4838	.data = .{ .imm = @intCast(u32, imm) },
4978	4839	});
4979	4840	},
4980	4841	.register => \|reg\| {
4981	4842	_ = try self.addInst(.{
4982	4843	.tag = .xor,
4983		.ops = (Mir.Ops(Register, Register){
	4844	.ops = Mir.Inst.Ops.encode(.{
4984	4845	.reg1 = registerAlias(cond_reg, abi_size),
4985	4846	.reg2 = registerAlias(reg, abi_size),
4986		}).encode(),
	4847	}),
4987	4848	.data = undefined,
4988	4849	});
4989	4850	},
...	...	@@ -5002,17 +4863,15 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
5002	4863
5003	4864	_ = try self.addInst(.{
5004	4865	.tag = .@"test",
5005		.ops = (Mir.Ops(Register, Register){
	4866	.ops = Mir.Inst.Ops.encode(.{
5006	4867	.reg1 = registerAlias(cond_reg, abi_size),
5007	4868	.reg2 = registerAlias(cond_reg, abi_size),
5008		}).encode(),
	4869	}),
5009	4870	.data = undefined,
5010	4871	});
5011	4872	return self.addInst(.{
5012	4873	.tag = .cond_jmp_eq_ne,
5013		.ops = (Mir.Ops(Register, Register){
5014		.flags = 0b00,
5015		}).encode(),
	4874	.ops = Mir.Inst.Ops.encode(.{}),
5016	4875	.data = .{ .inst = undefined },
5017	4876	});
5018	4877	},
...	...	@@ -5156,17 +5015,20 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
5156	5015	if (block_mcv == .none) {
5157	5016	block_data.mcv = switch (operand_mcv) {
5158	5017	.none, .dead, .unreach => unreachable,
5159		.register, .stack_offset, .memory => operand_mcv,
	5018	.stack_offset, .memory => operand_mcv,
5160	5019	.compare_flags_signed, .compare_flags_unsigned, .immediate => blk: {
5161	5020	const new_mcv = try self.allocRegOrMem(block, true);
5162	5021	try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
5163	5022	break :blk new_mcv;
5164	5023	},
5165		.avx_register => blk: {
5166		// TODO not needed; return operand_mcv ones we can transfer between XMM registers
5167		const new_mcv = try self.allocRegOrMem(block, false);
5168		try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
5169		break :blk new_mcv;
	5024	.register => blk: {
	5025	if (self.air.typeOfIndex(block).zigTypeTag() == .Float) {
	5026	// TODO not needed; return operand_mcv ones we can transfer between XMM registers
	5027	const new_mcv = try self.allocRegOrMem(block, false);
	5028	try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
	5029	break :blk new_mcv;
	5030	}
	5031	break :blk operand_mcv;
5170	5032	},
5171	5033	else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}),
5172	5034	};
...	...	@@ -5184,9 +5046,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
5184	5046	// Leave the jump offset undefined
5185	5047	const jmp_reloc = try self.addInst(.{
5186	5048	.tag = .jmp,
5187		.ops = (Mir.Ops(Register, Register){
5188		.flags = 0b00,
5189		}).encode(),
	5049	.ops = Mir.Inst.Ops.encode(.{}),
5190	5050	.data = .{ .inst = undefined },
5191	5051	});
5192	5052	block_data.relocs.appendAssumeCapacity(jmp_reloc);
...	...	@@ -5274,9 +5134,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5274	5134	};
5275	5135	_ = try self.addInst(.{
5276	5136	.tag = .push,
5277		.ops = (Mir.Ops(Register, Register){
5278		.flags = 0b10,
5279		}).encode(),
	5137	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b10 }),
5280	5138	.data = .{ .imm = n },
5281	5139	});
5282	5140	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
...	...	@@ -5285,9 +5143,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5285	5143	return self.fail("unrecognized register: '{s}'", .{reg_name});
5286	5144	_ = try self.addInst(.{
5287	5145	.tag = .push,
5288		.ops = (Mir.Ops(Register, Register){
5289		.reg1 = reg,
5290		}).encode(),
	5146	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5291	5147	.data = undefined,
5292	5148	});
5293	5149	} else return self.fail("TODO more push operands", .{});
...	...	@@ -5299,9 +5155,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5299	5155	return self.fail("unrecognized register: '{s}'", .{reg_name});
5300	5156	_ = try self.addInst(.{
5301	5157	.tag = .pop,
5302		.ops = (Mir.Ops(Register, Register){
5303		.reg1 = reg,
5304		}).encode(),
	5158	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5305	5159	.data = undefined,
5306	5160	});
5307	5161	} else return self.fail("TODO more pop operands", .{});
...	...	@@ -5365,7 +5219,6 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
5365	5219	.none => return,
5366	5220	.immediate => unreachable,
5367	5221	.register => \|reg\| return self.genSetReg(ty, reg, val),
5368		.avx_register => \|reg\| return self.genSetAvxReg(ty, reg, val),
5369	5222	.stack_offset => \|off\| return self.genSetStack(ty, off, val, .{}),
5370	5223	.memory => {
5371	5224	return self.fail("TODO implement setRegOrMem for memory", .{});
...	...	@@ -5396,9 +5249,6 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5396	5249	.register_overflow_unsigned,
5397	5250	.register_overflow_signed,
5398	5251	=> return self.fail("TODO genSetStackArg for register with overflow bit", .{}),
5399		.avx_register => {
5400		return self.fail("TODO genSetStackArg for AVX register", .{});
5401		},
5402	5252	.compare_flags_unsigned,
5403	5253	.compare_flags_signed,
5404	5254	=> {
...	...	@@ -5417,7 +5267,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5417	5267	});
5418	5268	_ = try self.addInst(.{
5419	5269	.tag = .mov_mem_imm,
5420		.ops = (Mir.Ops(Register, Register){
	5270	.ops = Mir.Inst.Ops.encode(.{
5421	5271	.reg1 = .rsp,
5422	5272	.flags = switch (abi_size) {
5423	5273	1 => 0b00,
...	...	@@ -5425,7 +5275,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5425	5275	4 => 0b10,
5426	5276	else => unreachable,
5427	5277	},
5428		}).encode(),
	5278	}),
5429	5279	.data = .{ .payload = payload },
5430	5280	});
5431	5281	},
...	...	@@ -5453,11 +5303,11 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5453	5303	.register => \|reg\| {
5454	5304	_ = try self.addInst(.{
5455	5305	.tag = .mov,
5456		.ops = (Mir.Ops(Register, Register){
	5306	.ops = Mir.Inst.Ops.encode(.{
5457	5307	.reg1 = .rsp,
5458	5308	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5459	5309	.flags = 0b10,
5460		}).encode(),
	5310	}),
5461	5311	.data = .{ .imm = @bitCast(u32, -stack_offset) },
5462	5312	});
5463	5313	},
...	...	@@ -5520,10 +5370,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5520	5370	};
5521	5371	_ = try self.addInst(.{
5522	5372	.tag = .cond_set_byte_overflow,
5523		.ops = (Mir.Ops(Register, Register){
	5373	.ops = Mir.Inst.Ops.encode(.{
5524	5374	.reg1 = tmp_reg.to8(),
5525	5375	.flags = flags,
5526		}).encode(),
	5376	}),
5527	5377	.data = undefined,
5528	5378	});
5529	5379
...	...	@@ -5550,7 +5400,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5550	5400	});
5551	5401	_ = try self.addInst(.{
5552	5402	.tag = .mov_mem_imm,
5553		.ops = (Mir.Ops(Register, Register){
	5403	.ops = Mir.Inst.Ops.encode(.{
5554	5404	.reg1 = base_reg,
5555	5405	.flags = switch (abi_size) {
5556	5406	1 => 0b00,
...	...	@@ -5558,7 +5408,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5558	5408	4 => 0b10,
5559	5409	else => unreachable,
5560	5410	},
5561		}).encode(),
	5411	}),
5562	5412	.data = .{ .payload = payload },
5563	5413	});
5564	5414	},
...	...	@@ -5572,10 +5422,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5572	5422	});
5573	5423	_ = try self.addInst(.{
5574	5424	.tag = .mov_mem_imm,
5575		.ops = (Mir.Ops(Register, Register){
	5425	.ops = Mir.Inst.Ops.encode(.{
5576	5426	.reg1 = base_reg,
5577	5427	.flags = 0b10,
5578		}).encode(),
	5428	}),
5579	5429	.data = .{ .payload = payload },
5580	5430	});
5581	5431	}
...	...	@@ -5586,10 +5436,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5586	5436	});
5587	5437	_ = try self.addInst(.{
5588	5438	.tag = .mov_mem_imm,
5589		.ops = (Mir.Ops(Register, Register){
	5439	.ops = Mir.Inst.Ops.encode(.{
5590	5440	.reg1 = base_reg,
5591	5441	.flags = 0b10,
5592		}).encode(),
	5442	}),
5593	5443	.data = .{ .payload = payload },
5594	5444	});
5595	5445	}
...	...	@@ -5605,65 +5455,64 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5605	5455	}
5606	5456
5607	5457	const base_reg = opts.dest_stack_base orelse .rbp;
5608		if (!math.isPowerOfTwo(abi_size)) {
5609		const reg_lock = self.register_manager.lockReg(reg);
5610		defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
5611
5612		const tmp_reg = try self.copyToTmpRegister(ty, mcv);
5613
5614		var next_offset = stack_offset;
5615		var remainder = abi_size;
5616		while (remainder > 0) {
5617		const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5618
5619		_ = try self.addInst(.{
5620		.tag = .mov,
5621		.ops = (Mir.Ops(Register, Register){
5622		.reg1 = base_reg,
5623		.reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5624		.flags = 0b10,
5625		}).encode(),
5626		.data = .{ .imm = @bitCast(u32, -next_offset) },
5627		});
5628	5458
5629		if (nearest_power_of_two > 1) {
5630		try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 });
5631		}
5632
5633		remainder -= nearest_power_of_two;
5634		next_offset -= nearest_power_of_two;
5635		}
5636		} else {
5637		_ = try self.addInst(.{
5638		.tag = .mov,
5639		.ops = (Mir.Ops(Register, Register){
5640		.reg1 = base_reg,
5641		.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5642		.flags = 0b10,
5643		}).encode(),
5644		.data = .{ .imm = @bitCast(u32, -stack_offset) },
5645		});
5646		}
5647		},
5648		.avx_register => \|reg\| {
5649		const base_reg = opts.dest_stack_base orelse .rbp;
5650	5459	switch (ty.zigTypeTag()) {
5651	5460	.Float => switch (ty.tag()) {
5652		.f32 => return self.fail("TODO genSetStack for AVX register for f32", .{}),
	5461	.f32 => return self.fail("TODO genSetStack for register for f32", .{}),
5653	5462	.f64 => {
5654	5463	_ = try self.addInst(.{
5655	5464	.tag = .mov_f64,
5656		.ops = (Mir.Ops(Register, AvxRegister){
	5465	.ops = Mir.Inst.Ops.encode(.{
5657	5466	.reg1 = base_reg,
5658	5467	.reg2 = reg.to128(),
5659	5468	.flags = 0b01,
5660		}).encode(),
	5469	}),
5661	5470	.data = .{ .imm = @bitCast(u32, -stack_offset) },
5662	5471	});
5663	5472	},
5664		else => return self.fail("TODO genSetStack for AVX register for type {}", .{ty.fmtDebug()}),
	5473	else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
	5474	},
	5475	else => {
	5476	if (!math.isPowerOfTwo(abi_size)) {
	5477	const reg_lock = self.register_manager.lockReg(reg);
	5478	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	5479
	5480	const tmp_reg = try self.copyToTmpRegister(ty, mcv);
	5481
	5482	var next_offset = stack_offset;
	5483	var remainder = abi_size;
	5484	while (remainder > 0) {
	5485	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
	5486
	5487	_ = try self.addInst(.{
	5488	.tag = .mov,
	5489	.ops = Mir.Inst.Ops.encode(.{
	5490	.reg1 = base_reg,
	5491	.reg2 = registerAlias(tmp_reg, nearest_power_of_two),
	5492	.flags = 0b10,
	5493	}),
	5494	.data = .{ .imm = @bitCast(u32, -next_offset) },
	5495	});
	5496
	5497	if (nearest_power_of_two > 1) {
	5498	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 });
	5499	}
	5500
	5501	remainder -= nearest_power_of_two;
	5502	next_offset -= nearest_power_of_two;
	5503	}
	5504	} else {
	5505	_ = try self.addInst(.{
	5506	.tag = .mov,
	5507	.ops = Mir.Inst.Ops.encode(.{
	5508	.reg1 = base_reg,
	5509	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
	5510	.flags = 0b10,
	5511	}),
	5512	.data = .{ .imm = @bitCast(u32, -stack_offset) },
	5513	});
	5514	}
5665	5515	},
5666		else => return self.fail("TODO genSetStack for AVX register for type {}", .{ty.fmtDebug()}),
5667	5516	}
5668	5517	},
5669	5518	.memory,
...	...	@@ -5742,20 +5591,20 @@ fn genInlineMemcpy(
5742	5591	.ptr_stack_offset, .stack_offset => \|off\| {
5743	5592	_ = try self.addInst(.{
5744	5593	.tag = .lea,
5745		.ops = (Mir.Ops(Register, Register){
	5594	.ops = Mir.Inst.Ops.encode(.{
5746	5595	.reg1 = dst_addr_reg.to64(),
5747	5596	.reg2 = opts.dest_stack_base orelse .rbp,
5748		}).encode(),
	5597	}),
5749	5598	.data = .{ .imm = @bitCast(u32, -off) },
5750	5599	});
5751	5600	},
5752	5601	.register => \|reg\| {
5753	5602	_ = try self.addInst(.{
5754	5603	.tag = .mov,
5755		.ops = (Mir.Ops(Register, Register){
	5604	.ops = Mir.Inst.Ops.encode(.{
5756	5605	.reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),
5757	5606	.reg2 = reg,
5758		}).encode(),
	5607	}),
5759	5608	.data = undefined,
5760	5609	});
5761	5610	},
...	...	@@ -5777,20 +5626,20 @@ fn genInlineMemcpy(
5777	5626	.ptr_stack_offset, .stack_offset => \|off\| {
5778	5627	_ = try self.addInst(.{
5779	5628	.tag = .lea,
5780		.ops = (Mir.Ops(Register, Register){
	5629	.ops = Mir.Inst.Ops.encode(.{
5781	5630	.reg1 = src_addr_reg.to64(),
5782	5631	.reg2 = opts.source_stack_base orelse .rbp,
5783		}).encode(),
	5632	}),
5784	5633	.data = .{ .imm = @bitCast(u32, -off) },
5785	5634	});
5786	5635	},
5787	5636	.register => \|reg\| {
5788	5637	_ = try self.addInst(.{
5789	5638	.tag = .mov,
5790		.ops = (Mir.Ops(Register, Register){
	5639	.ops = Mir.Inst.Ops.encode(.{
5791	5640	.reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),
5792	5641	.reg2 = reg,
5793		}).encode(),
	5642	}),
5794	5643	.data = undefined,
5795	5644	});
5796	5645	},
...	...	@@ -5810,18 +5659,14 @@ fn genInlineMemcpy(
5810	5659	// mov rcx, 0
5811	5660	_ = try self.addInst(.{
5812	5661	.tag = .mov,
5813		.ops = (Mir.Ops(Register, Register){
5814		.reg1 = .rcx,
5815		}).encode(),
	5662	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }),
5816	5663	.data = .{ .imm = 0 },
5817	5664	});
5818	5665
5819	5666	// mov rax, 0
5820	5667	_ = try self.addInst(.{
5821	5668	.tag = .mov,
5822		.ops = (Mir.Ops(Register, Register){
5823		.reg1 = .rax,
5824		}).encode(),
	5669	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5825	5670	.data = .{ .imm = 0 },
5826	5671	});
5827	5672
...	...	@@ -5829,70 +5674,62 @@ fn genInlineMemcpy(
5829	5674	// cmp count, 0
5830	5675	const loop_start = try self.addInst(.{
5831	5676	.tag = .cmp,
5832		.ops = (Mir.Ops(Register, Register){
5833		.reg1 = count_reg,
5834		}).encode(),
	5677	.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5835	5678	.data = .{ .imm = 0 },
5836	5679	});
5837	5680
5838	5681	// je end
5839	5682	const loop_reloc = try self.addInst(.{
5840	5683	.tag = .cond_jmp_eq_ne,
5841		.ops = (Mir.Ops(Register, Register){ .flags = 0b01 }).encode(),
	5684	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
5842	5685	.data = .{ .inst = undefined },
5843	5686	});
5844	5687
5845	5688	// mov tmp, [addr + rcx]
5846	5689	_ = try self.addInst(.{
5847	5690	.tag = .mov_scale_src,
5848		.ops = (Mir.Ops(Register, Register){
	5691	.ops = Mir.Inst.Ops.encode(.{
5849	5692	.reg1 = tmp_reg.to8(),
5850	5693	.reg2 = src_addr_reg,
5851		}).encode(),
	5694	}),
5852	5695	.data = .{ .imm = 0 },
5853	5696	});
5854	5697
5855	5698	// mov [stack_offset + rax], tmp
5856	5699	_ = try self.addInst(.{
5857	5700	.tag = .mov_scale_dst,
5858		.ops = (Mir.Ops(Register, Register){
	5701	.ops = Mir.Inst.Ops.encode(.{
5859	5702	.reg1 = dst_addr_reg,
5860	5703	.reg2 = tmp_reg.to8(),
5861		}).encode(),
	5704	}),
5862	5705	.data = .{ .imm = 0 },
5863	5706	});
5864	5707
5865	5708	// add rcx, 1
5866	5709	_ = try self.addInst(.{
5867	5710	.tag = .add,
5868		.ops = (Mir.Ops(Register, Register){
5869		.reg1 = .rcx,
5870		}).encode(),
	5711	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }),
5871	5712	.data = .{ .imm = 1 },
5872	5713	});
5873	5714
5874	5715	// add rax, 1
5875	5716	_ = try self.addInst(.{
5876	5717	.tag = .add,
5877		.ops = (Mir.Ops(Register, Register){
5878		.reg1 = .rax,
5879		}).encode(),
	5718	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5880	5719	.data = .{ .imm = 1 },
5881	5720	});
5882	5721
5883	5722	// sub count, 1
5884	5723	_ = try self.addInst(.{
5885	5724	.tag = .sub,
5886		.ops = (Mir.Ops(Register, Register){
5887		.reg1 = count_reg,
5888		}).encode(),
	5725	.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5889	5726	.data = .{ .imm = 1 },
5890	5727	});
5891	5728
5892	5729	// jmp loop
5893	5730	_ = try self.addInst(.{
5894	5731	.tag = .jmp,
5895		.ops = (Mir.Ops(Register, Register){ .flags = 0b00 }).encode(),
	5732	.ops = Mir.Inst.Ops.encode(.{}),
5896	5733	.data = .{ .inst = loop_start },
5897	5734	});
5898	5735
...	...	@@ -5924,20 +5761,20 @@ fn genInlineMemset(
5924	5761	.ptr_stack_offset, .stack_offset => \|off\| {
5925	5762	_ = try self.addInst(.{
5926	5763	.tag = .lea,
5927		.ops = (Mir.Ops(Register, Register){
	5764	.ops = Mir.Inst.Ops.encode(.{
5928	5765	.reg1 = addr_reg.to64(),
5929	5766	.reg2 = opts.dest_stack_base orelse .rbp,
5930		}).encode(),
	5767	}),
5931	5768	.data = .{ .imm = @bitCast(u32, -off) },
5932	5769	});
5933	5770	},
5934	5771	.register => \|reg\| {
5935	5772	_ = try self.addInst(.{
5936	5773	.tag = .mov,
5937		.ops = (Mir.Ops(Register, Register){
	5774	.ops = Mir.Inst.Ops.encode(.{
5938	5775	.reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
5939	5776	.reg2 = reg,
5940		}).encode(),
	5777	}),
5941	5778	.data = undefined,
5942	5779	});
5943	5780	},
...	...	@@ -5955,16 +5792,14 @@ fn genInlineMemset(
5955	5792	// cmp rax, -1
5956	5793	const loop_start = try self.addInst(.{
5957	5794	.tag = .cmp,
5958		.ops = (Mir.Ops(Register, Register){
5959		.reg1 = .rax,
5960		}).encode(),
	5795	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5961	5796	.data = .{ .imm = @bitCast(u32, @as(i32, -1)) },
5962	5797	});
5963	5798
5964	5799	// je end
5965	5800	const loop_reloc = try self.addInst(.{
5966	5801	.tag = .cond_jmp_eq_ne,
5967		.ops = (Mir.Ops(Register, Register){ .flags = 0b01 }).encode(),
	5802	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
5968	5803	.data = .{ .inst = undefined },
5969	5804	});
5970	5805
...	...	@@ -5980,9 +5815,7 @@ fn genInlineMemset(
5980	5815	});
5981	5816	_ = try self.addInst(.{
5982	5817	.tag = .mov_mem_index_imm,
5983		.ops = (Mir.Ops(Register, Register){
5984		.reg1 = addr_reg,
5985		}).encode(),
	5818	.ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg }),
5986	5819	.data = .{ .payload = payload },
5987	5820	});
5988	5821	},
...	...	@@ -5992,16 +5825,14 @@ fn genInlineMemset(
5992	5825	// sub rax, 1
5993	5826	_ = try self.addInst(.{
5994	5827	.tag = .sub,
5995		.ops = (Mir.Ops(Register, Register){
5996		.reg1 = .rax,
5997		}).encode(),
	5828	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5998	5829	.data = .{ .imm = 1 },
5999	5830	});
6000	5831
6001	5832	// jmp loop
6002	5833	_ = try self.addInst(.{
6003	5834	.tag = .jmp,
6004		.ops = (Mir.Ops(Register, Register){ .flags = 0b00 }).encode(),
	5835	.ops = Mir.Inst.Ops.encode(.{}),
6005	5836	.data = .{ .inst = loop_start },
6006	5837	});
6007	5838
...	...	@@ -6009,88 +5840,6 @@ fn genInlineMemset(
6009	5840	try self.performReloc(loop_reloc);
6010	5841	}
6011	5842
6012		fn genSetAvxReg(self: *Self, ty: Type, reg: AvxRegister, mcv: MCValue) InnerError!void {
6013		switch (mcv) {
6014		.dead => unreachable,
6015		.register_overflow_unsigned,
6016		.register_overflow_signed,
6017		=> unreachable,
6018		.stack_offset => \|off\| {
6019		if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
6020		return self.fail("stack offset too large", .{});
6021		}
6022
6023		switch (ty.zigTypeTag()) {
6024		.Float => {
6025		switch (ty.tag()) {
6026		.f32 => return self.fail("TODO genSetAvxReg from stack offset for f32", .{}),
6027		.f64 => {
6028		_ = try self.addInst(.{
6029		.tag = .mov_f64,
6030		.ops = (Mir.Ops(AvxRegister, Register){
6031		.reg1 = reg.to128(),
6032		.reg2 = .rbp,
6033		}).encode(),
6034		.data = .{ .imm = @bitCast(u32, -off) },
6035		});
6036		},
6037		else => return self.fail("TODO genSetAvxReg from stack offset for {}", .{ty.fmtDebug()}),
6038		}
6039		},
6040		else => return self.fail("TODO genSetAvxReg from stack offset for type {}", .{ty.fmtDebug()}),
6041		}
6042		},
6043		.avx_register => \|src_reg\| {
6044		switch (ty.zigTypeTag()) {
6045		.Float => {
6046		switch (ty.tag()) {
6047		.f32 => return self.fail("TODO genSetAvxReg from register for f32", .{}),
6048		.f64 => {
6049		_ = try self.addInst(.{
6050		.tag = .mov_f64,
6051		.ops = (Mir.Ops(AvxRegister, AvxRegister){
6052		.reg1 = reg.to128(),
6053		.reg2 = src_reg.to128(),
6054		.flags = 0b10,
6055		}).encode(),
6056		.data = undefined,
6057		});
6058		},
6059		else => return self.fail("TODO genSetAvxReg from register for {}", .{ty.fmtDebug()}),
6060		}
6061		},
6062		else => return self.fail("TODO genSetAvxReg from register for type {}", .{ty.fmtDebug()}),
6063		}
6064		},
6065		.memory => {
6066		switch (ty.zigTypeTag()) {
6067		.Float => {
6068		switch (ty.tag()) {
6069		.f32 => return self.fail("TODO genSetAvxReg from memory for f32", .{}),
6070		.f64 => {
6071		const base_reg = try self.register_manager.allocReg(null);
6072		try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
6073		_ = try self.addInst(.{
6074		.tag = .mov_f64,
6075		.ops = (Mir.Ops(AvxRegister, Register){
6076		.reg1 = reg.to128(),
6077		.reg2 = base_reg.to64(),
6078		}).encode(),
6079		.data = .{ .imm = 0 },
6080		});
6081		},
6082		else => return self.fail("TODO genSetAvxReg from memory for {}", .{ty.fmtDebug()}),
6083		}
6084		},
6085		else => return self.fail("TODO genSetAvxReg from memory for type {}", .{ty.fmtDebug()}),
6086		}
6087		},
6088		else => \|other\| {
6089		return self.fail("TODO genSetAvxReg from {}", .{other});
6090		},
6091		}
6092		}
6093
6094	5843	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
6095	5844	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6096	5845	switch (mcv) {
...	...	@@ -6098,17 +5847,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6098	5847	.register_overflow_unsigned,
6099	5848	.register_overflow_signed,
6100	5849	=> unreachable,
6101		.avx_register => unreachable,
6102	5850	.ptr_stack_offset => \|off\| {
6103	5851	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
6104	5852	return self.fail("stack offset too large", .{});
6105	5853	}
6106	5854	_ = try self.addInst(.{
6107	5855	.tag = .lea,
6108		.ops = (Mir.Ops(Register, Register){
	5856	.ops = Mir.Inst.Ops.encode(.{
6109	5857	.reg1 = registerAlias(reg, abi_size),
6110	5858	.reg2 = .rbp,
6111		}).encode(),
	5859	}),
6112	5860	.data = .{ .imm = @bitCast(u32, -off) },
6113	5861	});
6114	5862	},
...	...	@@ -6145,10 +5893,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6145	5893	};
6146	5894	_ = try self.addInst(.{
6147	5895	.tag = tag,
6148		.ops = (Mir.Ops(Register, Register){
	5896	.ops = Mir.Inst.Ops.encode(.{
6149	5897	.reg1 = reg.to8(),
6150	5898	.flags = flags,
6151		}).encode(),
	5899	}),
6152	5900	.data = undefined,
6153	5901	});
6154	5902	},
...	...	@@ -6158,10 +5906,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6158	5906	if (x == 0) {
6159	5907	_ = try self.addInst(.{
6160	5908	.tag = .xor,
6161		.ops = (Mir.Ops(Register, Register){
	5909	.ops = Mir.Inst.Ops.encode(.{
6162	5910	.reg1 = reg.to32(),
6163	5911	.reg2 = reg.to32(),
6164		}).encode(),
	5912	}),
6165	5913	.data = undefined,
6166	5914	});
6167	5915	return;
...	...	@@ -6170,9 +5918,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6170	5918	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
6171	5919	_ = try self.addInst(.{
6172	5920	.tag = .mov,
6173		.ops = (Mir.Ops(Register, Register){
6174		.reg1 = registerAlias(reg, abi_size),
6175		}).encode(),
	5921	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
6176	5922	.data = .{ .imm = @truncate(u32, x) },
6177	5923	});
6178	5924	return;
...	...	@@ -6187,9 +5933,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6187	5933	const payload = try self.addExtra(Mir.Imm64.encode(x));
6188	5934	_ = try self.addInst(.{
6189	5935	.tag = .movabs,
6190		.ops = (Mir.Ops(Register, Register){
6191		.reg1 = reg.to64(),
6192		}).encode(),
	5936	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }),
6193	5937	.data = .{ .payload = payload },
6194	5938	});
6195	5939	},
...	...	@@ -6198,40 +5942,60 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6198	5942	if (src_reg.id() == reg.id())
6199	5943	return;
6200	5944
6201		if (ty.zigTypeTag() == .Int) blk: {
6202		switch (ty.intInfo(self.target.*).signedness) {
	5945	switch (ty.zigTypeTag()) {
	5946	.Int => switch (ty.intInfo(self.target.*).signedness) {
6203	5947	.signed => {
6204		if (abi_size > 4) break :blk;
6205		_ = try self.addInst(.{
6206		.tag = .mov_sign_extend,
6207		.ops = (Mir.Ops(Register, Register){
6208		.reg1 = reg.to64(),
6209		.reg2 = registerAlias(src_reg, abi_size),
6210		}).encode(),
6211		.data = undefined,
6212		});
	5948	if (abi_size <= 4) {
	5949	_ = try self.addInst(.{
	5950	.tag = .mov_sign_extend,
	5951	.ops = Mir.Inst.Ops.encode(.{
	5952	.reg1 = reg.to64(),
	5953	.reg2 = registerAlias(src_reg, abi_size),
	5954	}),
	5955	.data = undefined,
	5956	});
	5957	return;
	5958	}
6213	5959	},
6214	5960	.unsigned => {
6215		if (abi_size > 2) break :blk;
	5961	if (abi_size <= 2) {
	5962	_ = try self.addInst(.{
	5963	.tag = .mov_zero_extend,
	5964	.ops = Mir.Inst.Ops.encode(.{
	5965	.reg1 = reg.to64(),
	5966	.reg2 = registerAlias(src_reg, abi_size),
	5967	}),
	5968	.data = undefined,
	5969	});
	5970	return;
	5971	}
	5972	},
	5973	},
	5974	.Float => switch (ty.tag()) {
	5975	.f32 => return self.fail("TODO genSetReg from register for f32", .{}),
	5976	.f64 => {
6216	5977	_ = try self.addInst(.{
6217		.tag = .mov_zero_extend,
6218		.ops = (Mir.Ops(Register, Register){
6219		.reg1 = reg.to64(),
6220		.reg2 = registerAlias(src_reg, abi_size),
6221		}).encode(),
	5978	.tag = .mov_f64,
	5979	.ops = Mir.Inst.Ops.encode(.{
	5980	.reg1 = reg.to128(),
	5981	.reg2 = src_reg.to128(),
	5982	.flags = 0b10,
	5983	}),
6222	5984	.data = undefined,
6223	5985	});
	5986	return;
6224	5987	},
6225		}
6226		return;
	5988	else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
	5989	},
	5990	else => {},
6227	5991	}
6228	5992
6229	5993	_ = try self.addInst(.{
6230	5994	.tag = .mov,
6231		.ops = (Mir.Ops(Register, Register){
	5995	.ops = Mir.Inst.Ops.encode(.{
6232	5996	.reg1 = registerAlias(reg, abi_size),
6233	5997	.reg2 = registerAlias(src_reg, abi_size),
6234		}).encode(),
	5998	}),
6235	5999	.data = undefined,
6236	6000	});
6237	6001	},
...	...	@@ -6241,107 +6005,148 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6241	6005	try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
6242	6006	_ = try self.addInst(.{
6243	6007	.tag = .mov,
6244		.ops = (Mir.Ops(Register, Register){
	6008	.ops = Mir.Inst.Ops.encode(.{
6245	6009	.reg1 = registerAlias(reg, abi_size),
6246	6010	.reg2 = reg.to64(),
6247	6011	.flags = 0b01,
6248		}).encode(),
	6012	}),
6249	6013	.data = .{ .imm = 0 },
6250	6014	});
6251	6015	},
6252		.memory => \|x\| {
6253		if (x <= math.maxInt(i32)) {
6254		// mov reg, [ds:imm32]
6255		_ = try self.addInst(.{
6256		.tag = .mov,
6257		.ops = (Mir.Ops(Register, Register){
6258		.reg1 = registerAlias(reg, abi_size),
6259		.flags = 0b01,
6260		}).encode(),
6261		.data = .{ .imm = @truncate(u32, x) },
6262		});
6263		} else {
6264		// If this is RAX, we can use a direct load.
6265		// Otherwise, we need to load the address, then indirectly load the value.
6266		if (reg.id() == 0) {
6267		// movabs rax, ds:moffs64
6268		const payload = try self.addExtra(Mir.Imm64.encode(x));
6269		_ = try self.addInst(.{
6270		.tag = .movabs,
6271		.ops = (Mir.Ops(Register, Register){
6272		.reg1 = .rax,
6273		.flags = 0b01, // imm64 will become moffs64
6274		}).encode(),
6275		.data = .{ .payload = payload },
6276		});
6277		} else {
6278		// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
6279		try self.genSetReg(ty, reg, MCValue{ .immediate = x });
6280
6281		// mov reg, [reg + 0x0]
	6016	.memory => \|x\| switch (ty.zigTypeTag()) {
	6017	.Float => {
	6018	switch (ty.tag()) {
	6019	.f32 => return self.fail("TODO genSetReg from memory for f32", .{}),
	6020	.f64 => {
	6021	const base_reg = try self.register_manager.allocReg(null);
	6022	try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
	6023	_ = try self.addInst(.{
	6024	.tag = .mov_f64,
	6025	.ops = Mir.Inst.Ops.encode(.{
	6026	.reg1 = reg.to128(),
	6027	.reg2 = base_reg.to64(),
	6028	}),
	6029	.data = .{ .imm = 0 },
	6030	});
	6031	},
	6032	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
	6033	}
	6034	},
	6035	else => {
	6036	if (x <= math.maxInt(i32)) {
	6037	// mov reg, [ds:imm32]
6282	6038	_ = try self.addInst(.{
6283	6039	.tag = .mov,
6284		.ops = (Mir.Ops(Register, Register){
	6040	.ops = Mir.Inst.Ops.encode(.{
6285	6041	.reg1 = registerAlias(reg, abi_size),
6286		.reg2 = reg.to64(),
6287	6042	.flags = 0b01,
6288		}).encode(),
6289		.data = .{ .imm = 0 },
	6043	}),
	6044	.data = .{ .imm = @truncate(u32, x) },
6290	6045	});
	6046	} else {
	6047	// If this is RAX, we can use a direct load.
	6048	// Otherwise, we need to load the address, then indirectly load the value.
	6049	if (reg.id() == 0) {
	6050	// movabs rax, ds:moffs64
	6051	const payload = try self.addExtra(Mir.Imm64.encode(x));
	6052	_ = try self.addInst(.{
	6053	.tag = .movabs,
	6054	.ops = Mir.Inst.Ops.encode(.{
	6055	.reg1 = .rax,
	6056	.flags = 0b01, // imm64 will become moffs64
	6057	}),
	6058	.data = .{ .payload = payload },
	6059	});
	6060	} else {
	6061	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
	6062	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
	6063
	6064	// mov reg, [reg + 0x0]
	6065	_ = try self.addInst(.{
	6066	.tag = .mov,
	6067	.ops = Mir.Inst.Ops.encode(.{
	6068	.reg1 = registerAlias(reg, abi_size),
	6069	.reg2 = reg.to64(),
	6070	.flags = 0b01,
	6071	}),
	6072	.data = .{ .imm = 0 },
	6073	});
	6074	}
6291	6075	}
6292		}
	6076	},
6293	6077	},
6294	6078	.stack_offset => \|off\| {
6295	6079	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
6296	6080	return self.fail("stack offset too large", .{});
6297	6081	}
6298	6082
6299		if (ty.zigTypeTag() == .Int) blk: {
6300		switch (ty.intInfo(self.target.*).signedness) {
	6083	switch (ty.zigTypeTag()) {
	6084	.Int => switch (ty.intInfo(self.target.*).signedness) {
6301	6085	.signed => {
6302		const flags: u2 = switch (abi_size) {
6303		1 => 0b01,
6304		2 => 0b10,
6305		4 => 0b11,
6306		else => break :blk,
6307		};
6308		_ = try self.addInst(.{
6309		.tag = .mov_sign_extend,
6310		.ops = (Mir.Ops(Register, Register){
6311		.reg1 = reg.to64(),
6312		.reg2 = .rbp,
6313		.flags = flags,
6314		}).encode(),
6315		.data = .{ .imm = @bitCast(u32, -off) },
6316		});
	6086	if (abi_size <= 4) {
	6087	const flags: u2 = switch (abi_size) {
	6088	1 => 0b01,
	6089	2 => 0b10,
	6090	4 => 0b11,
	6091	else => unreachable,
	6092	};
	6093	_ = try self.addInst(.{
	6094	.tag = .mov_sign_extend,
	6095	.ops = Mir.Inst.Ops.encode(.{
	6096	.reg1 = reg.to64(),
	6097	.reg2 = .rbp,
	6098	.flags = flags,
	6099	}),
	6100	.data = .{ .imm = @bitCast(u32, -off) },
	6101	});
	6102	return;
	6103	}
6317	6104	},
6318	6105	.unsigned => {
6319		const flags: u2 = switch (abi_size) {
6320		1 => 0b01,
6321		2 => 0b10,
6322		else => break :blk,
6323		};
	6106	if (abi_size <= 2) {
	6107	const flags: u2 = switch (abi_size) {
	6108	1 => 0b01,
	6109	2 => 0b10,
	6110	else => unreachable,
	6111	};
	6112	_ = try self.addInst(.{
	6113	.tag = .mov_zero_extend,
	6114	.ops = Mir.Inst.Ops.encode(.{
	6115	.reg1 = reg.to64(),
	6116	.reg2 = .rbp,
	6117	.flags = flags,
	6118	}),
	6119	.data = .{ .imm = @bitCast(u32, -off) },
	6120	});
	6121	return;
	6122	}
	6123	},
	6124	},
	6125	.Float => switch (ty.tag()) {
	6126	.f32 => return self.fail("TODO genSetReg from stack offset for f32", .{}),
	6127	.f64 => {
6324	6128	_ = try self.addInst(.{
6325		.tag = .mov_zero_extend,
6326		.ops = (Mir.Ops(Register, Register){
6327		.reg1 = reg.to64(),
	6129	.tag = .mov_f64,
	6130	.ops = Mir.Inst.Ops.encode(.{
	6131	.reg1 = reg.to128(),
6328	6132	.reg2 = .rbp,
6329		.flags = flags,
6330		}).encode(),
	6133	}),
6331	6134	.data = .{ .imm = @bitCast(u32, -off) },
6332	6135	});
	6136	return;
6333	6137	},
6334		}
6335		return;
	6138	else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
	6139	},
	6140	else => {},
6336	6141	}
6337	6142
6338	6143	_ = try self.addInst(.{
6339	6144	.tag = .mov,
6340		.ops = (Mir.Ops(Register, Register){
	6145	.ops = Mir.Inst.Ops.encode(.{
6341	6146	.reg1 = registerAlias(reg, abi_size),
6342	6147	.reg2 = .rbp,
6343	6148	.flags = 0b01,
6344		}).encode(),
	6149	}),
6345	6150	.data = .{ .imm = @bitCast(u32, -off) },
6346	6151	});
6347	6152	},
...	...	@@ -6408,14 +6213,14 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6408	6213	};
6409	6214	_ = try self.addInst(.{
6410	6215	.tag = .fld,
6411		.ops = (Mir.Ops(Register, Register){
	6216	.ops = Mir.Inst.Ops.encode(.{
	6217	.reg1 = .rbp,
6412	6218	.flags = switch (src_ty.abiSize(self.target.*)) {
6413	6219	4 => 0b01,
6414	6220	8 => 0b10,
6415	6221	else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
6416	6222	},
6417		.reg1 = .rbp,
6418		}).encode(),
	6223	}),
6419	6224	.data = .{ .imm = @bitCast(u32, -stack_offset) },
6420	6225	});
6421	6226
...	...	@@ -6423,15 +6228,15 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6423	6228	const stack_dst = try self.allocRegOrMem(inst, false);
6424	6229	_ = try self.addInst(.{
6425	6230	.tag = .fisttp,
6426		.ops = (Mir.Ops(Register, Register){
	6231	.ops = Mir.Inst.Ops.encode(.{
	6232	.reg1 = .rbp,
6427	6233	.flags = switch (dst_ty.abiSize(self.target.*)) {
6428	6234	1...2 => 0b00,
6429	6235	3...4 => 0b01,
6430	6236	5...8 => 0b10,
6431	6237	else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),
6432	6238	},
6433		.reg1 = .rbp,
6434		}).encode(),
	6239	}),
6435	6240	.data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) },
6436	6241	});
6437	6242
...	...	@@ -6527,11 +6332,11 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6527	6332	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
6528	6333	_ = try self.addInst(.{
6529	6334	.tag = .mov,
6530		.ops = (Mir.Ops(Register, Register){
	6335	.ops = Mir.Inst.Ops.encode(.{
6531	6336	.reg1 = reg,
6532	6337	.reg2 = reg,
6533	6338	.flags = 0b01,
6534		}).encode(),
	6339	}),
6535	6340	.data = .{ .imm = 0 },
6536	6341	});
6537	6342	break :blk MCValue{ .register = reg };
...	...	@@ -7095,22 +6900,11 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
7095	6900	return reg.to32();
7096	6901	} else if (size_bytes <= 8) {
7097	6902	return reg.to64();
7098		} else {
7099		unreachable;
7100		}
7101		}
7102
7103		/// Returns AVX register wide enough to hold at least `size_bytes`.
7104		fn avxRegisterAlias(reg: AvxRegister, size_bytes: u32) AvxRegister {
7105		if (size_bytes == 0) {
7106		unreachable; // should be comptime known
7107	6903	} else if (size_bytes <= 16) {
7108	6904	return reg.to128();
7109	6905	} else if (size_bytes <= 32) {
7110	6906	return reg.to256();
7111		} else {
7112		unreachable;
7113		}
	6907	} else unreachable;
7114	6908	}
7115	6909
7116	6910	/// Truncates the value in the register in place.

src/arch/x86_64/Emit.zig+383-389

...	...	@@ -25,9 +25,8 @@ const MCValue = @import("CodeGen.zig").MCValue;
25	25	const Mir = @import("Mir.zig");
26	26	const Module = @import("../../Module.zig");
27	27	const Instruction = bits.Instruction;
28		const GpRegister = bits.Register;
29		const AvxRegister = bits.AvxRegister;
30	28	const Type = @import("../../type.zig").Type;
	29	const Register = bits.Register;
31	30
32	31	mir: Mir,
33	32	bin_file: *link.File,
...	...	@@ -238,7 +237,7 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
238	237	fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
239	238	const tag = emit.mir.instructions.items(.tag)[inst];
240	239	assert(tag == .interrupt);
241		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	240	const ops = emit.mir.instructions.items(.ops)[inst].decode();
242	241	switch (ops.flags) {
243	242	0b00 => return lowerToZoEnc(.int3, emit.code),
244	243	else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}),
...	...	@@ -254,11 +253,11 @@ fn mirSyscall(emit: *Emit) InnerError!void {
254	253	}
255	254
256	255	fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
257		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	256	const ops = emit.mir.instructions.items(.ops)[inst].decode();
258	257	switch (ops.flags) {
259	258	0b00 => {
260	259	// PUSH/POP reg
261		return lowerToOEnc(tag, Register.reg(ops.reg1), emit.code);
	260	return lowerToOEnc(tag, ops.reg1, emit.code);
262	261	},
263	262	0b01 => {
264	263	// PUSH/POP r/m64
...	...	@@ -283,7 +282,7 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
283	282	}
284	283
285	284	fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
286		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	285	const ops = emit.mir.instructions.items(.ops)[inst].decode();
287	286	const payload = emit.mir.instructions.items(.data)[inst].payload;
288	287	const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data;
289	288	const regs = data.regs;
...	...	@@ -294,9 +293,9 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
294	293	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
295	294	.disp = @bitCast(u32, -@intCast(i32, disp)),
296	295	.base = ops.reg1,
297		}), Register.reg(reg.to64()), emit.code);
	296	}), reg.to64(), emit.code);
298	297	} else {
299		try lowerToRmEnc(.mov, Register.reg(reg.to64()), RegisterOrMemory.mem(.qword_ptr, .{
	298	try lowerToRmEnc(.mov, reg.to64(), RegisterOrMemory.mem(.qword_ptr, .{
300	299	.disp = @bitCast(u32, -@intCast(i32, disp)),
301	300	.base = ops.reg1,
302	301	}), emit.code);
...	...	@@ -306,7 +305,7 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
306	305	}
307	306
308	307	fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
309		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	308	const ops = emit.mir.instructions.items(.ops)[inst].decode();
310	309	switch (ops.flags) {
311	310	0b00 => {
312	311	const target = emit.mir.instructions.items(.data)[inst].inst;
...	...	@@ -335,7 +334,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
335	334	0b10 => {
336	335	// JMP/CALL r/m64
337	336	const imm = emit.mir.instructions.items(.data)[inst].imm;
338		return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	337	return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
339	338	.disp = imm,
340	339	.base = ops.reg1,
341	340	}), emit.code);
...	...	@@ -345,7 +344,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
345	344	}
346	345
347	346	fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
348		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	347	const ops = emit.mir.instructions.items(.ops)[inst].decode();
349	348	const target = emit.mir.instructions.items(.data)[inst].inst;
350	349	const tag = switch (mir_tag) {
351	350	.cond_jmp_greater_less => switch (ops.flags) {
...	...	@@ -377,7 +376,7 @@ fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerErr
377	376	}
378	377
379	378	fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
380		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	379	const ops = emit.mir.instructions.items(.ops)[inst].decode();
381	380	const tag = switch (mir_tag) {
382	381	.cond_set_byte_greater_less => switch (ops.flags) {
383	382	0b00 => Tag.setge,
...	...	@@ -407,9 +406,9 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne
407	406	}
408	407
409	408	fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
410		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	409	const ops = emit.mir.instructions.items(.ops)[inst].decode();
411	410	if (ops.flags == 0b00) {
412		return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code);
	411	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
413	412	}
414	413	const imm = emit.mir.instructions.items(.data)[inst].imm;
415	414	const ptr_size: Memory.PtrSize = switch (ops.flags) {
...	...	@@ -418,7 +417,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
418	417	0b10 => .dword_ptr,
419	418	0b11 => .qword_ptr,
420	419	};
421		return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(ptr_size, .{
	420	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ptr_size, .{
422	421	.disp = imm,
423	422	.base = ops.reg2,
424	423	}), emit.code);
...	...	@@ -427,7 +426,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
427	426	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
428	427	const tag = emit.mir.instructions.items(.tag)[inst];
429	428	assert(tag == .@"test");
430		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	429	const ops = emit.mir.instructions.items(.ops)[inst].decode();
431	430	switch (ops.flags) {
432	431	0b00 => {
433	432	if (ops.reg2 == .none) {
...	...	@@ -442,12 +441,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
442	441	return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
443	442	}
444	443	// TEST r/m64, r64
445		return lowerToMrEnc(
446		.@"test",
447		RegisterOrMemory.reg(ops.reg1),
448		Register.reg(ops.reg2),
449		emit.code,
450		);
	444	return lowerToMrEnc(.@"test", RegisterOrMemory.reg(ops.reg1), ops.reg2, emit.code);
451	445	},
452	446	else => return emit.fail("TODO more TEST alternatives", .{}),
453	447	}
...	...	@@ -456,7 +450,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
456	450	fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
457	451	const tag = emit.mir.instructions.items(.tag)[inst];
458	452	assert(tag == .ret);
459		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	453	const ops = emit.mir.instructions.items(.ops)[inst].decode();
460	454	switch (ops.flags) {
461	455	0b00 => {
462	456	// RETF imm16
...	...	@@ -480,7 +474,7 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
480	474	}
481	475
482	476	fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
483		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	477	const ops = emit.mir.instructions.items(.ops)[inst].decode();
484	478	switch (ops.flags) {
485	479	0b00 => {
486	480	if (ops.reg2 == .none) {
...	...	@@ -491,14 +485,14 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
491	485	}
492	486	// mov reg1, reg2
493	487	// RM
494		return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code);
	488	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
495	489	},
496	490	0b01 => {
497	491	// mov reg1, [reg2 + imm32]
498	492	// RM
499	493	const imm = emit.mir.instructions.items(.data)[inst].imm;
500		const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2;
501		return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	494	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
	495	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
502	496	.disp = imm,
503	497	.base = src_reg,
504	498	}), emit.code);
...	...	@@ -510,10 +504,10 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
510	504	// mov [reg1 + imm32], reg2
511	505	// MR
512	506	const imm = emit.mir.instructions.items(.data)[inst].imm;
513		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
	507	return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
514	508	.disp = imm,
515	509	.base = ops.reg1,
516		}), Register.reg(ops.reg2), emit.code);
	510	}), ops.reg2, emit.code);
517	511	},
518	512	0b11 => {
519	513	return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
...	...	@@ -522,7 +516,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
522	516	}
523	517
524	518	fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
525		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	519	const ops = emit.mir.instructions.items(.ops)[inst].decode();
526	520	assert(ops.reg2 == .none);
527	521	const payload = emit.mir.instructions.items(.data)[inst].payload;
528	522	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
...	...	@@ -539,19 +533,14 @@ fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
539	533	}
540	534
541	535	inline fn setRexWRegister(reg: Register) bool {
542		switch (reg) {
543		.avx_register => return false,
544		.register => \|r\| {
545		if (r.size() == 64) return true;
546		return switch (r) {
547		.ah, .bh, .ch, .dh => true,
548		else => false,
549		};
550		},
551		}
	536	if (reg.size() == 64) return true;
	537	return switch (reg) {
	538	.ah, .ch, .dh, .bh => true,
	539	else => false,
	540	};
552	541	}
553	542
554		inline fn immOpSize(u_imm: u32) u8 {
	543	inline fn immOpSize(u_imm: u32) u6 {
555	544	const imm = @bitCast(i32, u_imm);
556	545	if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
557	546	return 8;
...	...	@@ -563,7 +552,7 @@ inline fn immOpSize(u_imm: u32) u8 {
563	552	}
564	553
565	554	fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
566		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	555	const ops = emit.mir.instructions.items(.ops)[inst].decode();
567	556	const scale = ops.flags;
568	557	const imm = emit.mir.instructions.items(.data)[inst].imm;
569	558	// OP reg1, [reg2 + scale*rcx + imm32]
...	...	@@ -571,7 +560,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
571	560	.scale = scale,
572	561	.index = .rcx,
573	562	};
574		return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	563	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
575	564	.disp = imm,
576	565	.base = ops.reg2,
577	566	.scale_index = scale_index,
...	...	@@ -579,7 +568,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
579	568	}
580	569
581	570	fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
582		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	571	const ops = emit.mir.instructions.items(.ops)[inst].decode();
583	572	const scale = ops.flags;
584	573	const imm = emit.mir.instructions.items(.data)[inst].imm;
585	574	const scale_index = ScaleIndex{
...	...	@@ -595,15 +584,15 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
595	584	}), imm, emit.code);
596	585	}
597	586	// OP [reg1 + scale*rax + imm32], reg2
598		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
	587	return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
599	588	.disp = imm,
600	589	.base = ops.reg1,
601	590	.scale_index = scale_index,
602		}), Register.reg(ops.reg2), emit.code);
	591	}), ops.reg2, emit.code);
603	592	}
604	593
605	594	fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
606		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	595	const ops = emit.mir.instructions.items(.ops)[inst].decode();
607	596	const scale = ops.flags;
608	597	const payload = emit.mir.instructions.items(.data)[inst].payload;
609	598	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
...	...	@@ -620,7 +609,7 @@ fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
620	609	}
621	610
622	611	fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
623		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	612	const ops = emit.mir.instructions.items(.ops)[inst].decode();
624	613	assert(ops.reg2 == .none);
625	614	const payload = emit.mir.instructions.items(.data)[inst].payload;
626	615	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
...	...	@@ -645,27 +634,27 @@ fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!v
645	634	fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
646	635	const mir_tag = emit.mir.instructions.items(.tag)[inst];
647	636	assert(mir_tag == .mov_sign_extend);
648		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	637	const ops = emit.mir.instructions.items(.ops)[inst].decode();
649	638	const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
650	639	switch (ops.flags) {
651	640	0b00 => {
652	641	const tag: Tag = if (ops.reg2.size() == 32) .movsxd else .movsx;
653		return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code);
	642	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
654	643	},
655	644	0b01 => {
656		return lowerToRmEnc(.movsx, Register.reg(ops.reg1), RegisterOrMemory.mem(.byte_ptr, .{
	645	return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
657	646	.disp = imm,
658	647	.base = ops.reg2,
659	648	}), emit.code);
660	649	},
661	650	0b10 => {
662		return lowerToRmEnc(.movsx, Register.reg(ops.reg1), RegisterOrMemory.mem(.word_ptr, .{
	651	return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
663	652	.disp = imm,
664	653	.base = ops.reg2,
665	654	}), emit.code);
666	655	},
667	656	0b11 => {
668		return lowerToRmEnc(.movsxd, Register.reg(ops.reg1), RegisterOrMemory.mem(.dword_ptr, .{
	657	return lowerToRmEnc(.movsxd, ops.reg1, RegisterOrMemory.mem(.dword_ptr, .{
669	658	.disp = imm,
670	659	.base = ops.reg2,
671	660	}), emit.code);
...	...	@@ -676,20 +665,20 @@ fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
676	665	fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
677	666	const mir_tag = emit.mir.instructions.items(.tag)[inst];
678	667	assert(mir_tag == .mov_zero_extend);
679		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	668	const ops = emit.mir.instructions.items(.ops)[inst].decode();
680	669	const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
681	670	switch (ops.flags) {
682	671	0b00 => {
683		return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code);
	672	return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
684	673	},
685	674	0b01 => {
686		return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.mem(.byte_ptr, .{
	675	return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
687	676	.disp = imm,
688	677	.base = ops.reg2,
689	678	}), emit.code);
690	679	},
691	680	0b10 => {
692		return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.mem(.word_ptr, .{
	681	return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
693	682	.disp = imm,
694	683	.base = ops.reg2,
695	684	}), emit.code);
...	...	@@ -703,31 +692,46 @@ fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
703	692	fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
704	693	const tag = emit.mir.instructions.items(.tag)[inst];
705	694	assert(tag == .movabs);
706		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
707		const imm: u64 = if (ops.reg1.size() == 64) blk: {
708		const payload = emit.mir.instructions.items(.data)[inst].payload;
709		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
710		break :blk imm.decode();
711		} else emit.mir.instructions.items(.data)[inst].imm;
712		if (ops.flags == 0b00) {
713		// movabs reg, imm64
714		// OI
715		return lowerToOiEnc(.mov, Register.reg(ops.reg1), imm, emit.code);
716		}
717		if (ops.reg1 == .none) {
718		// movabs moffs64, rax
719		// TD
720		return lowerToTdEnc(.mov, imm, Register.reg(ops.reg2), emit.code);
	695	const ops = emit.mir.instructions.items(.ops)[inst].decode();
	696	switch (ops.flags) {
	697	0b00 => {
	698	const imm: u64 = if (ops.reg1.size() == 64) blk: {
	699	const payload = emit.mir.instructions.items(.data)[inst].payload;
	700	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
	701	break :blk imm.decode();
	702	} else emit.mir.instructions.items(.data)[inst].imm;
	703	// movabs reg, imm64
	704	// OI
	705	return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
	706	},
	707	0b01 => {
	708	if (ops.reg1 == .none) {
	709	const imm: u64 = if (ops.reg2.size() == 64) blk: {
	710	const payload = emit.mir.instructions.items(.data)[inst].payload;
	711	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
	712	break :blk imm.decode();
	713	} else emit.mir.instructions.items(.data)[inst].imm;
	714	// movabs moffs64, rax
	715	// TD
	716	return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
	717	}
	718	const imm: u64 = if (ops.reg1.size() == 64) blk: {
	719	const payload = emit.mir.instructions.items(.data)[inst].payload;
	720	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
	721	break :blk imm.decode();
	722	} else emit.mir.instructions.items(.data)[inst].imm;
	723	// movabs rax, moffs64
	724	// FD
	725	return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
	726	},
	727	else => return emit.fail("TODO unused variant: movabs 0b{b}", .{ops.flags}),
721	728	}
722		// movabs rax, moffs64
723		// FD
724		return lowerToFdEnc(.mov, Register.reg(ops.reg1), imm, emit.code);
725	729	}
726	730
727	731	fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
728	732	const tag = emit.mir.instructions.items(.tag)[inst];
729	733	assert(tag == .fisttp);
730		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	734	const ops = emit.mir.instructions.items(.ops)[inst].decode();
731	735
732	736	// the selecting between operand sizes for this particular `fisttp` instruction
733	737	// is done via opcode instead of the usual prefixes.
...	...	@@ -749,7 +753,7 @@ fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
749	753	fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
750	754	const tag = emit.mir.instructions.items(.tag)[inst];
751	755	assert(tag == .fld);
752		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	756	const ops = emit.mir.instructions.items(.ops)[inst].decode();
753	757
754	758	// the selecting between operand sizes for this particular `fisttp` instruction
755	759	// is done via opcode instead of the usual prefixes.
...	...	@@ -768,7 +772,7 @@ fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
768	772	}
769	773
770	774	fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
771		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	775	const ops = emit.mir.instructions.items(.ops)[inst].decode();
772	776	switch (ops.flags) {
773	777	0b00 => {
774	778	// sal reg1, 1
...	...	@@ -793,12 +797,11 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
793	797	}
794	798
795	799	fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
796		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	800	const ops = emit.mir.instructions.items(.ops)[inst].decode();
797	801	if (ops.reg1 != .none) {
798	802	assert(ops.reg2 == .none);
799	803	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
800	804	}
801		assert(ops.reg1 == .none);
802	805	assert(ops.reg2 != .none);
803	806	const imm = emit.mir.instructions.items(.data)[inst].imm;
804	807	const ptr_size: Memory.PtrSize = switch (ops.flags) {
...	...	@@ -816,27 +819,27 @@ fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
816	819	fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
817	820	const tag = emit.mir.instructions.items(.tag)[inst];
818	821	assert(tag == .imul_complex);
819		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	822	const ops = emit.mir.instructions.items(.ops)[inst].decode();
820	823	switch (ops.flags) {
821	824	0b00 => {
822		return lowerToRmEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code);
	825	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
823	826	},
824	827	0b01 => {
825	828	const imm = emit.mir.instructions.items(.data)[inst].imm;
826		const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2;
827		return lowerToRmEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.mem(.qword_ptr, .{
	829	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
	830	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
828	831	.disp = imm,
829	832	.base = src_reg,
830	833	}), emit.code);
831	834	},
832	835	0b10 => {
833	836	const imm = emit.mir.instructions.items(.data)[inst].imm;
834		return lowerToRmiEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), imm, emit.code);
	837	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code);
835	838	},
836	839	0b11 => {
837	840	const payload = emit.mir.instructions.items(.data)[inst].payload;
838	841	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
839		return lowerToRmiEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.mem(.qword_ptr, .{
	842	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
840	843	.disp = imm_pair.dest_off,
841	844	.base = ops.reg2,
842	845	}), imm_pair.operand, emit.code);
...	...	@@ -845,7 +848,7 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
845	848	}
846	849
847	850	fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
848		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	851	const ops = emit.mir.instructions.items(.ops)[inst].decode();
849	852	const tag: Tag = switch (ops.flags) {
850	853	0b00 => .cbw,
851	854	0b01 => .cwd,
...	...	@@ -858,17 +861,17 @@ fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
858	861	fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
859	862	const tag = emit.mir.instructions.items(.tag)[inst];
860	863	assert(tag == .lea);
861		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	864	const ops = emit.mir.instructions.items(.ops)[inst].decode();
862	865	switch (ops.flags) {
863	866	0b00 => {
864	867	// lea reg1, [reg2 + imm32]
865	868	// RM
866	869	const imm = emit.mir.instructions.items(.data)[inst].imm;
867		const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2;
	870	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
868	871	return lowerToRmEnc(
869	872	.lea,
870		Register.reg(ops.reg1),
871		RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	873	ops.reg1,
	874	RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
872	875	.disp = imm,
873	876	.base = src_reg,
874	877	}),
...	...	@@ -881,8 +884,8 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
881	884	const start_offset = emit.code.items.len;
882	885	try lowerToRmEnc(
883	886	.lea,
884		Register.reg(ops.reg1),
885		RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
	887	ops.reg1,
	888	RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
886	889	emit.code,
887	890	);
888	891	const end_offset = emit.code.items.len;
...	...	@@ -895,15 +898,15 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
895	898	0b10 => {
896	899	// lea reg, [rbp + rcx + imm32]
897	900	const imm = emit.mir.instructions.items(.data)[inst].imm;
898		const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2;
	901	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
899	902	const scale_index = ScaleIndex{
900	903	.scale = 0,
901	904	.index = .rcx,
902	905	};
903	906	return lowerToRmEnc(
904	907	.lea,
905		Register.reg(ops.reg1),
906		RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	908	ops.reg1,
	909	RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
907	910	.disp = imm,
908	911	.base = src_reg,
909	912	.scale_index = scale_index,
...	...	@@ -918,15 +921,15 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
918	921	fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
919	922	const tag = emit.mir.instructions.items(.tag)[inst];
920	923	assert(tag == .lea_pie);
921		const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]);
	924	const ops = emit.mir.instructions.items(.ops)[inst].decode();
922	925	const load_reloc = emit.mir.instructions.items(.data)[inst].load_reloc;
923	926
924	927	// lea reg1, [rip + reloc]
925	928	// RM
926	929	try lowerToRmEnc(
927	930	.lea,
928		Register.reg(ops.reg1),
929		RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
	931	ops.reg1,
	932	RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
930	933	emit.code,
931	934	);
932	935
...	...	@@ -962,78 +965,70 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
962	965	fn mirMovF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
963	966	const tag = emit.mir.instructions.items(.tag)[inst];
964	967	assert(tag == .mov_f64);
965		const ops = emit.mir.instructions.items(.ops)[inst];
966		const flags = @truncate(u2, ops);
	968	const ops = emit.mir.instructions.items(.ops)[inst].decode();
967	969
968		switch (flags) {
	970	switch (ops.flags) {
969	971	0b00 => {
970		const decoded = Mir.Ops(AvxRegister, GpRegister).decode(ops);
971	972	const imm = emit.mir.instructions.items(.data)[inst].imm;
972		return lowerToRmEnc(.vmovsd, Register.avxReg(decoded.reg1), RegisterOrMemory.mem(.qword_ptr, .{
	973	return lowerToVmEnc(.vmovsd, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
973	974	.disp = imm,
974		.base = decoded.reg2,
	975	.base = ops.reg2,
975	976	}), emit.code);
976	977	},
977	978	0b01 => {
978		const decoded = Mir.Ops(GpRegister, AvxRegister).decode(ops);
979	979	const imm = emit.mir.instructions.items(.data)[inst].imm;
980		return lowerToMrEnc(.vmovsd, RegisterOrMemory.mem(.qword_ptr, .{
	980	return lowerToMvEnc(.vmovsd, RegisterOrMemory.mem(.qword_ptr, .{
981	981	.disp = imm,
982		.base = decoded.reg1,
983		}), Register.avxReg(decoded.reg2), emit.code);
	982	.base = ops.reg1,
	983	}), ops.reg2, emit.code);
984	984	},
985	985	0b10 => {
986		const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops);
987	986	return lowerToRvmEnc(
988	987	.vmovsd,
989		Register.avxReg(decoded.reg1),
990		Register.avxReg(decoded.reg1),
991		RegisterOrMemory.avxReg(decoded.reg2),
	988	ops.reg1,
	989	ops.reg1,
	990	RegisterOrMemory.reg(ops.reg2),
992	991	emit.code,
993	992	);
994	993	},
995		else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}),
	994	else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
996	995	}
997	996	}
998	997
999	998	fn mirAddF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1000	999	const tag = emit.mir.instructions.items(.tag)[inst];
1001	1000	assert(tag == .add_f64);
1002		const ops = emit.mir.instructions.items(.ops)[inst];
1003		const flags = @truncate(u2, ops);
	1001	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1004	1002
1005		switch (flags) {
	1003	switch (ops.flags) {
1006	1004	0b00 => {
1007		const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops);
1008	1005	return lowerToRvmEnc(
1009	1006	.vaddsd,
1010		Register.avxReg(decoded.reg1),
1011		Register.avxReg(decoded.reg1),
1012		RegisterOrMemory.avxReg(decoded.reg2),
	1007	ops.reg1,
	1008	ops.reg1,
	1009	RegisterOrMemory.reg(ops.reg2),
1013	1010	emit.code,
1014	1011	);
1015	1012	},
1016		else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}),
	1013	else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
1017	1014	}
1018	1015	}
1019	1016
1020	1017	fn mirCmpF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1021	1018	const tag = emit.mir.instructions.items(.tag)[inst];
1022	1019	assert(tag == .cmp_f64);
1023		const ops = emit.mir.instructions.items(.ops)[inst];
1024		const flags = @truncate(u2, ops);
	1020	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1025	1021
1026		switch (flags) {
	1022	switch (ops.flags) {
1027	1023	0b00 => {
1028		const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops);
1029		return lowerToRmEnc(
	1024	return lowerToVmEnc(
1030	1025	.vucomisd,
1031		Register.avxReg(decoded.reg1),
1032		RegisterOrMemory.avxReg(decoded.reg2),
	1026	ops.reg1,
	1027	RegisterOrMemory.reg(ops.reg2),
1033	1028	emit.code,
1034	1029	);
1035	1030	},
1036		else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}),
	1031	else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
1037	1032	}
1038	1033	}
1039	1034
...	...	@@ -1277,6 +1272,18 @@ const Tag = enum {
1277	1272	vcmpsd,
1278	1273	vucomisd,
1279	1274
	1275	fn isAvx(tag: Tag) bool {
	1276	return switch (tag) {
	1277	.vmovsd,
	1278	.vaddsd,
	1279	.vcmpsd,
	1280	.vucomisd,
	1281	=> true,
	1282
	1283	else => false,
	1284	};
	1285	}
	1286
1280	1287	fn isSetCC(tag: Tag) bool {
1281	1288	return switch (tag) {
1282	1289	.seto,
...	...	@@ -1361,6 +1368,12 @@ const Encoding = enum {
1361	1368	/// OP r64, r/m64, imm32
1362	1369	rmi,
1363	1370
	1371	/// OP xmm1, xmm2/m64
	1372	vm,
	1373
	1374	/// OP m64, xmm1
	1375	mv,
	1376
1364	1377	/// OP xmm1, xmm2, xmm3/m64
1365	1378	rvm,
1366	1379
...	...	@@ -1389,7 +1402,7 @@ const OpCode = union(enum) {
1389	1402
1390	1403	fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1391	1404	assert(opc == .one_byte);
1392		encoder.opcode_withReg(opc.one_byte, reg.lowId());
	1405	encoder.opcode_withReg(opc.one_byte, reg.lowEnc());
1393	1406	}
1394	1407	};
1395	1408
...	...	@@ -1536,6 +1549,15 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1536	1549	.imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1537	1550	else => null,
1538	1551	},
	1552	.mv => return switch (tag) {
	1553	.vmovsd => OpCode.oneByte(0x11),
	1554	else => null,
	1555	},
	1556	.vm => return switch (tag) {
	1557	.vmovsd => OpCode.oneByte(0x10),
	1558	.vucomisd => OpCode.oneByte(0x2e),
	1559	else => null,
	1560	},
1539	1561	.rvm => return switch (tag) {
1540	1562	.vaddsd => OpCode.oneByte(0x58),
1541	1563	.vmovsd => OpCode.oneByte(0x10),
...	...	@@ -1647,11 +1669,11 @@ inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix {
1647	1669	} = .none,
1648	1670	} = blk: {
1649	1671	switch (enc) {
1650		.mr => switch (tag) {
	1672	.mv => switch (tag) {
1651	1673	.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1652	1674	else => return null,
1653	1675	},
1654		.rm => switch (tag) {
	1676	.vm => switch (tag) {
1655	1677	.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1656	1678	.vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true },
1657	1679	else => return null,
...	...	@@ -1697,61 +1719,50 @@ inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix {
1697	1719	} };
1698	1720	}
1699	1721
1700		const ScaleIndex = struct {
	1722	const ScaleIndex = packed struct {
1701	1723	scale: u2,
1702		index: GpRegister,
	1724	index: Register,
1703	1725	};
1704	1726
1705	1727	const Memory = struct {
1706		base: ?GpRegister,
	1728	base: ?Register,
1707	1729	rip: bool = false,
1708	1730	disp: u32,
1709	1731	ptr_size: PtrSize,
1710	1732	scale_index: ?ScaleIndex = null,
1711	1733
1712		const PtrSize = enum {
1713		byte_ptr,
1714		word_ptr,
1715		dword_ptr,
1716		qword_ptr,
	1734	const PtrSize = enum(u2) {
	1735	byte_ptr = 0b00,
	1736	word_ptr = 0b01,
	1737	dword_ptr = 0b10,
	1738	qword_ptr = 0b11,
1717	1739
1718		fn fromBits(in_bits: u64) PtrSize {
1719		return switch (in_bits) {
1720		8 => .byte_ptr,
1721		16 => .word_ptr,
1722		32 => .dword_ptr,
1723		64 => .qword_ptr,
1724		else => unreachable,
1725		};
	1740	fn new(bit_size: u64) PtrSize {
	1741	return @intToEnum(PtrSize, math.log2_int(u4, @intCast(u4, @divExact(bit_size, 8))));
1726	1742	}
1727	1743
1728	1744	/// Returns size in bits.
1729	1745	fn size(ptr_size: PtrSize) u64 {
1730		return switch (ptr_size) {
1731		.byte_ptr => 8,
1732		.word_ptr => 16,
1733		.dword_ptr => 32,
1734		.qword_ptr => 64,
1735		};
	1746	return 8 * (math.powi(u8, 2, @enumToInt(ptr_size)) catch unreachable);
1736	1747	}
1737	1748	};
1738	1749
1739	1750	fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
1740	1751	if (mem_op.base) \|base\| {
1741		const dst = base.lowId();
	1752	const dst = base.lowEnc();
1742	1753	const src = operand;
1743	1754	if (dst == 4 or mem_op.scale_index != null) {
1744	1755	if (mem_op.disp == 0 and dst != 5) {
1745	1756	encoder.modRm_SIBDisp0(src);
1746	1757	if (mem_op.scale_index) \|si\| {
1747		encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
	1758	encoder.sib_scaleIndexBase(si.scale, si.index.lowEnc(), dst);
1748	1759	} else {
1749	1760	encoder.sib_base(dst);
1750	1761	}
1751	1762	} else if (immOpSize(mem_op.disp) == 8) {
1752	1763	encoder.modRm_SIBDisp8(src);
1753	1764	if (mem_op.scale_index) \|si\| {
1754		encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
	1765	encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowEnc(), dst);
1755	1766	} else {
1756	1767	encoder.sib_baseDisp8(dst);
1757	1768	}
...	...	@@ -1759,7 +1770,7 @@ const Memory = struct {
1759	1770	} else {
1760	1771	encoder.modRm_SIBDisp32(src);
1761	1772	if (mem_op.scale_index) \|si\| {
1762		encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
	1773	encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowEnc(), dst);
1763	1774	} else {
1764	1775	encoder.sib_baseDisp32(dst);
1765	1776	}
...	...	@@ -1782,7 +1793,7 @@ const Memory = struct {
1782	1793	} else {
1783	1794	encoder.modRm_SIBDisp0(operand);
1784	1795	if (mem_op.scale_index) \|si\| {
1785		encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
	1796	encoder.sib_scaleIndexDisp32(si.scale, si.index.lowEnc());
1786	1797	} else {
1787	1798	encoder.sib_disp32();
1788	1799	}
...	...	@@ -1791,6 +1802,7 @@ const Memory = struct {
1791	1802	}
1792	1803	}
1793	1804
	1805	/// Returns size in bits.
1794	1806	fn size(memory: Memory) u64 {
1795	1807	return memory.ptr_size.size();
1796	1808	}
...	...	@@ -1805,55 +1817,17 @@ fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
1805	1817	}
1806	1818	}
1807	1819
1808		const Register = union(enum) {
1809		register: GpRegister,
1810		avx_register: AvxRegister,
1811
1812		fn reg(register: GpRegister) Register {
1813		return .{ .register = register };
1814		}
1815
1816		fn avxReg(register: AvxRegister) Register {
1817		return .{ .avx_register = register };
1818		}
1819
1820		fn lowId(register: Register) u3 {
1821		return switch (register) {
1822		.register => \|r\| r.lowId(),
1823		.avx_register => \|r\| r.lowId(),
1824		};
1825		}
1826
1827		fn size(register: Register) u64 {
1828		return switch (register) {
1829		.register => \|r\| r.size(),
1830		.avx_register => \|r\| r.size(),
1831		};
1832		}
1833
1834		fn isExtended(register: Register) bool {
1835		return switch (register) {
1836		.register => \|r\| r.isExtended(),
1837		.avx_register => \|r\| r.isExtended(),
1838		};
1839		}
1840		};
1841
1842	1820	const RegisterOrMemory = union(enum) {
1843	1821	register: Register,
1844	1822	memory: Memory,
1845	1823
1846		fn reg(register: GpRegister) RegisterOrMemory {
1847		return .{ .register = Register.reg(register) };
1848		}
1849
1850		fn avxReg(register: AvxRegister) RegisterOrMemory {
1851		return .{ .register = Register.avxReg(register) };
	1824	fn reg(register: Register) RegisterOrMemory {
	1825	return .{ .register = register };
1852	1826	}
1853	1827
1854	1828	fn mem(ptr_size: Memory.PtrSize, args: struct {
1855	1829	disp: u32,
1856		base: ?GpRegister = null,
	1830	base: ?Register = null,
1857	1831	scale_index: ?ScaleIndex = null,
1858	1832	}) RegisterOrMemory {
1859	1833	return .{
...	...	@@ -1877,6 +1851,7 @@ const RegisterOrMemory = union(enum) {
1877	1851	};
1878	1852	}
1879	1853
	1854	/// Returns size in bits.
1880	1855	fn size(reg_or_mem: RegisterOrMemory) u64 {
1881	1856	return switch (reg_or_mem) {
1882	1857	.register => \|reg\| reg.size(),
...	...	@@ -1886,6 +1861,7 @@ const RegisterOrMemory = union(enum) {
1886	1861	};
1887	1862
1888	1863	fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
	1864	assert(!tag.isAvx());
1889	1865	const opc = getOpCode(tag, .zo, false).?;
1890	1866	const encoder = try Encoder.init(code, 2);
1891	1867	switch (tag) {
...	...	@@ -1900,6 +1876,7 @@ fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
1900	1876	}
1901	1877
1902	1878	fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
	1879	assert(!tag.isAvx());
1903	1880	if (tag == .ret_far or tag == .ret_near) {
1904	1881	const encoder = try Encoder.init(code, 3);
1905	1882	const opc = getOpCode(tag, .i, false).?;
...	...	@@ -1917,6 +1894,7 @@ fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
1917	1894	}
1918	1895
1919	1896	fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
	1897	assert(!tag.isAvx());
1920	1898	const opc = getOpCode(tag, .o, false).?;
1921	1899	const encoder = try Encoder.init(code, 3);
1922	1900	if (reg.size() == 16) {
...	...	@@ -1930,6 +1908,7 @@ fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!voi
1930	1908	}
1931	1909
1932	1910	fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
	1911	assert(!tag.isAvx());
1933	1912	const opc = getOpCode(tag, .d, false).?;
1934	1913	const encoder = try Encoder.init(code, 6);
1935	1914	opc.encode(encoder);
...	...	@@ -1937,6 +1916,7 @@ fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
1937	1916	}
1938	1917
1939	1918	fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void {
	1919	assert(!tag.isAvx());
1940	1920	const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?;
1941	1921	const modrm_ext = getModRmExt(tag).?;
1942	1922	switch (reg_or_mem) {
...	...	@@ -1951,7 +1931,7 @@ fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *st
1951	1931	.b = reg.isExtended(),
1952	1932	});
1953	1933	opc.encode(encoder);
1954		encoder.modRm_direct(modrm_ext, reg.lowId());
	1934	encoder.modRm_direct(modrm_ext, reg.lowEnc());
1955	1935	},
1956	1936	.memory => \|mem_op\| {
1957	1937	const encoder = try Encoder.init(code, 8);
...	...	@@ -1992,6 +1972,7 @@ fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) I
1992	1972	}
1993	1973
1994	1974	fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void {
	1975	assert(!tag.isAvx());
1995	1976	const opc = if (td)
1996	1977	getOpCode(tag, .td, reg.size() == 8).?
1997	1978	else
...	...	@@ -2014,6 +1995,7 @@ fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8),
2014	1995	}
2015	1996
2016	1997	fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void {
	1998	assert(!tag.isAvx());
2017	1999	const opc = getOpCode(tag, .oi, reg.size() == 8).?;
2018	2000	const encoder = try Encoder.init(code, 10);
2019	2001	if (reg.size() == 16) {
...	...	@@ -2040,6 +2022,7 @@ fn lowerToMiXEnc(
2040	2022	enc: Encoding,
2041	2023	code: *std.ArrayList(u8),
2042	2024	) InnerError!void {
	2025	assert(!tag.isAvx());
2043	2026	const modrm_ext = getModRmExt(tag).?;
2044	2027	const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?;
2045	2028	switch (reg_or_mem) {
...	...	@@ -2056,7 +2039,7 @@ fn lowerToMiXEnc(
2056	2039	.b = dst_reg.isExtended(),
2057	2040	});
2058	2041	opc.encode(encoder);
2059		encoder.modRm_direct(modrm_ext, dst_reg.lowId());
	2042	encoder.modRm_direct(modrm_ext, dst_reg.lowEnc());
2060	2043	encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_reg.size());
2061	2044	},
2062	2045	.memory => \|dst_mem\| {
...	...	@@ -2095,84 +2078,43 @@ fn lowerToRmEnc(
2095	2078	reg_or_mem: RegisterOrMemory,
2096	2079	code: *std.ArrayList(u8),
2097	2080	) InnerError!void {
	2081	assert(!tag.isAvx());
2098	2082	const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8).?;
2099	2083	switch (reg_or_mem) {
2100	2084	.register => \|src_reg\| {
2101		const encoder: Encoder = blk: {
2102		switch (reg) {
2103		.register => {
2104		const encoder = try Encoder.init(code, 4);
2105		if (reg.size() == 16) {
2106		encoder.prefix16BitMode();
2107		}
2108		encoder.rex(.{
2109		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
2110		.r = reg.isExtended(),
2111		.b = src_reg.isExtended(),
2112		});
2113		break :blk encoder;
2114		},
2115		.avx_register => {
2116		const encoder = try Encoder.init(code, 5);
2117		var vex_prefix = getVexPrefix(tag, .rm).?;
2118		const vex = &vex_prefix.prefix;
2119		vex.rex(.{
2120		.r = reg.isExtended(),
2121		.b = src_reg.isExtended(),
2122		});
2123		encoder.vex(vex_prefix.prefix);
2124		break :blk encoder;
2125		},
2126		}
2127		};
	2085	const encoder = try Encoder.init(code, 5);
	2086	if (reg.size() == 16) {
	2087	encoder.prefix16BitMode();
	2088	}
	2089	encoder.rex(.{
	2090	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
	2091	.r = reg.isExtended(),
	2092	.b = src_reg.isExtended(),
	2093	});
2128	2094	opc.encode(encoder);
2129		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
	2095	encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2130	2096	},
2131	2097	.memory => \|src_mem\| {
2132		const encoder: Encoder = blk: {
2133		switch (reg) {
2134		.register => {
2135		const encoder = try Encoder.init(code, 9);
2136		if (reg.size() == 16) {
2137		encoder.prefix16BitMode();
2138		}
2139		if (src_mem.base) \|base\| {
2140		// TODO handle 32-bit base register - requires prefix 0x67
2141		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
2142		encoder.rex(.{
2143		.w = setRexWRegister(reg),
2144		.r = reg.isExtended(),
2145		.b = base.isExtended(),
2146		});
2147		} else {
2148		encoder.rex(.{
2149		.w = setRexWRegister(reg),
2150		.r = reg.isExtended(),
2151		});
2152		}
2153		break :blk encoder;
2154		},
2155		.avx_register => {
2156		const encoder = try Encoder.init(code, 10);
2157		var vex_prefix = getVexPrefix(tag, .rm).?;
2158		const vex = &vex_prefix.prefix;
2159		if (src_mem.base) \|base\| {
2160		vex.rex(.{
2161		.r = reg.isExtended(),
2162		.b = base.isExtended(),
2163		});
2164		} else {
2165		vex.rex(.{
2166		.r = reg.isExtended(),
2167		});
2168		}
2169		encoder.vex(vex_prefix.prefix);
2170		break :blk encoder;
2171		},
2172		}
2173		};
	2098	const encoder = try Encoder.init(code, 9);
	2099	if (reg.size() == 16) {
	2100	encoder.prefix16BitMode();
	2101	}
	2102	if (src_mem.base) \|base\| {
	2103	// TODO handle 32-bit base register - requires prefix 0x67
	2104	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	2105	encoder.rex(.{
	2106	.w = setRexWRegister(reg),
	2107	.r = reg.isExtended(),
	2108	.b = base.isExtended(),
	2109	});
	2110	} else {
	2111	encoder.rex(.{
	2112	.w = setRexWRegister(reg),
	2113	.r = reg.isExtended(),
	2114	});
	2115	}
2174	2116	opc.encode(encoder);
2175		src_mem.encode(encoder, reg.lowId());
	2117	src_mem.encode(encoder, reg.lowEnc());
2176	2118	},
2177	2119	}
2178	2120	}
...	...	@@ -2183,6 +2125,7 @@ fn lowerToMrEnc(
2183	2125	reg: Register,
2184	2126	code: *std.ArrayList(u8),
2185	2127	) InnerError!void {
	2128	assert(!tag.isAvx());
2186	2129	const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8).?;
2187	2130	switch (reg_or_mem) {
2188	2131	.register => \|dst_reg\| {
...	...	@@ -2196,53 +2139,27 @@ fn lowerToMrEnc(
2196	2139	.b = dst_reg.isExtended(),
2197	2140	});
2198	2141	opc.encode(encoder);
2199		encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
	2142	encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2200	2143	},
2201	2144	.memory => \|dst_mem\| {
2202		const encoder: Encoder = blk: {
2203		switch (reg) {
2204		.register => {
2205		const encoder = try Encoder.init(code, 9);
2206		if (reg.size() == 16) {
2207		encoder.prefix16BitMode();
2208		}
2209		if (dst_mem.base) \|base\| {
2210		encoder.rex(.{
2211		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2212		.r = reg.isExtended(),
2213		.b = base.isExtended(),
2214		});
2215		} else {
2216		encoder.rex(.{
2217		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2218		.r = reg.isExtended(),
2219		});
2220		}
2221		break :blk encoder;
2222		},
2223		.avx_register => {
2224		const encoder = try Encoder.init(code, 10);
2225		var vex_prefix = getVexPrefix(tag, .mr).?;
2226		const vex = &vex_prefix.prefix;
2227		if (dst_mem.base) \|base\| {
2228		vex.rex(.{
2229		.w = dst_mem.ptr_size == .qword_ptr,
2230		.r = reg.isExtended(),
2231		.b = base.isExtended(),
2232		});
2233		} else {
2234		vex.rex(.{
2235		.w = dst_mem.ptr_size == .qword_ptr,
2236		.r = reg.isExtended(),
2237		});
2238		}
2239		encoder.vex(vex_prefix.prefix);
2240		break :blk encoder;
2241		},
2242		}
2243		};
	2145	const encoder = try Encoder.init(code, 9);
	2146	if (reg.size() == 16) {
	2147	encoder.prefix16BitMode();
	2148	}
	2149	if (dst_mem.base) \|base\| {
	2150	encoder.rex(.{
	2151	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	2152	.r = reg.isExtended(),
	2153	.b = base.isExtended(),
	2154	});
	2155	} else {
	2156	encoder.rex(.{
	2157	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	2158	.r = reg.isExtended(),
	2159	});
	2160	}
2244	2161	opc.encode(encoder);
2245		dst_mem.encode(encoder, reg.lowId());
	2162	dst_mem.encode(encoder, reg.lowEnc());
2246	2163	},
2247	2164	}
2248	2165	}
...	...	@@ -2254,6 +2171,7 @@ fn lowerToRmiEnc(
2254	2171	imm: u32,
2255	2172	code: *std.ArrayList(u8),
2256	2173	) InnerError!void {
	2174	assert(!tag.isAvx());
2257	2175	const opc = getOpCode(tag, .rmi, false).?;
2258	2176	const encoder = try Encoder.init(code, 13);
2259	2177	if (reg.size() == 16) {
...	...	@@ -2267,7 +2185,7 @@ fn lowerToRmiEnc(
2267	2185	.b = src_reg.isExtended(),
2268	2186	});
2269	2187	opc.encode(encoder);
2270		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
	2188	encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2271	2189	},
2272	2190	.memory => \|src_mem\| {
2273	2191	if (src_mem.base) \|base\| {
...	...	@@ -2285,12 +2203,94 @@ fn lowerToRmiEnc(
2285	2203	});
2286	2204	}
2287	2205	opc.encode(encoder);
2288		src_mem.encode(encoder, reg.lowId());
	2206	src_mem.encode(encoder, reg.lowEnc());
2289	2207	},
2290	2208	}
2291	2209	encodeImm(encoder, imm, reg.size());
2292	2210	}
2293	2211
	2212	/// Also referred to as XM encoding in Intel manual.
	2213	fn lowerToVmEnc(
	2214	tag: Tag,
	2215	reg: Register,
	2216	reg_or_mem: RegisterOrMemory,
	2217	code: *std.ArrayList(u8),
	2218	) InnerError!void {
	2219	const opc = getOpCode(tag, .vm, false).?;
	2220	var vex_prefix = getVexPrefix(tag, .vm).?;
	2221	const vex = &vex_prefix.prefix;
	2222	switch (reg_or_mem) {
	2223	.register => \|src_reg\| {
	2224	const encoder = try Encoder.init(code, 5);
	2225	vex.rex(.{
	2226	.r = reg.isExtended(),
	2227	.b = src_reg.isExtended(),
	2228	});
	2229	encoder.vex(vex_prefix.prefix);
	2230	opc.encode(encoder);
	2231	encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
	2232	},
	2233	.memory => \|src_mem\| {
	2234	assert(src_mem.ptr_size == .qword_ptr);
	2235	const encoder = try Encoder.init(code, 10);
	2236	if (src_mem.base) \|base\| {
	2237	vex.rex(.{
	2238	.r = reg.isExtended(),
	2239	.b = base.isExtended(),
	2240	});
	2241	} else {
	2242	vex.rex(.{
	2243	.r = reg.isExtended(),
	2244	});
	2245	}
	2246	encoder.vex(vex_prefix.prefix);
	2247	opc.encode(encoder);
	2248	src_mem.encode(encoder, reg.lowEnc());
	2249	},
	2250	}
	2251	}
	2252
	2253	/// Usually referred to as MR encoding with V/V in Intel manual.
	2254	fn lowerToMvEnc(
	2255	tag: Tag,
	2256	reg_or_mem: RegisterOrMemory,
	2257	reg: Register,
	2258	code: *std.ArrayList(u8),
	2259	) InnerError!void {
	2260	const opc = getOpCode(tag, .mv, false).?;
	2261	var vex_prefix = getVexPrefix(tag, .mv).?;
	2262	const vex = &vex_prefix.prefix;
	2263	switch (reg_or_mem) {
	2264	.register => \|dst_reg\| {
	2265	const encoder = try Encoder.init(code, 4);
	2266	vex.rex(.{
	2267	.r = reg.isExtended(),
	2268	.b = dst_reg.isExtended(),
	2269	});
	2270	encoder.vex(vex_prefix.prefix);
	2271	opc.encode(encoder);
	2272	encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
	2273	},
	2274	.memory => \|dst_mem\| {
	2275	assert(dst_mem.ptr_size == .qword_ptr);
	2276	const encoder = try Encoder.init(code, 10);
	2277	if (dst_mem.base) \|base\| {
	2278	vex.rex(.{
	2279	.r = reg.isExtended(),
	2280	.b = base.isExtended(),
	2281	});
	2282	} else {
	2283	vex.rex(.{
	2284	.r = reg.isExtended(),
	2285	});
	2286	}
	2287	encoder.vex(vex_prefix.prefix);
	2288	opc.encode(encoder);
	2289	dst_mem.encode(encoder, reg.lowEnc());
	2290	},
	2291	}
	2292	}
	2293
2294	2294	fn lowerToRvmEnc(
2295	2295	tag: Tag,
2296	2296	reg1: Register,
...	...	@@ -2305,7 +2305,7 @@ fn lowerToRvmEnc(
2305	2305	.register => \|reg3\| {
2306	2306	if (vex_prefix.reg) \|vvvv\| {
2307	2307	switch (vvvv) {
2308		.nds => vex.reg(reg2.avx_register.id()),
	2308	.nds => vex.reg(reg2.enc()),
2309	2309	else => unreachable, // TODO
2310	2310	}
2311	2311	}
...	...	@@ -2316,7 +2316,7 @@ fn lowerToRvmEnc(
2316	2316	});
2317	2317	encoder.vex(vex_prefix.prefix);
2318	2318	opc.encode(encoder);
2319		encoder.modRm_direct(reg1.lowId(), reg3.lowId());
	2319	encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2320	2320	},
2321	2321	.memory => \|dst_mem\| {
2322	2322	_ = dst_mem;
...	...	@@ -2341,7 +2341,7 @@ fn lowerToRvmiEnc(
2341	2341	.register => \|reg3\| {
2342	2342	if (vex_prefix.reg) \|vvvv\| {
2343	2343	switch (vvvv) {
2344		.nds => vex.reg(reg2.avx_register.id()),
	2344	.nds => vex.reg(reg2.enc()),
2345	2345	else => unreachable, // TODO
2346	2346	}
2347	2347	}
...	...	@@ -2352,7 +2352,7 @@ fn lowerToRvmiEnc(
2352	2352	});
2353	2353	encoder.vex(vex_prefix.prefix);
2354	2354	opc.encode(encoder);
2355		encoder.modRm_direct(reg1.lowId(), reg3.lowId());
	2355	encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2356	2356	break :blk encoder;
2357	2357	},
2358	2358	.memory => \|dst_mem\| {
...	...	@@ -2490,23 +2490,23 @@ test "lower MI encoding" {
2490	2490	test "lower RM encoding" {
2491	2491	var emit = TestEmit.init();
2492	2492	defer emit.deinit();
2493		try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.reg(.rbx), emit.code());
	2493	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code());
2494	2494	try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx");
2495		try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
	2495	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
2496	2496	try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]");
2497		try lowerToRmEnc(.add, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
	2497	try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
2498	2498	try expectEqualHexStrings(
2499	2499	"\x4C\x03\x1C\x25\x00\x00\x00\x10",
2500	2500	emit.lowered(),
2501	2501	"add r11, qword ptr [ds:0x10000000]",
2502	2502	);
2503		try lowerToRmEnc(.add, Register.reg(.r12b), RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
	2503	try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
2504	2504	try expectEqualHexStrings(
2505	2505	"\x44\x02\x24\x25\x00\x00\x00\x10",
2506	2506	emit.lowered(),
2507	2507	"add r11b, byte ptr [ds:0x10000000]",
2508	2508	);
2509		try lowerToRmEnc(.sub, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{
	2509	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2510	2510	.disp = 0x10000000,
2511	2511	.base = .r13,
2512	2512	}), emit.code());
...	...	@@ -2515,7 +2515,7 @@ test "lower RM encoding" {
2515	2515	emit.lowered(),
2516	2516	"sub r11, qword ptr [r13 + 0x10000000]",
2517	2517	);
2518		try lowerToRmEnc(.sub, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{
	2518	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2519	2519	.disp = 0x10000000,
2520	2520	.base = .r12,
2521	2521	}), emit.code());
...	...	@@ -2524,14 +2524,14 @@ test "lower RM encoding" {
2524	2524	emit.lowered(),
2525	2525	"sub r11, qword ptr [r12 + 0x10000000]",
2526	2526	);
2527		try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{
	2527	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2528	2528	.disp = @bitCast(u32, @as(i32, -4)),
2529	2529	.base = .rbp,
2530	2530	}), emit.code());
2531	2531	try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]");
2532		try lowerToRmEnc(.lea, Register.reg(.rax), RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
	2532	try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2533	2533	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]");
2534		try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{
	2534	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2535	2535	.disp = @bitCast(u32, @as(i32, -8)),
2536	2536	.base = .rbp,
2537	2537	.scale_index = .{
...	...	@@ -2540,7 +2540,7 @@ test "lower RM encoding" {
2540	2540	},
2541	2541	}), emit.code());
2542	2542	try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
2543		try lowerToRmEnc(.mov, Register.reg(.eax), RegisterOrMemory.mem(.dword_ptr, .{
	2543	try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2544	2544	.disp = @bitCast(u32, @as(i32, -4)),
2545	2545	.base = .rbp,
2546	2546	.scale_index = .{
...	...	@@ -2549,7 +2549,7 @@ test "lower RM encoding" {
2549	2549	},
2550	2550	}), emit.code());
2551	2551	try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
2552		try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{
	2552	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2553	2553	.disp = @bitCast(u32, @as(i32, -8)),
2554	2554	.base = .rbp,
2555	2555	.scale_index = .{
...	...	@@ -2558,7 +2558,7 @@ test "lower RM encoding" {
2558	2558	},
2559	2559	}), emit.code());
2560	2560	try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
2561		try lowerToRmEnc(.mov, Register.reg(.r8b), RegisterOrMemory.mem(.byte_ptr, .{
	2561	try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
2562	2562	.disp = @bitCast(u32, @as(i32, -24)),
2563	2563	.base = .rsi,
2564	2564	.scale_index = .{
...	...	@@ -2567,7 +2567,7 @@ test "lower RM encoding" {
2567	2567	},
2568	2568	}), emit.code());
2569	2569	try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
2570		try lowerToRmEnc(.lea, Register.reg(.rsi), RegisterOrMemory.mem(.qword_ptr, .{
	2570	try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
2571	2571	.disp = 0,
2572	2572	.base = .rbp,
2573	2573	.scale_index = .{
...	...	@@ -2576,33 +2576,25 @@ test "lower RM encoding" {
2576	2576	},
2577	2577	}), emit.code());
2578	2578	try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
2579
2580		// AVX extension tests
2581		try lowerToRmEnc(.vmovsd, Register.avxReg(.xmm1), RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2582		try expectEqualHexStrings(
2583		"\xC5\xFB\x10\x0D\x10\x00\x00\x00",
2584		emit.lowered(),
2585		"vmovsd xmm1, qword ptr [rip + 0x10]",
2586		);
2587	2579	}
2588	2580
2589	2581	test "lower MR encoding" {
2590	2582	var emit = TestEmit.init();
2591	2583	defer emit.deinit();
2592		try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), Register.reg(.rbx), emit.code());
	2584	try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code());
2593	2585	try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx");
2594	2586	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2595	2587	.disp = @bitCast(u32, @as(i32, -4)),
2596	2588	.base = .rbp,
2597		}), Register.reg(.r11), emit.code());
	2589	}), .r11, emit.code());
2598	2590	try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11");
2599		try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), Register.reg(.r12b), emit.code());
	2591	try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code());
2600	2592	try expectEqualHexStrings(
2601	2593	"\x44\x00\x24\x25\x00\x00\x00\x10",
2602	2594	emit.lowered(),
2603	2595	"add byte ptr [ds:0x10000000], r12b",
2604	2596	);
2605		try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), Register.reg(.r12d), emit.code());
	2597	try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code());
2606	2598	try expectEqualHexStrings(
2607	2599	"\x44\x01\x24\x25\x00\x00\x00\x10",
2608	2600	emit.lowered(),
...	...	@@ -2611,61 +2603,53 @@ test "lower MR encoding" {
2611	2603	try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
2612	2604	.disp = 0x10000000,
2613	2605	.base = .r11,
2614		}), Register.reg(.r12), emit.code());
	2606	}), .r12, emit.code());
2615	2607	try expectEqualHexStrings(
2616	2608	"\x4D\x29\xA3\x00\x00\x00\x10",
2617	2609	emit.lowered(),
2618	2610	"sub qword ptr [r11 + 0x10000000], r12",
2619	2611	);
2620		try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), Register.reg(.r12), emit.code());
	2612	try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code());
2621	2613	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12");
2622
2623		// AVX extension tests
2624		try lowerToMrEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), Register.avxReg(.xmm1), emit.code());
2625		try expectEqualHexStrings(
2626		"\xC5\xFB\x11\x0D\x10\x00\x00\x00",
2627		emit.lowered(),
2628		"vmovsd qword ptr [rip + 0x10], xmm1",
2629		);
2630	2614	}
2631	2615
2632	2616	test "lower OI encoding" {
2633	2617	var emit = TestEmit.init();
2634	2618	defer emit.deinit();
2635		try lowerToOiEnc(.mov, Register.reg(.rax), 0x1000000000000000, emit.code());
	2619	try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code());
2636	2620	try expectEqualHexStrings(
2637	2621	"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
2638	2622	emit.lowered(),
2639	2623	"movabs rax, 0x1000000000000000",
2640	2624	);
2641		try lowerToOiEnc(.mov, Register.reg(.r11), 0x1000000000000000, emit.code());
	2625	try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code());
2642	2626	try expectEqualHexStrings(
2643	2627	"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
2644	2628	emit.lowered(),
2645	2629	"movabs r11, 0x1000000000000000",
2646	2630	);
2647		try lowerToOiEnc(.mov, Register.reg(.r11d), 0x10000000, emit.code());
	2631	try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code());
2648	2632	try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000");
2649		try lowerToOiEnc(.mov, Register.reg(.r11w), 0x1000, emit.code());
	2633	try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code());
2650	2634	try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000");
2651		try lowerToOiEnc(.mov, Register.reg(.r11b), 0x10, emit.code());
	2635	try lowerToOiEnc(.mov, .r11b, 0x10, emit.code());
2652	2636	try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10");
2653	2637	}
2654	2638
2655	2639	test "lower FD/TD encoding" {
2656	2640	var emit = TestEmit.init();
2657	2641	defer emit.deinit();
2658		try lowerToFdEnc(.mov, Register.reg(.rax), 0x1000000000000000, emit.code());
	2642	try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code());
2659	2643	try expectEqualHexStrings(
2660	2644	"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2661	2645	emit.lowered(),
2662	2646	"mov rax, ds:0x1000000000000000",
2663	2647	);
2664		try lowerToFdEnc(.mov, Register.reg(.eax), 0x10000000, emit.code());
	2648	try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code());
2665	2649	try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000");
2666		try lowerToFdEnc(.mov, Register.reg(.ax), 0x1000, emit.code());
	2650	try lowerToFdEnc(.mov, .ax, 0x1000, emit.code());
2667	2651	try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000");
2668		try lowerToFdEnc(.mov, Register.reg(.al), 0x10, emit.code());
	2652	try lowerToFdEnc(.mov, .al, 0x10, emit.code());
2669	2653	try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10");
2670	2654	}
2671	2655
...	...	@@ -2741,16 +2725,16 @@ test "lower M1 and MC encodings" {
2741	2725	test "lower O encoding" {
2742	2726	var emit = TestEmit.init();
2743	2727	defer emit.deinit();
2744		try lowerToOEnc(.pop, Register.reg(.r12), emit.code());
	2728	try lowerToOEnc(.pop, .r12, emit.code());
2745	2729	try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12");
2746		try lowerToOEnc(.push, Register.reg(.r12w), emit.code());
	2730	try lowerToOEnc(.push, .r12w, emit.code());
2747	2731	try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w");
2748	2732	}
2749	2733
2750	2734	test "lower RMI encoding" {
2751	2735	var emit = TestEmit.init();
2752	2736	defer emit.deinit();
2753		try lowerToRmiEnc(.imul, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{
	2737	try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2754	2738	.disp = @bitCast(u32, @as(i32, -8)),
2755	2739	.base = .rbp,
2756	2740	}), 0x10, emit.code());
...	...	@@ -2759,39 +2743,49 @@ test "lower RMI encoding" {
2759	2743	emit.lowered(),
2760	2744	"imul rax, qword ptr [rbp - 8], 0x10",
2761	2745	);
2762		try lowerToRmiEnc(.imul, Register.reg(.eax), RegisterOrMemory.mem(.dword_ptr, .{
	2746	try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2763	2747	.disp = @bitCast(u32, @as(i32, -4)),
2764	2748	.base = .rbp,
2765	2749	}), 0x10, emit.code());
2766	2750	try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
2767		try lowerToRmiEnc(.imul, Register.reg(.ax), RegisterOrMemory.mem(.word_ptr, .{
	2751	try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
2768	2752	.disp = @bitCast(u32, @as(i32, -2)),
2769	2753	.base = .rbp,
2770	2754	}), 0x10, emit.code());
2771	2755	try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
2772		try lowerToRmiEnc(.imul, Register.reg(.r12), RegisterOrMemory.reg(.r12), 0x10, emit.code());
	2756	try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code());
2773	2757	try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10");
2774		try lowerToRmiEnc(.imul, Register.reg(.r12w), RegisterOrMemory.reg(.r12w), 0x10, emit.code());
	2758	try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
2775	2759	try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
2776	2760	}
2777	2761
2778		test "lower to RVM encoding" {
	2762	test "lower MV encoding" {
2779	2763	var emit = TestEmit.init();
2780	2764	defer emit.deinit();
2781		try lowerToRvmEnc(
2782		.vaddsd,
2783		Register.avxReg(.xmm0),
2784		Register.avxReg(.xmm1),
2785		RegisterOrMemory.avxReg(.xmm2),
2786		emit.code(),
	2765	try lowerToMvEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), .xmm1, emit.code());
	2766	try expectEqualHexStrings(
	2767	"\xC5\xFB\x11\x0D\x10\x00\x00\x00",
	2768	emit.lowered(),
	2769	"vmovsd qword ptr [rip + 0x10], xmm1",
2787	2770	);
2788		try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2");
2789		try lowerToRvmEnc(
2790		.vaddsd,
2791		Register.avxReg(.xmm0),
2792		Register.avxReg(.xmm0),
2793		RegisterOrMemory.avxReg(.xmm1),
2794		emit.code(),
	2771	}
	2772
	2773	test "lower VM encoding" {
	2774	var emit = TestEmit.init();
	2775	defer emit.deinit();
	2776	try lowerToVmEnc(.vmovsd, .xmm1, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
	2777	try expectEqualHexStrings(
	2778	"\xC5\xFB\x10\x0D\x10\x00\x00\x00",
	2779	emit.lowered(),
	2780	"vmovsd xmm1, qword ptr [rip + 0x10]",
2795	2781	);
	2782	}
	2783
	2784	test "lower to RVM encoding" {
	2785	var emit = TestEmit.init();
	2786	defer emit.deinit();
	2787	try lowerToRvmEnc(.vaddsd, .xmm0, .xmm1, RegisterOrMemory.reg(.xmm2), emit.code());
	2788	try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2");
	2789	try lowerToRvmEnc(.vaddsd, .xmm0, .xmm0, RegisterOrMemory.reg(.xmm1), emit.code());
2796	2790	try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1");
2797	2791	}

src/arch/x86_64/Mir.zig+32-34

...	...	@@ -14,8 +14,7 @@ const assert = std.debug.assert;
14	14	const bits = @import("bits.zig");
15	15	const Air = @import("../../Air.zig");
16	16	const CodeGen = @import("CodeGen.zig");
17		const GpRegister = bits.Register;
18		const AvxRegister = bits.AvxRegister;
	17	const Register = bits.Register;
19	18
20	19	instructions: std.MultiArrayList(Inst).Slice,
21	20	/// The meaning of this data is determined by `Inst.Tag` value.
...	...	@@ -23,11 +22,7 @@ extra: []const u32,
23	22
24	23	pub const Inst = struct {
25	24	tag: Tag,
26		/// This is 3 fields, and the meaning of each depends on `tag`.
27		/// reg1: Register
28		/// reg2: Register
29		/// flags: u2
30		ops: u16,
	25	ops: Ops,
31	26	/// The meaning of this depends on `tag` and `ops`.
32	27	data: Data,
33	28
...	...	@@ -397,6 +392,36 @@ pub const Inst = struct {
397	392	/// The position of an MIR instruction within the `Mir` instructions array.
398	393	pub const Index = u32;
399	394
	395	pub const Ops = packed struct {
	396	reg1: u7,
	397	reg2: u7,
	398	flags: u2,
	399
	400	pub fn encode(vals: struct {
	401	reg1: Register = .none,
	402	reg2: Register = .none,
	403	flags: u2 = 0b00,
	404	}) Ops {
	405	return .{
	406	.reg1 = @enumToInt(vals.reg1),
	407	.reg2 = @enumToInt(vals.reg2),
	408	.flags = vals.flags,
	409	};
	410	}
	411
	412	pub fn decode(ops: Ops) struct {
	413	reg1: Register,
	414	reg2: Register,
	415	flags: u2,
	416	} {
	417	return .{
	418	.reg1 = @intToEnum(Register, ops.reg1),
	419	.reg2 = @intToEnum(Register, ops.reg2),
	420	.flags = ops.flags,
	421	};
	422	}
	423	};
	424
400	425	/// All instructions have a 4-byte payload, which is contained within
401	426	/// this union. `Tag` determines which union field is active, as well as
402	427	/// how to interpret the data within.
...	...	@@ -466,33 +491,6 @@ pub const DbgLineColumn = struct {
466	491	column: u32,
467	492	};
468	493
469		pub fn Ops(comptime Reg1: type, comptime Reg2: type) type {
470		return struct {
471		reg1: Reg1 = .none,
472		reg2: Reg2 = .none,
473		flags: u2 = 0b00,
474
475		pub fn encode(self: @This()) u16 {
476		var ops: u16 = 0;
477		ops \|= @intCast(u16, @enumToInt(self.reg1)) << 9;
478		ops \|= @intCast(u16, @enumToInt(self.reg2)) << 2;
479		ops \|= self.flags;
480		return ops;
481		}
482
483		pub fn decode(ops: u16) @This() {
484		const reg1 = @intToEnum(Reg1, @truncate(u7, ops >> 9));
485		const reg2 = @intToEnum(Reg2, @truncate(u7, ops >> 2));
486		const flags = @truncate(u2, ops);
487		return .{
488		.reg1 = reg1,
489		.reg2 = reg2,
490		.flags = flags,
491		};
492		}
493		};
494		}
495
496	494	pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
497	495	mir.instructions.deinit(gpa);
498	496	gpa.free(mir.extra);

src/arch/x86_64/abi.zig+11-6

...	...	@@ -3,7 +3,6 @@ const Type = @import("../../type.zig").Type;
3	3	const Target = std.Target;
4	4	const assert = std.debug.assert;
5	5	const Register = @import("bits.zig").Register;
6		const AvxRegister = @import("bits.zig").AvxRegister;
7	6
8	7	pub const Class = enum { integer, sse, sseup, x87, x87up, complex_x87, memory, none };
9	8
...	...	@@ -379,11 +378,17 @@ pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 };
379	378	/// the caller relinquishes control to a subroutine via call instruction (or similar).
380	379	/// In other words, these registers are free to use by the callee.
381	380	pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 };
382		pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
383		pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
384		pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
385
386		pub const avx_regs = [_]AvxRegister{
	381	pub const avx_regs = [_]Register{
387	382	.ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7,
388	383	.ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15,
389	384	};
	385	pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs;
	386
	387	// Masks for register manager
	388	const FreeRegInt = std.meta.Int(.unsigned, allocatable_registers.len);
	389	// TODO
	390	pub const gp_mask: FreeRegInt = 0x3fff;
	391	pub const avx_mask: FreeRegInt = 0x3fff_c000;
	392
	393	pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
	394	pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };

src/arch/x86_64/bits.zig+38-62

...	...	@@ -43,17 +43,36 @@ pub const Register = enum(u7) {
43	43	al, cl, dl, bl, ah, ch, dh, bh,
44	44	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,
45	45
46		// Pseudo, used only for MIR to signify that the
47		// operand is not a register but an immediate, etc.
	46	// 64-79, 256-bit registers.
	47	// id is int value - 64.
	48	ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,
	49	ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
	50
	51	// 80-95, 128-bit registers.
	52	// id is int value - 80.
	53	xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
	54	xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
	55
	56	// Pseudo-value for MIR instructions.
48	57	none,
49	58
	59	pub fn id(self: Register) u5 {
	60	return switch (@enumToInt(self)) {
	61	0...63 => @as(u5, @truncate(u4, @enumToInt(self))),
	62	64...79 => @truncate(u5, @enumToInt(self)),
	63	else => unreachable,
	64	};
	65	}
	66
50	67	/// Returns the bit-width of the register.
51		pub fn size(self: Register) u7 {
	68	pub fn size(self: Register) u9 {
52	69	return switch (@enumToInt(self)) {
53	70	0...15 => 64,
54	71	16...31 => 32,
55	72	32...47 => 16,
56		48...64 => 8,
	73	48...63 => 8,
	74	64...79 => 256,
	75	80...95 => 128,
57	76	else => unreachable,
58	77	};
59	78	}
...	...	@@ -72,15 +91,23 @@ pub const Register = enum(u7) {
72	91	/// an instruction (@see isExtended), and requires special handling. The
73	92	/// lower three bits are often embedded directly in instructions (such as
74	93	/// the B8 variant of moves), or used in R/M bytes.
75		pub fn id(self: Register) u4 {
	94	pub fn enc(self: Register) u4 {
76	95	return @truncate(u4, @enumToInt(self));
77	96	}
78	97
79		/// Like id, but only returns the lower 3 bits.
80		pub fn lowId(self: Register) u3 {
	98	/// Like enc, but only returns the lower 3 bits.
	99	pub fn lowEnc(self: Register) u3 {
81	100	return @truncate(u3, @enumToInt(self));
82	101	}
83	102
	103	pub fn to256(self: Register) Register {
	104	return @intToEnum(Register, @as(u8, self.id()) + 64);
	105	}
	106
	107	pub fn to128(self: Register) Register {
	108	return @intToEnum(Register, @as(u8, self.id()) + 80);
	109	}
	110
84	111	/// Convert from any register to its 64 bit alias.
85	112	pub fn to64(self: Register) Register {
86	113	return @intToEnum(Register, self.id());
...	...	@@ -126,57 +153,6 @@ pub const Register = enum(u7) {
126	153	}
127	154	};
128	155
129		/// AVX registers.
130		/// TODO missing dwarfLocOp implementation.
131		/// TODO add support for AVX-512
132		pub const AvxRegister = enum(u6) {
133		// 256-bit registers
134		ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,
135		ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
136
137		// 128-bit registers
138		xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
139		xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
140
141		// Pseudo, used only for MIR to signify that the
142		// operand is not a register but an immediate, etc.
143		none,
144
145		/// Returns the bit-width of the register.
146		pub fn size(self: AvxRegister) u9 {
147		return switch (@enumToInt(self)) {
148		0...15 => 256,
149		16...31 => 128,
150		else => unreachable,
151		};
152		}
153
154		/// Returns whether the register is extended.
155		pub fn isExtended(self: AvxRegister) bool {
156		return @enumToInt(self) & 0x08 != 0;
157		}
158
159		/// This returns the 4-bit register ID.
160		pub fn id(self: AvxRegister) u4 {
161		return @truncate(u4, @enumToInt(self));
162		}
163
164		/// Like id, but only returns the lower 3 bits.
165		pub fn lowId(self: AvxRegister) u3 {
166		return @truncate(u3, @enumToInt(self));
167		}
168
169		/// Convert from any register to its 256 bit alias.
170		pub fn to256(self: AvxRegister) AvxRegister {
171		return @intToEnum(AvxRegister, self.id());
172		}
173
174		/// Convert from any register to its 128 bit alias.
175		pub fn to128(self: AvxRegister) AvxRegister {
176		return @intToEnum(AvxRegister, @as(u8, self.id()) + 16);
177		}
178		};
179
180	156	// zig fmt: on
181	157
182	158	/// Encoding helper functions for x86_64 instructions
...	...	@@ -792,7 +768,7 @@ test "Encoder helpers - Vex prefix" {
792	768	{
793	769	stream.reset();
794	770	var vex_prefix = Encoder.Vex{};
795		vex_prefix.reg(AvxRegister.xmm15.id());
	771	vex_prefix.reg(Register.xmm15.id());
796	772	const nwritten = vex_prefix.write(writer);
797	773	try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]);
798	774	}
...	...	@@ -832,7 +808,7 @@ test "Encoder helpers - Vex prefix" {
832	808	vex.simd_prefix_66();
833	809	encoder.vex(vex); // use 64 bit operation
834	810	encoder.opcode_1byte(0x28);
835		encoder.modRm_direct(0, AvxRegister.xmm1.lowId());
	811	encoder.modRm_direct(0, Register.xmm1.lowId());
836	812	try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items);
837	813	}
838	814
...	...	@@ -846,10 +822,10 @@ test "Encoder helpers - Vex prefix" {
846	822	vex.simd_prefix_66();
847	823	vex.lead_opc_0f();
848	824	vex.rex(.{ .r = true });
849		vex.reg(AvxRegister.xmm1.id());
	825	vex.reg(Register.xmm1.id());
850	826	encoder.vex(vex);
851	827	encoder.opcode_1byte(0x16);
852		encoder.modRm_RIPDisp32(AvxRegister.xmm13.lowId());
	828	encoder.modRm_RIPDisp32(Register.xmm13.lowId());
853	829	encoder.disp32(0);
854	830	try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items);
855	831	}

src/codegen.zig+12-12

...	...	@@ -78,13 +78,13 @@ pub fn generateFunction(
78	78	debug_output: DebugInfoOutput,
79	79	) GenerateSymbolError!FnResult {
80	80	switch (bin_file.options.target.cpu.arch) {
81		// .arm,
82		// .armeb,
83		// => return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
84		// .aarch64,
85		// .aarch64_be,
86		// .aarch64_32,
87		// => return @import("arch/aarch64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
	81	.arm,
	82	.armeb,
	83	=> return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
	84	.aarch64,
	85	.aarch64_be,
	86	.aarch64_32,
	87	=> return @import("arch/aarch64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
88	88	//.arc => return Function(.arc).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
89	89	//.avr => return Function(.avr).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
90	90	//.bpfel => return Function(.bpfel).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
...	...	@@ -101,9 +101,9 @@ pub fn generateFunction(
101	101	//.r600 => return Function(.r600).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
102	102	//.amdgcn => return Function(.amdgcn).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
103	103	//.riscv32 => return Function(.riscv32).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
104		// .riscv64 => return @import("arch/riscv64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
	104	.riscv64 => return @import("arch/riscv64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
105	105	//.sparc => return Function(.sparc).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
106		// .sparc64 => return @import("arch/sparc64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
	106	.sparc64 => return @import("arch/sparc64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
107	107	//.sparcel => return Function(.sparcel).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
108	108	//.s390x => return Function(.s390x).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
109	109	//.tce => return Function(.tce).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
...	...	@@ -129,9 +129,9 @@ pub fn generateFunction(
129	129	//.renderscript32 => return Function(.renderscript32).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
130	130	//.renderscript64 => return Function(.renderscript64).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
131	131	//.ve => return Function(.ve).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
132		// .wasm32,
133		// .wasm64,
134		// => return @import("arch/wasm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
	132	.wasm32,
	133	.wasm64,
	134	=> return @import("arch/wasm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
135	135	else => @panic("Backend architectures that don't have good support yet are commented out, to improve compilation performance. If you are interested in one of these other backends feel free to uncomment them. Eventually these will be completed, but stage1 is slow and a memory hog."),
136	136	}
137	137	}

src/register_manager.zig+255-261

...	...	@@ -23,15 +23,10 @@ pub const AllocateRegistersError = error{
23	23	CodegenFail,
24	24	};
25	25
26		pub fn SpillFn(comptime Function: type, comptime Register: type) type {
27		return fn (*Function, Register, Air.Inst.Index) anyerror!void;
28		}
29
30	26	pub fn RegisterManager(
31	27	comptime Function: type,
32	28	comptime Register: type,
33	29	comptime tracked_registers: []const Register,
34		comptime spill_fn: SpillFn(Function, Register),
35	30	) type {
36	31	// architectures which do not have a concept of registers should
37	32	// refrain from using RegisterManager
...	...	@@ -52,7 +47,6 @@ pub fn RegisterManager(
52	47	allocated_registers: FreeRegInt = 0,
53	48	/// Tracks registers which are locked from being allocated
54	49	locked_registers: FreeRegInt = 0,
55		function: *Function,
56	50
57	51	const Self = @This();
58	52
...	...	@@ -61,8 +55,8 @@ pub fn RegisterManager(
61	55	const FreeRegInt = std.meta.Int(.unsigned, tracked_registers.len);
62	56	const ShiftInt = math.Log2Int(FreeRegInt);
63	57
64		fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) AllocateRegistersError!void {
65		return try spill_fn(self.function, reg, inst);
	58	fn getFunction(self: Self) Function {
	59	return @fieldParentPtr(Function, "register_manager", self);
66	60	}
67	61
68	62	fn getRegisterMask(reg: Register) ?FreeRegInt {
...	...	@@ -257,14 +251,14 @@ pub fn RegisterManager(
257	251	self.markRegUsed(reg);
258	252	} else {
259	253	const spilled_inst = self.registers[index];
260		try self.spillInstruction(reg, spilled_inst);
	254	try self.getFunction().spillInstruction(reg, spilled_inst);
261	255	}
262	256	self.registers[index] = inst;
263	257	} else {
264	258	// Don't track the register
265	259	if (!self.isRegFree(reg)) {
266	260	const spilled_inst = self.registers[index];
267		try self.spillInstruction(reg, spilled_inst);
	261	try self.getFunction().spillInstruction(reg, spilled_inst);
268	262	self.freeReg(reg);
269	263	}
270	264	}
...	...	@@ -299,7 +293,7 @@ pub fn RegisterManager(
299	293	// stack allocation.
300	294	const spilled_inst = self.registers[index];
301	295	self.registers[index] = tracked_inst;
302		try self.spillInstruction(reg, spilled_inst);
	296	try self.getFunction().spillInstruction(reg, spilled_inst);
303	297	} else {
304	298	self.getRegAssumeFree(reg, tracked_inst);
305	299	}
...	...	@@ -308,7 +302,7 @@ pub fn RegisterManager(
308	302	// Move the instruction that was previously there to a
309	303	// stack allocation.
310	304	const spilled_inst = self.registers[index];
311		try self.spillInstruction(reg, spilled_inst);
	305	try self.getFunction().spillInstruction(reg, spilled_inst);
312	306	self.freeReg(reg);
313	307	}
314	308	}
...	...	@@ -338,253 +332,253 @@ pub fn RegisterManager(
338	332	};
339	333	}
340	334
341		//const MockRegister1 = enum(u2) {
342		// r0,
343		// r1,
344		// r2,
345		// r3,
346
347		// pub fn id(reg: MockRegister1) u2 {
348		// return @enumToInt(reg);
349		// }
	335	const MockRegister1 = enum(u2) {
	336	r0,
	337	r1,
	338	r2,
	339	r3,
	340
	341	pub fn id(reg: MockRegister1) u2 {
	342	return @enumToInt(reg);
	343	}
	344
	345	const allocatable_registers = [_]MockRegister1{ .r2, .r3 };
	346	};
	347
	348	const MockRegister2 = enum(u2) {
	349	r0,
	350	r1,
	351	r2,
	352	r3,
	353
	354	pub fn id(reg: MockRegister2) u2 {
	355	return @enumToInt(reg);
	356	}
	357
	358	const allocatable_registers = [_]MockRegister2{ .r0, .r1, .r2, .r3 };
	359	};
	360
	361	fn MockFunction(comptime Register: type) type {
	362	return struct {
	363	allocator: Allocator,
	364	register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{},
	365	spilled: std.ArrayListUnmanaged(Register) = .{},
	366
	367	const Self = @This();
	368
	369	pub fn deinit(self: *Self) void {
	370	self.spilled.deinit(self.allocator);
	371	}
	372
	373	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
	374	_ = inst;
	375	try self.spilled.append(self.allocator, reg);
	376	}
	377
	378	pub fn genAdd(self: *Self, res: Register, lhs: Register, rhs: Register) !void {
	379	_ = self;
	380	_ = res;
	381	_ = lhs;
	382	_ = rhs;
	383	}
	384	};
	385	}
	386
	387	const MockFunction1 = MockFunction(MockRegister1);
	388	const MockFunction2 = MockFunction(MockRegister2);
	389
	390	test "default state" {
	391	const allocator = std.testing.allocator;
	392
	393	var function = MockFunction1{
	394	.allocator = allocator,
	395	};
	396	defer function.deinit();
	397
	398	try expect(!function.register_manager.isRegAllocated(.r2));
	399	try expect(!function.register_manager.isRegAllocated(.r3));
	400	try expect(function.register_manager.isRegFree(.r2));
	401	try expect(function.register_manager.isRegFree(.r3));
	402	}
	403
	404	test "tryAllocReg: no spilling" {
	405	const allocator = std.testing.allocator;
	406
	407	var function = MockFunction1{
	408	.allocator = allocator,
	409	};
	410	defer function.deinit();
	411
	412	const mock_instruction: Air.Inst.Index = 1;
	413
	414	try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction));
	415	try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction));
	416	try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction));
	417
	418	try expect(function.register_manager.isRegAllocated(.r2));
	419	try expect(function.register_manager.isRegAllocated(.r3));
	420	try expect(!function.register_manager.isRegFree(.r2));
	421	try expect(!function.register_manager.isRegFree(.r3));
	422
	423	function.register_manager.freeReg(.r2);
	424	function.register_manager.freeReg(.r3);
	425
	426	try expect(function.register_manager.isRegAllocated(.r2));
	427	try expect(function.register_manager.isRegAllocated(.r3));
	428	try expect(function.register_manager.isRegFree(.r2));
	429	try expect(function.register_manager.isRegFree(.r3));
	430	}
	431
	432	test "allocReg: spilling" {
	433	const allocator = std.testing.allocator;
350	434
351		// const allocatable_registers = [_]MockRegister1{ .r2, .r3 };
352		//};
353
354		//const MockRegister2 = enum(u2) {
355		// r0,
356		// r1,
357		// r2,
358		// r3,
359
360		// pub fn id(reg: MockRegister2) u2 {
361		// return @enumToInt(reg);
362		// }
363
364		// const allocatable_registers = [_]MockRegister2{ .r0, .r1, .r2, .r3 };
365		//};
366
367		//fn MockFunction(comptime Register: type) type {
368		// return struct {
369		// allocator: Allocator,
370		// register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{},
371		// spilled: std.ArrayListUnmanaged(Register) = .{},
372
373		// const Self = @This();
374
375		// pub fn deinit(self: *Self) void {
376		// self.spilled.deinit(self.allocator);
377		// }
378
379		// pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
380		// _ = inst;
381		// try self.spilled.append(self.allocator, reg);
382		// }
383
384		// pub fn genAdd(self: *Self, res: Register, lhs: Register, rhs: Register) !void {
385		// _ = self;
386		// _ = res;
387		// _ = lhs;
388		// _ = rhs;
389		// }
390		// };
391		//}
392
393		//const MockFunction1 = MockFunction(MockRegister1);
394		//const MockFunction2 = MockFunction(MockRegister2);
395
396		//test "default state" {
397		// const allocator = std.testing.allocator;
398
399		// var function = MockFunction1{
400		// .allocator = allocator,
401		// };
402		// defer function.deinit();
403
404		// try expect(!function.register_manager.isRegAllocated(.r2));
405		// try expect(!function.register_manager.isRegAllocated(.r3));
406		// try expect(function.register_manager.isRegFree(.r2));
407		// try expect(function.register_manager.isRegFree(.r3));
408		//}
409
410		//test "tryAllocReg: no spilling" {
411		// const allocator = std.testing.allocator;
412
413		// var function = MockFunction1{
414		// .allocator = allocator,
415		// };
416		// defer function.deinit();
417
418		// const mock_instruction: Air.Inst.Index = 1;
419
420		// try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction));
421		// try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction));
422		// try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction));
423
424		// try expect(function.register_manager.isRegAllocated(.r2));
425		// try expect(function.register_manager.isRegAllocated(.r3));
426		// try expect(!function.register_manager.isRegFree(.r2));
427		// try expect(!function.register_manager.isRegFree(.r3));
428
429		// function.register_manager.freeReg(.r2);
430		// function.register_manager.freeReg(.r3);
431
432		// try expect(function.register_manager.isRegAllocated(.r2));
433		// try expect(function.register_manager.isRegAllocated(.r3));
434		// try expect(function.register_manager.isRegFree(.r2));
435		// try expect(function.register_manager.isRegFree(.r3));
436		//}
437
438		//test "allocReg: spilling" {
439		// const allocator = std.testing.allocator;
440
441		// var function = MockFunction1{
442		// .allocator = allocator,
443		// };
444		// defer function.deinit();
445
446		// const mock_instruction: Air.Inst.Index = 1;
447
448		// try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));
449		// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
450
451		// // Spill a register
452		// try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));
453		// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
454
455		// // No spilling necessary
456		// function.register_manager.freeReg(.r3);
457		// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
458		// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
459
460		// // Locked registers
461		// function.register_manager.freeReg(.r3);
462		// {
463		// const lock = function.register_manager.lockReg(.r2);
464		// defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
465
466		// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
467		// }
468		// try expect(!function.register_manager.lockedRegsExist());
469		//}
470
471		//test "tryAllocRegs" {
472		// const allocator = std.testing.allocator;
473
474		// var function = MockFunction2{
475		// .allocator = allocator,
476		// };
477		// defer function.deinit();
478
479		// try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
480
481		// try expect(function.register_manager.isRegAllocated(.r0));
482		// try expect(function.register_manager.isRegAllocated(.r1));
483		// try expect(function.register_manager.isRegAllocated(.r2));
484		// try expect(!function.register_manager.isRegAllocated(.r3));
485
486		// // Locked registers
487		// function.register_manager.freeReg(.r0);
488		// function.register_manager.freeReg(.r2);
489		// function.register_manager.freeReg(.r3);
490		// {
491		// const lock = function.register_manager.lockReg(.r1);
492		// defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
493
494		// try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
495		// }
496		// try expect(!function.register_manager.lockedRegsExist());
497
498		// try expect(function.register_manager.isRegAllocated(.r0));
499		// try expect(function.register_manager.isRegAllocated(.r1));
500		// try expect(function.register_manager.isRegAllocated(.r2));
501		// try expect(function.register_manager.isRegAllocated(.r3));
502		//}
503
504		//test "allocRegs: normal usage" {
505		// // TODO: convert this into a decltest once that is supported
506
507		// const allocator = std.testing.allocator;
508
509		// var function = MockFunction2{
510		// .allocator = allocator,
511		// };
512		// defer function.deinit();
513
514		// {
515		// const result_reg: MockRegister2 = .r1;
516
517		// // The result register is known and fixed at this point, we
518		// // don't want to accidentally allocate lhs or rhs to the
519		// // result register, this is why we lock it.
520		// //
521		// // Using defer unlock right after lock is a good idea in
522		// // most cases as you probably are using the locked registers
523		// // in the remainder of this scope and don't need to use it
524		// // after the end of this scope. However, in some situations,
525		// // it may make sense to manually unlock registers before the
526		// // end of the scope when you are certain that they don't
527		// // contain any valuable data anymore and can be reused. For an
528		// // example of that, see `selectively reducing register
529		// // pressure`.
530		// const lock = function.register_manager.lockReg(result_reg);
531		// defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
532
533		// const regs = try function.register_manager.allocRegs(2, .{ null, null });
534		// try function.genAdd(result_reg, regs[0], regs[1]);
535		// }
536		//}
537
538		//test "allocRegs: selectively reducing register pressure" {
539		// // TODO: convert this into a decltest once that is supported
540
541		// const allocator = std.testing.allocator;
542
543		// var function = MockFunction2{
544		// .allocator = allocator,
545		// };
546		// defer function.deinit();
547
548		// {
549		// const result_reg: MockRegister2 = .r1;
550
551		// const lock = function.register_manager.lockReg(result_reg);
552
553		// // Here, we don't defer unlock because we manually unlock
554		// // after genAdd
555		// const regs = try function.register_manager.allocRegs(2, .{ null, null });
556
557		// try function.genAdd(result_reg, regs[0], regs[1]);
558		// function.register_manager.unlockReg(lock.?);
559
560		// const extra_summand_reg = try function.register_manager.allocReg(null);
561		// try function.genAdd(result_reg, result_reg, extra_summand_reg);
562		// }
563		//}
564
565		//test "getReg" {
566		// const allocator = std.testing.allocator;
567
568		// var function = MockFunction1{
569		// .allocator = allocator,
570		// };
571		// defer function.deinit();
572
573		// const mock_instruction: Air.Inst.Index = 1;
574
575		// try function.register_manager.getReg(.r3, mock_instruction);
576
577		// try expect(!function.register_manager.isRegAllocated(.r2));
578		// try expect(function.register_manager.isRegAllocated(.r3));
579		// try expect(function.register_manager.isRegFree(.r2));
580		// try expect(!function.register_manager.isRegFree(.r3));
581
582		// // Spill r3
583		// try function.register_manager.getReg(.r3, mock_instruction);
584
585		// try expect(!function.register_manager.isRegAllocated(.r2));
586		// try expect(function.register_manager.isRegAllocated(.r3));
587		// try expect(function.register_manager.isRegFree(.r2));
588		// try expect(!function.register_manager.isRegFree(.r3));
589		// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r3}, function.spilled.items);
590		//}
	435	var function = MockFunction1{
	436	.allocator = allocator,
	437	};
	438	defer function.deinit();
	439
	440	const mock_instruction: Air.Inst.Index = 1;
	441
	442	try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));
	443	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
	444
	445	// Spill a register
	446	try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));
	447	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
	448
	449	// No spilling necessary
	450	function.register_manager.freeReg(.r3);
	451	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
	452	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
	453
	454	// Locked registers
	455	function.register_manager.freeReg(.r3);
	456	{
	457	const lock = function.register_manager.lockReg(.r2);
	458	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
	459
	460	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
	461	}
	462	try expect(!function.register_manager.lockedRegsExist());
	463	}
	464
	465	test "tryAllocRegs" {
	466	const allocator = std.testing.allocator;
	467
	468	var function = MockFunction2{
	469	.allocator = allocator,
	470	};
	471	defer function.deinit();
	472
	473	try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
	474
	475	try expect(function.register_manager.isRegAllocated(.r0));
	476	try expect(function.register_manager.isRegAllocated(.r1));
	477	try expect(function.register_manager.isRegAllocated(.r2));
	478	try expect(!function.register_manager.isRegAllocated(.r3));
	479
	480	// Locked registers
	481	function.register_manager.freeReg(.r0);
	482	function.register_manager.freeReg(.r2);
	483	function.register_manager.freeReg(.r3);
	484	{
	485	const lock = function.register_manager.lockReg(.r1);
	486	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
	487
	488	try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
	489	}
	490	try expect(!function.register_manager.lockedRegsExist());
	491
	492	try expect(function.register_manager.isRegAllocated(.r0));
	493	try expect(function.register_manager.isRegAllocated(.r1));
	494	try expect(function.register_manager.isRegAllocated(.r2));
	495	try expect(function.register_manager.isRegAllocated(.r3));
	496	}
	497
	498	test "allocRegs: normal usage" {
	499	// TODO: convert this into a decltest once that is supported
	500
	501	const allocator = std.testing.allocator;
	502
	503	var function = MockFunction2{
	504	.allocator = allocator,
	505	};
	506	defer function.deinit();
	507
	508	{
	509	const result_reg: MockRegister2 = .r1;
	510
	511	// The result register is known and fixed at this point, we
	512	// don't want to accidentally allocate lhs or rhs to the
	513	// result register, this is why we lock it.
	514	//
	515	// Using defer unlock right after lock is a good idea in
	516	// most cases as you probably are using the locked registers
	517	// in the remainder of this scope and don't need to use it
	518	// after the end of this scope. However, in some situations,
	519	// it may make sense to manually unlock registers before the
	520	// end of the scope when you are certain that they don't
	521	// contain any valuable data anymore and can be reused. For an
	522	// example of that, see `selectively reducing register
	523	// pressure`.
	524	const lock = function.register_manager.lockReg(result_reg);
	525	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
	526
	527	const regs = try function.register_manager.allocRegs(2, .{ null, null });
	528	try function.genAdd(result_reg, regs[0], regs[1]);
	529	}
	530	}
	531
	532	test "allocRegs: selectively reducing register pressure" {
	533	// TODO: convert this into a decltest once that is supported
	534
	535	const allocator = std.testing.allocator;
	536
	537	var function = MockFunction2{
	538	.allocator = allocator,
	539	};
	540	defer function.deinit();
	541
	542	{
	543	const result_reg: MockRegister2 = .r1;
	544
	545	const lock = function.register_manager.lockReg(result_reg);
	546
	547	// Here, we don't defer unlock because we manually unlock
	548	// after genAdd
	549	const regs = try function.register_manager.allocRegs(2, .{ null, null });
	550
	551	try function.genAdd(result_reg, regs[0], regs[1]);
	552	function.register_manager.unlockReg(lock.?);
	553
	554	const extra_summand_reg = try function.register_manager.allocReg(null);
	555	try function.genAdd(result_reg, result_reg, extra_summand_reg);
	556	}
	557	}
	558
	559	test "getReg" {
	560	const allocator = std.testing.allocator;
	561
	562	var function = MockFunction1{
	563	.allocator = allocator,
	564	};
	565	defer function.deinit();
	566
	567	const mock_instruction: Air.Inst.Index = 1;
	568
	569	try function.register_manager.getReg(.r3, mock_instruction);
	570
	571	try expect(!function.register_manager.isRegAllocated(.r2));
	572	try expect(function.register_manager.isRegAllocated(.r3));
	573	try expect(function.register_manager.isRegFree(.r2));
	574	try expect(!function.register_manager.isRegFree(.r3));
	575
	576	// Spill r3
	577	try function.register_manager.getReg(.r3, mock_instruction);
	578
	579	try expect(!function.register_manager.isRegAllocated(.r2));
	580	try expect(function.register_manager.isRegAllocated(.r3));
	581	try expect(function.register_manager.isRegFree(.r2));
	582	try expect(!function.register_manager.isRegFree(.r3));
	583	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r3}, function.spilled.items);
	584	}