Commit 3901b6fb000b909e90347f6910ea6315ee03e220 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2022-01-15 22:27:56+01:00
committer	GitHub <noreply@github.com> 2022-01-15 22:27:56+01:00
log	3901b6fb000b909e90347f6910ea6315ee03e220
tree	2ff3fac79d429e6442c755717905a7a79a38f910
parent	e23f7c01ee91321bd7845397d45f189a3e161806
parent	3af4e28dda7e4537255516fd1d575c2a34d904e7
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10598 from ziglang/stage2-x86_64-imm-sizes

stage2: rename Isel back to Emit, and fix immediate casts in Emit for x86 64

6 files changed, 2191 insertions(+), 2141 deletions(-)

CMakeLists.txt
src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Isel.zig
    Mir.zig
test
- stage2
  - x86_64.zig

CMakeLists.txt+1-1

...	...	@@ -578,7 +578,7 @@ set(ZIG_STAGE2_SOURCES
578	578	"${CMAKE_SOURCE_DIR}/src/arch/wasm/Emit.zig"
579	579	"${CMAKE_SOURCE_DIR}/src/arch/wasm/Mir.zig"
580	580	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/CodeGen.zig"
581		"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Isel.zig"
	581	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Emit.zig"
582	582	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Mir.zig"
583	583	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
584	584	"${CMAKE_SOURCE_DIR}/src/clang.zig"

src/arch/x86_64/CodeGen.zig+53-49

...	...	@@ -17,7 +17,7 @@ const DW = std.dwarf;
17	17	const ErrorMsg = Module.ErrorMsg;
18	18	const FnResult = @import("../../codegen.zig").FnResult;
19	19	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
20		const Isel = @import("Isel.zig");
	20	const Emit = @import("Emit.zig");
21	21	const Liveness = @import("../../Liveness.zig");
22	22	const Mir = @import("Mir.zig");
23	23	const Module = @import("../../Module.zig");
...	...	@@ -309,7 +309,7 @@ pub fn generate(
309	309	};
310	310	defer mir.deinit(bin_file.allocator);
311	311
312		var isel = Isel{
	312	var emit = Emit{
313	313	.mir = mir,
314	314	.bin_file = bin_file,
315	315	.debug_output = debug_output,
...	...	@@ -320,9 +320,9 @@ pub fn generate(
320	320	.prev_di_line = module_fn.lbrace_line,
321	321	.prev_di_column = module_fn.lbrace_column,
322	322	};
323		defer isel.deinit();
324		isel.lowerMir() catch \|err\| switch (err) {
325		error.IselFail => return FnResult{ .fail = isel.err_msg.? },
	323	defer emit.deinit();
	324	emit.lowerMir() catch \|err\| switch (err) {
	325	error.EmitFail => return FnResult{ .fail = emit.err_msg.? },
326	326	else => \|e\| return e,
327	327	};
328	328
...	...	@@ -497,14 +497,14 @@ fn gen(self: *Self) InnerError!void {
497	497	.ops = (Mir.Ops{
498	498	.reg1 = .rsp,
499	499	}).encode(),
500		.data = .{ .imm = @intCast(i32, aligned_stack_end) + stack_adjustment },
	500	.data = .{ .imm = @bitCast(u32, @intCast(i32, aligned_stack_end) + stack_adjustment) },
501	501	});
502	502	self.mir_instructions.set(backpatch_stack_add, .{
503	503	.tag = .add,
504	504	.ops = (Mir.Ops{
505	505	.reg1 = .rsp,
506	506	}).encode(),
507		.data = .{ .imm = @intCast(i32, aligned_stack_end) + stack_adjustment },
	507	.data = .{ .imm = @bitCast(u32, @intCast(i32, aligned_stack_end) + stack_adjustment) },
508	508	});
509	509	}
510	510	} else {
...	...	@@ -1347,7 +1347,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1347	1347	.reg2 = .rbp,
1348	1348	.flags = 0b01,
1349	1349	}).encode(),
1350		.data = .{ .imm = -@intCast(i32, off + 16) },
	1350	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off + 16)) },
1351	1351	});
1352	1352	// add addr, offset
1353	1353	_ = try self.addInst(.{
...	...	@@ -1555,7 +1555,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
1555	1555	try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
1556	1556
1557	1557	return self.genInlineMemcpy(
1558		-@intCast(i32, off + abi_size),
	1558	@bitCast(u32, -@intCast(i32, off + abi_size)),
1559	1559	registerAlias(addr_reg, @divExact(reg.size(), 8)),
1560	1560	count_reg.to64(),
1561	1561	tmp_reg.to8(),
...	...	@@ -1637,7 +1637,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
1637	1637	// introduce new MIR tag specifically for mov [reg + 0], imm
1638	1638	const payload = try self.addExtra(Mir.ImmPair{
1639	1639	.dest_off = 0,
1640		.operand = @bitCast(i32, @intCast(u32, imm)),
	1640	.operand = @truncate(u32, imm),
1641	1641	});
1642	1642	_ = try self.addInst(.{
1643	1643	.tag = .mov_mem_imm,
...	...	@@ -1821,11 +1821,11 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
1821	1821	const dst_ty = self.air.typeOfIndex(inst);
1822	1822	const air_tags = self.air.instructions.items(.tag);
1823	1823	switch (air_tags[inst]) {
1824		.add, .addwrap => try self.genBinMathOpMir(.add, dst_ty, dst_mcv, src_mcv),
1825		.bool_or, .bit_or => try self.genBinMathOpMir(.@"or", dst_ty, dst_mcv, src_mcv),
1826		.bool_and, .bit_and => try self.genBinMathOpMir(.@"and", dst_ty, dst_mcv, src_mcv),
1827		.sub, .subwrap => try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, src_mcv),
1828		.xor, .not => try self.genBinMathOpMir(.xor, dst_ty, dst_mcv, src_mcv),
	1824	.add, .addwrap => try self.genBinMathOpMir(.add, dst_ty, .unsigned, dst_mcv, src_mcv),
	1825	.bool_or, .bit_or => try self.genBinMathOpMir(.@"or", dst_ty, .unsigned, dst_mcv, src_mcv),
	1826	.bool_and, .bit_and => try self.genBinMathOpMir(.@"and", dst_ty, .unsigned, dst_mcv, src_mcv),
	1827	.sub, .subwrap => try self.genBinMathOpMir(.sub, dst_ty, .unsigned, dst_mcv, src_mcv),
	1828	.xor, .not => try self.genBinMathOpMir(.xor, dst_ty, .unsigned, dst_mcv, src_mcv),
1829	1829	.mul, .mulwrap => try self.genIMulOpMir(dst_ty, dst_mcv, src_mcv),
1830	1830	else => unreachable,
1831	1831	}
...	...	@@ -1837,6 +1837,7 @@ fn genBinMathOpMir(
1837	1837	self: *Self,
1838	1838	mir_tag: Mir.Inst.Tag,
1839	1839	dst_ty: Type,
	1840	signedness: std.builtin.Signedness,
1840	1841	dst_mcv: MCValue,
1841	1842	src_mcv: MCValue,
1842	1843	) !void {
...	...	@@ -1856,11 +1857,16 @@ fn genBinMathOpMir(
1856	1857	.ptr_stack_offset => unreachable,
1857	1858	.ptr_embedded_in_code => unreachable,
1858	1859	.register => \|src_reg\| {
	1860	// TODO think more carefully about this: is this actually correct?
	1861	const reg_size = if (mir_tag == .cmp and signedness == .signed)
	1862	@divExact(dst_reg.size(), 8)
	1863	else
	1864	@divExact(src_reg.size(), 8);
1859	1865	_ = try self.addInst(.{
1860	1866	.tag = mir_tag,
1861	1867	.ops = (Mir.Ops{
1862		.reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)),
1863		.reg2 = src_reg,
	1868	.reg1 = registerAlias(dst_reg, reg_size),
	1869	.reg2 = registerAlias(src_reg, reg_size),
1864	1870	}).encode(),
1865	1871	.data = undefined,
1866	1872	});
...	...	@@ -1872,7 +1878,7 @@ fn genBinMathOpMir(
1872	1878	.ops = (Mir.Ops{
1873	1879	.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
1874	1880	}).encode(),
1875		.data = .{ .imm = @intCast(i32, imm) },
	1881	.data = .{ .imm = @truncate(u32, imm) },
1876	1882	});
1877	1883	},
1878	1884	.embedded_in_code, .memory => {
...	...	@@ -1891,7 +1897,7 @@ fn genBinMathOpMir(
1891	1897	.reg2 = .rbp,
1892	1898	.flags = 0b01,
1893	1899	}).encode(),
1894		.data = .{ .imm = -@intCast(i32, adj_off) },
	1900	.data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
1895	1901	});
1896	1902	},
1897	1903	.compare_flags_unsigned => {
...	...	@@ -1926,7 +1932,7 @@ fn genBinMathOpMir(
1926	1932	.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
1927	1933	.flags = 0b10,
1928	1934	}).encode(),
1929		.data = .{ .imm = -@intCast(i32, adj_off) },
	1935	.data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
1930	1936	});
1931	1937	},
1932	1938	.immediate => \|imm\| {
...	...	@@ -1947,8 +1953,8 @@ fn genBinMathOpMir(
1947	1953	else => unreachable,
1948	1954	};
1949	1955	const payload = try self.addExtra(Mir.ImmPair{
1950		.dest_off = -@intCast(i32, adj_off),
1951		.operand = @bitCast(i32, @intCast(u32, imm)),
	1956	.dest_off = @bitCast(u32, -@intCast(i32, adj_off)),
	1957	.operand = @truncate(u32, imm),
1952	1958	});
1953	1959	_ = try self.addInst(.{
1954	1960	.tag = tag,
...	...	@@ -2015,7 +2021,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
2015	2021	.reg2 = dst_reg.to32(),
2016	2022	.flags = 0b10,
2017	2023	}).encode(),
2018		.data = .{ .imm = @intCast(i32, imm) },
	2024	.data = .{ .imm = @truncate(u32, imm) },
2019	2025	});
2020	2026	} else {
2021	2027	// TODO verify we don't spill and assign to the same register as dst_mcv
...	...	@@ -2088,7 +2094,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
2088	2094	const mcv = self.args[arg_index];
2089	2095	const payload = try self.addExtra(Mir.ArgDbgInfo{
2090	2096	.air_inst = inst,
2091		.arg_index = @intCast(u32, arg_index), // TODO can arg_index: u32?
	2097	.arg_index = @truncate(u32, arg_index), // TODO can arg_index: u32?
2092	2098	});
2093	2099	_ = try self.addInst(.{
2094	2100	.tag = .arg_dbg_info,
...	...	@@ -2196,7 +2202,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2196	2202	.ops = (Mir.Ops{
2197	2203	.flags = 0b01,
2198	2204	}).encode(),
2199		.data = .{ .imm = @bitCast(i32, got_addr) },
	2205	.data = .{ .imm = @truncate(u32, got_addr) },
2200	2206	});
2201	2207	} else if (func_value.castTag(.extern_fn)) \|_\| {
2202	2208	return self.fail("TODO implement calling extern functions", .{});
...	...	@@ -2446,7 +2452,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2446	2452	// This instruction supports only signed 32-bit immediates at most.
2447	2453	const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
2448	2454
2449		try self.genBinMathOpMir(.cmp, ty, dst_mcv, src_mcv);
	2455	try self.genBinMathOpMir(.cmp, ty, signedness, dst_mcv, src_mcv);
2450	2456	break :result switch (signedness) {
2451	2457	.signed => MCValue{ .compare_flags_signed = op },
2452	2458	.unsigned => MCValue{ .compare_flags_unsigned = op },
...	...	@@ -2669,7 +2675,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2669	2675	}
2670	2676
2671	2677	fn isNull(self: *Self, ty: Type, operand: MCValue) !MCValue {
2672		try self.genBinMathOpMir(.cmp, ty, operand, MCValue{ .immediate = 0 });
	2678	try self.genBinMathOpMir(.cmp, ty, .unsigned, operand, MCValue{ .immediate = 0 });
2673	2679	return MCValue{ .compare_flags_unsigned = .eq };
2674	2680	}
2675	2681
...	...	@@ -2686,7 +2692,7 @@ fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
2686	2692	return MCValue{ .immediate = 0 }; // always false
2687	2693	} else if (!payload_type.hasCodeGenBits()) {
2688	2694	if (err_type.abiSize(self.target.*) <= 8) {
2689		try self.genBinMathOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 });
	2695	try self.genBinMathOpMir(.cmp, err_type, .unsigned, operand, MCValue{ .immediate = 0 });
2690	2696	return MCValue{ .compare_flags_unsigned = .gt };
2691	2697	} else {
2692	2698	return self.fail("TODO isErr for errors with size larger than register size", .{});
...	...	@@ -3121,8 +3127,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3121	3127	// offset from rbp, which is at the top of the stack frame.
3122	3128	// mov [rbp+offset], immediate
3123	3129	const payload = try self.addExtra(Mir.ImmPair{
3124		.dest_off = -@intCast(i32, adj_off),
3125		.operand = @bitCast(i32, @intCast(u32, x_big)),
	3130	.dest_off = @bitCast(u32, -@intCast(i32, adj_off)),
	3131	.operand = @truncate(u32, x_big),
3126	3132	});
3127	3133	_ = try self.addInst(.{
3128	3134	.tag = .mov_mem_imm,
...	...	@@ -3147,8 +3153,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3147	3153	// insted just use two 32 bit writes to avoid register allocation
3148	3154	{
3149	3155	const payload = try self.addExtra(Mir.ImmPair{
3150		.dest_off = negative_offset + 4,
3151		.operand = @bitCast(i32, @truncate(u32, x_big >> 32)),
	3156	.dest_off = @bitCast(u32, negative_offset + 4),
	3157	.operand = @truncate(u32, x_big >> 32),
3152	3158	});
3153	3159	_ = try self.addInst(.{
3154	3160	.tag = .mov_mem_imm,
...	...	@@ -3161,8 +3167,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3161	3167	}
3162	3168	{
3163	3169	const payload = try self.addExtra(Mir.ImmPair{
3164		.dest_off = negative_offset,
3165		.operand = @bitCast(i32, @truncate(u32, x_big)),
	3170	.dest_off = @bitCast(u32, negative_offset),
	3171	.operand = @truncate(u32, x_big),
3166	3172	});
3167	3173	_ = try self.addInst(.{
3168	3174	.tag = .mov_mem_imm,
...	...	@@ -3192,7 +3198,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3192	3198	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
3193	3199	.flags = 0b10,
3194	3200	}).encode(),
3195		.data = .{ .imm = -@intCast(i32, adj_off) },
	3201	.data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
3196	3202	});
3197	3203	},
3198	3204	.memory, .embedded_in_code => {
...	...	@@ -3228,14 +3234,14 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3228	3234	.reg1 = addr_reg.to64(),
3229	3235	.reg2 = .rbp,
3230	3236	}).encode(),
3231		.data = .{ .imm = -@intCast(i32, off + abi_size) },
	3237	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off + abi_size)) },
3232	3238	});
3233	3239
3234	3240	// TODO allow for abi_size to be u64
3235	3241	try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
3236	3242
3237	3243	return self.genInlineMemcpy(
3238		-@intCast(i32, stack_offset + abi_size),
	3244	@bitCast(u32, -@intCast(i32, stack_offset + abi_size)),
3239	3245	addr_reg.to64(),
3240	3246	count_reg.to64(),
3241	3247	tmp_reg.to8(),
...	...	@@ -3246,7 +3252,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3246	3252
3247	3253	fn genInlineMemcpy(
3248	3254	self: *Self,
3249		stack_offset: i32,
	3255	stack_offset: u32,
3250	3256	addr_reg: Register,
3251	3257	count_reg: Register,
3252	3258	tmp_reg: Register,
...	...	@@ -3361,7 +3367,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3361	3367	.reg1 = registerAlias(reg, @intCast(u32, ptr_abi_size)),
3362	3368	.reg2 = .rbp,
3363	3369	}).encode(),
3364		.data = .{ .imm = -@intCast(i32, off) },
	3370	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
3365	3371	});
3366	3372	},
3367	3373	.ptr_embedded_in_code => unreachable,
...	...	@@ -3378,7 +3384,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3378	3384	else => unreachable,
3379	3385	}
3380	3386	},
3381		.compare_flags_unsigned => \|op\| {
	3387	.compare_flags_unsigned,
	3388	.compare_flags_signed,
	3389	=> \|op\| {
3382	3390	const tag: Mir.Inst.Tag = switch (op) {
3383	3391	.gte, .gt, .lt, .lte => .cond_set_byte_above_below,
3384	3392	.eq, .neq => .cond_set_byte_eq_ne,
...	...	@@ -3400,10 +3408,6 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3400	3408	.data = undefined,
3401	3409	});
3402	3410	},
3403		.compare_flags_signed => \|op\| {
3404		_ = op;
3405		return self.fail("TODO set register with compare flags value (signed)", .{});
3406		},
3407	3411	.immediate => \|x\| {
3408	3412	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
3409	3413	// register is the fastest way to zero a register.
...	...	@@ -3426,7 +3430,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3426	3430	.ops = (Mir.Ops{
3427	3431	.reg1 = registerAlias(reg, @intCast(u32, abi_size)),
3428	3432	}).encode(),
3429		.data = .{ .imm = @intCast(i32, x) },
	3433	.data = .{ .imm = @truncate(u32, x) },
3430	3434	});
3431	3435	return;
3432	3436	}
...	...	@@ -3441,7 +3445,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3441	3445	_ = try self.addInst(.{
3442	3446	.tag = .movabs,
3443	3447	.ops = (Mir.Ops{
3444		.reg1 = reg,
	3448	.reg1 = reg.to64(),
3445	3449	}).encode(),
3446	3450	.data = .{ .payload = payload },
3447	3451	});
...	...	@@ -3482,7 +3486,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3482	3486	.reg1 = reg,
3483	3487	.flags = 0b10,
3484	3488	}).encode(),
3485		.data = .{ .got_entry = @intCast(u32, x) },
	3489	.data = .{ .got_entry = @truncate(u32, x) },
3486	3490	});
3487	3491	// MOV reg, [reg]
3488	3492	_ = try self.addInst(.{
...	...	@@ -3502,7 +3506,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3502	3506	.reg1 = reg,
3503	3507	.flags = 0b01,
3504	3508	}).encode(),
3505		.data = .{ .imm = @intCast(i32, x) },
	3509	.data = .{ .imm = @truncate(u32, x) },
3506	3510	});
3507	3511	} else {
3508	3512	// If this is RAX, we can use a direct load.
...	...	@@ -3561,7 +3565,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3561	3565	.reg2 = .rbp,
3562	3566	.flags = 0b01,
3563	3567	}).encode(),
3564		.data = .{ .imm = -@intCast(i32, off) },
	3568	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
3565	3569	});
3566	3570	},
3567	3571	}

src/arch/x86_64/Emit.zig created+2070

...	...	@@ -0,0 +1,2070 @@
	1	//! This file contains the functionality for lowering x86_64 MIR into
	2	//! machine code
	3
	4	const Emit = @This();
	5
	6	const std = @import("std");
	7	const assert = std.debug.assert;
	8	const bits = @import("bits.zig");
	9	const leb128 = std.leb;
	10	const link = @import("../../link.zig");
	11	const log = std.log.scoped(.codegen);
	12	const math = std.math;
	13	const mem = std.mem;
	14	const testing = std.testing;
	15
	16	const Air = @import("../../Air.zig");
	17	const Allocator = mem.Allocator;
	18	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
	19	const DW = std.dwarf;
	20	const Encoder = bits.Encoder;
	21	const ErrorMsg = Module.ErrorMsg;
	22	const MCValue = @import("CodeGen.zig").MCValue;
	23	const Mir = @import("Mir.zig");
	24	const Module = @import("../../Module.zig");
	25	const Instruction = bits.Instruction;
	26	const Register = bits.Register;
	27	const Type = @import("../../type.zig").Type;
	28
	29	mir: Mir,
	30	bin_file: *link.File,
	31	debug_output: DebugInfoOutput,
	32	target: *const std.Target,
	33	err_msg: ?*ErrorMsg = null,
	34	src_loc: Module.SrcLoc,
	35	code: *std.ArrayList(u8),
	36
	37	prev_di_line: u32,
	38	prev_di_column: u32,
	39	/// Relative to the beginning of `code`.
	40	prev_di_pc: usize,
	41
	42	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
	43	relocs: std.ArrayListUnmanaged(Reloc) = .{},
	44
	45	const InnerError = error{
	46	OutOfMemory,
	47	EmitFail,
	48	};
	49
	50	const Reloc = struct {
	51	/// Offset of the instruction.
	52	source: usize,
	53	/// Target of the relocation.
	54	target: Mir.Inst.Index,
	55	/// Offset of the relocation within the instruction.
	56	offset: usize,
	57	/// Length of the instruction.
	58	length: u5,
	59	};
	60
	61	pub fn lowerMir(emit: *Emit) InnerError!void {
	62	const mir_tags = emit.mir.instructions.items(.tag);
	63
	64	for (mir_tags) \|tag, index\| {
	65	const inst = @intCast(u32, index);
	66	try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
	67	switch (tag) {
	68	.adc => try emit.mirArith(.adc, inst),
	69	.add => try emit.mirArith(.add, inst),
	70	.sub => try emit.mirArith(.sub, inst),
	71	.xor => try emit.mirArith(.xor, inst),
	72	.@"and" => try emit.mirArith(.@"and", inst),
	73	.@"or" => try emit.mirArith(.@"or", inst),
	74	.sbb => try emit.mirArith(.sbb, inst),
	75	.cmp => try emit.mirArith(.cmp, inst),
	76	.mov => try emit.mirArith(.mov, inst),
	77
	78	.adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
	79	.add_mem_imm => try emit.mirArithMemImm(.add, inst),
	80	.sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
	81	.xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
	82	.and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
	83	.or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
	84	.sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
	85	.cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
	86	.mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
	87
	88	.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
	89	.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
	90	.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
	91	.xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),
	92	.and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),
	93	.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
	94	.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
	95	.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
	96	.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
	97
	98	.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
	99	.add_scale_dst => try emit.mirArithScaleDst(.add, inst),
	100	.sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),
	101	.xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),
	102	.and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),
	103	.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
	104	.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
	105	.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
	106	.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
	107
	108	.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
	109	.add_scale_imm => try emit.mirArithScaleImm(.add, inst),
	110	.sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
	111	.xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
	112	.and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
	113	.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
	114	.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
	115	.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
	116	.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
	117
	118	.movabs => try emit.mirMovabs(inst),
	119
	120	.lea => try emit.mirLea(inst),
	121
	122	.imul_complex => try emit.mirIMulComplex(inst),
	123
	124	.push => try emit.mirPushPop(.push, inst),
	125	.pop => try emit.mirPushPop(.pop, inst),
	126
	127	.jmp => try emit.mirJmpCall(.jmp_near, inst),
	128	.call => try emit.mirJmpCall(.call_near, inst),
	129
	130	.cond_jmp_greater_less,
	131	.cond_jmp_above_below,
	132	.cond_jmp_eq_ne,
	133	=> try emit.mirCondJmp(tag, inst),
	134
	135	.cond_set_byte_greater_less,
	136	.cond_set_byte_above_below,
	137	.cond_set_byte_eq_ne,
	138	=> try emit.mirCondSetByte(tag, inst),
	139
	140	.ret => try emit.mirRet(inst),
	141
	142	.syscall => try emit.mirSyscall(),
	143
	144	.@"test" => try emit.mirTest(inst),
	145
	146	.brk => try emit.mirBrk(),
	147	.nop => try emit.mirNop(),
	148
	149	.call_extern => try emit.mirCallExtern(inst),
	150
	151	.dbg_line => try emit.mirDbgLine(inst),
	152	.dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
	153	.dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
	154	.arg_dbg_info => try emit.mirArgDbgInfo(inst),
	155
	156	.push_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
	157	.pop_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
	158
	159	else => {
	160	return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {s}", .{tag});
	161	},
	162	}
	163	}
	164
	165	try emit.fixupRelocs();
	166	}
	167
	168	pub fn deinit(emit: *Emit) void {
	169	emit.relocs.deinit(emit.bin_file.allocator);
	170	emit.code_offset_mapping.deinit(emit.bin_file.allocator);
	171	emit.* = undefined;
	172	}
	173
	174	fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
	175	@setCold(true);
	176	assert(emit.err_msg == null);
	177	emit.err_msg = try ErrorMsg.create(emit.bin_file.allocator, emit.src_loc, format, args);
	178	return error.EmitFail;
	179	}
	180
	181	fn failWithLoweringError(emit: *Emit, err: LoweringError) InnerError {
	182	return switch (err) {
	183	error.RaxOperandExpected => emit.fail("Register.rax expected as destination operand", .{}),
	184	error.OperandSizeMismatch => emit.fail("operand size mismatch", .{}),
	185	else => \|e\| e,
	186	};
	187	}
	188
	189	fn fixupRelocs(emit: *Emit) InnerError!void {
	190	// TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
	191	// This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
	192	// possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
	193	// until the entire decl is correctly emitted with all JMP/CALL instructions within range.
	194	for (emit.relocs.items) \|reloc\| {
	195	const target = emit.code_offset_mapping.get(reloc.target) orelse
	196	return emit.fail("JMP/CALL relocation target not found!", .{});
	197	const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
	198	mem.writeIntLittle(i32, emit.code.items[reloc.offset..][0..4], disp);
	199	}
	200	}
	201
	202	fn mirBrk(emit: *Emit) InnerError!void {
	203	return lowerToZoEnc(.brk, emit.code) catch \|err\| emit.failWithLoweringError(err);
	204	}
	205
	206	fn mirNop(emit: *Emit) InnerError!void {
	207	return lowerToZoEnc(.nop, emit.code) catch \|err\| emit.failWithLoweringError(err);
	208	}
	209
	210	fn mirSyscall(emit: *Emit) InnerError!void {
	211	return lowerToZoEnc(.syscall, emit.code) catch \|err\| emit.failWithLoweringError(err);
	212	}
	213
	214	fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	215	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	216	switch (ops.flags) {
	217	0b00 => {
	218	// PUSH/POP reg
	219	return lowerToOEnc(tag, ops.reg1, emit.code) catch \|err\| emit.failWithLoweringError(err);
	220	},
	221	0b01 => {
	222	// PUSH/POP r/m64
	223	const imm = emit.mir.instructions.items(.data)[inst].imm;
	224	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	225	16 => .word_ptr,
	226	else => .qword_ptr,
	227	};
	228	return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
	229	.disp = imm,
	230	.base = ops.reg1,
	231	}), emit.code) catch \|err\| emit.failWithLoweringError(err);
	232	},
	233	0b10 => {
	234	// PUSH imm32
	235	assert(tag == .push);
	236	const imm = emit.mir.instructions.items(.data)[inst].imm;
	237	return lowerToIEnc(.push, imm, emit.code) catch \|err\|
	238	emit.failWithLoweringError(err);
	239	},
	240	0b11 => unreachable,
	241	}
	242	}
	243	fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	244	const callee_preserved_regs = bits.callee_preserved_regs;
	245	const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
	246	if (tag == .push) {
	247	for (callee_preserved_regs) \|reg, i\| {
	248	if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
	249	lowerToOEnc(.push, reg, emit.code) catch \|err\|
	250	return emit.failWithLoweringError(err);
	251	}
	252	} else {
	253	// pop in the reverse direction
	254	var i = callee_preserved_regs.len;
	255	while (i > 0) : (i -= 1) {
	256	const reg = callee_preserved_regs[i - 1];
	257	if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
	258	lowerToOEnc(.pop, reg, emit.code) catch \|err\|
	259	return emit.failWithLoweringError(err);
	260	}
	261	}
	262	}
	263
	264	fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	265	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	266	switch (ops.flags) {
	267	0b00 => {
	268	const target = emit.mir.instructions.items(.data)[inst].inst;
	269	const source = emit.code.items.len;
	270	lowerToDEnc(tag, 0, emit.code) catch \|err\|
	271	return emit.failWithLoweringError(err);
	272	try emit.relocs.append(emit.bin_file.allocator, .{
	273	.source = source,
	274	.target = target,
	275	.offset = emit.code.items.len - 4,
	276	.length = 5,
	277	});
	278	},
	279	0b01 => {
	280	if (ops.reg1 == .none) {
	281	// JMP/CALL [imm]
	282	const imm = emit.mir.instructions.items(.data)[inst].imm;
	283	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	284	16 => .word_ptr,
	285	else => .qword_ptr,
	286	};
	287	return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code) catch \|err\|
	288	emit.failWithLoweringError(err);
	289	}
	290	// JMP/CALL reg
	291	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code) catch \|err\| emit.failWithLoweringError(err);
	292	},
	293	0b10 => {
	294	// JMP/CALL r/m64
	295	const imm = emit.mir.instructions.items(.data)[inst].imm;
	296	return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	297	.disp = imm,
	298	.base = ops.reg1,
	299	}), emit.code) catch \|err\| emit.failWithLoweringError(err);
	300	},
	301	0b11 => return emit.fail("TODO unused JMP/CALL variant 0b11", .{}),
	302	}
	303	}
	304
	305	fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	306	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	307	const target = emit.mir.instructions.items(.data)[inst].inst;
	308	const tag = switch (mir_tag) {
	309	.cond_jmp_greater_less => switch (ops.flags) {
	310	0b00 => Tag.jge,
	311	0b01 => Tag.jg,
	312	0b10 => Tag.jl,
	313	0b11 => Tag.jle,
	314	},
	315	.cond_jmp_above_below => switch (ops.flags) {
	316	0b00 => Tag.jae,
	317	0b01 => Tag.ja,
	318	0b10 => Tag.jb,
	319	0b11 => Tag.jbe,
	320	},
	321	.cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
	322	0b0 => Tag.jne,
	323	0b1 => Tag.je,
	324	},
	325	else => unreachable,
	326	};
	327	const source = emit.code.items.len;
	328	lowerToDEnc(tag, 0, emit.code) catch \|err\|
	329	return emit.failWithLoweringError(err);
	330	try emit.relocs.append(emit.bin_file.allocator, .{
	331	.source = source,
	332	.target = target,
	333	.offset = emit.code.items.len - 4,
	334	.length = 6,
	335	});
	336	}
	337
	338	fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	339	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	340	const tag = switch (mir_tag) {
	341	.cond_set_byte_greater_less => switch (ops.flags) {
	342	0b00 => Tag.setge,
	343	0b01 => Tag.setg,
	344	0b10 => Tag.setl,
	345	0b11 => Tag.setle,
	346	},
	347	.cond_set_byte_above_below => switch (ops.flags) {
	348	0b00 => Tag.setae,
	349	0b01 => Tag.seta,
	350	0b10 => Tag.setb,
	351	0b11 => Tag.setbe,
	352	},
	353	.cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
	354	0b0 => Tag.setne,
	355	0b1 => Tag.sete,
	356	},
	357	else => unreachable,
	358	};
	359	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code) catch \|err\|
	360	emit.failWithLoweringError(err);
	361	}
	362
	363	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	364	const tag = emit.mir.instructions.items(.tag)[inst];
	365	assert(tag == .@"test");
	366	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	367	switch (ops.flags) {
	368	0b00 => {
	369	if (ops.reg2 == .none) {
	370	// TEST r/m64, imm32
	371	// MI
	372	const imm = emit.mir.instructions.items(.data)[inst].imm;
	373	if (ops.reg1.to64() == .rax) {
	374	// TEST rax, imm32
	375	// I
	376	return lowerToIEnc(.@"test", imm, emit.code) catch \|err\|
	377	emit.failWithLoweringError(err);
	378	}
	379	return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
	380	emit.failWithLoweringError(err);
	381	}
	382	// TEST r/m64, r64
	383	return emit.fail("TODO TEST r/m64, r64", .{});
	384	},
	385	else => return emit.fail("TODO more TEST alternatives", .{}),
	386	}
	387	}
	388
	389	fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	390	const tag = emit.mir.instructions.items(.tag)[inst];
	391	assert(tag == .ret);
	392	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	393	switch (ops.flags) {
	394	0b00 => {
	395	// RETF imm16
	396	// I
	397	const imm = emit.mir.instructions.items(.data)[inst].imm;
	398	return lowerToIEnc(.ret_far, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	399	},
	400	0b01 => {
	401	return lowerToZoEnc(.ret_far, emit.code) catch \|err\| emit.failWithLoweringError(err);
	402	},
	403	0b10 => {
	404	// RET imm16
	405	// I
	406	const imm = emit.mir.instructions.items(.data)[inst].imm;
	407	return lowerToIEnc(.ret_near, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	408	},
	409	0b11 => {
	410	return lowerToZoEnc(.ret_near, emit.code) catch \|err\| emit.failWithLoweringError(err);
	411	},
	412	}
	413	}
	414
	415	fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	416	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	417	switch (ops.flags) {
	418	0b00 => {
	419	if (ops.reg2 == .none) {
	420	// mov reg1, imm32
	421	// MI
	422	const imm = emit.mir.instructions.items(.data)[inst].imm;
	423	return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
	424	emit.failWithLoweringError(err);
	425	}
	426	// mov reg1, reg2
	427	// RM
	428	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
	429	emit.failWithLoweringError(err);
	430	},
	431	0b01 => {
	432	// mov reg1, [reg2 + imm32]
	433	// RM
	434	const imm = emit.mir.instructions.items(.data)[inst].imm;
	435	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	436	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	437	.disp = imm,
	438	.base = src_reg,
	439	}), emit.code) catch \|err\| emit.failWithLoweringError(err);
	440	},
	441	0b10 => {
	442	if (ops.reg2 == .none) {
	443	return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
	444	}
	445	// mov [reg1 + imm32], reg2
	446	// MR
	447	const imm = emit.mir.instructions.items(.data)[inst].imm;
	448	return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
	449	.disp = imm,
	450	.base = ops.reg1,
	451	}), ops.reg2, emit.code) catch \|err\| emit.failWithLoweringError(err);
	452	},
	453	0b11 => {
	454	return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
	455	},
	456	}
	457	}
	458
	459	fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	460	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	461	assert(ops.reg2 == .none);
	462	const payload = emit.mir.instructions.items(.data)[inst].payload;
	463	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	464	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	465	0b00 => .byte_ptr,
	466	0b01 => .word_ptr,
	467	0b10 => .dword_ptr,
	468	0b11 => .qword_ptr,
	469	};
	470	return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
	471	.disp = imm_pair.dest_off,
	472	.base = ops.reg1,
	473	}), imm_pair.operand, emit.code) catch \|err\| emit.failWithLoweringError(err);
	474	}
	475
	476	inline fn setRexWRegister(reg: Register) bool {
	477	if (reg.size() == 64) return true;
	478	return switch (reg) {
	479	.ah, .bh, .ch, .dh => true,
	480	else => false,
	481	};
	482	}
	483
	484	inline fn immOpSize(u_imm: u32) u8 {
	485	const imm = @bitCast(i32, u_imm);
	486	if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
	487	return 8;
	488	}
	489	if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
	490	return 16;
	491	}
	492	return 32;
	493	}
	494
	495	fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	496	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	497	const scale = ops.flags;
	498	const imm = emit.mir.instructions.items(.data)[inst].imm;
	499	// OP reg1, [reg2 + scale*rcx + imm32]
	500	const scale_index = ScaleIndex{
	501	.scale = scale,
	502	.index = .rcx,
	503	};
	504	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	505	.disp = imm,
	506	.base = ops.reg2,
	507	.scale_index = scale_index,
	508	}), emit.code) catch \|err\| emit.failWithLoweringError(err);
	509	}
	510
	511	fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	512	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	513	const scale = ops.flags;
	514	const imm = emit.mir.instructions.items(.data)[inst].imm;
	515	const scale_index = ScaleIndex{
	516	.scale = scale,
	517	.index = .rax,
	518	};
	519	if (ops.reg2 == .none) {
	520	// OP qword ptr [reg1 + scale*rax + 0], imm32
	521	return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
	522	.disp = 0,
	523	.base = ops.reg1,
	524	.scale_index = scale_index,
	525	}), imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	526	}
	527	// OP [reg1 + scale*rax + imm32], reg2
	528	return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
	529	.disp = imm,
	530	.base = ops.reg1,
	531	.scale_index = scale_index,
	532	}), ops.reg2, emit.code) catch \|err\| emit.failWithLoweringError(err);
	533	}
	534
	535	fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	536	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	537	const scale = ops.flags;
	538	const payload = emit.mir.instructions.items(.data)[inst].payload;
	539	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	540	const scale_index = ScaleIndex{
	541	.scale = scale,
	542	.index = .rax,
	543	};
	544	// OP qword ptr [reg1 + scale*rax + imm32], imm32
	545	return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
	546	.disp = imm_pair.dest_off,
	547	.base = ops.reg1,
	548	.scale_index = scale_index,
	549	}), imm_pair.operand, emit.code) catch \|err\| emit.failWithLoweringError(err);
	550	}
	551
	552	fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	553	const tag = emit.mir.instructions.items(.tag)[inst];
	554	assert(tag == .movabs);
	555	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	556	const imm: u64 = if (ops.reg1.size() == 64) blk: {
	557	const payload = emit.mir.instructions.items(.data)[inst].payload;
	558	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
	559	break :blk imm.decode();
	560	} else emit.mir.instructions.items(.data)[inst].imm;
	561	if (ops.flags == 0b00) {
	562	// movabs reg, imm64
	563	// OI
	564	return lowerToOiEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	565	}
	566	if (ops.reg1 == .none) {
	567	// movabs moffs64, rax
	568	// TD
	569	return lowerToTdEnc(.mov, imm, ops.reg2, emit.code) catch \|err\| emit.failWithLoweringError(err);
	570	}
	571	// movabs rax, moffs64
	572	// FD
	573	return lowerToFdEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	574	}
	575
	576	fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	577	const tag = emit.mir.instructions.items(.tag)[inst];
	578	assert(tag == .imul_complex);
	579	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	580	switch (ops.flags) {
	581	0b00 => {
	582	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
	583	emit.failWithLoweringError(err);
	584	},
	585	0b10 => {
	586	const imm = emit.mir.instructions.items(.data)[inst].imm;
	587	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code) catch \|err\|
	588	emit.failWithLoweringError(err);
	589	},
	590	else => return emit.fail("TODO implement imul", .{}),
	591	}
	592	}
	593
	594	fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	595	const tag = emit.mir.instructions.items(.tag)[inst];
	596	assert(tag == .lea);
	597	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	598	switch (ops.flags) {
	599	0b00 => {
	600	// lea reg1, [reg2 + imm32]
	601	// RM
	602	const imm = emit.mir.instructions.items(.data)[inst].imm;
	603	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	604	return lowerToRmEnc(
	605	.lea,
	606	ops.reg1,
	607	RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	608	.disp = imm,
	609	.base = src_reg,
	610	}),
	611	emit.code,
	612	) catch \|err\| emit.failWithLoweringError(err);
	613	},
	614	0b01 => {
	615	// lea reg1, [rip + imm32]
	616	// RM
	617	const start_offset = emit.code.items.len;
	618	lowerToRmEnc(
	619	.lea,
	620	ops.reg1,
	621	RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
	622	emit.code,
	623	) catch \|err\| return emit.failWithLoweringError(err);
	624	const end_offset = emit.code.items.len;
	625	// Backpatch the displacement
	626	const payload = emit.mir.instructions.items(.data)[inst].payload;
	627	const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
	628	const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
	629	mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
	630	},
	631	0b10 => {
	632	// lea reg1, [rip + reloc]
	633	// RM
	634	lowerToRmEnc(
	635	.lea,
	636	ops.reg1,
	637	RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
	638	emit.code,
	639	) catch \|err\| return emit.failWithLoweringError(err);
	640	const end_offset = emit.code.items.len;
	641	const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
	642	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	643	// TODO I think the reloc might be in the wrong place.
	644	const decl = macho_file.active_decl.?;
	645	try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
	646	.offset = @intCast(u32, end_offset - 4),
	647	.target = .{ .local = got_entry },
	648	.addend = 0,
	649	.subtractor = null,
	650	.pcrel = true,
	651	.length = 2,
	652	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	653	});
	654	} else {
	655	return emit.fail(
	656	"TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
	657	.{},
	658	);
	659	}
	660	},
	661	0b11 => {
	662	// lea reg, [rbp + rcx + imm32]
	663	const imm = emit.mir.instructions.items(.data)[inst].imm;
	664	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	665	const scale_index = ScaleIndex{
	666	.scale = 0,
	667	.index = .rcx,
	668	};
	669	return lowerToRmEnc(
	670	.lea,
	671	ops.reg1,
	672	RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
	673	.disp = imm,
	674	.base = src_reg,
	675	.scale_index = scale_index,
	676	}),
	677	emit.code,
	678	) catch \|err\| emit.failWithLoweringError(err);
	679	},
	680	}
	681	}
	682
	683	fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	684	const tag = emit.mir.instructions.items(.tag)[inst];
	685	assert(tag == .call_extern);
	686	const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
	687	const offset = blk: {
	688	// callq
	689	lowerToDEnc(.call_near, 0, emit.code) catch \|err\|
	690	return emit.failWithLoweringError(err);
	691	break :blk @intCast(u32, emit.code.items.len) - 4;
	692	};
	693	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	694	// Add relocation to the decl.
	695	try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
	696	.offset = offset,
	697	.target = .{ .global = n_strx },
	698	.addend = 0,
	699	.subtractor = null,
	700	.pcrel = true,
	701	.length = 2,
	702	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	703	});
	704	} else {
	705	return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
	706	}
	707	}
	708
	709	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	710	const tag = emit.mir.instructions.items(.tag)[inst];
	711	assert(tag == .dbg_line);
	712	const payload = emit.mir.instructions.items(.data)[inst].payload;
	713	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
	714	try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
	715	}
	716
	717	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
	718	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
	719	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
	720	switch (emit.debug_output) {
	721	.dwarf => \|dbg_out\| {
	722	// TODO Look into using the DWARF special opcodes to compress this data.
	723	// It lets you emit single-byte opcodes that add different numbers to
	724	// both the PC and the line number at the same time.
	725	try dbg_out.dbg_line.ensureUnusedCapacity(11);
	726	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
	727	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
	728	if (delta_line != 0) {
	729	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
	730	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	731	}
	732	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
	733	emit.prev_di_pc = emit.code.items.len;
	734	emit.prev_di_line = line;
	735	emit.prev_di_column = column;
	736	emit.prev_di_pc = emit.code.items.len;
	737	},
	738	.plan9 => \|dbg_out\| {
	739	if (delta_pc <= 0) return; // only do this when the pc changes
	740	// we have already checked the target in the linker to make sure it is compatable
	741	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
	742
	743	// increasing the line number
	744	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
	745	// increasing the pc
	746	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
	747	if (d_pc_p9 > 0) {
	748	// minus one because if its the last one, we want to leave space to change the line which is one quanta
	749	var diff = @divExact(d_pc_p9, quant) - quant;
	750	while (diff > 0) {
	751	if (diff < 64) {
	752	try dbg_out.dbg_line.append(@intCast(u8, diff + 128));
	753	diff = 0;
	754	} else {
	755	try dbg_out.dbg_line.append(@intCast(u8, 64 + 128));
	756	diff -= 64;
	757	}
	758	}
	759	if (dbg_out.pcop_change_index.*) \|pci\|
	760	dbg_out.dbg_line.items[pci] += 1;
	761	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
	762	} else if (d_pc_p9 == 0) {
	763	// we don't need to do anything, because adding the quant does it for us
	764	} else unreachable;
	765	if (dbg_out.start_line.* == null)
	766	dbg_out.start_line.* = emit.prev_di_line;
	767	dbg_out.end_line.* = line;
	768	// only do this if the pc changed
	769	emit.prev_di_line = line;
	770	emit.prev_di_column = column;
	771	emit.prev_di_pc = emit.code.items.len;
	772	},
	773	.none => {},
	774	}
	775	}
	776
	777	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	778	const tag = emit.mir.instructions.items(.tag)[inst];
	779	assert(tag == .dbg_prologue_end);
	780	switch (emit.debug_output) {
	781	.dwarf => \|dbg_out\| {
	782	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
	783	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	784	},
	785	.plan9 => {},
	786	.none => {},
	787	}
	788	}
	789
	790	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	791	const tag = emit.mir.instructions.items(.tag)[inst];
	792	assert(tag == .dbg_epilogue_begin);
	793	switch (emit.debug_output) {
	794	.dwarf => \|dbg_out\| {
	795	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
	796	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	797	},
	798	.plan9 => {},
	799	.none => {},
	800	}
	801	}
	802
	803	fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	804	const tag = emit.mir.instructions.items(.tag)[inst];
	805	assert(tag == .arg_dbg_info);
	806	const payload = emit.mir.instructions.items(.data)[inst].payload;
	807	const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
	808	const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
	809	try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
	810	}
	811
	812	fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
	813	const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
	814	const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
	815	const name = zir.nullTerminatedString(ty_str.str);
	816	const name_with_null = name.ptr[0 .. name.len + 1];
	817	const ty = emit.mir.function.air.getRefType(ty_str.ty);
	818
	819	switch (mcv) {
	820	.register => \|reg\| {
	821	switch (emit.debug_output) {
	822	.dwarf => \|dbg_out\| {
	823	try dbg_out.dbg_info.ensureUnusedCapacity(3);
	824	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
	825	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
	826	1, // ULEB128 dwarf expression length
	827	reg.dwarfLocOp(),
	828	});
	829	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
	830	try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
	831	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
	832	},
	833	.plan9 => {},
	834	.none => {},
	835	}
	836	},
	837	.stack_offset => {
	838	switch (emit.debug_output) {
	839	.dwarf => {},
	840	.plan9 => {},
	841	.none => {},
	842	}
	843	},
	844	else => {},
	845	}
	846	}
	847
	848	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
	849	/// after codegen for this symbol is done.
	850	fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
	851	switch (emit.debug_output) {
	852	.dwarf => \|dbg_out\| {
	853	assert(ty.hasCodeGenBits());
	854	const index = dbg_out.dbg_info.items.len;
	855	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
	856
	857	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
	858	if (!gop.found_existing) {
	859	gop.value_ptr.* = .{
	860	.off = undefined,
	861	.relocs = .{},
	862	};
	863	}
	864	try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
	865	},
	866	.plan9 => {},
	867	.none => {},
	868	}
	869	}
	870
	871	const Tag = enum {
	872	adc,
	873	add,
	874	sub,
	875	xor,
	876	@"and",
	877	@"or",
	878	sbb,
	879	cmp,
	880	mov,
	881	lea,
	882	jmp_near,
	883	call_near,
	884	push,
	885	pop,
	886	@"test",
	887	brk,
	888	nop,
	889	imul,
	890	syscall,
	891	ret_near,
	892	ret_far,
	893	jo,
	894	jno,
	895	jb,
	896	jbe,
	897	jc,
	898	jnae,
	899	jnc,
	900	jae,
	901	je,
	902	jz,
	903	jne,
	904	jnz,
	905	jna,
	906	jnb,
	907	jnbe,
	908	ja,
	909	js,
	910	jns,
	911	jpe,
	912	jp,
	913	jpo,
	914	jnp,
	915	jnge,
	916	jl,
	917	jge,
	918	jnl,
	919	jle,
	920	jng,
	921	jg,
	922	jnle,
	923	seto,
	924	setno,
	925	setb,
	926	setc,
	927	setnae,
	928	setnb,
	929	setnc,
	930	setae,
	931	sete,
	932	setz,
	933	setne,
	934	setnz,
	935	setbe,
	936	setna,
	937	seta,
	938	setnbe,
	939	sets,
	940	setns,
	941	setp,
	942	setpe,
	943	setnp,
	944	setop,
	945	setl,
	946	setnge,
	947	setnl,
	948	setge,
	949	setle,
	950	setng,
	951	setnle,
	952	setg,
	953
	954	fn isSetCC(tag: Tag) bool {
	955	return switch (tag) {
	956	.seto,
	957	.setno,
	958	.setb,
	959	.setc,
	960	.setnae,
	961	.setnb,
	962	.setnc,
	963	.setae,
	964	.sete,
	965	.setz,
	966	.setne,
	967	.setnz,
	968	.setbe,
	969	.setna,
	970	.seta,
	971	.setnbe,
	972	.sets,
	973	.setns,
	974	.setp,
	975	.setpe,
	976	.setnp,
	977	.setop,
	978	.setl,
	979	.setnge,
	980	.setnl,
	981	.setge,
	982	.setle,
	983	.setng,
	984	.setnle,
	985	.setg,
	986	=> true,
	987	else => false,
	988	};
	989	}
	990	};
	991
	992	const Encoding = enum {
	993	/// OP
	994	zo,
	995
	996	/// OP rel32
	997	d,
	998
	999	/// OP r/m64
	1000	m,
	1001
	1002	/// OP r64
	1003	o,
	1004
	1005	/// OP imm32
	1006	i,
	1007
	1008	/// OP r/m64, imm32
	1009	mi,
	1010
	1011	/// OP r/m64, r64
	1012	mr,
	1013
	1014	/// OP r64, r/m64
	1015	rm,
	1016
	1017	/// OP r64, imm64
	1018	oi,
	1019
	1020	/// OP al/ax/eax/rax, moffs
	1021	fd,
	1022
	1023	/// OP moffs, al/ax/eax/rax
	1024	td,
	1025
	1026	/// OP r64, r/m64, imm32
	1027	rmi,
	1028	};
	1029
	1030	const OpCode = union(enum) {
	1031	one_byte: u8,
	1032	two_byte: struct { _1: u8, _2: u8 },
	1033
	1034	fn oneByte(opc: u8) OpCode {
	1035	return .{ .one_byte = opc };
	1036	}
	1037
	1038	fn twoByte(opc1: u8, opc2: u8) OpCode {
	1039	return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
	1040	}
	1041
	1042	fn encode(opc: OpCode, encoder: Encoder) void {
	1043	switch (opc) {
	1044	.one_byte => \|v\| encoder.opcode_1byte(v),
	1045	.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
	1046	}
	1047	}
	1048
	1049	fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
	1050	assert(opc == .one_byte);
	1051	encoder.opcode_withReg(opc.one_byte, reg.lowId());
	1052	}
	1053	};
	1054
	1055	inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
	1056	switch (enc) {
	1057	.zo => return switch (tag) {
	1058	.ret_near => OpCode.oneByte(0xc3),
	1059	.ret_far => OpCode.oneByte(0xcb),
	1060	.brk => OpCode.oneByte(0xcc),
	1061	.nop => OpCode.oneByte(0x90),
	1062	.syscall => OpCode.twoByte(0x0f, 0x05),
	1063	else => null,
	1064	},
	1065	.d => return switch (tag) {
	1066	.jmp_near => OpCode.oneByte(0xe9),
	1067	.call_near => OpCode.oneByte(0xe8),
	1068	.jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
	1069	.jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
	1070	.jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
	1071	.jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
	1072	.je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
	1073	.jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
	1074	.jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
	1075	.jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
	1076	.js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
	1077	.jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
	1078	.jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
	1079	.jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
	1080	.jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
	1081	.jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
	1082	.jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
	1083	.jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
	1084	else => null,
	1085	},
	1086	.m => return switch (tag) {
	1087	.jmp_near, .call_near, .push => OpCode.oneByte(0xff),
	1088	.pop => OpCode.oneByte(0x8f),
	1089	.seto => OpCode.twoByte(0x0f, 0x90),
	1090	.setno => OpCode.twoByte(0x0f, 0x91),
	1091	.setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
	1092	.setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
	1093	.sete, .setz => OpCode.twoByte(0x0f, 0x94),
	1094	.setne, .setnz => OpCode.twoByte(0x0f, 0x95),
	1095	.setbe, .setna => OpCode.twoByte(0x0f, 0x96),
	1096	.seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
	1097	.sets => OpCode.twoByte(0x0f, 0x98),
	1098	.setns => OpCode.twoByte(0x0f, 0x99),
	1099	.setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
	1100	.setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
	1101	.setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
	1102	.setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
	1103	.setle, .setng => OpCode.twoByte(0x0f, 0x9e),
	1104	.setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
	1105	else => null,
	1106	},
	1107	.o => return switch (tag) {
	1108	.push => OpCode.oneByte(0x50),
	1109	.pop => OpCode.oneByte(0x58),
	1110	else => null,
	1111	},
	1112	.i => return switch (tag) {
	1113	.push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
	1114	.@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
	1115	.ret_near => OpCode.oneByte(0xc2),
	1116	.ret_far => OpCode.oneByte(0xca),
	1117	else => null,
	1118	},
	1119	.mi => return switch (tag) {
	1120	.adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
	1121	.mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
	1122	.@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
	1123	else => null,
	1124	},
	1125	.mr => return switch (tag) {
	1126	.adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
	1127	.add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
	1128	.sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
	1129	.xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
	1130	.@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
	1131	.@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
	1132	.sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
	1133	.cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
	1134	.mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
	1135	else => null,
	1136	},
	1137	.rm => return switch (tag) {
	1138	.adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
	1139	.add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
	1140	.sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
	1141	.xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
	1142	.@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
	1143	.@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
	1144	.sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
	1145	.cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
	1146	.mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
	1147	.lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
	1148	.imul => OpCode.twoByte(0x0f, 0xaf),
	1149	else => null,
	1150	},
	1151	.oi => return switch (tag) {
	1152	.mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
	1153	else => null,
	1154	},
	1155	.fd => return switch (tag) {
	1156	.mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
	1157	else => null,
	1158	},
	1159	.td => return switch (tag) {
	1160	.mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
	1161	else => null,
	1162	},
	1163	.rmi => return switch (tag) {
	1164	.imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
	1165	else => null,
	1166	},
	1167	}
	1168	}
	1169
	1170	inline fn getModRmExt(tag: Tag) ?u3 {
	1171	return switch (tag) {
	1172	.adc => 0x2,
	1173	.add => 0x0,
	1174	.sub => 0x5,
	1175	.xor => 0x6,
	1176	.@"and" => 0x4,
	1177	.@"or" => 0x1,
	1178	.sbb => 0x3,
	1179	.cmp => 0x7,
	1180	.mov => 0x0,
	1181	.jmp_near => 0x4,
	1182	.call_near => 0x2,
	1183	.push => 0x6,
	1184	.pop => 0x0,
	1185	.@"test" => 0x0,
	1186	.seto,
	1187	.setno,
	1188	.setb,
	1189	.setc,
	1190	.setnae,
	1191	.setnb,
	1192	.setnc,
	1193	.setae,
	1194	.sete,
	1195	.setz,
	1196	.setne,
	1197	.setnz,
	1198	.setbe,
	1199	.setna,
	1200	.seta,
	1201	.setnbe,
	1202	.sets,
	1203	.setns,
	1204	.setp,
	1205	.setpe,
	1206	.setnp,
	1207	.setop,
	1208	.setl,
	1209	.setnge,
	1210	.setnl,
	1211	.setge,
	1212	.setle,
	1213	.setng,
	1214	.setnle,
	1215	.setg,
	1216	=> 0x0,
	1217	else => null,
	1218	};
	1219	}
	1220
	1221	const ScaleIndex = struct {
	1222	scale: u2,
	1223	index: Register,
	1224	};
	1225
	1226	const Memory = struct {
	1227	base: ?Register,
	1228	rip: bool = false,
	1229	disp: u32,
	1230	ptr_size: PtrSize,
	1231	scale_index: ?ScaleIndex = null,
	1232
	1233	const PtrSize = enum {
	1234	byte_ptr,
	1235	word_ptr,
	1236	dword_ptr,
	1237	qword_ptr,
	1238
	1239	fn fromBits(in_bits: u64) PtrSize {
	1240	return switch (in_bits) {
	1241	8 => .byte_ptr,
	1242	16 => .word_ptr,
	1243	32 => .dword_ptr,
	1244	64 => .qword_ptr,
	1245	else => unreachable,
	1246	};
	1247	}
	1248
	1249	/// Returns size in bits.
	1250	fn size(ptr_size: PtrSize) u64 {
	1251	return switch (ptr_size) {
	1252	.byte_ptr => 8,
	1253	.word_ptr => 16,
	1254	.dword_ptr => 32,
	1255	.qword_ptr => 64,
	1256	};
	1257	}
	1258	};
	1259
	1260	fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
	1261	if (mem_op.base) \|base\| {
	1262	const dst = base.lowId();
	1263	const src = operand;
	1264	if (dst == 4 or mem_op.scale_index != null) {
	1265	if (mem_op.disp == 0 and dst != 5) {
	1266	encoder.modRm_SIBDisp0(src);
	1267	if (mem_op.scale_index) \|si\| {
	1268	encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
	1269	} else {
	1270	encoder.sib_base(dst);
	1271	}
	1272	} else if (immOpSize(mem_op.disp) == 8) {
	1273	encoder.modRm_SIBDisp8(src);
	1274	if (mem_op.scale_index) \|si\| {
	1275	encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
	1276	} else {
	1277	encoder.sib_baseDisp8(dst);
	1278	}
	1279	encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
	1280	} else {
	1281	encoder.modRm_SIBDisp32(src);
	1282	if (mem_op.scale_index) \|si\| {
	1283	encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
	1284	} else {
	1285	encoder.sib_baseDisp32(dst);
	1286	}
	1287	encoder.disp32(@bitCast(i32, mem_op.disp));
	1288	}
	1289	} else {
	1290	if (mem_op.disp == 0) {
	1291	encoder.modRm_indirectDisp0(src, dst);
	1292	} else if (immOpSize(mem_op.disp) == 8) {
	1293	encoder.modRm_indirectDisp8(src, dst);
	1294	encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
	1295	} else {
	1296	encoder.modRm_indirectDisp32(src, dst);
	1297	encoder.disp32(@bitCast(i32, mem_op.disp));
	1298	}
	1299	}
	1300	} else {
	1301	if (mem_op.rip) {
	1302	encoder.modRm_RIPDisp32(operand);
	1303	} else {
	1304	encoder.modRm_SIBDisp0(operand);
	1305	if (mem_op.scale_index) \|si\| {
	1306	encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
	1307	} else {
	1308	encoder.sib_disp32();
	1309	}
	1310	}
	1311	encoder.disp32(@bitCast(i32, mem_op.disp));
	1312	}
	1313	}
	1314	};
	1315
	1316	fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
	1317	switch (size) {
	1318	8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
	1319	16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
	1320	32, 64 => encoder.imm32(@bitCast(i32, imm)),
	1321	else => unreachable,
	1322	}
	1323	}
	1324
	1325	const RegisterOrMemory = union(enum) {
	1326	register: Register,
	1327	memory: Memory,
	1328
	1329	fn reg(register: Register) RegisterOrMemory {
	1330	return .{ .register = register };
	1331	}
	1332
	1333	fn mem(ptr_size: Memory.PtrSize, args: struct {
	1334	disp: u32,
	1335	base: ?Register = null,
	1336	scale_index: ?ScaleIndex = null,
	1337	}) RegisterOrMemory {
	1338	return .{
	1339	.memory = .{
	1340	.base = args.base,
	1341	.disp = args.disp,
	1342	.ptr_size = ptr_size,
	1343	.scale_index = args.scale_index,
	1344	},
	1345	};
	1346	}
	1347
	1348	fn rip(ptr_size: Memory.PtrSize, disp: u32) RegisterOrMemory {
	1349	return .{
	1350	.memory = .{
	1351	.base = null,
	1352	.rip = true,
	1353	.disp = disp,
	1354	.ptr_size = ptr_size,
	1355	},
	1356	};
	1357	}
	1358	};
	1359
	1360	const LoweringError = error{
	1361	OutOfMemory,
	1362	OperandSizeMismatch,
	1363	RaxOperandExpected,
	1364	};
	1365
	1366	fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
	1367	const opc = getOpCode(tag, .zo, false).?;
	1368	const encoder = try Encoder.init(code, 1);
	1369	opc.encode(encoder);
	1370	}
	1371
	1372	fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
	1373	if (tag == .ret_far or tag == .ret_near) {
	1374	const encoder = try Encoder.init(code, 3);
	1375	const opc = getOpCode(tag, .i, false).?;
	1376	opc.encode(encoder);
	1377	encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
	1378	return;
	1379	}
	1380	const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
	1381	const encoder = try Encoder.init(code, 5);
	1382	if (immOpSize(imm) == 16) {
	1383	encoder.prefix16BitMode();
	1384	}
	1385	opc.encode(encoder);
	1386	encodeImm(encoder, imm, immOpSize(imm));
	1387	}
	1388
	1389	fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
	1390	if (reg.size() != 16 and reg.size() != 64) {
	1391	return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
	1392	}
	1393	const opc = getOpCode(tag, .o, false).?;
	1394	const encoder = try Encoder.init(code, 3);
	1395	if (reg.size() == 16) {
	1396	encoder.prefix16BitMode();
	1397	}
	1398	encoder.rex(.{
	1399	.w = false,
	1400	.b = reg.isExtended(),
	1401	});
	1402	opc.encodeWithReg(encoder, reg);
	1403	}
	1404
	1405	fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
	1406	const opc = getOpCode(tag, .d, false).?;
	1407	const encoder = try Encoder.init(code, 6);
	1408	opc.encode(encoder);
	1409	encoder.imm32(@bitCast(i32, imm));
	1410	}
	1411
	1412	fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
	1413	const opc = getOpCode(tag, .m, false).?;
	1414	const modrm_ext = getModRmExt(tag).?;
	1415	switch (reg_or_mem) {
	1416	.register => \|reg\| {
	1417	const op_size_mismatch = blk: {
	1418	if (tag.isSetCC() and reg.size() == 8)
	1419	break :blk false;
	1420	break :blk reg.size() != 64 and reg.size() != 16;
	1421	};
	1422	if (op_size_mismatch) {
	1423	return error.OperandSizeMismatch;
	1424	}
	1425	const encoder = try Encoder.init(code, 4);
	1426	if (reg.size() == 16) {
	1427	encoder.prefix16BitMode();
	1428	}
	1429	encoder.rex(.{
	1430	.w = switch (reg) {
	1431	.ah, .bh, .ch, .dh => true,
	1432	else => false,
	1433	},
	1434	.b = reg.isExtended(),
	1435	});
	1436	opc.encode(encoder);
	1437	encoder.modRm_direct(modrm_ext, reg.lowId());
	1438	},
	1439	.memory => \|mem_op\| {
	1440	if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
	1441	return error.OperandSizeMismatch;
	1442	}
	1443	const encoder = try Encoder.init(code, 8);
	1444	if (mem_op.ptr_size == .word_ptr) {
	1445	encoder.prefix16BitMode();
	1446	}
	1447	if (mem_op.base) \|base\| {
	1448	if (base.size() != 64) {
	1449	return error.OperandSizeMismatch;
	1450	}
	1451	encoder.rex(.{
	1452	.w = false,
	1453	.b = base.isExtended(),
	1454	});
	1455	}
	1456	opc.encode(encoder);
	1457	mem_op.encode(encoder, modrm_ext);
	1458	},
	1459	}
	1460	}
	1461
	1462	fn lowerToTdEnc(tag: Tag, moffs: u64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
	1463	return lowerToTdFdEnc(tag, reg, moffs, code, true);
	1464	}
	1465
	1466	fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) LoweringError!void {
	1467	return lowerToTdFdEnc(tag, reg, moffs, code, false);
	1468	}
	1469
	1470	fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
	1471	if (reg.lowId() != Register.rax.lowId()) {
	1472	return error.RaxOperandExpected;
	1473	}
	1474	const opc = if (td)
	1475	getOpCode(tag, .td, reg.size() == 8).?
	1476	else
	1477	getOpCode(tag, .fd, reg.size() == 8).?;
	1478	const encoder = try Encoder.init(code, 10);
	1479	if (reg.size() == 16) {
	1480	encoder.prefix16BitMode();
	1481	}
	1482	encoder.rex(.{
	1483	.w = setRexWRegister(reg),
	1484	});
	1485	opc.encode(encoder);
	1486	switch (reg.size()) {
	1487	8 => encoder.imm8(@bitCast(i8, @truncate(u8, moffs))),
	1488	16 => encoder.imm16(@bitCast(i16, @truncate(u16, moffs))),
	1489	32 => encoder.imm32(@bitCast(i32, @truncate(u32, moffs))),
	1490	64 => encoder.imm64(moffs),
	1491	else => unreachable,
	1492	}
	1493	}
	1494
	1495	fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) LoweringError!void {
	1496	const opc = getOpCode(tag, .oi, reg.size() == 8).?;
	1497	const encoder = try Encoder.init(code, 10);
	1498	if (reg.size() == 16) {
	1499	encoder.prefix16BitMode();
	1500	}
	1501	encoder.rex(.{
	1502	.w = setRexWRegister(reg),
	1503	.b = reg.isExtended(),
	1504	});
	1505	opc.encodeWithReg(encoder, reg);
	1506	switch (reg.size()) {
	1507	8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
	1508	16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
	1509	32 => encoder.imm32(@bitCast(i32, @truncate(u32, imm))),
	1510	64 => encoder.imm64(imm),
	1511	else => unreachable,
	1512	}
	1513	}
	1514
	1515	fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
	1516	const modrm_ext = getModRmExt(tag).?;
	1517	switch (reg_or_mem) {
	1518	.register => \|dst_reg\| {
	1519	const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
	1520	const encoder = try Encoder.init(code, 7);
	1521	if (dst_reg.size() == 16) {
	1522	// 0x66 prefix switches to the non-default size; here we assume a switch from
	1523	// the default 32bits to 16bits operand-size.
	1524	// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
	1525	encoder.prefix16BitMode();
	1526	}
	1527	encoder.rex(.{
	1528	.w = setRexWRegister(dst_reg),
	1529	.b = dst_reg.isExtended(),
	1530	});
	1531	opc.encode(encoder);
	1532	encoder.modRm_direct(modrm_ext, dst_reg.lowId());
	1533	encodeImm(encoder, imm, dst_reg.size());
	1534	},
	1535	.memory => \|dst_mem\| {
	1536	const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
	1537	const encoder = try Encoder.init(code, 12);
	1538	if (dst_mem.ptr_size == .word_ptr) {
	1539	encoder.prefix16BitMode();
	1540	}
	1541	if (dst_mem.base) \|base\| {
	1542	if (base.size() != 64) {
	1543	return error.OperandSizeMismatch;
	1544	}
	1545	encoder.rex(.{
	1546	.w = dst_mem.ptr_size == .qword_ptr,
	1547	.b = base.isExtended(),
	1548	});
	1549	} else {
	1550	encoder.rex(.{
	1551	.w = dst_mem.ptr_size == .qword_ptr,
	1552	});
	1553	}
	1554	opc.encode(encoder);
	1555	dst_mem.encode(encoder, modrm_ext);
	1556	encodeImm(encoder, imm, dst_mem.ptr_size.size());
	1557	},
	1558	}
	1559	}
	1560
	1561	fn lowerToRmEnc(
	1562	tag: Tag,
	1563	reg: Register,
	1564	reg_or_mem: RegisterOrMemory,
	1565	code: *std.ArrayList(u8),
	1566	) LoweringError!void {
	1567	const opc = getOpCode(tag, .rm, reg.size() == 8).?;
	1568	switch (reg_or_mem) {
	1569	.register => \|src_reg\| {
	1570	if (reg.size() != src_reg.size()) {
	1571	return error.OperandSizeMismatch;
	1572	}
	1573	const encoder = try Encoder.init(code, 3);
	1574	encoder.rex(.{
	1575	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
	1576	.r = reg.isExtended(),
	1577	.b = src_reg.isExtended(),
	1578	});
	1579	opc.encode(encoder);
	1580	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
	1581	},
	1582	.memory => \|src_mem\| {
	1583	if (reg.size() != src_mem.ptr_size.size()) {
	1584	return error.OperandSizeMismatch;
	1585	}
	1586	const encoder = try Encoder.init(code, 9);
	1587	if (reg.size() == 16) {
	1588	encoder.prefix16BitMode();
	1589	}
	1590	if (src_mem.base) \|base\| {
	1591	// TODO handle 32-bit base register - requires prefix 0x67
	1592	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	1593	if (base.size() != 64) {
	1594	return error.OperandSizeMismatch;
	1595	}
	1596	encoder.rex(.{
	1597	.w = setRexWRegister(reg),
	1598	.r = reg.isExtended(),
	1599	.b = base.isExtended(),
	1600	});
	1601	} else {
	1602	encoder.rex(.{
	1603	.w = setRexWRegister(reg),
	1604	.r = reg.isExtended(),
	1605	});
	1606	}
	1607	opc.encode(encoder);
	1608	src_mem.encode(encoder, reg.lowId());
	1609	},
	1610	}
	1611	}
	1612
	1613	fn lowerToMrEnc(
	1614	tag: Tag,
	1615	reg_or_mem: RegisterOrMemory,
	1616	reg: Register,
	1617	code: *std.ArrayList(u8),
	1618	) LoweringError!void {
	1619	const opc = getOpCode(tag, .mr, reg.size() == 8).?;
	1620	switch (reg_or_mem) {
	1621	.register => \|dst_reg\| {
	1622	if (dst_reg.size() != reg.size()) {
	1623	return error.OperandSizeMismatch;
	1624	}
	1625	const encoder = try Encoder.init(code, 3);
	1626	encoder.rex(.{
	1627	.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
	1628	.r = reg.isExtended(),
	1629	.b = dst_reg.isExtended(),
	1630	});
	1631	opc.encode(encoder);
	1632	encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
	1633	},
	1634	.memory => \|dst_mem\| {
	1635	if (dst_mem.ptr_size.size() != reg.size()) {
	1636	return error.OperandSizeMismatch;
	1637	}
	1638	const encoder = try Encoder.init(code, 9);
	1639	if (reg.size() == 16) {
	1640	encoder.prefix16BitMode();
	1641	}
	1642	if (dst_mem.base) \|base\| {
	1643	if (base.size() != 64) {
	1644	return error.OperandSizeMismatch;
	1645	}
	1646	encoder.rex(.{
	1647	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	1648	.r = reg.isExtended(),
	1649	.b = base.isExtended(),
	1650	});
	1651	} else {
	1652	encoder.rex(.{
	1653	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	1654	.r = reg.isExtended(),
	1655	});
	1656	}
	1657	opc.encode(encoder);
	1658	dst_mem.encode(encoder, reg.lowId());
	1659	},
	1660	}
	1661	}
	1662
	1663	fn lowerToRmiEnc(
	1664	tag: Tag,
	1665	reg: Register,
	1666	reg_or_mem: RegisterOrMemory,
	1667	imm: u32,
	1668	code: *std.ArrayList(u8),
	1669	) LoweringError!void {
	1670	if (reg.size() == 8) {
	1671	return error.OperandSizeMismatch;
	1672	}
	1673	const opc = getOpCode(tag, .rmi, false).?;
	1674	const encoder = try Encoder.init(code, 13);
	1675	if (reg.size() == 16) {
	1676	encoder.prefix16BitMode();
	1677	}
	1678	switch (reg_or_mem) {
	1679	.register => \|src_reg\| {
	1680	if (reg.size() != src_reg.size()) {
	1681	return error.OperandSizeMismatch;
	1682	}
	1683	encoder.rex(.{
	1684	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
	1685	.r = reg.isExtended(),
	1686	.b = src_reg.isExtended(),
	1687	});
	1688	opc.encode(encoder);
	1689	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
	1690	},
	1691	.memory => \|src_mem\| {
	1692	if (src_mem.base) \|base\| {
	1693	// TODO handle 32-bit base register - requires prefix 0x67
	1694	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	1695	if (base.size() != 64) {
	1696	return error.OperandSizeMismatch;
	1697	}
	1698	if (src_mem.ptr_size == .byte_ptr) {
	1699	return error.OperandSizeMismatch;
	1700	}
	1701	encoder.rex(.{
	1702	.w = setRexWRegister(reg),
	1703	.r = reg.isExtended(),
	1704	.b = base.isExtended(),
	1705	});
	1706	} else {
	1707	encoder.rex(.{
	1708	.w = setRexWRegister(reg),
	1709	.r = reg.isExtended(),
	1710	});
	1711	}
	1712	opc.encode(encoder);
	1713	src_mem.encode(encoder, reg.lowId());
	1714	},
	1715	}
	1716	encodeImm(encoder, imm, reg.size());
	1717	}
	1718
	1719	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
	1720	assert(expected.len > 0);
	1721	if (mem.eql(u8, expected, given)) return;
	1722	const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
	1723	defer testing.allocator.free(expected_fmt);
	1724	const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
	1725	defer testing.allocator.free(given_fmt);
	1726	const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
	1727	var padding = try testing.allocator.alloc(u8, idx + 5);
	1728	defer testing.allocator.free(padding);
	1729	mem.set(u8, padding, ' ');
	1730	std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
	1731	assembly,
	1732	expected_fmt,
	1733	given_fmt,
	1734	padding,
	1735	});
	1736	return error.TestFailed;
	1737	}
	1738
	1739	const TestEmit = struct {
	1740	code_buffer: std.ArrayList(u8),
	1741	next: usize = 0,
	1742
	1743	fn init() TestEmit {
	1744	return .{
	1745	.code_buffer = std.ArrayList(u8).init(testing.allocator),
	1746	};
	1747	}
	1748
	1749	fn deinit(emit: *TestEmit) void {
	1750	emit.code_buffer.deinit();
	1751	emit.next = undefined;
	1752	}
	1753
	1754	fn code(emit: TestEmit) std.ArrayList(u8) {
	1755	emit.next = emit.code_buffer.items.len;
	1756	return &emit.code_buffer;
	1757	}
	1758
	1759	fn lowered(emit: TestEmit) []const u8 {
	1760	return emit.code_buffer.items[emit.next..];
	1761	}
	1762	};
	1763
	1764	test "lower MI encoding" {
	1765	var emit = TestEmit.init();
	1766	defer emit.deinit();
	1767	try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, emit.code());
	1768	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", emit.lowered(), "mov rax, 0x10");
	1769	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, emit.code());
	1770	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", emit.lowered(), "mov dword ptr [r11 + 0], 0x10");
	1771	try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{
	1772	.disp = @bitCast(u32, @as(i32, -8)),
	1773	.base = .rdx,
	1774	}), 0x10, emit.code());
	1775	try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", emit.lowered(), "add dword ptr [rdx - 8], 0x10");
	1776	try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
	1777	.disp = 0x10000000,
	1778	.base = .r11,
	1779	}), 0x10, emit.code());
	1780	try expectEqualHexStrings(
	1781	"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
	1782	emit.lowered(),
	1783	"sub dword ptr [r11 + 0x10000000], 0x10",
	1784	);
	1785	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, emit.code());
	1786	try expectEqualHexStrings(
	1787	"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
	1788	emit.lowered(),
	1789	"and dword ptr [ds:0x10000000], 0x10",
	1790	);
	1791	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
	1792	.disp = 0x10000000,
	1793	.base = .r12,
	1794	}), 0x10, emit.code());
	1795	try expectEqualHexStrings(
	1796	"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
	1797	emit.lowered(),
	1798	"and dword ptr [r12 + 0x10000000], 0x10",
	1799	);
	1800	try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, emit.code());
	1801	try expectEqualHexStrings(
	1802	"\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
	1803	emit.lowered(),
	1804	"mov qword ptr [rip + 0x10], 0x10",
	1805	);
	1806	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
	1807	.disp = @bitCast(u32, @as(i32, -8)),
	1808	.base = .rbp,
	1809	}), 0x10, emit.code());
	1810	try expectEqualHexStrings(
	1811	"\x48\xc7\x45\xf8\x10\x00\x00\x00",
	1812	emit.lowered(),
	1813	"mov qword ptr [rbp - 8], 0x10",
	1814	);
	1815	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{
	1816	.disp = @bitCast(u32, @as(i32, -2)),
	1817	.base = .rbp,
	1818	}), 0x10, emit.code());
	1819	try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", emit.lowered(), "mov word ptr [rbp - 2], 0x10");
	1820	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{
	1821	.disp = @bitCast(u32, @as(i32, -1)),
	1822	.base = .rbp,
	1823	}), 0x10, emit.code());
	1824	try expectEqualHexStrings("\xC6\x45\xFF\x10", emit.lowered(), "mov byte ptr [rbp - 1], 0x10");
	1825	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
	1826	.disp = 0x10000000,
	1827	.scale_index = .{
	1828	.scale = 1,
	1829	.index = .rcx,
	1830	},
	1831	}), 0x10, emit.code());
	1832	try expectEqualHexStrings(
	1833	"\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
	1834	emit.lowered(),
	1835	"mov qword ptr [rcx*2 + 0x10000000], 0x10",
	1836	);
	1837	}
	1838
	1839	test "lower RM encoding" {
	1840	var emit = TestEmit.init();
	1841	defer emit.deinit();
	1842	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code());
	1843	try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx");
	1844	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
	1845	try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]");
	1846	try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
	1847	try expectEqualHexStrings(
	1848	"\x4C\x03\x1C\x25\x00\x00\x00\x10",
	1849	emit.lowered(),
	1850	"add r11, qword ptr [ds:0x10000000]",
	1851	);
	1852	try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
	1853	try expectEqualHexStrings(
	1854	"\x44\x02\x24\x25\x00\x00\x00\x10",
	1855	emit.lowered(),
	1856	"add r11b, byte ptr [ds:0x10000000]",
	1857	);
	1858	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
	1859	.disp = 0x10000000,
	1860	.base = .r13,
	1861	}), emit.code());
	1862	try expectEqualHexStrings(
	1863	"\x4D\x2B\x9D\x00\x00\x00\x10",
	1864	emit.lowered(),
	1865	"sub r11, qword ptr [r13 + 0x10000000]",
	1866	);
	1867	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
	1868	.disp = 0x10000000,
	1869	.base = .r12,
	1870	}), emit.code());
	1871	try expectEqualHexStrings(
	1872	"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
	1873	emit.lowered(),
	1874	"sub r11, qword ptr [r12 + 0x10000000]",
	1875	);
	1876	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
	1877	.disp = @bitCast(u32, @as(i32, -4)),
	1878	.base = .rbp,
	1879	}), emit.code());
	1880	try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]");
	1881	try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
	1882	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]");
	1883	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
	1884	.disp = @bitCast(u32, @as(i32, -8)),
	1885	.base = .rbp,
	1886	.scale_index = .{
	1887	.scale = 0,
	1888	.index = .rcx,
	1889	},
	1890	}), emit.code());
	1891	try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
	1892	try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
	1893	.disp = @bitCast(u32, @as(i32, -4)),
	1894	.base = .rbp,
	1895	.scale_index = .{
	1896	.scale = 2,
	1897	.index = .rdx,
	1898	},
	1899	}), emit.code());
	1900	try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
	1901	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
	1902	.disp = @bitCast(u32, @as(i32, -8)),
	1903	.base = .rbp,
	1904	.scale_index = .{
	1905	.scale = 3,
	1906	.index = .rcx,
	1907	},
	1908	}), emit.code());
	1909	try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
	1910	try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
	1911	.disp = @bitCast(u32, @as(i32, -24)),
	1912	.base = .rsi,
	1913	.scale_index = .{
	1914	.scale = 0,
	1915	.index = .rcx,
	1916	},
	1917	}), emit.code());
	1918	try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
	1919	try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
	1920	.disp = 0,
	1921	.base = .rbp,
	1922	.scale_index = .{
	1923	.scale = 0,
	1924	.index = .rcx,
	1925	},
	1926	}), emit.code());
	1927	try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
	1928	}
	1929
	1930	test "lower MR encoding" {
	1931	var emit = TestEmit.init();
	1932	defer emit.deinit();
	1933	try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code());
	1934	try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx");
	1935	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
	1936	.disp = @bitCast(u32, @as(i32, -4)),
	1937	.base = .rbp,
	1938	}), .r11, emit.code());
	1939	try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11");
	1940	try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code());
	1941	try expectEqualHexStrings(
	1942	"\x44\x00\x24\x25\x00\x00\x00\x10",
	1943	emit.lowered(),
	1944	"add byte ptr [ds:0x10000000], r12b",
	1945	);
	1946	try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code());
	1947	try expectEqualHexStrings(
	1948	"\x44\x01\x24\x25\x00\x00\x00\x10",
	1949	emit.lowered(),
	1950	"add dword ptr [ds:0x10000000], r12d",
	1951	);
	1952	try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
	1953	.disp = 0x10000000,
	1954	.base = .r11,
	1955	}), .r12, emit.code());
	1956	try expectEqualHexStrings(
	1957	"\x4D\x29\xA3\x00\x00\x00\x10",
	1958	emit.lowered(),
	1959	"sub qword ptr [r11 + 0x10000000], r12",
	1960	);
	1961	try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code());
	1962	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12");
	1963	}
	1964
	1965	test "lower OI encoding" {
	1966	var emit = TestEmit.init();
	1967	defer emit.deinit();
	1968	try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code());
	1969	try expectEqualHexStrings(
	1970	"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
	1971	emit.lowered(),
	1972	"movabs rax, 0x1000000000000000",
	1973	);
	1974	try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code());
	1975	try expectEqualHexStrings(
	1976	"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
	1977	emit.lowered(),
	1978	"movabs r11, 0x1000000000000000",
	1979	);
	1980	try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code());
	1981	try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000");
	1982	try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code());
	1983	try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000");
	1984	try lowerToOiEnc(.mov, .r11b, 0x10, emit.code());
	1985	try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10");
	1986	}
	1987
	1988	test "lower FD/TD encoding" {
	1989	var emit = TestEmit.init();
	1990	defer emit.deinit();
	1991	try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code());
	1992	try expectEqualHexStrings(
	1993	"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
	1994	emit.lowered(),
	1995	"mov rax, ds:0x1000000000000000",
	1996	);
	1997	try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code());
	1998	try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000");
	1999	try lowerToFdEnc(.mov, .ax, 0x1000, emit.code());
	2000	try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000");
	2001	try lowerToFdEnc(.mov, .al, 0x10, emit.code());
	2002	try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10");
	2003	}
	2004
	2005	test "lower M encoding" {
	2006	var emit = TestEmit.init();
	2007	defer emit.deinit();
	2008	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), emit.code());
	2009	try expectEqualHexStrings("\x41\xFF\xE4", emit.lowered(), "jmp r12");
	2010	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), emit.code());
	2011	try expectEqualHexStrings("\x66\x41\xFF\xE4", emit.lowered(), "jmp r12w");
	2012	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
	2013	try expectEqualHexStrings("\x41\xFF\x24\x24", emit.lowered(), "jmp qword ptr [r12]");
	2014	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
	2015	try expectEqualHexStrings("\x66\x41\xFF\x24\x24", emit.lowered(), "jmp word ptr [r12]");
	2016	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), emit.code());
	2017	try expectEqualHexStrings("\x41\xFF\x64\x24\x10", emit.lowered(), "jmp qword ptr [r12 + 0x10]");
	2018	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
	2019	.disp = 0x1000,
	2020	.base = .r12,
	2021	}), emit.code());
	2022	try expectEqualHexStrings(
	2023	"\x41\xFF\xA4\x24\x00\x10\x00\x00",
	2024	emit.lowered(),
	2025	"jmp qword ptr [r12 + 0x1000]",
	2026	);
	2027	try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
	2028	try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [rip + 0x10]");
	2029	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), emit.code());
	2030	try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [ds:0x10]");
	2031	try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), emit.code());
	2032	try expectEqualHexStrings("\x41\x0F\x97\xC3", emit.lowered(), "seta r11b");
	2033	}
	2034
	2035	test "lower O encoding" {
	2036	var emit = TestEmit.init();
	2037	defer emit.deinit();
	2038	try lowerToOEnc(.pop, .r12, emit.code());
	2039	try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12");
	2040	try lowerToOEnc(.push, .r12w, emit.code());
	2041	try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w");
	2042	}
	2043
	2044	test "lower RMI encoding" {
	2045	var emit = TestEmit.init();
	2046	defer emit.deinit();
	2047	try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
	2048	.disp = @bitCast(u32, @as(i32, -8)),
	2049	.base = .rbp,
	2050	}), 0x10, emit.code());
	2051	try expectEqualHexStrings(
	2052	"\x48\x69\x45\xF8\x10\x00\x00\x00",
	2053	emit.lowered(),
	2054	"imul rax, qword ptr [rbp - 8], 0x10",
	2055	);
	2056	try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
	2057	.disp = @bitCast(u32, @as(i32, -4)),
	2058	.base = .rbp,
	2059	}), 0x10, emit.code());
	2060	try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
	2061	try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
	2062	.disp = @bitCast(u32, @as(i32, -2)),
	2063	.base = .rbp,
	2064	}), 0x10, emit.code());
	2065	try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
	2066	try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code());
	2067	try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10");
	2068	try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
	2069	try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
	2070	}

src/arch/x86_64/Isel.zig deleted-2088

...	...	@@ -1,2088 +0,0 @@
1		//! This file contains the functionality for lowering x86_64 MIR into
2		//! machine code
3
4		const Isel = @This();
5
6		const std = @import("std");
7		const assert = std.debug.assert;
8		const bits = @import("bits.zig");
9		const leb128 = std.leb;
10		const link = @import("../../link.zig");
11		const log = std.log.scoped(.codegen);
12		const math = std.math;
13		const mem = std.mem;
14		const testing = std.testing;
15
16		const Air = @import("../../Air.zig");
17		const Allocator = mem.Allocator;
18		const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
19		const DW = std.dwarf;
20		const Encoder = bits.Encoder;
21		const ErrorMsg = Module.ErrorMsg;
22		const MCValue = @import("CodeGen.zig").MCValue;
23		const Mir = @import("Mir.zig");
24		const Module = @import("../../Module.zig");
25		const Instruction = bits.Instruction;
26		const Register = bits.Register;
27		const Type = @import("../../type.zig").Type;
28
29		mir: Mir,
30		bin_file: *link.File,
31		debug_output: DebugInfoOutput,
32		target: *const std.Target,
33		err_msg: ?*ErrorMsg = null,
34		src_loc: Module.SrcLoc,
35		code: *std.ArrayList(u8),
36
37		prev_di_line: u32,
38		prev_di_column: u32,
39		/// Relative to the beginning of `code`.
40		prev_di_pc: usize,
41
42		code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
43		relocs: std.ArrayListUnmanaged(Reloc) = .{},
44
45		const InnerError = error{
46		OutOfMemory,
47		Overflow,
48		IselFail,
49		};
50
51		const Reloc = struct {
52		/// Offset of the instruction.
53		source: u64,
54		/// Target of the relocation.
55		target: Mir.Inst.Index,
56		/// Offset of the relocation within the instruction.
57		offset: u64,
58		/// Length of the instruction.
59		length: u5,
60		};
61
62		pub fn lowerMir(isel: *Isel) InnerError!void {
63		const mir_tags = isel.mir.instructions.items(.tag);
64
65		for (mir_tags) \|tag, index\| {
66		const inst = @intCast(u32, index);
67		try isel.code_offset_mapping.putNoClobber(isel.bin_file.allocator, inst, isel.code.items.len);
68		switch (tag) {
69		.adc => try isel.mirArith(.adc, inst),
70		.add => try isel.mirArith(.add, inst),
71		.sub => try isel.mirArith(.sub, inst),
72		.xor => try isel.mirArith(.xor, inst),
73		.@"and" => try isel.mirArith(.@"and", inst),
74		.@"or" => try isel.mirArith(.@"or", inst),
75		.sbb => try isel.mirArith(.sbb, inst),
76		.cmp => try isel.mirArith(.cmp, inst),
77		.mov => try isel.mirArith(.mov, inst),
78
79		.adc_mem_imm => try isel.mirArithMemImm(.adc, inst),
80		.add_mem_imm => try isel.mirArithMemImm(.add, inst),
81		.sub_mem_imm => try isel.mirArithMemImm(.sub, inst),
82		.xor_mem_imm => try isel.mirArithMemImm(.xor, inst),
83		.and_mem_imm => try isel.mirArithMemImm(.@"and", inst),
84		.or_mem_imm => try isel.mirArithMemImm(.@"or", inst),
85		.sbb_mem_imm => try isel.mirArithMemImm(.sbb, inst),
86		.cmp_mem_imm => try isel.mirArithMemImm(.cmp, inst),
87		.mov_mem_imm => try isel.mirArithMemImm(.mov, inst),
88
89		.adc_scale_src => try isel.mirArithScaleSrc(.adc, inst),
90		.add_scale_src => try isel.mirArithScaleSrc(.add, inst),
91		.sub_scale_src => try isel.mirArithScaleSrc(.sub, inst),
92		.xor_scale_src => try isel.mirArithScaleSrc(.xor, inst),
93		.and_scale_src => try isel.mirArithScaleSrc(.@"and", inst),
94		.or_scale_src => try isel.mirArithScaleSrc(.@"or", inst),
95		.sbb_scale_src => try isel.mirArithScaleSrc(.sbb, inst),
96		.cmp_scale_src => try isel.mirArithScaleSrc(.cmp, inst),
97		.mov_scale_src => try isel.mirArithScaleSrc(.mov, inst),
98
99		.adc_scale_dst => try isel.mirArithScaleDst(.adc, inst),
100		.add_scale_dst => try isel.mirArithScaleDst(.add, inst),
101		.sub_scale_dst => try isel.mirArithScaleDst(.sub, inst),
102		.xor_scale_dst => try isel.mirArithScaleDst(.xor, inst),
103		.and_scale_dst => try isel.mirArithScaleDst(.@"and", inst),
104		.or_scale_dst => try isel.mirArithScaleDst(.@"or", inst),
105		.sbb_scale_dst => try isel.mirArithScaleDst(.sbb, inst),
106		.cmp_scale_dst => try isel.mirArithScaleDst(.cmp, inst),
107		.mov_scale_dst => try isel.mirArithScaleDst(.mov, inst),
108
109		.adc_scale_imm => try isel.mirArithScaleImm(.adc, inst),
110		.add_scale_imm => try isel.mirArithScaleImm(.add, inst),
111		.sub_scale_imm => try isel.mirArithScaleImm(.sub, inst),
112		.xor_scale_imm => try isel.mirArithScaleImm(.xor, inst),
113		.and_scale_imm => try isel.mirArithScaleImm(.@"and", inst),
114		.or_scale_imm => try isel.mirArithScaleImm(.@"or", inst),
115		.sbb_scale_imm => try isel.mirArithScaleImm(.sbb, inst),
116		.cmp_scale_imm => try isel.mirArithScaleImm(.cmp, inst),
117		.mov_scale_imm => try isel.mirArithScaleImm(.mov, inst),
118
119		.movabs => try isel.mirMovabs(inst),
120
121		.lea => try isel.mirLea(inst),
122
123		.imul_complex => try isel.mirIMulComplex(inst),
124
125		.push => try isel.mirPushPop(.push, inst),
126		.pop => try isel.mirPushPop(.pop, inst),
127
128		.jmp => try isel.mirJmpCall(.jmp_near, inst),
129		.call => try isel.mirJmpCall(.call_near, inst),
130
131		.cond_jmp_greater_less,
132		.cond_jmp_above_below,
133		.cond_jmp_eq_ne,
134		=> try isel.mirCondJmp(tag, inst),
135
136		.cond_set_byte_greater_less,
137		.cond_set_byte_above_below,
138		.cond_set_byte_eq_ne,
139		=> try isel.mirCondSetByte(tag, inst),
140
141		.ret => try isel.mirRet(inst),
142
143		.syscall => try isel.mirSyscall(),
144
145		.@"test" => try isel.mirTest(inst),
146
147		.brk => try isel.mirBrk(),
148		.nop => try isel.mirNop(),
149
150		.call_extern => try isel.mirCallExtern(inst),
151
152		.dbg_line => try isel.mirDbgLine(inst),
153		.dbg_prologue_end => try isel.mirDbgPrologueEnd(inst),
154		.dbg_epilogue_begin => try isel.mirDbgEpilogueBegin(inst),
155		.arg_dbg_info => try isel.mirArgDbgInfo(inst),
156
157		.push_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
158		.pop_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
159
160		else => {
161		return isel.fail("Implement MIR->Isel lowering for x86_64 for pseudo-inst: {s}", .{tag});
162		},
163		}
164		}
165
166		try isel.fixupRelocs();
167		}
168
169		pub fn deinit(isel: *Isel) void {
170		isel.relocs.deinit(isel.bin_file.allocator);
171		isel.code_offset_mapping.deinit(isel.bin_file.allocator);
172		isel.* = undefined;
173		}
174
175		fn fail(isel: *Isel, comptime format: []const u8, args: anytype) InnerError {
176		@setCold(true);
177		assert(isel.err_msg == null);
178		isel.err_msg = try ErrorMsg.create(isel.bin_file.allocator, isel.src_loc, format, args);
179		return error.IselFail;
180		}
181
182		fn failWithLoweringError(isel: *Isel, err: LoweringError) InnerError {
183		return switch (err) {
184		error.RaxOperandExpected => isel.fail("Register.rax expected as destination operand", .{}),
185		error.OperandSizeMismatch => isel.fail("operand size mismatch", .{}),
186		else => \|e\| e,
187		};
188		}
189
190		fn fixupRelocs(isel: *Isel) InnerError!void {
191		// TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
192		// This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
193		// possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
194		// until the entire decl is correctly emitted with all JMP/CALL instructions within range.
195		for (isel.relocs.items) \|reloc\| {
196		const offset = try math.cast(usize, reloc.offset);
197		const target = isel.code_offset_mapping.get(reloc.target) orelse
198		return isel.fail("JMP/CALL relocation target not found!", .{});
199		const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
200		mem.writeIntLittle(i32, isel.code.items[offset..][0..4], disp);
201		}
202		}
203
204		fn mirBrk(isel: *Isel) InnerError!void {
205		return lowerToZoEnc(.brk, isel.code) catch \|err\| isel.failWithLoweringError(err);
206		}
207
208		fn mirNop(isel: *Isel) InnerError!void {
209		return lowerToZoEnc(.nop, isel.code) catch \|err\| isel.failWithLoweringError(err);
210		}
211
212		fn mirSyscall(isel: *Isel) InnerError!void {
213		return lowerToZoEnc(.syscall, isel.code) catch \|err\| isel.failWithLoweringError(err);
214		}
215
216		fn mirPushPop(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
217		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
218		switch (ops.flags) {
219		0b00 => {
220		// PUSH/POP reg
221		return lowerToOEnc(tag, ops.reg1, isel.code) catch \|err\| isel.failWithLoweringError(err);
222		},
223		0b01 => {
224		// PUSH/POP r/m64
225		const imm = isel.mir.instructions.items(.data)[inst].imm;
226		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
227		16 => .word_ptr,
228		else => .qword_ptr,
229		};
230		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
231		.disp = imm,
232		.base = ops.reg1,
233		}), isel.code) catch \|err\| isel.failWithLoweringError(err);
234		},
235		0b10 => {
236		// PUSH imm32
237		assert(tag == .push);
238		const imm = isel.mir.instructions.items(.data)[inst].imm;
239		return lowerToIEnc(.push, imm, isel.code) catch \|err\|
240		isel.failWithLoweringError(err);
241		},
242		0b11 => unreachable,
243		}
244		}
245		fn mirPushPopRegsFromCalleePreservedRegs(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
246		const callee_preserved_regs = bits.callee_preserved_regs;
247		const regs = isel.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
248		if (tag == .push) {
249		for (callee_preserved_regs) \|reg, i\| {
250		if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
251		lowerToOEnc(.push, reg, isel.code) catch \|err\|
252		return isel.failWithLoweringError(err);
253		}
254		} else {
255		// pop in the reverse direction
256		var i = callee_preserved_regs.len;
257		while (i > 0) : (i -= 1) {
258		const reg = callee_preserved_regs[i - 1];
259		if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
260		lowerToOEnc(.pop, reg, isel.code) catch \|err\|
261		return isel.failWithLoweringError(err);
262		}
263		}
264		}
265
266		fn mirJmpCall(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
267		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
268		switch (ops.flags) {
269		0b00 => {
270		const target = isel.mir.instructions.items(.data)[inst].inst;
271		const source = isel.code.items.len;
272		lowerToDEnc(tag, 0, isel.code) catch \|err\|
273		return isel.failWithLoweringError(err);
274		try isel.relocs.append(isel.bin_file.allocator, .{
275		.source = source,
276		.target = target,
277		.offset = isel.code.items.len - 4,
278		.length = 5,
279		});
280		},
281		0b01 => {
282		if (ops.reg1 == .none) {
283		// JMP/CALL [imm]
284		const imm = isel.mir.instructions.items(.data)[inst].imm;
285		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
286		16 => .word_ptr,
287		else => .qword_ptr,
288		};
289		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), isel.code) catch \|err\|
290		isel.failWithLoweringError(err);
291		}
292		// JMP/CALL reg
293		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), isel.code) catch \|err\| isel.failWithLoweringError(err);
294		},
295		0b10 => {
296		// JMP/CALL r/m64
297		const imm = isel.mir.instructions.items(.data)[inst].imm;
298		return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
299		.disp = imm,
300		.base = ops.reg1,
301		}), isel.code) catch \|err\| isel.failWithLoweringError(err);
302		},
303		0b11 => return isel.fail("TODO unused JMP/CALL variant 0b11", .{}),
304		}
305		}
306
307		fn mirCondJmp(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
308		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
309		const target = isel.mir.instructions.items(.data)[inst].inst;
310		const tag = switch (mir_tag) {
311		.cond_jmp_greater_less => switch (ops.flags) {
312		0b00 => Tag.jge,
313		0b01 => Tag.jg,
314		0b10 => Tag.jl,
315		0b11 => Tag.jle,
316		},
317		.cond_jmp_above_below => switch (ops.flags) {
318		0b00 => Tag.jae,
319		0b01 => Tag.ja,
320		0b10 => Tag.jb,
321		0b11 => Tag.jbe,
322		},
323		.cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
324		0b0 => Tag.jne,
325		0b1 => Tag.je,
326		},
327		else => unreachable,
328		};
329		const source = isel.code.items.len;
330		lowerToDEnc(tag, 0, isel.code) catch \|err\|
331		return isel.failWithLoweringError(err);
332		try isel.relocs.append(isel.bin_file.allocator, .{
333		.source = source,
334		.target = target,
335		.offset = isel.code.items.len - 4,
336		.length = 6,
337		});
338		}
339
340		fn mirCondSetByte(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
341		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
342		const tag = switch (mir_tag) {
343		.cond_set_byte_greater_less => switch (ops.flags) {
344		0b00 => Tag.setge,
345		0b01 => Tag.setg,
346		0b10 => Tag.setl,
347		0b11 => Tag.setle,
348		},
349		.cond_set_byte_above_below => switch (ops.flags) {
350		0b00 => Tag.setae,
351		0b01 => Tag.seta,
352		0b10 => Tag.setb,
353		0b11 => Tag.setbe,
354		},
355		.cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
356		0b0 => Tag.setne,
357		0b1 => Tag.sete,
358		},
359		else => unreachable,
360		};
361		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), isel.code) catch \|err\|
362		isel.failWithLoweringError(err);
363		}
364
365		fn mirTest(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
366		const tag = isel.mir.instructions.items(.tag)[inst];
367		assert(tag == .@"test");
368		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
369		switch (ops.flags) {
370		0b00 => {
371		if (ops.reg2 == .none) {
372		// TEST r/m64, imm32
373		// MI
374		const imm = isel.mir.instructions.items(.data)[inst].imm;
375		if (ops.reg1.to64() == .rax) {
376		// TEST rax, imm32
377		// I
378		return lowerToIEnc(.@"test", imm, isel.code) catch \|err\|
379		isel.failWithLoweringError(err);
380		}
381		return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch \|err\|
382		isel.failWithLoweringError(err);
383		}
384		// TEST r/m64, r64
385		return isel.fail("TODO TEST r/m64, r64", .{});
386		},
387		else => return isel.fail("TODO more TEST alternatives", .{}),
388		}
389		}
390
391		fn mirRet(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
392		const tag = isel.mir.instructions.items(.tag)[inst];
393		assert(tag == .ret);
394		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
395		switch (ops.flags) {
396		0b00 => {
397		// RETF imm16
398		// I
399		const imm = isel.mir.instructions.items(.data)[inst].imm;
400		return lowerToIEnc(.ret_far, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
401		},
402		0b01 => {
403		return lowerToZoEnc(.ret_far, isel.code) catch \|err\| isel.failWithLoweringError(err);
404		},
405		0b10 => {
406		// RET imm16
407		// I
408		const imm = isel.mir.instructions.items(.data)[inst].imm;
409		return lowerToIEnc(.ret_near, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
410		},
411		0b11 => {
412		return lowerToZoEnc(.ret_near, isel.code) catch \|err\| isel.failWithLoweringError(err);
413		},
414		}
415		}
416
417		fn mirArith(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
418		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
419		switch (ops.flags) {
420		0b00 => {
421		if (ops.reg2 == .none) {
422		// mov reg1, imm32
423		// MI
424		const imm = isel.mir.instructions.items(.data)[inst].imm;
425		return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch \|err\|
426		isel.failWithLoweringError(err);
427		}
428		// mov reg1, reg2
429		// RM
430		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch \|err\|
431		isel.failWithLoweringError(err);
432		},
433		0b01 => {
434		// mov reg1, [reg2 + imm32]
435		// RM
436		const imm = isel.mir.instructions.items(.data)[inst].imm;
437		const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
438		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
439		.disp = imm,
440		.base = src_reg,
441		}), isel.code) catch \|err\| isel.failWithLoweringError(err);
442		},
443		0b10 => {
444		if (ops.reg2 == .none) {
445		return isel.fail("TODO unused variant: mov reg1, none, 0b10", .{});
446		}
447		// mov [reg1 + imm32], reg2
448		// MR
449		const imm = isel.mir.instructions.items(.data)[inst].imm;
450		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
451		.disp = imm,
452		.base = ops.reg1,
453		}), ops.reg2, isel.code) catch \|err\| isel.failWithLoweringError(err);
454		},
455		0b11 => {
456		return isel.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
457		},
458		}
459		}
460
461		fn mirArithMemImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
462		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
463		assert(ops.reg2 == .none);
464		const payload = isel.mir.instructions.items(.data)[inst].payload;
465		const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
466		const ptr_size: Memory.PtrSize = switch (ops.flags) {
467		0b00 => .byte_ptr,
468		0b01 => .word_ptr,
469		0b10 => .dword_ptr,
470		0b11 => .qword_ptr,
471		};
472		return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
473		.disp = imm_pair.dest_off,
474		.base = ops.reg1,
475		}), imm_pair.operand, isel.code) catch \|err\| isel.failWithLoweringError(err);
476		}
477
478		inline fn setRexWRegister(reg: Register) bool {
479		if (reg.size() == 64) return true;
480		return switch (reg) {
481		.ah, .bh, .ch, .dh => true,
482		else => false,
483		};
484		}
485
486		inline fn immOpSize(imm: i64) u8 {
487		blk: {
488		_ = math.cast(i8, imm) catch break :blk;
489		return 8;
490		}
491		blk: {
492		_ = math.cast(i16, imm) catch break :blk;
493		return 16;
494		}
495		blk: {
496		_ = math.cast(i32, imm) catch break :blk;
497		return 32;
498		}
499		return 64;
500		}
501
502		fn mirArithScaleSrc(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
503		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
504		const scale = ops.flags;
505		const imm = isel.mir.instructions.items(.data)[inst].imm;
506		// OP reg1, [reg2 + scale*rcx + imm32]
507		const scale_index = ScaleIndex{
508		.scale = scale,
509		.index = .rcx,
510		};
511		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
512		.disp = imm,
513		.base = ops.reg2,
514		.scale_index = scale_index,
515		}), isel.code) catch \|err\| isel.failWithLoweringError(err);
516		}
517
518		fn mirArithScaleDst(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
519		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
520		const scale = ops.flags;
521		const imm = isel.mir.instructions.items(.data)[inst].imm;
522		const scale_index = ScaleIndex{
523		.scale = scale,
524		.index = .rax,
525		};
526		if (ops.reg2 == .none) {
527		// OP qword ptr [reg1 + scale*rax + 0], imm32
528		return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
529		.disp = 0,
530		.base = ops.reg1,
531		.scale_index = scale_index,
532		}), imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
533		}
534		// OP [reg1 + scale*rax + imm32], reg2
535		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
536		.disp = imm,
537		.base = ops.reg1,
538		.scale_index = scale_index,
539		}), ops.reg2, isel.code) catch \|err\| isel.failWithLoweringError(err);
540		}
541
542		fn mirArithScaleImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
543		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
544		const scale = ops.flags;
545		const payload = isel.mir.instructions.items(.data)[inst].payload;
546		const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
547		const scale_index = ScaleIndex{
548		.scale = scale,
549		.index = .rax,
550		};
551		// OP qword ptr [reg1 + scale*rax + imm32], imm32
552		return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
553		.disp = imm_pair.dest_off,
554		.base = ops.reg1,
555		.scale_index = scale_index,
556		}), imm_pair.operand, isel.code) catch \|err\| isel.failWithLoweringError(err);
557		}
558
559		fn mirMovabs(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
560		const tag = isel.mir.instructions.items(.tag)[inst];
561		assert(tag == .movabs);
562		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
563		const imm: i64 = if (ops.reg1.size() == 64) blk: {
564		const payload = isel.mir.instructions.items(.data)[inst].payload;
565		const imm = isel.mir.extraData(Mir.Imm64, payload).data;
566		break :blk @bitCast(i64, imm.decode());
567		} else isel.mir.instructions.items(.data)[inst].imm;
568		if (ops.flags == 0b00) {
569		// movabs reg, imm64
570		// OI
571		return lowerToOiEnc(.mov, ops.reg1, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
572		}
573		if (ops.reg1 == .none) {
574		// movabs moffs64, rax
575		// TD
576		return lowerToTdEnc(.mov, imm, ops.reg2, isel.code) catch \|err\| isel.failWithLoweringError(err);
577		}
578		// movabs rax, moffs64
579		// FD
580		return lowerToFdEnc(.mov, ops.reg1, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
581		}
582
583		fn mirIMulComplex(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
584		const tag = isel.mir.instructions.items(.tag)[inst];
585		assert(tag == .imul_complex);
586		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
587		switch (ops.flags) {
588		0b00 => {
589		return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch \|err\|
590		isel.failWithLoweringError(err);
591		},
592		0b10 => {
593		const imm = isel.mir.instructions.items(.data)[inst].imm;
594		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, isel.code) catch \|err\|
595		isel.failWithLoweringError(err);
596		},
597		else => return isel.fail("TODO implement imul", .{}),
598		}
599		}
600
601		fn mirLea(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
602		const tag = isel.mir.instructions.items(.tag)[inst];
603		assert(tag == .lea);
604		const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
605		switch (ops.flags) {
606		0b00 => {
607		// lea reg1, [reg2 + imm32]
608		// RM
609		const imm = isel.mir.instructions.items(.data)[inst].imm;
610		const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
611		return lowerToRmEnc(
612		.lea,
613		ops.reg1,
614		RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
615		.disp = imm,
616		.base = src_reg,
617		}),
618		isel.code,
619		) catch \|err\| isel.failWithLoweringError(err);
620		},
621		0b01 => {
622		// lea reg1, [rip + imm32]
623		// RM
624		const start_offset = isel.code.items.len;
625		lowerToRmEnc(
626		.lea,
627		ops.reg1,
628		RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
629		isel.code,
630		) catch \|err\| return isel.failWithLoweringError(err);
631		const end_offset = isel.code.items.len;
632		// Backpatch the displacement
633		const payload = isel.mir.instructions.items(.data)[inst].payload;
634		const imm = isel.mir.extraData(Mir.Imm64, payload).data.decode();
635		const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
636		mem.writeIntLittle(i32, isel.code.items[end_offset - 4 ..][0..4], disp);
637		},
638		0b10 => {
639		// lea reg1, [rip + reloc]
640		// RM
641		lowerToRmEnc(
642		.lea,
643		ops.reg1,
644		RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
645		isel.code,
646		) catch \|err\| return isel.failWithLoweringError(err);
647		const end_offset = isel.code.items.len;
648		const got_entry = isel.mir.instructions.items(.data)[inst].got_entry;
649		if (isel.bin_file.cast(link.File.MachO)) \|macho_file\| {
650		// TODO I think the reloc might be in the wrong place.
651		const decl = macho_file.active_decl.?;
652		try decl.link.macho.relocs.append(isel.bin_file.allocator, .{
653		.offset = @intCast(u32, end_offset - 4),
654		.target = .{ .local = got_entry },
655		.addend = 0,
656		.subtractor = null,
657		.pcrel = true,
658		.length = 2,
659		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
660		});
661		} else {
662		return isel.fail(
663		"TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
664		.{},
665		);
666		}
667		},
668		0b11 => {
669		// lea reg, [rbp + rcx + imm32]
670		const imm = isel.mir.instructions.items(.data)[inst].imm;
671		const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
672		const scale_index = ScaleIndex{
673		.scale = 0,
674		.index = .rcx,
675		};
676		return lowerToRmEnc(
677		.lea,
678		ops.reg1,
679		RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
680		.disp = imm,
681		.base = src_reg,
682		.scale_index = scale_index,
683		}),
684		isel.code,
685		) catch \|err\| isel.failWithLoweringError(err);
686		},
687		}
688		}
689
690		fn mirCallExtern(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
691		const tag = isel.mir.instructions.items(.tag)[inst];
692		assert(tag == .call_extern);
693		const n_strx = isel.mir.instructions.items(.data)[inst].extern_fn;
694		const offset = blk: {
695		// callq
696		lowerToDEnc(.call_near, 0, isel.code) catch \|err\|
697		return isel.failWithLoweringError(err);
698		break :blk @intCast(u32, isel.code.items.len) - 4;
699		};
700		if (isel.bin_file.cast(link.File.MachO)) \|macho_file\| {
701		// Add relocation to the decl.
702		try macho_file.active_decl.?.link.macho.relocs.append(isel.bin_file.allocator, .{
703		.offset = offset,
704		.target = .{ .global = n_strx },
705		.addend = 0,
706		.subtractor = null,
707		.pcrel = true,
708		.length = 2,
709		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
710		});
711		} else {
712		return isel.fail("TODO implement call_extern for linking backends different than MachO", .{});
713		}
714		}
715
716		fn mirDbgLine(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
717		const tag = isel.mir.instructions.items(.tag)[inst];
718		assert(tag == .dbg_line);
719		const payload = isel.mir.instructions.items(.data)[inst].payload;
720		const dbg_line_column = isel.mir.extraData(Mir.DbgLineColumn, payload).data;
721		try isel.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
722		}
723
724		fn dbgAdvancePCAndLine(isel: *Isel, line: u32, column: u32) InnerError!void {
725		const delta_line = @intCast(i32, line) - @intCast(i32, isel.prev_di_line);
726		const delta_pc: usize = isel.code.items.len - isel.prev_di_pc;
727		switch (isel.debug_output) {
728		.dwarf => \|dbg_out\| {
729		// TODO Look into using the DWARF special opcodes to compress this data.
730		// It lets you emit single-byte opcodes that add different numbers to
731		// both the PC and the line number at the same time.
732		try dbg_out.dbg_line.ensureUnusedCapacity(11);
733		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
734		leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
735		if (delta_line != 0) {
736		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
737		leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
738		}
739		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
740		isel.prev_di_pc = isel.code.items.len;
741		isel.prev_di_line = line;
742		isel.prev_di_column = column;
743		isel.prev_di_pc = isel.code.items.len;
744		},
745		.plan9 => \|dbg_out\| {
746		if (delta_pc <= 0) return; // only do this when the pc changes
747		// we have already checked the target in the linker to make sure it is compatable
748		const quant = @import("../../link/Plan9/aout.zig").getPCQuant(isel.target.cpu.arch) catch unreachable;
749
750		// increasing the line number
751		try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
752		// increasing the pc
753		const d_pc_p9 = @intCast(i64, delta_pc) - quant;
754		if (d_pc_p9 > 0) {
755		// minus one because if its the last one, we want to leave space to change the line which is one quanta
756		var diff = @divExact(d_pc_p9, quant) - quant;
757		while (diff > 0) {
758		if (diff < 64) {
759		try dbg_out.dbg_line.append(@intCast(u8, diff + 128));
760		diff = 0;
761		} else {
762		try dbg_out.dbg_line.append(@intCast(u8, 64 + 128));
763		diff -= 64;
764		}
765		}
766		if (dbg_out.pcop_change_index.*) \|pci\|
767		dbg_out.dbg_line.items[pci] += 1;
768		dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
769		} else if (d_pc_p9 == 0) {
770		// we don't need to do anything, because adding the quant does it for us
771		} else unreachable;
772		if (dbg_out.start_line.* == null)
773		dbg_out.start_line.* = isel.prev_di_line;
774		dbg_out.end_line.* = line;
775		// only do this if the pc changed
776		isel.prev_di_line = line;
777		isel.prev_di_column = column;
778		isel.prev_di_pc = isel.code.items.len;
779		},
780		.none => {},
781		}
782		}
783
784		fn mirDbgPrologueEnd(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
785		const tag = isel.mir.instructions.items(.tag)[inst];
786		assert(tag == .dbg_prologue_end);
787		switch (isel.debug_output) {
788		.dwarf => \|dbg_out\| {
789		try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
790		try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
791		},
792		.plan9 => {},
793		.none => {},
794		}
795		}
796
797		fn mirDbgEpilogueBegin(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
798		const tag = isel.mir.instructions.items(.tag)[inst];
799		assert(tag == .dbg_epilogue_begin);
800		switch (isel.debug_output) {
801		.dwarf => \|dbg_out\| {
802		try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
803		try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
804		},
805		.plan9 => {},
806		.none => {},
807		}
808		}
809
810		fn mirArgDbgInfo(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
811		const tag = isel.mir.instructions.items(.tag)[inst];
812		assert(tag == .arg_dbg_info);
813		const payload = isel.mir.instructions.items(.data)[inst].payload;
814		const arg_dbg_info = isel.mir.extraData(Mir.ArgDbgInfo, payload).data;
815		const mcv = isel.mir.function.args[arg_dbg_info.arg_index];
816		try isel.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
817		}
818
819		fn genArgDbgInfo(isel: *Isel, inst: Air.Inst.Index, mcv: MCValue) !void {
820		const ty_str = isel.mir.function.air.instructions.items(.data)[inst].ty_str;
821		const zir = &isel.mir.function.mod_fn.owner_decl.getFileScope().zir;
822		const name = zir.nullTerminatedString(ty_str.str);
823		const name_with_null = name.ptr[0 .. name.len + 1];
824		const ty = isel.mir.function.air.getRefType(ty_str.ty);
825
826		switch (mcv) {
827		.register => \|reg\| {
828		switch (isel.debug_output) {
829		.dwarf => \|dbg_out\| {
830		try dbg_out.dbg_info.ensureUnusedCapacity(3);
831		dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
832		dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
833		1, // ULEB128 dwarf expression length
834		reg.dwarfLocOp(),
835		});
836		try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
837		try isel.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
838		dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
839		},
840		.plan9 => {},
841		.none => {},
842		}
843		},
844		.stack_offset => {
845		switch (isel.debug_output) {
846		.dwarf => {},
847		.plan9 => {},
848		.none => {},
849		}
850		},
851		else => {},
852		}
853		}
854
855		/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
856		/// after codegen for this symbol is done.
857		fn addDbgInfoTypeReloc(isel: *Isel, ty: Type) !void {
858		switch (isel.debug_output) {
859		.dwarf => \|dbg_out\| {
860		assert(ty.hasCodeGenBits());
861		const index = dbg_out.dbg_info.items.len;
862		try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
863
864		const gop = try dbg_out.dbg_info_type_relocs.getOrPut(isel.bin_file.allocator, ty);
865		if (!gop.found_existing) {
866		gop.value_ptr.* = .{
867		.off = undefined,
868		.relocs = .{},
869		};
870		}
871		try gop.value_ptr.relocs.append(isel.bin_file.allocator, @intCast(u32, index));
872		},
873		.plan9 => {},
874		.none => {},
875		}
876		}
877
878		const Tag = enum {
879		adc,
880		add,
881		sub,
882		xor,
883		@"and",
884		@"or",
885		sbb,
886		cmp,
887		mov,
888		lea,
889		jmp_near,
890		call_near,
891		push,
892		pop,
893		@"test",
894		brk,
895		nop,
896		imul,
897		syscall,
898		ret_near,
899		ret_far,
900		jo,
901		jno,
902		jb,
903		jbe,
904		jc,
905		jnae,
906		jnc,
907		jae,
908		je,
909		jz,
910		jne,
911		jnz,
912		jna,
913		jnb,
914		jnbe,
915		ja,
916		js,
917		jns,
918		jpe,
919		jp,
920		jpo,
921		jnp,
922		jnge,
923		jl,
924		jge,
925		jnl,
926		jle,
927		jng,
928		jg,
929		jnle,
930		seto,
931		setno,
932		setb,
933		setc,
934		setnae,
935		setnb,
936		setnc,
937		setae,
938		sete,
939		setz,
940		setne,
941		setnz,
942		setbe,
943		setna,
944		seta,
945		setnbe,
946		sets,
947		setns,
948		setp,
949		setpe,
950		setnp,
951		setop,
952		setl,
953		setnge,
954		setnl,
955		setge,
956		setle,
957		setng,
958		setnle,
959		setg,
960
961		fn isSetCC(tag: Tag) bool {
962		return switch (tag) {
963		.seto,
964		.setno,
965		.setb,
966		.setc,
967		.setnae,
968		.setnb,
969		.setnc,
970		.setae,
971		.sete,
972		.setz,
973		.setne,
974		.setnz,
975		.setbe,
976		.setna,
977		.seta,
978		.setnbe,
979		.sets,
980		.setns,
981		.setp,
982		.setpe,
983		.setnp,
984		.setop,
985		.setl,
986		.setnge,
987		.setnl,
988		.setge,
989		.setle,
990		.setng,
991		.setnle,
992		.setg,
993		=> true,
994		else => false,
995		};
996		}
997		};
998
999		const Encoding = enum {
1000		/// OP
1001		zo,
1002
1003		/// OP rel32
1004		d,
1005
1006		/// OP r/m64
1007		m,
1008
1009		/// OP r64
1010		o,
1011
1012		/// OP imm32
1013		i,
1014
1015		/// OP r/m64, imm32
1016		mi,
1017
1018		/// OP r/m64, r64
1019		mr,
1020
1021		/// OP r64, r/m64
1022		rm,
1023
1024		/// OP r64, imm64
1025		oi,
1026
1027		/// OP al/ax/eax/rax, moffs
1028		fd,
1029
1030		/// OP moffs, al/ax/eax/rax
1031		td,
1032
1033		/// OP r64, r/m64, imm32
1034		rmi,
1035		};
1036
1037		const OpCode = union(enum) {
1038		one_byte: u8,
1039		two_byte: struct { _1: u8, _2: u8 },
1040
1041		fn oneByte(opc: u8) OpCode {
1042		return .{ .one_byte = opc };
1043		}
1044
1045		fn twoByte(opc1: u8, opc2: u8) OpCode {
1046		return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
1047		}
1048
1049		fn encode(opc: OpCode, encoder: Encoder) void {
1050		switch (opc) {
1051		.one_byte => \|v\| encoder.opcode_1byte(v),
1052		.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
1053		}
1054		}
1055
1056		fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1057		assert(opc == .one_byte);
1058		encoder.opcode_withReg(opc.one_byte, reg.lowId());
1059		}
1060		};
1061
1062		inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1063		switch (enc) {
1064		.zo => return switch (tag) {
1065		.ret_near => OpCode.oneByte(0xc3),
1066		.ret_far => OpCode.oneByte(0xcb),
1067		.brk => OpCode.oneByte(0xcc),
1068		.nop => OpCode.oneByte(0x90),
1069		.syscall => OpCode.twoByte(0x0f, 0x05),
1070		else => null,
1071		},
1072		.d => return switch (tag) {
1073		.jmp_near => OpCode.oneByte(0xe9),
1074		.call_near => OpCode.oneByte(0xe8),
1075		.jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
1076		.jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
1077		.jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
1078		.jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
1079		.je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
1080		.jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
1081		.jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
1082		.jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
1083		.js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
1084		.jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
1085		.jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
1086		.jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
1087		.jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
1088		.jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
1089		.jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
1090		.jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
1091		else => null,
1092		},
1093		.m => return switch (tag) {
1094		.jmp_near, .call_near, .push => OpCode.oneByte(0xff),
1095		.pop => OpCode.oneByte(0x8f),
1096		.seto => OpCode.twoByte(0x0f, 0x90),
1097		.setno => OpCode.twoByte(0x0f, 0x91),
1098		.setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
1099		.setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
1100		.sete, .setz => OpCode.twoByte(0x0f, 0x94),
1101		.setne, .setnz => OpCode.twoByte(0x0f, 0x95),
1102		.setbe, .setna => OpCode.twoByte(0x0f, 0x96),
1103		.seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
1104		.sets => OpCode.twoByte(0x0f, 0x98),
1105		.setns => OpCode.twoByte(0x0f, 0x99),
1106		.setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
1107		.setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
1108		.setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
1109		.setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
1110		.setle, .setng => OpCode.twoByte(0x0f, 0x9e),
1111		.setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1112		else => null,
1113		},
1114		.o => return switch (tag) {
1115		.push => OpCode.oneByte(0x50),
1116		.pop => OpCode.oneByte(0x58),
1117		else => null,
1118		},
1119		.i => return switch (tag) {
1120		.push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
1121		.@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
1122		.ret_near => OpCode.oneByte(0xc2),
1123		.ret_far => OpCode.oneByte(0xca),
1124		else => null,
1125		},
1126		.mi => return switch (tag) {
1127		.adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
1128		.mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
1129		.@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1130		else => null,
1131		},
1132		.mr => return switch (tag) {
1133		.adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
1134		.add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
1135		.sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
1136		.xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
1137		.@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
1138		.@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
1139		.sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1140		.cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1141		.mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1142		else => null,
1143		},
1144		.rm => return switch (tag) {
1145		.adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
1146		.add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
1147		.sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
1148		.xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
1149		.@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1150		.@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
1151		.sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
1152		.cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
1153		.mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
1154		.lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1155		.imul => OpCode.twoByte(0x0f, 0xaf),
1156		else => null,
1157		},
1158		.oi => return switch (tag) {
1159		.mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
1160		else => null,
1161		},
1162		.fd => return switch (tag) {
1163		.mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
1164		else => null,
1165		},
1166		.td => return switch (tag) {
1167		.mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
1168		else => null,
1169		},
1170		.rmi => return switch (tag) {
1171		.imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1172		else => null,
1173		},
1174		}
1175		}
1176
1177		inline fn getModRmExt(tag: Tag) ?u3 {
1178		return switch (tag) {
1179		.adc => 0x2,
1180		.add => 0x0,
1181		.sub => 0x5,
1182		.xor => 0x6,
1183		.@"and" => 0x4,
1184		.@"or" => 0x1,
1185		.sbb => 0x3,
1186		.cmp => 0x7,
1187		.mov => 0x0,
1188		.jmp_near => 0x4,
1189		.call_near => 0x2,
1190		.push => 0x6,
1191		.pop => 0x0,
1192		.@"test" => 0x0,
1193		.seto,
1194		.setno,
1195		.setb,
1196		.setc,
1197		.setnae,
1198		.setnb,
1199		.setnc,
1200		.setae,
1201		.sete,
1202		.setz,
1203		.setne,
1204		.setnz,
1205		.setbe,
1206		.setna,
1207		.seta,
1208		.setnbe,
1209		.sets,
1210		.setns,
1211		.setp,
1212		.setpe,
1213		.setnp,
1214		.setop,
1215		.setl,
1216		.setnge,
1217		.setnl,
1218		.setge,
1219		.setle,
1220		.setng,
1221		.setnle,
1222		.setg,
1223		=> 0x0,
1224		else => null,
1225		};
1226		}
1227
1228		const ScaleIndex = struct {
1229		scale: u2,
1230		index: Register,
1231		};
1232
1233		const Memory = struct {
1234		base: ?Register,
1235		rip: bool = false,
1236		disp: i32,
1237		ptr_size: PtrSize,
1238		scale_index: ?ScaleIndex = null,
1239
1240		const PtrSize = enum {
1241		byte_ptr,
1242		word_ptr,
1243		dword_ptr,
1244		qword_ptr,
1245
1246		fn fromBits(in_bits: u64) PtrSize {
1247		return switch (in_bits) {
1248		8 => .byte_ptr,
1249		16 => .word_ptr,
1250		32 => .dword_ptr,
1251		64 => .qword_ptr,
1252		else => unreachable,
1253		};
1254		}
1255
1256		/// Returns size in bits.
1257		fn size(ptr_size: PtrSize) u64 {
1258		return switch (ptr_size) {
1259		.byte_ptr => 8,
1260		.word_ptr => 16,
1261		.dword_ptr => 32,
1262		.qword_ptr => 64,
1263		};
1264		}
1265		};
1266
1267		fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
1268		if (mem_op.base) \|base\| {
1269		const dst = base.lowId();
1270		const src = operand;
1271		if (dst == 4 or mem_op.scale_index != null) {
1272		if (mem_op.disp == 0 and dst != 5) {
1273		encoder.modRm_SIBDisp0(src);
1274		if (mem_op.scale_index) \|si\| {
1275		encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
1276		} else {
1277		encoder.sib_base(dst);
1278		}
1279		} else if (immOpSize(mem_op.disp) == 8) {
1280		encoder.modRm_SIBDisp8(src);
1281		if (mem_op.scale_index) \|si\| {
1282		encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
1283		} else {
1284		encoder.sib_baseDisp8(dst);
1285		}
1286		encoder.disp8(@intCast(i8, mem_op.disp));
1287		} else {
1288		encoder.modRm_SIBDisp32(src);
1289		if (mem_op.scale_index) \|si\| {
1290		encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
1291		} else {
1292		encoder.sib_baseDisp32(dst);
1293		}
1294		encoder.disp32(mem_op.disp);
1295		}
1296		} else {
1297		if (mem_op.disp == 0) {
1298		encoder.modRm_indirectDisp0(src, dst);
1299		} else if (immOpSize(mem_op.disp) == 8) {
1300		encoder.modRm_indirectDisp8(src, dst);
1301		encoder.disp8(@intCast(i8, mem_op.disp));
1302		} else {
1303		encoder.modRm_indirectDisp32(src, dst);
1304		encoder.disp32(mem_op.disp);
1305		}
1306		}
1307		} else {
1308		if (mem_op.rip) {
1309		encoder.modRm_RIPDisp32(operand);
1310		} else {
1311		encoder.modRm_SIBDisp0(operand);
1312		if (mem_op.scale_index) \|si\| {
1313		encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
1314		} else {
1315		encoder.sib_disp32();
1316		}
1317		}
1318		encoder.disp32(mem_op.disp);
1319		}
1320		}
1321		};
1322
1323		fn encodeImm(encoder: Encoder, imm: i32, size: u64) void {
1324		switch (size) {
1325		8 => encoder.imm8(@intCast(i8, imm)),
1326		16 => encoder.imm16(@intCast(i16, imm)),
1327		32, 64 => encoder.imm32(imm),
1328		else => unreachable,
1329		}
1330		}
1331
1332		const RegisterOrMemory = union(enum) {
1333		register: Register,
1334		memory: Memory,
1335
1336		fn reg(register: Register) RegisterOrMemory {
1337		return .{ .register = register };
1338		}
1339
1340		fn mem(ptr_size: Memory.PtrSize, args: struct {
1341		disp: i32,
1342		base: ?Register = null,
1343		scale_index: ?ScaleIndex = null,
1344		}) RegisterOrMemory {
1345		return .{
1346		.memory = .{
1347		.base = args.base,
1348		.disp = args.disp,
1349		.ptr_size = ptr_size,
1350		.scale_index = args.scale_index,
1351		},
1352		};
1353		}
1354
1355		fn rip(ptr_size: Memory.PtrSize, disp: i32) RegisterOrMemory {
1356		return .{
1357		.memory = .{
1358		.base = null,
1359		.rip = true,
1360		.disp = disp,
1361		.ptr_size = ptr_size,
1362		},
1363		};
1364		}
1365		};
1366
1367		const LoweringError = error{
1368		OutOfMemory,
1369		Overflow,
1370		OperandSizeMismatch,
1371		RaxOperandExpected,
1372		};
1373
1374		fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
1375		const opc = getOpCode(tag, .zo, false).?;
1376		const encoder = try Encoder.init(code, 1);
1377		opc.encode(encoder);
1378		}
1379
1380		fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1381		if (tag == .ret_far or tag == .ret_near) {
1382		const encoder = try Encoder.init(code, 3);
1383		const opc = getOpCode(tag, .i, false).?;
1384		opc.encode(encoder);
1385		encoder.imm16(@intCast(i16, imm));
1386		return;
1387		}
1388		const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
1389		const encoder = try Encoder.init(code, 5);
1390		if (immOpSize(imm) == 16) {
1391		encoder.prefix16BitMode();
1392		}
1393		opc.encode(encoder);
1394		encodeImm(encoder, imm, immOpSize(imm));
1395		}
1396
1397		fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1398		if (reg.size() != 16 and reg.size() != 64) {
1399		return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
1400		}
1401		const opc = getOpCode(tag, .o, false).?;
1402		const encoder = try Encoder.init(code, 3);
1403		if (reg.size() == 16) {
1404		encoder.prefix16BitMode();
1405		}
1406		encoder.rex(.{
1407		.w = false,
1408		.b = reg.isExtended(),
1409		});
1410		opc.encodeWithReg(encoder, reg);
1411		}
1412
1413		fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1414		const opc = getOpCode(tag, .d, false).?;
1415		const encoder = try Encoder.init(code, 6);
1416		opc.encode(encoder);
1417		encoder.imm32(imm);
1418		}
1419
1420		fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
1421		const opc = getOpCode(tag, .m, false).?;
1422		const modrm_ext = getModRmExt(tag).?;
1423		switch (reg_or_mem) {
1424		.register => \|reg\| {
1425		const op_size_mismatch = blk: {
1426		if (tag.isSetCC() and reg.size() == 8)
1427		break :blk false;
1428		break :blk reg.size() != 64 and reg.size() != 16;
1429		};
1430		if (op_size_mismatch) {
1431		return error.OperandSizeMismatch;
1432		}
1433		const encoder = try Encoder.init(code, 4);
1434		if (reg.size() == 16) {
1435		encoder.prefix16BitMode();
1436		}
1437		encoder.rex(.{
1438		.w = switch (reg) {
1439		.ah, .bh, .ch, .dh => true,
1440		else => false,
1441		},
1442		.b = reg.isExtended(),
1443		});
1444		opc.encode(encoder);
1445		encoder.modRm_direct(modrm_ext, reg.lowId());
1446		},
1447		.memory => \|mem_op\| {
1448		if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
1449		return error.OperandSizeMismatch;
1450		}
1451		const encoder = try Encoder.init(code, 8);
1452		if (mem_op.ptr_size == .word_ptr) {
1453		encoder.prefix16BitMode();
1454		}
1455		if (mem_op.base) \|base\| {
1456		if (base.size() != 64) {
1457		return error.OperandSizeMismatch;
1458		}
1459		encoder.rex(.{
1460		.w = false,
1461		.b = base.isExtended(),
1462		});
1463		}
1464		opc.encode(encoder);
1465		mem_op.encode(encoder, modrm_ext);
1466		},
1467		}
1468		}
1469
1470		fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1471		return lowerToTdFdEnc(tag, reg, moffs, code, true);
1472		}
1473
1474		fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) LoweringError!void {
1475		return lowerToTdFdEnc(tag, reg, moffs, code, false);
1476		}
1477
1478		fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
1479		if (reg.lowId() != Register.rax.lowId()) {
1480		return error.RaxOperandExpected;
1481		}
1482		if (reg.size() != immOpSize(moffs)) {
1483		return error.OperandSizeMismatch;
1484		}
1485		const opc = if (td)
1486		getOpCode(tag, .td, reg.size() == 8).?
1487		else
1488		getOpCode(tag, .fd, reg.size() == 8).?;
1489		const encoder = try Encoder.init(code, 10);
1490		if (reg.size() == 16) {
1491		encoder.prefix16BitMode();
1492		}
1493		encoder.rex(.{
1494		.w = setRexWRegister(reg),
1495		});
1496		opc.encode(encoder);
1497		switch (reg.size()) {
1498		8 => {
1499		const moffs8 = try math.cast(i8, moffs);
1500		encoder.imm8(moffs8);
1501		},
1502		16 => {
1503		const moffs16 = try math.cast(i16, moffs);
1504		encoder.imm16(moffs16);
1505		},
1506		32 => {
1507		const moffs32 = try math.cast(i32, moffs);
1508		encoder.imm32(moffs32);
1509		},
1510		64 => {
1511		encoder.imm64(@bitCast(u64, moffs));
1512		},
1513		else => unreachable,
1514		}
1515		}
1516
1517		fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) LoweringError!void {
1518		if (reg.size() != immOpSize(imm)) {
1519		return error.OperandSizeMismatch;
1520		}
1521		const opc = getOpCode(tag, .oi, reg.size() == 8).?;
1522		const encoder = try Encoder.init(code, 10);
1523		if (reg.size() == 16) {
1524		encoder.prefix16BitMode();
1525		}
1526		encoder.rex(.{
1527		.w = setRexWRegister(reg),
1528		.b = reg.isExtended(),
1529		});
1530		opc.encodeWithReg(encoder, reg);
1531		switch (reg.size()) {
1532		8 => {
1533		const imm8 = try math.cast(i8, imm);
1534		encoder.imm8(imm8);
1535		},
1536		16 => {
1537		const imm16 = try math.cast(i16, imm);
1538		encoder.imm16(imm16);
1539		},
1540		32 => {
1541		const imm32 = try math.cast(i32, imm);
1542		encoder.imm32(imm32);
1543		},
1544		64 => {
1545		encoder.imm64(@bitCast(u64, imm));
1546		},
1547		else => unreachable,
1548		}
1549		}
1550
1551		fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1552		const modrm_ext = getModRmExt(tag).?;
1553		switch (reg_or_mem) {
1554		.register => \|dst_reg\| {
1555		const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
1556		const encoder = try Encoder.init(code, 7);
1557		if (dst_reg.size() == 16) {
1558		// 0x66 prefix switches to the non-default size; here we assume a switch from
1559		// the default 32bits to 16bits operand-size.
1560		// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
1561		encoder.prefix16BitMode();
1562		}
1563		encoder.rex(.{
1564		.w = setRexWRegister(dst_reg),
1565		.b = dst_reg.isExtended(),
1566		});
1567		opc.encode(encoder);
1568		encoder.modRm_direct(modrm_ext, dst_reg.lowId());
1569		encodeImm(encoder, imm, dst_reg.size());
1570		},
1571		.memory => \|dst_mem\| {
1572		const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
1573		const encoder = try Encoder.init(code, 12);
1574		if (dst_mem.ptr_size == .word_ptr) {
1575		encoder.prefix16BitMode();
1576		}
1577		if (dst_mem.base) \|base\| {
1578		if (base.size() != 64) {
1579		return error.OperandSizeMismatch;
1580		}
1581		encoder.rex(.{
1582		.w = dst_mem.ptr_size == .qword_ptr,
1583		.b = base.isExtended(),
1584		});
1585		} else {
1586		encoder.rex(.{
1587		.w = dst_mem.ptr_size == .qword_ptr,
1588		});
1589		}
1590		opc.encode(encoder);
1591		dst_mem.encode(encoder, modrm_ext);
1592		encodeImm(encoder, imm, dst_mem.ptr_size.size());
1593		},
1594		}
1595		}
1596
1597		fn lowerToRmEnc(
1598		tag: Tag,
1599		reg: Register,
1600		reg_or_mem: RegisterOrMemory,
1601		code: *std.ArrayList(u8),
1602		) LoweringError!void {
1603		const opc = getOpCode(tag, .rm, reg.size() == 8).?;
1604		switch (reg_or_mem) {
1605		.register => \|src_reg\| {
1606		if (reg.size() != src_reg.size()) {
1607		return error.OperandSizeMismatch;
1608		}
1609		const encoder = try Encoder.init(code, 3);
1610		encoder.rex(.{
1611		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
1612		.r = reg.isExtended(),
1613		.b = src_reg.isExtended(),
1614		});
1615		opc.encode(encoder);
1616		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1617		},
1618		.memory => \|src_mem\| {
1619		if (reg.size() != src_mem.ptr_size.size()) {
1620		return error.OperandSizeMismatch;
1621		}
1622		const encoder = try Encoder.init(code, 9);
1623		if (reg.size() == 16) {
1624		encoder.prefix16BitMode();
1625		}
1626		if (src_mem.base) \|base\| {
1627		// TODO handle 32-bit base register - requires prefix 0x67
1628		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1629		if (base.size() != 64) {
1630		return error.OperandSizeMismatch;
1631		}
1632		encoder.rex(.{
1633		.w = setRexWRegister(reg),
1634		.r = reg.isExtended(),
1635		.b = base.isExtended(),
1636		});
1637		} else {
1638		encoder.rex(.{
1639		.w = setRexWRegister(reg),
1640		.r = reg.isExtended(),
1641		});
1642		}
1643		opc.encode(encoder);
1644		src_mem.encode(encoder, reg.lowId());
1645		},
1646		}
1647		}
1648
1649		fn lowerToMrEnc(
1650		tag: Tag,
1651		reg_or_mem: RegisterOrMemory,
1652		reg: Register,
1653		code: *std.ArrayList(u8),
1654		) LoweringError!void {
1655		const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1656		switch (reg_or_mem) {
1657		.register => \|dst_reg\| {
1658		if (dst_reg.size() != reg.size()) {
1659		return error.OperandSizeMismatch;
1660		}
1661		const encoder = try Encoder.init(code, 3);
1662		encoder.rex(.{
1663		.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
1664		.r = reg.isExtended(),
1665		.b = dst_reg.isExtended(),
1666		});
1667		opc.encode(encoder);
1668		encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1669		},
1670		.memory => \|dst_mem\| {
1671		if (dst_mem.ptr_size.size() != reg.size()) {
1672		return error.OperandSizeMismatch;
1673		}
1674		const encoder = try Encoder.init(code, 9);
1675		if (reg.size() == 16) {
1676		encoder.prefix16BitMode();
1677		}
1678		if (dst_mem.base) \|base\| {
1679		if (base.size() != 64) {
1680		return error.OperandSizeMismatch;
1681		}
1682		encoder.rex(.{
1683		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1684		.r = reg.isExtended(),
1685		.b = base.isExtended(),
1686		});
1687		} else {
1688		encoder.rex(.{
1689		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1690		.r = reg.isExtended(),
1691		});
1692		}
1693		opc.encode(encoder);
1694		dst_mem.encode(encoder, reg.lowId());
1695		},
1696		}
1697		}
1698
1699		fn lowerToRmiEnc(
1700		tag: Tag,
1701		reg: Register,
1702		reg_or_mem: RegisterOrMemory,
1703		imm: i32,
1704		code: *std.ArrayList(u8),
1705		) LoweringError!void {
1706		if (reg.size() == 8) {
1707		return error.OperandSizeMismatch;
1708		}
1709		const opc = getOpCode(tag, .rmi, false).?;
1710		const encoder = try Encoder.init(code, 13);
1711		if (reg.size() == 16) {
1712		encoder.prefix16BitMode();
1713		}
1714		switch (reg_or_mem) {
1715		.register => \|src_reg\| {
1716		if (reg.size() != src_reg.size()) {
1717		return error.OperandSizeMismatch;
1718		}
1719		encoder.rex(.{
1720		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
1721		.r = reg.isExtended(),
1722		.b = src_reg.isExtended(),
1723		});
1724		opc.encode(encoder);
1725		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1726		},
1727		.memory => \|src_mem\| {
1728		if (src_mem.base) \|base\| {
1729		// TODO handle 32-bit base register - requires prefix 0x67
1730		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1731		if (base.size() != 64) {
1732		return error.OperandSizeMismatch;
1733		}
1734		if (src_mem.ptr_size == .byte_ptr) {
1735		return error.OperandSizeMismatch;
1736		}
1737		encoder.rex(.{
1738		.w = setRexWRegister(reg),
1739		.r = reg.isExtended(),
1740		.b = base.isExtended(),
1741		});
1742		} else {
1743		encoder.rex(.{
1744		.w = setRexWRegister(reg),
1745		.r = reg.isExtended(),
1746		});
1747		}
1748		opc.encode(encoder);
1749		src_mem.encode(encoder, reg.lowId());
1750		},
1751		}
1752		encodeImm(encoder, imm, reg.size());
1753		}
1754
1755		fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
1756		assert(expected.len > 0);
1757		if (mem.eql(u8, expected, given)) return;
1758		const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
1759		defer testing.allocator.free(expected_fmt);
1760		const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
1761		defer testing.allocator.free(given_fmt);
1762		const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
1763		var padding = try testing.allocator.alloc(u8, idx + 5);
1764		defer testing.allocator.free(padding);
1765		mem.set(u8, padding, ' ');
1766		std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
1767		assembly,
1768		expected_fmt,
1769		given_fmt,
1770		padding,
1771		});
1772		return error.TestFailed;
1773		}
1774
1775		const TestIsel = struct {
1776		code_buffer: std.ArrayList(u8),
1777		next: usize = 0,
1778
1779		fn init() TestIsel {
1780		return .{
1781		.code_buffer = std.ArrayList(u8).init(testing.allocator),
1782		};
1783		}
1784
1785		fn deinit(isel: *TestIsel) void {
1786		isel.code_buffer.deinit();
1787		isel.next = undefined;
1788		}
1789
1790		fn code(isel: TestIsel) std.ArrayList(u8) {
1791		isel.next = isel.code_buffer.items.len;
1792		return &isel.code_buffer;
1793		}
1794
1795		fn lowered(isel: TestIsel) []const u8 {
1796		return isel.code_buffer.items[isel.next..];
1797		}
1798		};
1799
1800		test "lower MI encoding" {
1801		var isel = TestIsel.init();
1802		defer isel.deinit();
1803		try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, isel.code());
1804		try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", isel.lowered(), "mov rax, 0x10");
1805		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, isel.code());
1806		try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", isel.lowered(), "mov dword ptr [r11 + 0], 0x10");
1807		try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = -8, .base = .rdx }), 0x10, isel.code());
1808		try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", isel.lowered(), "add dword ptr [rdx - 8], 0x10");
1809		try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
1810		.disp = 0x10000000,
1811		.base = .r11,
1812		}), 0x10, isel.code());
1813		try expectEqualHexStrings(
1814		"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1815		isel.lowered(),
1816		"sub dword ptr [r11 + 0x10000000], 0x10",
1817		);
1818		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, isel.code());
1819		try expectEqualHexStrings(
1820		"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1821		isel.lowered(),
1822		"and dword ptr [ds:0x10000000], 0x10",
1823		);
1824		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
1825		.disp = 0x10000000,
1826		.base = .r12,
1827		}), 0x10, isel.code());
1828		try expectEqualHexStrings(
1829		"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1830		isel.lowered(),
1831		"and dword ptr [r12 + 0x10000000], 0x10",
1832		);
1833		try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, isel.code());
1834		try expectEqualHexStrings(
1835		"\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1836		isel.lowered(),
1837		"mov qword ptr [rip + 0x10], 0x10",
1838		);
1839		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = -8, .base = .rbp }), 0x10, isel.code());
1840		try expectEqualHexStrings(
1841		"\x48\xc7\x45\xf8\x10\x00\x00\x00",
1842		isel.lowered(),
1843		"mov qword ptr [rbp - 8], 0x10",
1844		);
1845		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{ .disp = -2, .base = .rbp }), 0x10, isel.code());
1846		try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", isel.lowered(), "mov word ptr [rbp - 2], 0x10");
1847		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{ .disp = -1, .base = .rbp }), 0x10, isel.code());
1848		try expectEqualHexStrings("\xC6\x45\xFF\x10", isel.lowered(), "mov byte ptr [rbp - 1], 0x10");
1849		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1850		.disp = 0x10000000,
1851		.scale_index = .{
1852		.scale = 1,
1853		.index = .rcx,
1854		},
1855		}), 0x10, isel.code());
1856		try expectEqualHexStrings(
1857		"\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
1858		isel.lowered(),
1859		"mov qword ptr [rcx*2 + 0x10000000], 0x10",
1860		);
1861		}
1862
1863		test "lower RM encoding" {
1864		var isel = TestIsel.init();
1865		defer isel.deinit();
1866		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), isel.code());
1867		try expectEqualHexStrings("\x48\x8b\xc3", isel.lowered(), "mov rax, rbx");
1868		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), isel.code());
1869		try expectEqualHexStrings("\x49\x8b\x03", isel.lowered(), "mov rax, qword ptr [r11 + 0]");
1870		try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), isel.code());
1871		try expectEqualHexStrings(
1872		"\x4C\x03\x1C\x25\x00\x00\x00\x10",
1873		isel.lowered(),
1874		"add r11, qword ptr [ds:0x10000000]",
1875		);
1876		try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), isel.code());
1877		try expectEqualHexStrings(
1878		"\x44\x02\x24\x25\x00\x00\x00\x10",
1879		isel.lowered(),
1880		"add r11b, byte ptr [ds:0x10000000]",
1881		);
1882		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1883		.disp = 0x10000000,
1884		.base = .r13,
1885		}), isel.code());
1886		try expectEqualHexStrings(
1887		"\x4D\x2B\x9D\x00\x00\x00\x10",
1888		isel.lowered(),
1889		"sub r11, qword ptr [r13 + 0x10000000]",
1890		);
1891		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1892		.disp = 0x10000000,
1893		.base = .r12,
1894		}), isel.code());
1895		try expectEqualHexStrings(
1896		"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
1897		isel.lowered(),
1898		"sub r11, qword ptr [r12 + 0x10000000]",
1899		);
1900		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = -4, .base = .rbp }), isel.code());
1901		try expectEqualHexStrings("\x48\x8B\x45\xFC", isel.lowered(), "mov rax, qword ptr [rbp - 4]");
1902		try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), isel.code());
1903		try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", isel.lowered(), "lea rax, [rip + 0x10]");
1904		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1905		.disp = -8,
1906		.base = .rbp,
1907		.scale_index = .{
1908		.scale = 0,
1909		.index = .rcx,
1910		},
1911		}), isel.code());
1912		try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", isel.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
1913		try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
1914		.disp = -4,
1915		.base = .rbp,
1916		.scale_index = .{
1917		.scale = 2,
1918		.index = .rdx,
1919		},
1920		}), isel.code());
1921		try expectEqualHexStrings("\x8B\x44\x95\xFC", isel.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
1922		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1923		.disp = -8,
1924		.base = .rbp,
1925		.scale_index = .{
1926		.scale = 3,
1927		.index = .rcx,
1928		},
1929		}), isel.code());
1930		try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", isel.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
1931		try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
1932		.disp = -24,
1933		.base = .rsi,
1934		.scale_index = .{
1935		.scale = 0,
1936		.index = .rcx,
1937		},
1938		}), isel.code());
1939		try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", isel.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
1940		try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
1941		.disp = 0,
1942		.base = .rbp,
1943		.scale_index = .{
1944		.scale = 0,
1945		.index = .rcx,
1946		},
1947		}), isel.code());
1948		try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", isel.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
1949		}
1950
1951		test "lower MR encoding" {
1952		var isel = TestIsel.init();
1953		defer isel.deinit();
1954		try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, isel.code());
1955		try expectEqualHexStrings("\x48\x89\xd8", isel.lowered(), "mov rax, rbx");
1956		try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = -4, .base = .rbp }), .r11, isel.code());
1957		try expectEqualHexStrings("\x4c\x89\x5d\xfc", isel.lowered(), "mov qword ptr [rbp - 4], r11");
1958		try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, isel.code());
1959		try expectEqualHexStrings(
1960		"\x44\x00\x24\x25\x00\x00\x00\x10",
1961		isel.lowered(),
1962		"add byte ptr [ds:0x10000000], r12b",
1963		);
1964		try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, isel.code());
1965		try expectEqualHexStrings(
1966		"\x44\x01\x24\x25\x00\x00\x00\x10",
1967		isel.lowered(),
1968		"add dword ptr [ds:0x10000000], r12d",
1969		);
1970		try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
1971		.disp = 0x10000000,
1972		.base = .r11,
1973		}), .r12, isel.code());
1974		try expectEqualHexStrings(
1975		"\x4D\x29\xA3\x00\x00\x00\x10",
1976		isel.lowered(),
1977		"sub qword ptr [r11 + 0x10000000], r12",
1978		);
1979		try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, isel.code());
1980		try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", isel.lowered(), "mov qword ptr [rip + 0x10], r12");
1981		}
1982
1983		test "lower OI encoding" {
1984		var isel = TestIsel.init();
1985		defer isel.deinit();
1986		try lowerToOiEnc(.mov, .rax, 0x1000000000000000, isel.code());
1987		try expectEqualHexStrings(
1988		"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
1989		isel.lowered(),
1990		"movabs rax, 0x1000000000000000",
1991		);
1992		try lowerToOiEnc(.mov, .r11, 0x1000000000000000, isel.code());
1993		try expectEqualHexStrings(
1994		"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
1995		isel.lowered(),
1996		"movabs r11, 0x1000000000000000",
1997		);
1998		try lowerToOiEnc(.mov, .r11d, 0x10000000, isel.code());
1999		try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", isel.lowered(), "mov r11d, 0x10000000");
2000		try lowerToOiEnc(.mov, .r11w, 0x1000, isel.code());
2001		try expectEqualHexStrings("\x66\x41\xBB\x00\x10", isel.lowered(), "mov r11w, 0x1000");
2002		try lowerToOiEnc(.mov, .r11b, 0x10, isel.code());
2003		try expectEqualHexStrings("\x41\xB3\x10", isel.lowered(), "mov r11b, 0x10");
2004		}
2005
2006		test "lower FD/TD encoding" {
2007		var isel = TestIsel.init();
2008		defer isel.deinit();
2009		try lowerToFdEnc(.mov, .rax, 0x1000000000000000, isel.code());
2010		try expectEqualHexStrings(
2011		"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2012		isel.lowered(),
2013		"mov rax, ds:0x1000000000000000",
2014		);
2015		try lowerToFdEnc(.mov, .eax, 0x10000000, isel.code());
2016		try expectEqualHexStrings("\xa1\x00\x00\x00\x10", isel.lowered(), "mov eax, ds:0x10000000");
2017		try lowerToFdEnc(.mov, .ax, 0x1000, isel.code());
2018		try expectEqualHexStrings("\x66\xa1\x00\x10", isel.lowered(), "mov ax, ds:0x1000");
2019		try lowerToFdEnc(.mov, .al, 0x10, isel.code());
2020		try expectEqualHexStrings("\xa0\x10", isel.lowered(), "mov al, ds:0x10");
2021		}
2022
2023		test "lower M encoding" {
2024		var isel = TestIsel.init();
2025		defer isel.deinit();
2026		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), isel.code());
2027		try expectEqualHexStrings("\x41\xFF\xE4", isel.lowered(), "jmp r12");
2028		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), isel.code());
2029		try expectEqualHexStrings("\x66\x41\xFF\xE4", isel.lowered(), "jmp r12w");
2030		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), isel.code());
2031		try expectEqualHexStrings("\x41\xFF\x24\x24", isel.lowered(), "jmp qword ptr [r12]");
2032		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), isel.code());
2033		try expectEqualHexStrings("\x66\x41\xFF\x24\x24", isel.lowered(), "jmp word ptr [r12]");
2034		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), isel.code());
2035		try expectEqualHexStrings("\x41\xFF\x64\x24\x10", isel.lowered(), "jmp qword ptr [r12 + 0x10]");
2036		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2037		.disp = 0x1000,
2038		.base = .r12,
2039		}), isel.code());
2040		try expectEqualHexStrings(
2041		"\x41\xFF\xA4\x24\x00\x10\x00\x00",
2042		isel.lowered(),
2043		"jmp qword ptr [r12 + 0x1000]",
2044		);
2045		try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), isel.code());
2046		try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [rip + 0x10]");
2047		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), isel.code());
2048		try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [ds:0x10]");
2049		try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), isel.code());
2050		try expectEqualHexStrings("\x41\x0F\x97\xC3", isel.lowered(), "seta r11b");
2051		}
2052
2053		test "lower O encoding" {
2054		var isel = TestIsel.init();
2055		defer isel.deinit();
2056		try lowerToOEnc(.pop, .r12, isel.code());
2057		try expectEqualHexStrings("\x41\x5c", isel.lowered(), "pop r12");
2058		try lowerToOEnc(.push, .r12w, isel.code());
2059		try expectEqualHexStrings("\x66\x41\x54", isel.lowered(), "push r12w");
2060		}
2061
2062		test "lower RMI encoding" {
2063		var isel = TestIsel.init();
2064		defer isel.deinit();
2065		try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2066		.disp = -8,
2067		.base = .rbp,
2068		}), 0x10, isel.code());
2069		try expectEqualHexStrings(
2070		"\x48\x69\x45\xF8\x10\x00\x00\x00",
2071		isel.lowered(),
2072		"imul rax, qword ptr [rbp - 8], 0x10",
2073		);
2074		try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2075		.disp = -4,
2076		.base = .rbp,
2077		}), 0x10, isel.code());
2078		try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", isel.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
2079		try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
2080		.disp = -2,
2081		.base = .rbp,
2082		}), 0x10, isel.code());
2083		try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", isel.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
2084		try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, isel.code());
2085		try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", isel.lowered(), "imul r12, r12, 0x10");
2086		try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, isel.code());
2087		try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", isel.lowered(), "imul r12w, r12w, 0x10");
2088		}

src/arch/x86_64/Mir.zig+3-3

...	...	@@ -302,7 +302,7 @@ pub const Inst = struct {
302	302	/// Another instruction.
303	303	inst: Index,
304	304	/// A 32-bit immediate value.
305		imm: i32,
	305	imm: u32,
306	306	/// An extern function.
307	307	/// Index into the linker's string table.
308	308	extern_fn: u32,
...	...	@@ -324,8 +324,8 @@ pub const Inst = struct {
324	324	};
325	325
326	326	pub const ImmPair = struct {
327		dest_off: i32,
328		operand: i32,
	327	dest_off: u32,
	328	operand: u32,
329	329	};
330	330
331	331	pub const Imm64 = struct {

test/stage2/x86_64.zig+64

...	...	@@ -1700,6 +1700,36 @@ pub fn addCases(ctx: *TestContext) !void {
1700	1700	\\ if (!ok) unreachable;
1701	1701	\\}
1702	1702	, "");
	1703	case.addCompareOutput(
	1704	\\pub fn main() void {
	1705	\\ var x: u16 = undefined;
	1706	\\ set(&x);
	1707	\\ assert(x == 123);
	1708	\\}
	1709	\\
	1710	\\fn set(x: *u16) void {
	1711	\\ x.* = 123;
	1712	\\}
	1713	\\
	1714	\\fn assert(ok: bool) void {
	1715	\\ if (!ok) unreachable;
	1716	\\}
	1717	, "");
	1718	case.addCompareOutput(
	1719	\\pub fn main() void {
	1720	\\ var x: u8 = undefined;
	1721	\\ set(&x);
	1722	\\ assert(x == 123);
	1723	\\}
	1724	\\
	1725	\\fn set(x: *u8) void {
	1726	\\ x.* = 123;
	1727	\\}
	1728	\\
	1729	\\fn assert(ok: bool) void {
	1730	\\ if (!ok) unreachable;
	1731	\\}
	1732	, "");
1703	1733	}
1704	1734
1705	1735	{
...	...	@@ -1737,6 +1767,40 @@ pub fn addCases(ctx: *TestContext) !void {
1737	1767	\\ if (!ok) unreachable;
1738	1768	\\}
1739	1769	, "");
	1770	case.addCompareOutput(
	1771	\\pub fn main() void {
	1772	\\ var x: u8 = undefined;
	1773	\\ const maybe_x = byPtr(&x);
	1774	\\ assert(maybe_x != null);
	1775	\\ maybe_x.?.* = 255;
	1776	\\ assert(x == 255);
	1777	\\}
	1778	\\
	1779	\\fn byPtr(x: u8) ?u8 {
	1780	\\ return x;
	1781	\\}
	1782	\\
	1783	\\fn assert(ok: bool) void {
	1784	\\ if (!ok) unreachable;
	1785	\\}
	1786	, "");
	1787	case.addCompareOutput(
	1788	\\pub fn main() void {
	1789	\\ var x: i8 = undefined;
	1790	\\ const maybe_x = byPtr(&x);
	1791	\\ assert(maybe_x != null);
	1792	\\ maybe_x.?.* = -1;
	1793	\\ assert(x == -1);
	1794	\\}
	1795	\\
	1796	\\fn byPtr(x: i8) ?i8 {
	1797	\\ return x;
	1798	\\}
	1799	\\
	1800	\\fn assert(ok: bool) void {
	1801	\\ if (!ok) unreachable;
	1802	\\}
	1803	, "");
1740	1804	}
1741	1805
1742	1806	{