Commit 0409f9e0244aebab5c47f0ec24114e101c3f54e6 | zig

author	gracefu <81774659+gracefuu@users.noreply.github.com> 2021-04-11 16:09:47+08:00
committer	gracefu <81774659+gracefuu@users.noreply.github.com> 2021-04-16 15:21:17+08:00
log	0409f9e0244aebab5c47f0ec24114e101c3f54e6
tree	5452e71c410bd83ae219643e35e19bf48b98d4ee
parent	613f39eb622d341d036ef418b19778d1f04d5a47
signature	Commit is signed but in an unrecognized format.

stage2 x86_64: simplify inst encoder to a set of dumb helper fns

2 files changed, 694 insertions(+), 630 deletions(-)

src
- codegen.zig
- codegen
  - x86_64.zig

src/codegen.zig+292-304

...	...	@@ -20,6 +20,8 @@ const build_options = @import("build_options");
20	20	const LazySrcLoc = Module.LazySrcLoc;
21	21	const RegisterManager = @import("register_manager.zig").RegisterManager;
22	22
	23	const X8664Encoder = @import("codegen/x86_64.zig").Encoder;
	24
23	25	/// The codegen-related data that is stored in `ir.Inst.Block` instructions.
24	26	pub const BlockData = struct {
25	27	relocs: std.ArrayListUnmanaged(Reloc) = undefined,
...	...	@@ -1617,9 +1619,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1617	1619	///
1618	1620	/// opcode \| operand shape
1619	1621	/// --------+----------------------
1620		/// 80 /opx \| r/m8, imm8
1621		/// 81 /opx \| r/m16/32/64, imm16/32
1622		/// 83 /opx \| r/m16/32/64, imm8
	1622	/// 80 /opx \| r/m8, imm8
	1623	/// 81 /opx \| r/m16/32/64, imm16/32
	1624	/// 83 /opx \| r/m16/32/64, imm8
1623	1625	///
1624	1626	/// "mr"-style instructions use the low bits of opcode to indicate shape of instruction:
1625	1627	///
...	...	@@ -1634,12 +1636,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1634	1636	///
1635	1637	/// opcode \| operand shape
1636	1638	/// -------+-------------------------
1637		/// mr + 0 \| r/m8, r8
1638		/// mr + 1 \| r/m16/32/64, r16/32/64
1639		/// mr + 2 \| r8, r/m8
1640		/// mr + 3 \| r16/32/64, r/m16/32/64
1641		/// mr + 4 \| AL, imm8
1642		/// mr + 5 \| rAX, imm16/32
	1639	/// mr + 0 \| r/m8, r8
	1640	/// mr + 1 \| r/m16/32/64, r16/32/64
	1641	/// mr + 2 \| r8, r/m8
	1642	/// mr + 3 \| r16/32/64, r/m16/32/64
	1643	/// mr + 4 \| AL, imm8
	1644	/// mr + 5 \| rAX, imm16/32
1643	1645	///
1644	1646	/// TODO: rotates and shifts share the same structure, so we can potentially implement them
1645	1647	/// at a later date with very similar code.
...	...	@@ -1656,12 +1658,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1656	1658	///
1657	1659	/// opcode \| operand shape
1658	1660	/// --------+------------------
1659		/// c0 /opx \| r/m8, imm8
1660		/// c1 /opx \| r/m16/32/64, imm8
1661		/// d0 /opx \| r/m8, 1
1662		/// d1 /opx \| r/m16/32/64, 1
1663		/// d2 /opx \| r/m8, CL (for context, CL is register 1)
1664		/// d3 /opx \| r/m16/32/64, CL (for context, CL is register 1)
	1661	/// c0 /opx \| r/m8, imm8
	1662	/// c1 /opx \| r/m16/32/64, imm8
	1663	/// d0 /opx \| r/m8, 1
	1664	/// d1 /opx \| r/m16/32/64, 1
	1665	/// d2 /opx \| r/m8, CL (for context, CL is register 1)
	1666	/// d3 /opx \| r/m16/32/64, CL (for context, CL is register 1)
1665	1667	fn genX8664BinMathCode(
1666	1668	self: *Self,
1667	1669	src: LazySrcLoc,
...	...	@@ -1687,77 +1689,84 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1687	1689	.ptr_stack_offset => unreachable,
1688	1690	.ptr_embedded_in_code => unreachable,
1689	1691	.register => \|src_reg\| {
1690		// register, register use mr + 1 addressing mode: r/m16/32/64, r16/32/64
1691		try self.encodeX8664Instruction(src, Instruction{
1692		.operand_size_64 = dst_ty.abiSize(self.target.*) == 64,
1693		.primary_opcode_1b = mr + 1,
1694		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1695		// https://github.com/ziglang/zig/issues/6515
1696		.modrm = @as(
1697		?Instruction.ModrmEffectiveAddress,
1698		Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1699		),
1700		.reg = src_reg,
	1692	// for register, register use mr + 1
	1693	// addressing mode: r/m16/32/64, r16/32/64
	1694	const operand_size = dst_ty.abiSize(self.target.*);
	1695	const encoder = try X8664Encoder.init(self.code, 3);
	1696	encoder.rex(.{
	1697	.w = operand_size == 64,
	1698	.r = src_reg.isExtended(),
	1699	.b = dst_reg.isExtended(),
1701	1700	});
	1701	encoder.opcode_1byte(mr + 1);
	1702	encoder.modRm_direct(
	1703	src_reg.low_id(),
	1704	dst_reg.low_id(),
	1705	);
1702	1706	},
1703	1707	.immediate => \|imm\| {
1704	1708	// register, immediate use opx = 81 or 83 addressing modes:
1705	1709	// opx = 81: r/m16/32/64, imm16/32
1706	1710	// opx = 83: r/m16/32/64, imm8
1707		const imm32 = @intCast(u31, imm); // This case must be handled before calling genX8664BinMathCode.
1708		if (imm32 <= math.maxInt(u7)) {
1709		try self.encodeX8664Instruction(src, Instruction{
1710		.operand_size_64 = dst_ty.abiSize(self.target.*) == 64,
1711		.primary_opcode_1b = 0x83,
1712		.opcode_extension = opx,
1713		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1714		// https://github.com/ziglang/zig/issues/6515
1715		.modrm = @as(
1716		?Instruction.ModrmEffectiveAddress,
1717		Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1718		),
1719		.immediate_bytes = 1,
1720		.immediate = imm32,
	1711	const imm32 = @intCast(i32, imm); // This case must be handled before calling genX8664BinMathCode.
	1712	if (imm32 <= math.maxInt(i8)) {
	1713	const operand_size = dst_ty.abiSize(self.target.*);
	1714	const encoder = try X8664Encoder.init(self.code, 4);
	1715	encoder.rex(.{
	1716	.w = operand_size == 64,
	1717	.b = dst_reg.isExtended(),
1721	1718	});
	1719	encoder.opcode_1byte(0x83);
	1720	encoder.modRm_direct(
	1721	opx,
	1722	dst_reg.low_id(),
	1723	);
	1724	encoder.imm8(@intCast(i8, imm32));
1722	1725	} else {
1723		try self.encodeX8664Instruction(src, Instruction{
1724		.operand_size_64 = dst_ty.abiSize(self.target.*) == 64,
1725		.primary_opcode_1b = 0x81,
1726		.opcode_extension = opx,
1727		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1728		// https://github.com/ziglang/zig/issues/6515
1729		.modrm = @as(
1730		?Instruction.ModrmEffectiveAddress,
1731		Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1732		),
1733		.immediate_bytes = 4,
1734		.immediate = imm32,
	1726	const operand_size = dst_ty.abiSize(self.target.*);
	1727	const encoder = try X8664Encoder.init(self.code, 7);
	1728	encoder.rex(.{
	1729	.w = operand_size == 64,
	1730	.b = dst_reg.isExtended(),
1735	1731	});
	1732	encoder.opcode_1byte(0x81);
	1733	encoder.modRm_direct(
	1734	opx,
	1735	dst_reg.low_id(),
	1736	);
	1737	encoder.imm32(@intCast(i32, imm32));
1736	1738	}
1737	1739	},
1738	1740	.embedded_in_code, .memory => {
1739	1741	return self.fail(src, "TODO implement x86 ADD/SUB/CMP source memory", .{});
1740	1742	},
1741	1743	.stack_offset => \|off\| {
	1744	// register, indirect use mr + 3
	1745	// addressing mode: r16/32/64, r/m16/32/64
1742	1746	const abi_size = dst_ty.abiSize(self.target.*);
1743	1747	const adj_off = off + abi_size;
1744	1748	if (off > math.maxInt(i32)) {
1745	1749	return self.fail(src, "stack offset too large", .{});
1746	1750	}
1747		try self.encodeX8664Instruction(src, Instruction{
1748		.operand_size_64 = abi_size == 64,
1749		.primary_opcode_1b = mr + 0x3,
1750		.reg = dst_reg,
1751		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1752		// https://github.com/ziglang/zig/issues/6515
1753		.modrm = @as(
1754		?Instruction.ModrmEffectiveAddress,
1755		Instruction.ModrmEffectiveAddress{ .mem_disp = .{
1756		.reg = Register.ebp,
1757		.disp = -@intCast(i32, adj_off),
1758		} },
1759		),
	1751	const encoder = try X8664Encoder.init(self.code, 7);
	1752	encoder.rex(.{
	1753	.w = abi_size == 64,
	1754	.r = dst_reg.isExtended(),
1760	1755	});
	1756	encoder.opcode_1byte(mr + 3);
	1757	if (adj_off <= std.math.maxInt(i8)) {
	1758	encoder.modRm_indirectDisp8(
	1759	dst_reg.low_id(),
	1760	Register.ebp.low_id(),
	1761	);
	1762	encoder.disp8(-@intCast(i8, adj_off));
	1763	} else {
	1764	encoder.modRm_indirectDisp32(
	1765	dst_reg.low_id(),
	1766	Register.ebp.low_id(),
	1767	);
	1768	encoder.disp32(-@intCast(i32, adj_off));
	1769	}
1761	1770	},
1762	1771	.compare_flags_unsigned => {
1763	1772	return self.fail(src, "TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
...	...	@@ -1825,17 +1834,18 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1825	1834	//
1826	1835	// Use the following imul opcode
1827	1836	// 0F AF /r: IMUL r32/64, r/m32/64
1828		try self.encodeX8664Instruction(src, Instruction{
1829		.operand_size_64 = dst_ty.abiSize(self.target.*) == 64,
1830		.primary_opcode_2b = 0xaf,
1831		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1832		// https://github.com/ziglang/zig/issues/6515
1833		.modrm = @as(
1834		?Instruction.ModrmEffectiveAddress,
1835		Instruction.ModrmEffectiveAddress{ .reg = src_reg },
1836		),
1837		.reg = dst_reg,
	1837	const abi_size = dst_ty.abiSize(self.target.*);
	1838	const encoder = try X8664Encoder.init(self.code, 4);
	1839	encoder.rex(.{
	1840	.w = abi_size == 64,
	1841	.r = dst_reg.isExtended(),
	1842	.b = src_reg.isExtended(),
1838	1843	});
	1844	encoder.opcode_2byte(0x0f, 0xaf);
	1845	encoder.modRm_direct(
	1846	dst_reg.low_id(),
	1847	src_reg.low_id(),
	1848	);
1839	1849	},
1840	1850	.immediate => \|imm\| {
1841	1851	// register, immediate:
...	...	@@ -1853,33 +1863,33 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1853	1863	// 2) perform register,register mul
1854	1864	// 0F AF /r: IMUL r32/64, r/m32/64
1855	1865	if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
1856		try self.encodeX8664Instruction(src, Instruction{
1857		.operand_size_64 = dst_ty.abiSize(self.target.*) == 64,
1858		.primary_opcode_1b = 0x6B,
1859		.reg = dst_reg,
1860		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1861		// https://github.com/ziglang/zig/issues/6515
1862		.modrm = @as(
1863		?Instruction.ModrmEffectiveAddress,
1864		Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1865		),
1866		.immediate_bytes = 1,
1867		.immediate = imm,
	1866	const abi_size = dst_ty.abiSize(self.target.*);
	1867	const encoder = try X8664Encoder.init(self.code, 4);
	1868	encoder.rex(.{
	1869	.w = abi_size == 64,
	1870	.r = dst_reg.isExtended(),
	1871	.b = dst_reg.isExtended(),
1868	1872	});
	1873	encoder.opcode_1byte(0x6B);
	1874	encoder.modRm_direct(
	1875	dst_reg.low_id(),
	1876	dst_reg.low_id(),
	1877	);
	1878	encoder.imm8(@intCast(i8, imm));
1869	1879	} else if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
1870		try self.encodeX8664Instruction(src, Instruction{
1871		.operand_size_64 = dst_ty.abiSize(self.target.*) == 64,
1872		.primary_opcode_1b = 0x69,
1873		.reg = dst_reg,
1874		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1875		// https://github.com/ziglang/zig/issues/6515
1876		.modrm = @as(
1877		?Instruction.ModrmEffectiveAddress,
1878		Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1879		),
1880		.immediate_bytes = 4,
1881		.immediate = imm,
	1880	const abi_size = dst_ty.abiSize(self.target.*);
	1881	const encoder = try X8664Encoder.init(self.code, 7);
	1882	encoder.rex(.{
	1883	.w = abi_size == 64,
	1884	.r = dst_reg.isExtended(),
	1885	.b = dst_reg.isExtended(),
1882	1886	});
	1887	encoder.opcode_1byte(0x69);
	1888	encoder.modRm_direct(
	1889	dst_reg.low_id(),
	1890	dst_reg.low_id(),
	1891	);
	1892	encoder.imm32(@intCast(i32, imm));
1883	1893	} else {
1884	1894	const src_reg = try self.copyToTmpRegister(src, dst_ty, src_mcv);
1885	1895	return self.genX8664Imul(src, dst_ty, dst_mcv, MCValue{ .register = src_reg });
...	...	@@ -1910,17 +1920,18 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1910	1920	// register, register
1911	1921	// Use the following imul opcode
1912	1922	// 0F AF /r: IMUL r32/64, r/m32/64
1913		try self.encodeX8664Instruction(src, Instruction{
1914		.operand_size_64 = dst_ty.abiSize(self.target.*) == 64,
1915		.primary_opcode_2b = 0xaf,
1916		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1917		// https://github.com/ziglang/zig/issues/6515
1918		.modrm = @as(
1919		?Instruction.ModrmEffectiveAddress,
1920		Instruction.ModrmEffectiveAddress{ .reg = src_reg },
1921		),
1922		.reg = dst_reg,
	1923	const abi_size = dst_ty.abiSize(self.target.*);
	1924	const encoder = try X8664Encoder.init(self.code, 4);
	1925	encoder.rex(.{
	1926	.w = abi_size == 64,
	1927	.r = dst_reg.isExtended(),
	1928	.b = src_reg.isExtended(),
1923	1929	});
	1930	encoder.opcode_2byte(0x0f, 0xaf);
	1931	encoder.modRm_direct(
	1932	dst_reg.low_id(),
	1933	src_reg.low_id(),
	1934	);
1924	1935	// copy dst_reg back out
1925	1936	return self.genSetStack(src, dst_ty, off, MCValue{ .register = dst_reg });
1926	1937	},
...	...	@@ -1950,20 +1961,29 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1950	1961	if (off > math.maxInt(i32)) {
1951	1962	return self.fail(src, "stack offset too large", .{});
1952	1963	}
1953		try self.encodeX8664Instruction(src, Instruction{
1954		.operand_size_64 = abi_size == 64,
1955		.primary_opcode_1b = opcode,
1956		.reg = reg,
1957		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
1958		// https://github.com/ziglang/zig/issues/6515
1959		.modrm = @as(
1960		?Instruction.ModrmEffectiveAddress,
1961		Instruction.ModrmEffectiveAddress{ .mem_disp = .{
1962		.reg = Register.ebp,
1963		.disp = -@intCast(i32, adj_off),
1964		} },
1965		),
	1964
	1965	const i_adj_off = -@intCast(i32, adj_off);
	1966	const encoder = try X8664Encoder.init(self.code, 7);
	1967	encoder.rex(.{
	1968	.w = abi_size == 64,
	1969	.r = reg.isExtended(),
1966	1970	});
	1971	encoder.opcode_1byte(opcode);
	1972	if (i_adj_off < std.math.maxInt(i8)) {
	1973	// example: 48 89 55 7f mov QWORD PTR [rbp+0x7f],rdx
	1974	encoder.modRm_indirectDisp8(
	1975	reg.low_id(),
	1976	Register.ebp.low_id(),
	1977	);
	1978	encoder.disp8(@intCast(i8, i_adj_off));
	1979	} else {
	1980	// example: 48 89 95 80 00 00 00 mov QWORD PTR [rbp+0x80],rdx
	1981	encoder.modRm_indirectDisp32(
	1982	reg.low_id(),
	1983	Register.ebp.low_id(),
	1984	);
	1985	encoder.disp32(i_adj_off);
	1986	}
1967	1987	}
1968	1988
1969	1989	fn genArgDbgInfo(self: Self, inst: ir.Inst.Arg, mcv: MCValue) !void {
...	...	@@ -2630,25 +2650,20 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2630	2650	},
2631	2651	.register => \|reg\| blk: {
2632	2652	// test reg, 1
2633		try self.encodeX8664Instruction(inst.base.src, Instruction{
	2653	// TODO detect al, ax, eax
	2654	const encoder = try X8664Encoder.init(self.code, 4);
	2655	encoder.rex(.{
2634	2656	// TODO audit this codegen: we force w = true here to make
2635	2657	// the value affect the big register
2636		.operand_size_64 = true,
2637
2638		.primary_opcode_1b = 0xf6, // f6/0 is TEST r/m8, imm8
2639		.opcode_extension = 0,
2640
2641		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
2642		// https://github.com/ziglang/zig/issues/6515
2643		// TODO detect al, ax, eax, there's another opcode 0xa8 for that
2644		.modrm = @as(
2645		?Instruction.ModrmEffectiveAddress,
2646		Instruction.ModrmEffectiveAddress{ .reg = reg },
2647		),
2648
2649		.immediate_bytes = 1,
2650		.immediate = 1,
	2658	.w = true,
	2659	.b = reg.isExtended(),
2651	2660	});
	2661	encoder.opcode_1byte(0xf6);
	2662	encoder.modRm_direct(
	2663	0,
	2664	reg.low_id(),
	2665	);
	2666	encoder.disp8(1);
2652	2667	break :blk 0x84;
2653	2668	},
2654	2669	else => return self.fail(inst.base.src, "TODO implement condbr {s} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
...	...	@@ -3170,39 +3185,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3170	3185	}
3171	3186	}
3172	3187
3173		/// Encodes a REX prefix as specified, and appends it to the instruction
3174		/// stream. This only modifies the instruction stream if at least one bit
3175		/// is set true, which has a few implications:
3176		///
3177		/// * The length of the instruction buffer will be modified if the
3178		/// resulting REX is meaningful, but will remain the same if it is not.
3179		/// * Deliberately inserting a "meaningless REX" requires explicit usage of
3180		/// 0x40, and cannot be done via this function.
3181		/// W => 64 bit mode
3182		/// R => extension to the MODRM.reg field
3183		/// X => extension to the SIB.index field
3184		/// B => extension to the MODRM.rm field or the SIB.base field
3185		fn rex(self: *Self, arg: struct { b: bool = false, w: bool = false, x: bool = false, r: bool = false }) void {
3186		comptime assert(arch == .x86_64);
3187		// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB.
3188		var value: u8 = 0x40;
3189		if (arg.b) {
3190		value \|= 0x1;
3191		}
3192		if (arg.x) {
3193		value \|= 0x2;
3194		}
3195		if (arg.r) {
3196		value \|= 0x4;
3197		}
3198		if (arg.w) {
3199		value \|= 0x8;
3200		}
3201		if (value != 0x40) {
3202		self.code.appendAssumeCapacity(value);
3203		}
3204		}
3205
3206	3188	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
3207	3189	fn setRegOrMem(self: *Self, src: LazySrcLoc, ty: Type, loc: MCValue, val: MCValue) !void {
3208	3190	switch (loc) {
...	...	@@ -3750,27 +3732,25 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3750	3732	}
3751	3733	},
3752	3734	.compare_flags_unsigned => \|op\| {
3753		try self.encodeX8664Instruction(src, Instruction{
3754		// TODO audit this codegen: we force w = true here to make
3755		// the value affect the big register
3756		.operand_size_64 = true,
3757
3758		.primary_opcode_2b = switch (op) {
3759		.gte => 0x93,
3760		.gt => 0x97,
3761		.neq => 0x95,
3762		.lt => 0x92,
3763		.lte => 0x96,
3764		.eq => 0x94,
3765		},
3766
3767		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
3768		// https://github.com/ziglang/zig/issues/6515
3769		.modrm = @as(
3770		?Instruction.ModrmEffectiveAddress,
3771		Instruction.ModrmEffectiveAddress{ .reg = reg },
3772		),
	3735	const encoder = try X8664Encoder.init(self.code, 7);
	3736	// TODO audit this codegen: we force w = true here to make
	3737	// the value affect the big register
	3738	encoder.rex(.{
	3739	.w = true,
	3740	.b = reg.isExtended(),
3773	3741	});
	3742	encoder.opcode_2byte(0x0f, switch (op) {
	3743	.gte => 0x93,
	3744	.gt => 0x97,
	3745	.neq => 0x95,
	3746	.lt => 0x92,
	3747	.lte => 0x96,
	3748	.eq => 0x94,
	3749	});
	3750	encoder.modRm_direct(
	3751	0,
	3752	reg.low_id(),
	3753	);
3774	3754	},
3775	3755	.compare_flags_signed => \|op\| {
3776	3756	return self.fail(src, "TODO set register with compare flags value (signed)", .{});
...	...	@@ -3780,34 +3760,43 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3780	3760	// register is the fastest way to zero a register.
3781	3761	if (x == 0) {
3782	3762	// The encoding for `xor r32, r32` is `0x31 /r`.
	3763	const encoder = try X8664Encoder.init(self.code, 3);
	3764
	3765	// If we're accessing e.g. r8d, we need to use a REX prefix before the actual operation. Since
	3766	// this is a 32-bit operation, the W flag is set to zero. X is also zero, as we're not using a SIB.
	3767	// Both R and B are set, as we're extending, in effect, the register bits and the operand.
	3768	encoder.rex(.{
	3769	.r = reg.isExtended(),
	3770	.b = reg.isExtended(),
	3771	});
	3772	encoder.opcode_1byte(0x31);
3783	3773	// Section 3.1.1.1 of the Intel x64 Manual states that "/r indicates that the
3784	3774	// ModR/M byte of the instruction contains a register operand and an r/m operand."
3785		try self.encodeX8664Instruction(src, Instruction{
3786		.primary_opcode_1b = 0x31,
3787
3788		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
3789		// https://github.com/ziglang/zig/issues/6515
3790		.reg = @as(?Register, reg),
3791		.modrm = @as(
3792		?Instruction.ModrmEffectiveAddress,
3793		Instruction.ModrmEffectiveAddress{ .reg = reg },
3794		),
3795		});
	3775	encoder.modRm_direct(
	3776	reg.low_id(),
	3777	reg.low_id(),
	3778	);
	3779
3796	3780	return;
3797	3781	}
3798		if (x <= math.maxInt(u32)) {
	3782	if (x <= math.maxInt(i32)) {
3799	3783	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
3800	3784	//
3801	3785	// The encoding for `mov IMM32 -> REG` is (0xB8 + R) IMM.
3802		try self.encodeX8664Instruction(src, Instruction{
3803		// B8 + R
3804		.primary_opcode_1b = 0xB8,
3805		.opcode_reg = @as(?Register, reg),
3806
3807		// IMM32
3808		.immediate_bytes = 4,
3809		.immediate = x,
	3786
	3787	const encoder = try X8664Encoder.init(self.code, 6);
	3788	// Just as with XORing, we need a REX prefix. This time though, we only
	3789	// need the B bit set, as we're extending the opcode's register field,
	3790	// and there is no Mod R/M byte.
	3791	encoder.rex(.{
	3792	.b = reg.isExtended(),
3810	3793	});
	3794	encoder.opcode_withReg(0xB8, reg.low_id());
	3795
	3796	// no ModR/M byte
	3797
	3798	// IMM
	3799	encoder.imm32(@intCast(i32, x));
3811	3800	return;
3812	3801	}
3813	3802	// Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls
...	...	@@ -3817,37 +3806,40 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3817	3806	// This encoding is, in fact, the same as the one used for 32-bit loads. The only
3818	3807	// difference is that we set REX.W before the instruction, which extends the load to
3819	3808	// 64-bit and uses the full bit-width of the register.
3820		try self.encodeX8664Instruction(src, Instruction{
3821		.operand_size_64 = true,
3822		// B8 + R
3823		.primary_opcode_1b = 0xB8,
3824		.opcode_reg = @as(?Register, reg),
3825
3826		// IMM64
3827		.immediate_bytes = 8,
3828		.immediate = x,
3829		});
	3809	{
	3810	const encoder = try X8664Encoder.init(self.code, 10);
	3811	encoder.rex(.{
	3812	.w = true,
	3813	.b = reg.isExtended(),
	3814	});
	3815	encoder.opcode_withReg(0xB8, reg.low_id());
	3816	encoder.imm64(x);
	3817	}
3830	3818	},
3831	3819	.embedded_in_code => \|code_offset\| {
	3820	// We need the offset from RIP in a signed i32 twos complement.
	3821	// The instruction is 7 bytes long and RIP points to the next instruction.
	3822
3832	3823	// 64-bit LEA is encoded as REX.W 8D /r.
3833		const rip = self.code.items.len;
	3824	const rip = self.code.items.len + 7;
3834	3825	const big_offset = @intCast(i64, code_offset) - @intCast(i64, rip);
3835	3826	const offset = @intCast(i32, big_offset);
3836		try self.encodeX8664Instruction(src, Instruction{
3837		.operand_size_64 = true,
3838
3839		// LEA
3840		.primary_opcode_1b = 0x8D,
	3827	const encoder = try X8664Encoder.init(self.code, 7);
3841	3828
3842		.reg = reg,
3843
3844		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
3845		// https://github.com/ziglang/zig/issues/6515
3846		.modrm = @as(
3847		?Instruction.ModrmEffectiveAddress,
3848		Instruction.ModrmEffectiveAddress{ .disp32 = @bitCast(i32, offset) },
3849		),
	3829	// byte 1, always exists because w = true
	3830	encoder.rex(.{
	3831	.w = true,
	3832	.r = reg.isExtended(),
3850	3833	});
	3834	// byte 2
	3835	encoder.opcode_1byte(0x8D);
	3836	// byte 3
	3837	encoder.modRm_RIPDisp32(reg.low_id());
	3838	// byte 4-7
	3839	encoder.disp32(offset);
	3840
	3841	// Double check that we haven't done any math errors
	3842	assert(rip == self.code.items.len);
3851	3843	},
3852	3844	.register => \|src_reg\| {
3853	3845	// If the registers are the same, nothing to do.
...	...	@@ -3855,20 +3847,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3855	3847	return;
3856	3848
3857	3849	// This is a variant of 8B /r.
3858		try self.encodeX8664Instruction(src, Instruction{
3859		.operand_size_64 = ty.abiSize(self.target.*) == 64,
3860
3861		.primary_opcode_1b = 0x8B,
3862
3863		.reg = reg,
3864
3865		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
3866		// https://github.com/ziglang/zig/issues/6515
3867		.modrm = @as(
3868		?Instruction.ModrmEffectiveAddress,
3869		Instruction.ModrmEffectiveAddress{ .reg = src_reg },
3870		),
	3850	const abi_size = ty.abiSize(self.target.*);
	3851	const encoder = try X8664Encoder.init(self.code, 3);
	3852	encoder.rex(.{
	3853	.w = abi_size == 64,
	3854	.r = reg.isExtended(),
	3855	.b = src_reg.isExtended(),
3871	3856	});
	3857	encoder.opcode_1byte(0x8B);
	3858	encoder.modRm_direct(reg.low_id(), src_reg.low_id());
3872	3859	},
3873	3860	.memory => \|x\| {
3874	3861	if (self.bin_file.options.pie) {
...	...	@@ -3886,32 +3873,28 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3886	3873	return self.fail(src, "TODO implement genSetReg for PIE GOT indirection on this platform", .{});
3887	3874	}
3888	3875
	3876	const abi_size = ty.abiSize(self.target.*);
	3877	const encoder = try X8664Encoder.init(self.code, 7);
3889	3878	// LEA reg, [<offset>]
3890		// manually do this instruction to make sure the offset into the disp32 field won't change.
3891		try self.code.ensureCapacity(self.code.items.len + 7);
3892		self.rex(.{ .w = ty.abiSize(self.target.*) == 64, .r = reg.isExtended() });
3893		self.code.appendSliceAssumeCapacity(&[_]u8{
3894		0x8D,
3895		0x05 \| (@as(u8, reg.id() & 0b111) << 3),
	3879	// TODO: Check if this breaks on macho if abi_size != 64 and reg is not extended
	3880	// this causes rex byte to be omitted, which might mean the offset (+3) above is wrong.
	3881	encoder.rex(.{
	3882	.w = abi_size == 64,
	3883	.r = reg.isExtended(),
3896	3884	});
3897		mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), 0);
	3885	encoder.opcode_1byte(0x8D);
	3886	encoder.modRm_RIPDisp32(reg.low_id());
	3887	encoder.disp32(0);
3898	3888
3899	3889	// MOV reg, [reg]
3900		try self.encodeX8664Instruction(src, Instruction{
3901		.operand_size_64 = ty.abiSize(self.target.*) == 64,
3902
3903		.primary_opcode_1b = 0x8B,
3904
3905		.reg = reg,
3906
3907		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
3908		// https://github.com/ziglang/zig/issues/6515
3909		.modrm = @as(
3910		?Instruction.ModrmEffectiveAddress,
3911		Instruction.ModrmEffectiveAddress{ .mem = reg },
3912		),
	3890	encoder.rex(.{
	3891	.w = abi_size == 64,
	3892	.r = reg.isExtended(),
	3893	.b = reg.isExtended(),
3913	3894	});
3914		} else if (x <= math.maxInt(u32)) {
	3895	encoder.opcode_1byte(0x8B);
	3896	encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
	3897	} else if (x <= math.maxInt(i32)) {
3915	3898	// Moving from memory to a register is a variant of `8B /r`.
3916	3899	// Since we're using 64-bit moves, we require a REX.
3917	3900	// This variant also requires a SIB, as it would otherwise be RIP-relative.
...	...	@@ -3919,14 +3902,18 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3919	3902	// The SIB must be 0x25, to indicate a disp32 with no scaled index.
3920	3903	// 0b00RRR100, where RRR is the lower three bits of the register ID.
3921	3904	// The instruction is thus eight bytes; REX 0x8B 0b00RRR100 0x25 followed by a four-byte disp32.
3922		try self.code.ensureCapacity(self.code.items.len + 8);
3923		self.rex(.{ .w = ty.abiSize(self.target.*) == 64, .r = reg.isExtended() });
3924		self.code.appendSliceAssumeCapacity(&[_]u8{
3925		0x8B,
3926		0x04 \| (@as(u8, reg.id() & 0b111) << 3), // R
3927		0x25,
	3905	const abi_size = ty.abiSize(self.target.*);
	3906	const encoder = try X8664Encoder.init(self.code, 8);
	3907	encoder.rex(.{
	3908	.w = abi_size == 64,
	3909	.r = reg.isExtended(),
3928	3910	});
3929		mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), @intCast(u32, x));
	3911	encoder.opcode_1byte(0x8B);
	3912	// effective address = [SIB]
	3913	encoder.modRm_SIBDisp0(reg.low_id());
	3914	// SIB = disp32
	3915	encoder.sib_disp32();
	3916	encoder.disp32(@intCast(i32, x));
3930	3917	} else {
3931	3918	// If this is RAX, we can use a direct load; otherwise, we need to load the address, then indirectly load
3932	3919	// the value.
...	...	@@ -3935,12 +3922,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3935	3922	// moffs64* is a 64-bit offset "relative to segment base", which really just means the
3936	3923	// absolute address for all practical purposes.
3937	3924
3938		try self.encodeX8664Instruction(src, Instruction{
3939		.operand_size_64 = true,
3940		.primary_opcode_1b = 0xa1,
3941		.immediate_bytes = 8,
3942		.immediate = x,
	3925	const encoder = try X8664Encoder.init(self.code, 10);
	3926	encoder.rex(.{
	3927	.w = true,
3943	3928	});
	3929	encoder.opcode_1byte(0xA1);
	3930	encoder.writeIntLittle(u64, x);
3944	3931	} else {
3945	3932	// This requires two instructions; a move imm as used above, followed by an indirect load using the register
3946	3933	// as the address and the register as the destination.
...	...	@@ -3957,17 +3944,17 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3957	3944	// Now, the register contains the address of the value to load into it
3958	3945	// Currently, we're only allowing 64-bit registers, so we need the `REX.W 8B /r` variant.
3959	3946	// TODO: determine whether to allow other sized registers, and if so, handle them properly.
3960		try self.encodeX8664Instruction(src, Instruction{
3961		.operand_size_64 = ty.abiSize(self.target.*) == 64,
3962		.primary_opcode_1b = 0x8B,
3963		.reg = reg,
3964		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
3965		// https://github.com/ziglang/zig/issues/6515
3966		.modrm = @as(
3967		?Instruction.ModrmEffectiveAddress,
3968		Instruction.ModrmEffectiveAddress{ .mem = reg },
3969		),
	3947
	3948	// mov reg, [reg]
	3949	const abi_size = ty.abiSize(self.target.*);
	3950	const encoder = try X8664Encoder.init(self.code, 3);
	3951	encoder.rex(.{
	3952	.w = abi_size == 64,
	3953	.r = reg.isExtended(),
	3954	.b = reg.isExtended(),
3970	3955	});
	3956	encoder.opcode_1byte(0x8B);
	3957	encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
3971	3958	}
3972	3959	}
3973	3960	},
...	...	@@ -3978,20 +3965,21 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3978	3965	return self.fail(src, "stack offset too large", .{});
3979	3966	}
3980	3967	const ioff = -@intCast(i32, off);
3981		try self.encodeX8664Instruction(src, Instruction{
3982		.operand_size_64 = ty.abiSize(self.target.*) == 64,
3983		.primary_opcode_1b = 0x8B,
3984		.reg = reg,
3985		// TODO: Explicit optional wrap due to stage 1 miscompilation :(
3986		// https://github.com/ziglang/zig/issues/6515
3987		.modrm = @as(
3988		?Instruction.ModrmEffectiveAddress,
3989		Instruction.ModrmEffectiveAddress{ .mem_disp = .{
3990		.reg = Register.ebp,
3991		.disp = ioff,
3992		} },
3993		),
	3968	const encoder = try X8664Encoder.init(self.code, 3);
	3969	encoder.rex(.{
	3970	.w = abi_size == 64,
	3971	.r = reg.isExtended(),
3994	3972	});
	3973	encoder.opcode_1byte(0x8B);
	3974	if (std.math.minInt(i8) <= ioff and ioff <= std.math.maxInt(i8)) {
	3975	// Example: 48 8b 4d 7f mov rcx,QWORD PTR [rbp+0x7f]
	3976	encoder.modRm_indirectDisp8(reg.low_id(), Register.ebp.low_id());
	3977	encoder.disp8(@intCast(i8, ioff));
	3978	} else {
	3979	// Example: 48 8b 8d 80 00 00 00 mov rcx,QWORD PTR [rbp+0x80]
	3980	encoder.modRm_indirectDisp32(reg.low_id(), Register.ebp.low_id());
	3981	encoder.disp32(ioff);
	3982	}
3995	3983	},
3996	3984	},
3997	3985	else => return self.fail(src, "TODO implement getSetReg for {}", .{self.target.cpu.arch}),

src/codegen/x86_64.zig+402-326

...	...	@@ -3,6 +3,7 @@ const testing = std.testing;
3	3	const mem = std.mem;
4	4	const assert = std.debug.assert;
5	5	const ArrayList = std.ArrayList;
	6	const Allocator = std.mem.Allocator;
6	7	const Type = @import("../Type.zig");
7	8	const DW = std.dwarf;
8	9
...	...	@@ -145,51 +146,57 @@ pub const callee_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8
145	146	pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
146	147	pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
147	148
148		/// Represents an unencoded x86 instruction.
	149	/// Encoding helper functions for x86_64 instructions
149	150	///
150		/// Roughly based on the table headings at http://ref.x86asm.net/coder64.html
151		pub const Instruction = struct {
152		/// Opcode prefix, needed for certain rare ops (e.g. MOVSS)
153		opcode_prefix: ?u8 = null,
154
155		/// One-byte primary opcode
156		primary_opcode_1b: ?u8 = null,
157		/// Two-byte primary opcode (always prefixed with 0f)
158		primary_opcode_2b: ?u8 = null,
159		// TODO: Support 3-byte opcodes
160
161		/// Secondary opcode
162		secondary_opcode: ?u8 = null,
163
164		/// Opcode extension (to be placed in the ModR/M byte in place of reg)
165		opcode_extension: ?u3 = null,
166
167		/// Legacy prefixes to use with this instruction
168		/// Most of the time, this field will be 0 and no prefixes are added.
169		/// Otherwise, a prefix will be added for each field set.
170		legacy_prefixes: LegacyPrefixes = .{},
171
172		/// 64-bit operand size
173		operand_size_64: bool = false,
174
175		/// The opcode-reg field,
176		/// stored in the 3 least significant bits of the opcode
177		/// on certain instructions + REX if extended
178		opcode_reg: ?Register = null,
179
180		/// The reg field
181		reg: ?Register = null,
182		/// The mod + r/m field
183		modrm: ?ModrmEffectiveAddress = null,
184		/// Location of the 3rd operand, if applicable
185		sib: ?SibEffectiveAddress = null,
186
187		/// Number of bytes of immediate
188		immediate_bytes: u8 = 0,
189		/// The value of the immediate
190		immediate: u64 = 0,
191
192		/// See legacy_prefixes
	151	/// Many of these helpers do very little, but they can help make things
	152	/// slightly more readable with more descriptive field names / function names.
	153	///
	154	/// Some of them also have asserts to ensure that we aren't doing dumb things.
	155	/// For example, trying to use register 4 (esp) in an indirect modr/m byte is illegal,
	156	/// you need to encode it with an SIB byte.
	157	///
	158	/// Note that ALL of these helper functions will assume capacity,
	159	/// so ensure that the `code` has sufficient capacity before using them.
	160	/// The `init` method is the recommended way to ensure capacity.
	161	pub const Encoder = struct {
	162	/// Non-owning reference to the code array
	163	code: *ArrayList(u8),
	164
	165	const Self = @This();
	166
	167	/// Wrap `code` in Encoder to make it easier to call these helper functions
	168	///
	169	/// maximum_inst_size should contain the maximum number of bytes
	170	/// that the encoded instruction will take.
	171	/// This is because the helper functions will assume capacity
	172	/// in order to avoid bounds checking.
	173	pub fn init(code: *ArrayList(u8), maximum_inst_size: u8) !Self {
	174	try code.ensureCapacity(code.items.len + maximum_inst_size);
	175	return Self{ .code = code };
	176	}
	177
	178	/// Directly write a number to the code array with big endianness
	179	pub fn writeIntBig(self: Self, comptime T: type, value: T) void {
	180	mem.writeIntBig(
	181	T,
	182	self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
	183	value,
	184	);
	185	}
	186
	187	/// Directly write a number to the code array with little endianness
	188	pub fn writeIntLittle(self: Self, comptime T: type, value: T) void {
	189	mem.writeIntLittle(
	190	T,
	191	self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
	192	value,
	193	);
	194	}
	195
	196	// --------
	197	// Prefixes
	198	// --------
	199
193	200	pub const LegacyPrefixes = packed struct {
194	201	/// LOCK
195	202	prefix_f0: bool = false,
...	...	@@ -212,322 +219,391 @@ pub const Instruction = struct {
212	219	/// Branch taken
213	220	prefix_3e: bool = false,
214	221
215		/// Operand size override
	222	/// Operand size override (enables 16 bit operation)
216	223	prefix_66: bool = false,
217	224
218		/// Address size override
	225	/// Address size override (enables 16 bit address size)
219	226	prefix_67: bool = false,
220	227
221	228	padding: u5 = 0,
222	229	};
223	230
224		/// Encodes an effective address for the Mod + R/M part of the ModR/M byte
225		///
226		/// Note that depending on the instruction, not all effective addresses are allowed.
227		///
228		/// Examples:
229		/// eax: .reg = .eax
230		/// [eax]: .mem = .eax
231		/// [eax + 8]: .mem_disp = .{ .reg = .eax, .disp = 8 }
232		/// [eax - 8]: .mem_disp = .{ .reg = .eax, .disp = -8 }
233		/// [55]: .disp32 = 55
234		pub const ModrmEffectiveAddress = union(enum) {
235		reg: Register,
236		mem: Register,
237		mem_disp: struct {
238		reg: Register,
239		disp: i32,
240		},
241		disp32: u32,
242
243		pub fn isExtended(self: @This()) bool {
244		return switch (self) {
245		.reg => \|reg\| reg.isExtended(),
246		.mem => \|memea\| memea.isExtended(),
247		.mem_disp => \|mem_disp\| mem_disp.reg.isExtended(),
248		.disp32 => false,
249		};
250		}
251		};
252
253		/// Encodes an effective address for the SIB byte
254		///
255		/// Note that depending on the instruction, not all effective addresses are allowed.
256		///
257		/// Examples:
258		/// [eax + ebx * 2]: .base_index = .{ .base = .eax, .index = .ebx, .scale = 2 }
259		/// [eax]: .base_index = .{ .base = .eax, .index = null, .scale = 1 }
260		/// [ebx * 2 + 256]: .index_disp = .{ .index = .ebx, .scale = 2, .disp = 256 }
261		/// [[ebp] + ebx * 2 + 8]: .ebp_index_disp = .{ .index = .ebx, .scale = 2, .disp = 8 }
262		pub const SibEffectiveAddress = union(enum) {
263		base_index: struct {
264		base: Register,
265		index: ?Register,
266		scale: u8, // 1, 2, 4, or 8
267		},
268		index_disp: struct {
269		index: ?Register,
270		scale: u8, // 1, 2, 4, or 8
271		disp: u32,
272		},
273		ebp_index_disp: struct {
274		index: ?Register,
275		scale: u8, // 1, 2, 4, or 8
276		disp: u32,
277		},
278
279		pub fn baseIsExtended(self: @This()) bool {
280		return switch (self) {
281		.base_index => \|base_index\| base_index.base.isExtended(),
282		.index_disp, .ebp_index_disp => false,
283		};
284		}
285
286		pub fn indexIsExtended(self: @This()) bool {
287		return switch (self) {
288		.base_index => \|base_index\| if (base_index.index) \|idx\| idx.isExtended() else false,
289		.index_disp => \|index_disp\| if (index_disp.index) \|idx\| idx.isExtended() else false,
290		.ebp_index_disp => \|ebp_index_disp\| if (ebp_index_disp.index) \|idx\| idx.isExtended() else false,
291		};
292		}
293		};
294
295		/// Writes the encoded Instruction to the code ArrayList
296		pub fn encodeInto(inst: Instruction, code: *ArrayList(u8)) !void {
297		// We need to write the following, in that order:
298		// - Legacy prefixes (0 to 13 bytes)
299		// - REX prefix (0 to 1 byte)
300		// - Opcode (1, 2, or 3 bytes)
301		// - ModR/M (0 or 1 byte)
302		// - SIB (0 or 1 byte)
303		// - Displacement (0, 1, 2, or 4 bytes)
304		// - Immediate (0, 1, 2, 4, or 8 bytes)
305
306		// By this calculation, an instruction could be up to 31 bytes long (will probably not happen)
307		try code.ensureCapacity(code.items.len + 31);
308
309		// Legacy prefixes
310		if (@bitCast(u16, inst.legacy_prefixes) != 0) {
	231	/// Encodes legacy prefixes
	232	pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) void {
	233	if (@bitCast(u16, prefixes) != 0) {
311	234	// Hopefully this path isn't taken very often, so we'll do it the slow way for now
312	235
313	236	// LOCK
314		if (inst.legacy_prefixes.prefix_f0) code.appendAssumeCapacity(0xf0);
	237	if (prefixes.prefix_f0) self.code.appendAssumeCapacity(0xf0);
315	238	// REPNZ, REPNE, REP, Scalar Double-precision
316		if (inst.legacy_prefixes.prefix_f2) code.appendAssumeCapacity(0xf2);
	239	if (prefixes.prefix_f2) self.code.appendAssumeCapacity(0xf2);
317	240	// REPZ, REPE, REP, Scalar Single-precision
318		if (inst.legacy_prefixes.prefix_f3) code.appendAssumeCapacity(0xf3);
	241	if (prefixes.prefix_f3) self.code.appendAssumeCapacity(0xf3);
319	242
320	243	// CS segment override or Branch not taken
321		if (inst.legacy_prefixes.prefix_2e) code.appendAssumeCapacity(0x2e);
	244	if (prefixes.prefix_2e) self.code.appendAssumeCapacity(0x2e);
322	245	// DS segment override
323		if (inst.legacy_prefixes.prefix_36) code.appendAssumeCapacity(0x36);
	246	if (prefixes.prefix_36) self.code.appendAssumeCapacity(0x36);
324	247	// ES segment override
325		if (inst.legacy_prefixes.prefix_26) code.appendAssumeCapacity(0x26);
	248	if (prefixes.prefix_26) self.code.appendAssumeCapacity(0x26);
326	249	// FS segment override
327		if (inst.legacy_prefixes.prefix_64) code.appendAssumeCapacity(0x64);
	250	if (prefixes.prefix_64) self.code.appendAssumeCapacity(0x64);
328	251	// GS segment override
329		if (inst.legacy_prefixes.prefix_65) code.appendAssumeCapacity(0x65);
	252	if (prefixes.prefix_65) self.code.appendAssumeCapacity(0x65);
330	253
331	254	// Branch taken
332		if (inst.legacy_prefixes.prefix_3e) code.appendAssumeCapacity(0x3e);
	255	if (prefixes.prefix_3e) self.code.appendAssumeCapacity(0x3e);
333	256
334	257	// Operand size override
335		if (inst.legacy_prefixes.prefix_66) code.appendAssumeCapacity(0x66);
	258	if (prefixes.prefix_66) self.code.appendAssumeCapacity(0x66);
336	259
337	260	// Address size override
338		if (inst.legacy_prefixes.prefix_67) code.appendAssumeCapacity(0x67);
	261	if (prefixes.prefix_67) self.code.appendAssumeCapacity(0x67);
339	262	}
	263	}
340	264
341		// REX prefix
342		//
343		// A REX prefix has the following form:
344		// 0b0100_WRXB
345		// 0100: fixed bits
346		// W: stands for "wide", indicates that the instruction uses 64-bit operands.
347		// R, X, and B each contain the 4th bit of a register
348		// these have to be set when using registers 8-15.
349		// R: stands for "reg", extends the reg field in the ModR/M byte.
350		// X: stands for "index", extends the index field in the SIB byte.
351		// B: stands for "base", extends either the r/m field in the ModR/M byte,
352		// the base field in the SIB byte,
353		// or the opcode reg field in the Opcode byte.
354		{
355		var value: u8 = 0x40;
356		if (inst.opcode_reg) \|opcode_reg\| {
357		if (opcode_reg.isExtended()) {
358		value \|= 0x1;
359		}
360		}
361		if (inst.modrm) \|modrm\| {
362		if (modrm.isExtended()) {
363		value \|= 0x1;
364		}
365		}
366		if (inst.sib) \|sib\| {
367		if (sib.baseIsExtended()) {
368		value \|= 0x1;
369		}
370		if (sib.indexIsExtended()) {
371		value \|= 0x2;
372		}
373		}
374		if (inst.reg) \|reg\| {
375		if (reg.isExtended()) {
376		value \|= 0x4;
377		}
378		}
379		if (inst.operand_size_64) {
380		value \|= 0x8;
381		}
382		if (value != 0x40) {
383		code.appendAssumeCapacity(value);
384		}
385		}
	265	/// Use 16 bit operand size
	266	///
	267	/// Note that this flag is overridden by REX.W, if both are present.
	268	pub fn prefix16BitMode(self: Self) void {
	269	self.code.appendAssumeCapacity(0x66);
	270	}
386	271
387		// Opcode
388		if (inst.primary_opcode_1b) \|opcode\| {
389		var value = opcode;
390		if (inst.opcode_reg) \|opcode_reg\| {
391		value \|= opcode_reg.low_id();
392		}
393		code.appendAssumeCapacity(value);
394		} else if (inst.primary_opcode_2b) \|opcode\| {
395		code.appendAssumeCapacity(0x0f);
396		var value = opcode;
397		if (inst.opcode_reg) \|opcode_reg\| {
398		value \|= opcode_reg.low_id();
399		}
400		code.appendAssumeCapacity(value);
401		}
	272	/// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB
	273	pub const Rex = struct {
	274	/// Wide, enables 64-bit operation
	275	w: bool = false,
	276	/// Extends the reg field in the ModR/M byte
	277	r: bool = false,
	278	/// Extends the index field in the SIB byte
	279	x: bool = false,
	280	/// Extends the r/m field in the ModR/M byte,
	281	/// or the base field in the SIB byte,
	282	/// or the reg field in the Opcode byte
	283	b: bool = false,
	284	};
402	285
403		var disp8: ?u8 = null;
404		var disp16: ?u16 = null;
405		var disp32: ?u32 = null;
406
407		// ModR/M
408		//
409		// Example ModR/M byte:
410		// c7: ModR/M byte that contains:
411		// 11 000 111:
412		// ^ ^ ^
413		// mod \| \|
414		// reg \|
415		// r/m
416		// where mod = 11 indicates that both operands are registers,
417		// reg = 000 indicates that the first operand is register EAX
418		// r/m = 111 indicates that the second operand is register EDI (since mod = 11)
419		if (inst.modrm != null or inst.reg != null or inst.opcode_extension != null) {
420		var value: u8 = 0;
421
422		// mod + rm
423		if (inst.modrm) \|modrm\| {
424		switch (modrm) {
425		.reg => \|reg\| {
426		value \|= reg.low_id();
427		value \|= 0b11_000_000;
428		},
429		.mem => \|memea\| {
430		assert(memea.low_id() != 4 and memea.low_id() != 5);
431		value \|= memea.low_id();
432		// value \|= 0b00_000_000;
433		},
434		.mem_disp => \|mem_disp\| {
435		assert(mem_disp.reg.low_id() != 4);
436		value \|= mem_disp.reg.low_id();
437		if (mem_disp.disp < 128) {
438		// Use 1 byte of displacement
439		value \|= 0b01_000_000;
440		disp8 = @bitCast(u8, @intCast(i8, mem_disp.disp));
441		} else {
442		// Use all 4 bytes of displacement
443		value \|= 0b10_000_000;
444		disp32 = @bitCast(u32, mem_disp.disp);
445		}
446		},
447		.disp32 => \|d\| {
448		value \|= 0b00_000_101;
449		disp32 = d;
450		},
451		}
452		}
453
454		// reg
455		if (inst.reg) \|reg\| {
456		value \|= @as(u8, reg.low_id()) << 3;
457		} else if (inst.opcode_extension) \|ext\| {
458		value \|= @as(u8, ext) << 3;
459		}
460
461		code.appendAssumeCapacity(value);
462		}
	286	/// Encodes a REX prefix byte given all the fields
	287	///
	288	/// Use this byte whenever you need 64 bit operation,
	289	/// or one of reg, index, r/m, base, or opcode-reg might be extended.
	290	///
	291	/// See struct `Rex` for a description of each field.
	292	///
	293	/// Does not add a prefix byte if none of the fields are set!
	294	pub fn rex(self: Self, byte: Rex) void {
	295	var value: u8 = 0b0100_0000;
463	296
464		// SIB
465		{
466		if (inst.sib) \|sib\| {
467		return error.TODOSIBByteForX8664;
468		}
469		}
	297	if (byte.w) value \|= 0b1000;
	298	if (byte.r) value \|= 0b0100;
	299	if (byte.x) value \|= 0b0010;
	300	if (byte.b) value \|= 0b0001;
470	301
471		// Displacement
472		//
473		// The size of the displacement depends on the instruction used and is very fragile.
474		// The bytes are simply written in LE order.
475		{
476
477		// These writes won't fail because we ensured capacity earlier.
478		if (disp8) \|d\|
479		code.appendAssumeCapacity(d)
480		else if (disp16) \|d\|
481		mem.writeIntLittle(u16, code.addManyAsArrayAssumeCapacity(2), d)
482		else if (disp32) \|d\|
483		mem.writeIntLittle(u32, code.addManyAsArrayAssumeCapacity(4), d);
	302	if (value != 0b0100_0000) {
	303	self.code.appendAssumeCapacity(value);
484	304	}
	305	}
485	306
486		// Immediate
487		//
488		// The size of the immediate depends on the instruction used and is very fragile.
489		// The bytes are simply written in LE order.
490		{
491		// These writes won't fail because we ensured capacity earlier.
492		if (inst.immediate_bytes == 1)
493		code.appendAssumeCapacity(@intCast(u8, inst.immediate))
494		else if (inst.immediate_bytes == 2)
495		mem.writeIntLittle(u16, code.addManyAsArrayAssumeCapacity(2), @intCast(u16, inst.immediate))
496		else if (inst.immediate_bytes == 4)
497		mem.writeIntLittle(u32, code.addManyAsArrayAssumeCapacity(4), @intCast(u32, inst.immediate))
498		else if (inst.immediate_bytes == 8)
499		mem.writeIntLittle(u64, code.addManyAsArrayAssumeCapacity(8), inst.immediate);
500		}
	307	// ------
	308	// Opcode
	309	// ------
	310
	311	/// Encodes a 1 byte opcode
	312	pub fn opcode_1byte(self: Self, opcode: u8) void {
	313	self.code.appendAssumeCapacity(opcode);
	314	}
	315
	316	/// Encodes a 2 byte opcode
	317	///
	318	/// e.g. IMUL has the opcode 0x0f 0xaf, so you use
	319	///
	320	/// encoder.opcode_2byte(0x0f, 0xaf);
	321	pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) void {
	322	self.code.appendAssumeCapacity(prefix);
	323	self.code.appendAssumeCapacity(opcode);
	324	}
	325
	326	/// Encodes a 1 byte opcode with a reg field
	327	///
	328	/// Remember to add a REX prefix byte if reg is extended!
	329	pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) void {
	330	assert(opcode & 0b111 == 0);
	331	self.code.appendAssumeCapacity(opcode \| reg);
	332	}
	333
	334	// ------
	335	// ModR/M
	336	// ------
	337
	338	/// Construct a ModR/M byte given all the fields
	339	///
	340	/// Remember to add a REX prefix byte if reg or rm are extended!
	341	pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) void {
	342	self.code.appendAssumeCapacity(
	343	@as(u8, mod) << 6 \| @as(u8, reg_or_opx) << 3 \| rm,
	344	);
	345	}
	346
	347	/// Construct a ModR/M byte using direct r/m addressing
	348	/// r/m effective address: r/m
	349	///
	350	/// Note reg's effective address is always just reg for the ModR/M byte.
	351	/// Remember to add a REX prefix byte if reg or rm are extended!
	352	pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) void {
	353	self.modRm(0b11, reg_or_opx, rm);
	354	}
	355
	356	/// Construct a ModR/M byte using indirect r/m addressing
	357	/// r/m effective address: [r/m]
	358	///
	359	/// Note reg's effective address is always just reg for the ModR/M byte.
	360	/// Remember to add a REX prefix byte if reg or rm are extended!
	361	pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) void {
	362	assert(rm != 4 and rm != 5);
	363	self.modRm(0b00, reg_or_opx, rm);
	364	}
	365
	366	/// Construct a ModR/M byte using indirect SIB addressing
	367	/// r/m effective address: [SIB]
	368	///
	369	/// Note reg's effective address is always just reg for the ModR/M byte.
	370	/// Remember to add a REX prefix byte if reg or rm are extended!
	371	pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) void {
	372	self.modRm(0b00, reg_or_opx, 0b100);
	373	}
	374
	375	/// Construct a ModR/M byte using RIP-relative addressing
	376	/// r/m effective address: [RIP + disp32]
	377	///
	378	/// Note reg's effective address is always just reg for the ModR/M byte.
	379	/// Remember to add a REX prefix byte if reg or rm are extended!
	380	pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) void {
	381	self.modRm(0b00, reg_or_opx, 0b101);
	382	}
	383
	384	/// Construct a ModR/M byte using indirect r/m with a 8bit displacement
	385	/// r/m effective address: [r/m + disp8]
	386	///
	387	/// Note reg's effective address is always just reg for the ModR/M byte.
	388	/// Remember to add a REX prefix byte if reg or rm are extended!
	389	pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) void {
	390	assert(rm != 4);
	391	self.modRm(0b01, reg_or_opx, rm);
	392	}
	393
	394	/// Construct a ModR/M byte using indirect SIB with a 8bit displacement
	395	/// r/m effective address: [SIB + disp8]
	396	///
	397	/// Note reg's effective address is always just reg for the ModR/M byte.
	398	/// Remember to add a REX prefix byte if reg or rm are extended!
	399	pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) void {
	400	self.modRm(0b01, reg_or_opx, 0b100);
	401	}
	402
	403	/// Construct a ModR/M byte using indirect r/m with a 32bit displacement
	404	/// r/m effective address: [r/m + disp32]
	405	///
	406	/// Note reg's effective address is always just reg for the ModR/M byte.
	407	/// Remember to add a REX prefix byte if reg or rm are extended!
	408	pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) void {
	409	assert(rm != 4);
	410	self.modRm(0b10, reg_or_opx, rm);
	411	}
	412
	413	/// Construct a ModR/M byte using indirect SIB with a 32bit displacement
	414	/// r/m effective address: [SIB + disp32]
	415	///
	416	/// Note reg's effective address is always just reg for the ModR/M byte.
	417	/// Remember to add a REX prefix byte if reg or rm are extended!
	418	pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) void {
	419	self.modRm(0b10, reg_or_opx, 0b100);
	420	}
	421
	422	// ---
	423	// SIB
	424	// ---
	425
	426	/// Construct a SIB byte given all the fields
	427	///
	428	/// Remember to add a REX prefix byte if index or base are extended!
	429	pub fn sib(self: Self, scale: u2, index: u3, base: u3) void {
	430	self.code.appendAssumeCapacity(
	431	@as(u8, scale) << 6 \| @as(u8, index) << 3 \| base,
	432	);
	433	}
	434
	435	/// Construct a SIB byte with scale * index + base, no frills.
	436	/// r/m effective address: [base + scale * index]
	437	///
	438	/// Remember to add a REX prefix byte if index or base are extended!
	439	pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) void {
	440	assert(base != 5);
	441
	442	self.sib(scale, index, base);
	443	}
	444
	445	/// Construct a SIB byte with scale * index + disp32
	446	/// r/m effective address: [scale * index + disp32]
	447	///
	448	/// Remember to add a REX prefix byte if index or base are extended!
	449	pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) void {
	450	assert(index != 4);
	451
	452	// scale is actually ignored
	453	// index = 4 means no index
	454	// base = 5 means no base, if mod == 0.
	455	self.sib(scale, index, 5);
	456	}
	457
	458	/// Construct a SIB byte with just base
	459	/// r/m effective address: [base]
	460	///
	461	/// Remember to add a REX prefix byte if index or base are extended!
	462	pub fn sib_base(self: Self, base: u3) void {
	463	assert(base != 5);
	464
	465	// scale is actually ignored
	466	// index = 4 means no index
	467	self.sib(0, 4, base);
	468	}
	469
	470	/// Construct a SIB byte with just disp32
	471	/// r/m effective address: [disp32]
	472	///
	473	/// Remember to add a REX prefix byte if index or base are extended!
	474	pub fn sib_disp32(self: Self) void {
	475	// scale is actually ignored
	476	// index = 4 means no index
	477	// base = 5 means no base, if mod == 0.
	478	self.sib(0, 4, 5);
	479	}
	480
	481	/// Construct a SIB byte with scale * index + base + disp8
	482	/// r/m effective address: [base + scale * index + disp8]
	483	///
	484	/// Remember to add a REX prefix byte if index or base are extended!
	485	pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) void {
	486	self.sib(scale, index, base);
	487	}
	488
	489	/// Construct a SIB byte with base + disp8, no index
	490	/// r/m effective address: [base + disp8]
	491	///
	492	/// Remember to add a REX prefix byte if index or base are extended!
	493	pub fn sib_baseDisp8(self: Self, base: u3) void {
	494	// scale is ignored
	495	// index = 4 means no index
	496	self.sib(0, 4, base);
	497	}
	498
	499	/// Construct a SIB byte with scale * index + base + disp32
	500	/// r/m effective address: [base + scale * index + disp32]
	501	///
	502	/// Remember to add a REX prefix byte if index or base are extended!
	503	pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) void {
	504	self.sib(scale, index, base);
	505	}
	506
	507	/// Construct a SIB byte with base + disp32, no index
	508	/// r/m effective address: [base + disp32]
	509	///
	510	/// Remember to add a REX prefix byte if index or base are extended!
	511	pub fn sib_baseDisp32(self: Self, base: u3) void {
	512	// scale is ignored
	513	// index = 4 means no index
	514	self.sib(0, 4, base);
	515	}
	516
	517	// -------------------------
	518	// Trivial (no bit fiddling)
	519	// -------------------------
	520
	521	/// Encode an 8 bit immediate
	522	///
	523	/// It is sign-extended to 64 bits by the cpu.
	524	pub fn imm8(self: Self, imm: i8) void {
	525	self.code.appendAssumeCapacity(@bitCast(u8, imm));
	526	}
	527
	528	/// Encode an 8 bit displacement
	529	///
	530	/// It is sign-extended to 64 bits by the cpu.
	531	pub fn disp8(self: Self, disp: i8) void {
	532	self.code.appendAssumeCapacity(@bitCast(u8, disp));
	533	}
	534
	535	/// Encode an 16 bit immediate
	536	///
	537	/// It is sign-extended to 64 bits by the cpu.
	538	pub fn imm16(self: Self, imm: i16) void {
	539	self.writeIntLittle(i16, imm);
	540	}
	541
	542	/// Encode an 32 bit immediate
	543	///
	544	/// It is sign-extended to 64 bits by the cpu.
	545	pub fn imm32(self: Self, imm: i32) void {
	546	self.writeIntLittle(i32, imm);
	547	}
	548
	549	/// Encode an 32 bit displacement
	550	///
	551	/// It is sign-extended to 64 bits by the cpu.
	552	pub fn disp32(self: Self, disp: i32) void {
	553	self.writeIntLittle(i32, disp);
	554	}
	555
	556	/// Encode an 64 bit immediate
	557	///
	558	/// It is sign-extended to 64 bits by the cpu.
	559	pub fn imm64(self: Self, imm: u64) void {
	560	self.writeIntLittle(u64, imm);
501	561	}
502	562	};
503	563
504		fn expectEncoded(inst: Instruction, expected: []const u8) !void {
	564	test "x86_64 Encoder helpers" {
505	565	var code = ArrayList(u8).init(testing.allocator);
506	566	defer code.deinit();
507		try inst.encodeInto(&code);
508		testing.expectEqualSlices(u8, expected, code.items);
509		}
510	567
511		test "x86_64 Instruction.encodeInto" {
512	568	// simple integer multiplication
513	569
514	570	// imul eax,edi
515	571	// 0faf c7
516		try expectEncoded(Instruction{
517		.primary_opcode_2b = 0xaf, // imul
518		.reg = .eax, // destination
519		.modrm = .{ .reg = .edi }, // source
520		}, &[_]u8{ 0x0f, 0xaf, 0xc7 });
	572	{
	573	try code.resize(0);
	574	const encoder = try Encoder.init(&code, 4);
	575	encoder.rex(.{
	576	.r = Register.eax.isExtended(),
	577	.b = Register.edi.isExtended(),
	578	});
	579	encoder.opcode_2byte(0x0f, 0xaf);
	580	encoder.modRm_direct(
	581	Register.eax.low_id(),
	582	Register.edi.low_id(),
	583	);
	584
	585	testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);
	586	}
521	587
522	588	// simple mov
523	589
524	590	// mov eax,edi
525	591	// 89 f8
526		try expectEncoded(Instruction{
527		.primary_opcode_1b = 0x89, // mov (with rm as destination)
528		.reg = .edi, // source
529		.modrm = .{ .reg = .eax }, // destination
530		}, &[_]u8{ 0x89, 0xf8 });
	592	{
	593	try code.resize(0);
	594	const encoder = try Encoder.init(&code, 3);
	595	encoder.rex(.{
	596	.r = Register.edi.isExtended(),
	597	.b = Register.eax.isExtended(),
	598	});
	599	encoder.opcode_1byte(0x89);
	600	encoder.modRm_direct(
	601	Register.edi.low_id(),
	602	Register.eax.low_id(),
	603	);
	604
	605	testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);
	606	}
531	607
532	608	// signed integer addition of 32-bit sign extended immediate to 64 bit register
533	609
...	...	@@ -542,19 +618,19 @@ test "x86_64 Instruction.encodeInto" {
542	618	// : 000 <-- opcode_extension = 0 because opcode extension is /0. /0 specifies ADD
543	619	// : 001 <-- 001 is rcx
544	620	// ffffff7f : 2147483647
545		try expectEncoded(Instruction{
546		// REX.W +
547		.operand_size_64 = true,
548		// 81
549		.primary_opcode_1b = 0x81,
550		// /0
551		.opcode_extension = 0,
552		// rcx
553		.modrm = .{ .reg = .rcx },
554		// immediate
555		.immediate_bytes = 4,
556		.immediate = 2147483647,
557		}, &[_]u8{ 0x48, 0x81, 0xc1, 0xff, 0xff, 0xff, 0x7f });
	621	{
	622	try code.resize(0);
	623	const encoder = try Encoder.init(&code, 7);
	624	encoder.rex(.{ .w = true }); // use 64 bit operation
	625	encoder.opcode_1byte(0x81);
	626	encoder.modRm_direct(
	627	0,
	628	Register.rcx.low_id(),
	629	);
	630	encoder.imm32(2147483647);
	631
	632	testing.expectEqualSlices(u8, &[_]u8{ 0x48, 0x81, 0xc1, 0xff, 0xff, 0xff, 0x7f }, code.items);
	633	}
558	634	}
559	635
560	636	// TODO add these registers to the enum and populate dwarfLocOp