Commit 817fb263b533f0a24476cabe43a6ee5826113d8d | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-03-05 17:40:53+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2023-03-11 20:05:49+01:00
log	817fb263b533f0a24476cabe43a6ee5826113d8d
tree	115e23cd751165d1b7a791ba6d5290caea949a11
parent	4ea2f441df36cec61e1017f4d795d4037326c98c

x86_64: downstream table-driven instruction encoder

7 files changed, 2597 insertions(+), 3237 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Mir.zig
    bits.zig
    encoder.zig
    encodings.zig

src/arch/x86_64/CodeGen.zig+73-102

...	...	@@ -303,7 +303,12 @@ pub fn generate(
303	303	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
304	304	error.CodegenFail => return Result{ .fail = function.err_msg.? },
305	305	error.OutOfRegisters => return Result{
306		.fail = try ErrorMsg.create(bin_file.allocator, src_loc, "CodeGen ran out of registers. This is a bug in the Zig compiler.", .{}),
	306	.fail = try ErrorMsg.create(
	307	bin_file.allocator,
	308	src_loc,
	309	"CodeGen ran out of registers. This is a bug in the Zig compiler.",
	310	.{},
	311	),
307	312	},
308	313	else => \|e\| return e,
309	314	};
...	...	@@ -342,6 +347,20 @@ pub fn generate(
342	347	defer emit.deinit();
343	348	emit.lowerMir() catch \|err\| switch (err) {
344	349	error.EmitFail => return Result{ .fail = emit.err_msg.? },
	350	error.InvalidInstruction, error.CannotEncode => \|e\| {
	351	const msg = switch (e) {
	352	error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
	353	error.CannotEncode => "CodeGen failed to encode the instruction.",
	354	};
	355	return Result{
	356	.fail = try ErrorMsg.create(
	357	bin_file.allocator,
	358	src_loc,
	359	"{s} This is a bug in the Zig compiler.",
	360	.{msg},
	361	),
	362	};
	363	},
345	364	else => \|e\| return e,
346	365	};
347	366
...	...	@@ -1687,7 +1706,7 @@ fn genIntMulDivOpMir(
1687	1706	else => unreachable,
1688	1707	},
1689	1708	}),
1690		.data = .{ .imm = @bitCast(u32, -off) },
	1709	.data = .{ .disp = -off },
1691	1710	});
1692	1711	},
1693	1712	else => unreachable,
...	...	@@ -2191,7 +2210,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2191	2210	.reg2 = .rbp,
2192	2211	.flags = 0b01,
2193	2212	}),
2194		.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
	2213	.data = .{ .disp = -@intCast(i32, off) },
2195	2214	});
2196	2215	},
2197	2216	else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
...	...	@@ -2275,7 +2294,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2275	2294	.reg1 = addr_reg.to64(),
2276	2295	.reg2 = .rbp,
2277	2296	}),
2278		.data = .{ .imm = @bitCast(u32, -off) },
	2297	.data = .{ .disp = -off },
2279	2298	});
2280	2299	},
2281	2300	.stack_offset => \|off\| {
...	...	@@ -2286,7 +2305,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2286	2305	.reg1 = addr_reg.to64(),
2287	2306	.reg2 = .rbp,
2288	2307	}),
2289		.data = .{ .imm = @bitCast(u32, -off) },
	2308	.data = .{ .disp = -off },
2290	2309	});
2291	2310	},
2292	2311	.memory, .linker_load => {
...	...	@@ -2352,7 +2371,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2352	2371	.reg2 = dst_mcv.register,
2353	2372	.flags = 0b01,
2354	2373	}),
2355		.data = .{ .imm = 0 },
	2374	.data = .{ .disp = 0 },
2356	2375	});
2357	2376	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
2358	2377	}
...	...	@@ -2615,7 +2634,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2615	2634	.reg2 = reg,
2616	2635	.flags = 0b01,
2617	2636	}),
2618		.data = .{ .imm = 0 },
	2637	.data = .{ .disp = 0 },
2619	2638	});
2620	2639	},
2621	2640	.stack_offset => \|off\| {
...	...	@@ -2842,7 +2861,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2842	2861	.reg2 = addr_reg.to64(),
2843	2862	.flags = 0b01,
2844	2863	}),
2845		.data = .{ .imm = 0 },
	2864	.data = .{ .disp = 0 },
2846	2865	});
2847	2866
2848	2867	const new_ptr = MCValue{ .register = addr_reg.to64() };
...	...	@@ -2903,7 +2922,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2903	2922	.reg2 = tmp_reg,
2904	2923	.flags = 0b01,
2905	2924	}),
2906		.data = .{ .imm = 0 },
	2925	.data = .{ .disp = 0 },
2907	2926	});
2908	2927	return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2909	2928	}
...	...	@@ -3542,25 +3561,13 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3542	3561	if (intrinsicsAllowed(self.target.*, dst_ty)) {
3543	3562	const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3544	3563	.f32 => switch (mir_tag) {
3545		.add => if (hasAvxSupport(self.target.*))
3546		Mir.Inst.Tag.add_f32_avx
3547		else
3548		Mir.Inst.Tag.add_f32_sse,
3549		.cmp => if (hasAvxSupport(self.target.*))
3550		Mir.Inst.Tag.cmp_f32_avx
3551		else
3552		Mir.Inst.Tag.cmp_f32_sse,
	3564	.add => Mir.Inst.Tag.add_f32,
	3565	.cmp => Mir.Inst.Tag.cmp_f32,
3553	3566	else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
3554	3567	},
3555	3568	.f64 => switch (mir_tag) {
3556		.add => if (hasAvxSupport(self.target.*))
3557		Mir.Inst.Tag.add_f64_avx
3558		else
3559		Mir.Inst.Tag.add_f64_sse,
3560		.cmp => if (hasAvxSupport(self.target.*))
3561		Mir.Inst.Tag.cmp_f64_avx
3562		else
3563		Mir.Inst.Tag.cmp_f64_sse,
	3569	.add => Mir.Inst.Tag.add_f64,
	3570	.cmp => Mir.Inst.Tag.cmp_f64,
3564	3571	else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
3565	3572	},
3566	3573	else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
...	...	@@ -3618,7 +3625,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3618	3625	.reg2 = .rbp,
3619	3626	.flags = 0b01,
3620	3627	}),
3621		.data = .{ .imm = @bitCast(u32, -off) },
	3628	.data = .{ .disp = -off },
3622	3629	});
3623	3630	},
3624	3631	}
...	...	@@ -3644,7 +3651,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3644	3651	.reg2 = registerAlias(src_reg, abi_size),
3645	3652	.flags = 0b10,
3646	3653	}),
3647		.data = .{ .imm = @bitCast(u32, -off) },
	3654	.data = .{ .disp = -off },
3648	3655	});
3649	3656	},
3650	3657	.immediate => \|imm\| {
...	...	@@ -3665,7 +3672,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3665	3672	else => unreachable,
3666	3673	};
3667	3674	const payload = try self.addExtra(Mir.ImmPair{
3668		.dest_off = @bitCast(u32, -off),
	3675	.dest_off = -off,
3669	3676	.operand = @truncate(u32, imm),
3670	3677	});
3671	3678	_ = try self.addInst(.{
...	...	@@ -3756,7 +3763,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3756	3763	.reg2 = .rbp,
3757	3764	.flags = 0b01,
3758	3765	}),
3759		.data = .{ .imm = @bitCast(u32, -off) },
	3766	.data = .{ .disp = -off },
3760	3767	});
3761	3768	},
3762	3769	.memory => {
...	...	@@ -5360,7 +5367,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5360	5367	// offset from rbp, which is at the top of the stack frame.
5361	5368	// mov [rbp+offset], immediate
5362	5369	const payload = try self.addExtra(Mir.ImmPair{
5363		.dest_off = @bitCast(u32, -stack_offset),
	5370	.dest_off = -stack_offset,
5364	5371	.operand = @truncate(u32, imm),
5365	5372	});
5366	5373	_ = try self.addInst(.{
...	...	@@ -5400,14 +5407,8 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5400	5407	.Float => {
5401	5408	if (intrinsicsAllowed(self.target.*, ty)) {
5402	5409	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5403		.f32 => if (hasAvxSupport(self.target.*))
5404		Mir.Inst.Tag.mov_f32_avx
5405		else
5406		Mir.Inst.Tag.mov_f32_sse,
5407		.f64 => if (hasAvxSupport(self.target.*))
5408		Mir.Inst.Tag.mov_f64_avx
5409		else
5410		Mir.Inst.Tag.mov_f64_sse,
	5410	.f32 => Mir.Inst.Tag.mov_f32,
	5411	.f64 => Mir.Inst.Tag.mov_f64,
5411	5412	else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
5412	5413	};
5413	5414	_ = try self.addInst(.{
...	...	@@ -5421,7 +5422,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5421	5422	.reg2 = reg.to128(),
5422	5423	.flags = 0b01,
5423	5424	}),
5424		.data = .{ .imm = @bitCast(u32, -stack_offset) },
	5425	.data = .{ .disp = -stack_offset },
5425	5426	});
5426	5427	return;
5427	5428	}
...	...	@@ -5436,7 +5437,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5436	5437	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5437	5438	.flags = 0b10,
5438	5439	}),
5439		.data = .{ .imm = @bitCast(u32, -stack_offset) },
	5440	.data = .{ .disp = -stack_offset },
5440	5441	});
5441	5442	},
5442	5443	}
...	...	@@ -5516,7 +5517,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5516	5517	0 => {
5517	5518	assert(ty.isError());
5518	5519	const payload = try self.addExtra(Mir.ImmPair{
5519		.dest_off = @bitCast(u32, -stack_offset),
	5520	.dest_off = -stack_offset,
5520	5521	.operand = @truncate(u32, x_big),
5521	5522	});
5522	5523	_ = try self.addInst(.{
...	...	@@ -5530,7 +5531,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5530	5531	},
5531	5532	1, 2, 4 => {
5532	5533	const payload = try self.addExtra(Mir.ImmPair{
5533		.dest_off = @bitCast(u32, -stack_offset),
	5534	.dest_off = -stack_offset,
5534	5535	.operand = @truncate(u32, x_big),
5535	5536	});
5536	5537	_ = try self.addInst(.{
...	...	@@ -5552,7 +5553,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5552	5553	// insted just use two 32 bit writes to avoid register allocation
5553	5554	{
5554	5555	const payload = try self.addExtra(Mir.ImmPair{
5555		.dest_off = @bitCast(u32, -stack_offset + 4),
	5556	.dest_off = -stack_offset + 4,
5556	5557	.operand = @truncate(u32, x_big >> 32),
5557	5558	});
5558	5559	_ = try self.addInst(.{
...	...	@@ -5566,7 +5567,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5566	5567	}
5567	5568	{
5568	5569	const payload = try self.addExtra(Mir.ImmPair{
5569		.dest_off = @bitCast(u32, -stack_offset),
	5570	.dest_off = -stack_offset,
5570	5571	.operand = @truncate(u32, x_big),
5571	5572	});
5572	5573	_ = try self.addInst(.{
...	...	@@ -5595,14 +5596,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5595	5596	.Float => {
5596	5597	if (intrinsicsAllowed(self.target.*, ty)) {
5597	5598	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5598		.f32 => if (hasAvxSupport(self.target.*))
5599		Mir.Inst.Tag.mov_f32_avx
5600		else
5601		Mir.Inst.Tag.mov_f32_sse,
5602		.f64 => if (hasAvxSupport(self.target.*))
5603		Mir.Inst.Tag.mov_f64_avx
5604		else
5605		Mir.Inst.Tag.mov_f64_sse,
	5599	.f32 => Mir.Inst.Tag.mov_f32,
	5600	.f64 => Mir.Inst.Tag.mov_f64,
5606	5601	else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
5607	5602	};
5608	5603	_ = try self.addInst(.{
...	...	@@ -5616,7 +5611,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5616	5611	.reg2 = reg.to128(),
5617	5612	.flags = 0b01,
5618	5613	}),
5619		.data = .{ .imm = @bitCast(u32, -stack_offset) },
	5614	.data = .{ .disp = -stack_offset },
5620	5615	});
5621	5616	return;
5622	5617	}
...	...	@@ -5691,7 +5686,7 @@ fn genInlineMemcpyRegisterRegister(
5691	5686	.reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5692	5687	.flags = 0b10,
5693	5688	}),
5694		.data = .{ .imm = @bitCast(u32, -next_offset) },
	5689	.data = .{ .disp = -next_offset },
5695	5690	});
5696	5691
5697	5692	if (nearest_power_of_two > 1) {
...	...	@@ -5711,7 +5706,7 @@ fn genInlineMemcpyRegisterRegister(
5711	5706	.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
5712	5707	.flags = 0b10,
5713	5708	}),
5714		.data = .{ .imm = @bitCast(u32, -offset) },
	5709	.data = .{ .disp = -offset },
5715	5710	});
5716	5711	}
5717	5712	}
...	...	@@ -5758,7 +5753,7 @@ fn genInlineMemcpy(
5758	5753	.reg1 = dst_addr_reg.to64(),
5759	5754	.reg2 = opts.dest_stack_base orelse .rbp,
5760	5755	}),
5761		.data = .{ .imm = @bitCast(u32, -off) },
	5756	.data = .{ .disp = -off },
5762	5757	});
5763	5758	},
5764	5759	.register => \|reg\| {
...	...	@@ -5787,7 +5782,7 @@ fn genInlineMemcpy(
5787	5782	.reg1 = src_addr_reg.to64(),
5788	5783	.reg2 = opts.source_stack_base orelse .rbp,
5789	5784	}),
5790		.data = .{ .imm = @bitCast(u32, -off) },
	5785	.data = .{ .disp = -off },
5791	5786	});
5792	5787	},
5793	5788	.register => \|reg\| {
...	...	@@ -5911,7 +5906,7 @@ fn genInlineMemset(
5911	5906	.reg1 = addr_reg.to64(),
5912	5907	.reg2 = opts.dest_stack_base orelse .rbp,
5913	5908	}),
5914		.data = .{ .imm = @bitCast(u32, -off) },
	5909	.data = .{ .disp = -off },
5915	5910	});
5916	5911	},
5917	5912	.register => \|reg\| {
...	...	@@ -5998,7 +5993,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5998	5993	.reg1 = registerAlias(reg, abi_size),
5999	5994	.reg2 = .rbp,
6000	5995	}),
6001		.data = .{ .imm = @bitCast(u32, -off) },
	5996	.data = .{ .disp = -off },
6002	5997	});
6003	5998	},
6004	5999	.unreach, .none => return, // Nothing to do.
...	...	@@ -6097,14 +6092,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6097	6092	.Float => {
6098	6093	if (intrinsicsAllowed(self.target.*, ty)) {
6099	6094	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6100		.f32 => if (hasAvxSupport(self.target.*))
6101		Mir.Inst.Tag.mov_f32_avx
6102		else
6103		Mir.Inst.Tag.mov_f32_sse,
6104		.f64 => if (hasAvxSupport(self.target.*))
6105		Mir.Inst.Tag.mov_f64_avx
6106		else
6107		Mir.Inst.Tag.mov_f64_sse,
	6095	.f32 => Mir.Inst.Tag.mov_f32,
	6096	.f64 => Mir.Inst.Tag.mov_f64,
6108	6097	else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
6109	6098	};
6110	6099	_ = try self.addInst(.{
...	...	@@ -6141,14 +6130,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6141	6130
6142	6131	if (intrinsicsAllowed(self.target.*, ty)) {
6143	6132	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6144		.f32 => if (hasAvxSupport(self.target.*))
6145		Mir.Inst.Tag.mov_f32_avx
6146		else
6147		Mir.Inst.Tag.mov_f32_sse,
6148		.f64 => if (hasAvxSupport(self.target.*))
6149		Mir.Inst.Tag.mov_f64_avx
6150		else
6151		Mir.Inst.Tag.mov_f64_sse,
	6133	.f32 => Mir.Inst.Tag.mov_f32,
	6134	.f64 => Mir.Inst.Tag.mov_f64,
6152	6135	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6153	6136	};
6154	6137
...	...	@@ -6162,7 +6145,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6162	6145	else => unreachable,
6163	6146	},
6164	6147	}),
6165		.data = .{ .imm = 0 },
	6148	.data = .{ .disp = 0 },
6166	6149	});
6167	6150	return;
6168	6151	}
...	...	@@ -6178,7 +6161,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6178	6161	.reg2 = reg.to64(),
6179	6162	.flags = 0b01,
6180	6163	}),
6181		.data = .{ .imm = 0 },
	6164	.data = .{ .disp = 0 },
6182	6165	});
6183	6166	},
6184	6167	}
...	...	@@ -6190,14 +6173,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6190	6173
6191	6174	if (intrinsicsAllowed(self.target.*, ty)) {
6192	6175	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6193		.f32 => if (hasAvxSupport(self.target.*))
6194		Mir.Inst.Tag.mov_f32_avx
6195		else
6196		Mir.Inst.Tag.mov_f32_sse,
6197		.f64 => if (hasAvxSupport(self.target.*))
6198		Mir.Inst.Tag.mov_f64_avx
6199		else
6200		Mir.Inst.Tag.mov_f64_sse,
	6176	.f32 => Mir.Inst.Tag.mov_f32,
	6177	.f64 => Mir.Inst.Tag.mov_f64,
6201	6178	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6202	6179	};
6203	6180
...	...	@@ -6211,7 +6188,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6211	6188	else => unreachable,
6212	6189	},
6213	6190	}),
6214		.data = .{ .imm = 0 },
	6191	.data = .{ .disp = 0 },
6215	6192	});
6216	6193	return;
6217	6194	}
...	...	@@ -6255,7 +6232,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6255	6232	.reg2 = reg.to64(),
6256	6233	.flags = 0b01,
6257	6234	}),
6258		.data = .{ .imm = 0 },
	6235	.data = .{ .disp = 0 },
6259	6236	});
6260	6237	}
6261	6238	}
...	...	@@ -6283,7 +6260,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6283	6260	.reg2 = .rbp,
6284	6261	.flags = flags,
6285	6262	}),
6286		.data = .{ .imm = @bitCast(u32, -off) },
	6263	.data = .{ .disp = -off },
6287	6264	});
6288	6265	return;
6289	6266	}
...	...	@@ -6302,7 +6279,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6302	6279	.reg2 = .rbp,
6303	6280	.flags = flags,
6304	6281	}),
6305		.data = .{ .imm = @bitCast(u32, -off) },
	6282	.data = .{ .disp = -off },
6306	6283	});
6307	6284	return;
6308	6285	}
...	...	@@ -6311,14 +6288,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6311	6288	.Float => {
6312	6289	if (intrinsicsAllowed(self.target.*, ty)) {
6313	6290	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6314		.f32 => if (hasAvxSupport(self.target.*))
6315		Mir.Inst.Tag.mov_f32_avx
6316		else
6317		Mir.Inst.Tag.mov_f32_sse,
6318		.f64 => if (hasAvxSupport(self.target.*))
6319		Mir.Inst.Tag.mov_f64_avx
6320		else
6321		Mir.Inst.Tag.mov_f64_sse,
	6291	.f32 => Mir.Inst.Tag.mov_f32,
	6292	.f64 => Mir.Inst.Tag.mov_f64,
6322	6293	else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
6323	6294	};
6324	6295	_ = try self.addInst(.{
...	...	@@ -6331,7 +6302,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6331	6302	else => unreachable,
6332	6303	},
6333	6304	}),
6334		.data = .{ .imm = @bitCast(u32, -off) },
	6305	.data = .{ .disp = -off },
6335	6306	});
6336	6307	return;
6337	6308	}
...	...	@@ -6347,7 +6318,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6347	6318	.reg2 = .rbp,
6348	6319	.flags = 0b01,
6349	6320	}),
6350		.data = .{ .imm = @bitCast(u32, -off) },
	6321	.data = .{ .disp = -off },
6351	6322	});
6352	6323	},
6353	6324	}
...	...	@@ -6436,7 +6407,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6436	6407	else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
6437	6408	},
6438	6409	}),
6439		.data = .{ .imm = @bitCast(u32, -stack_offset) },
	6410	.data = .{ .disp = -stack_offset },
6440	6411	});
6441	6412
6442	6413	// convert
...	...	@@ -6452,7 +6423,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6452	6423	else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),
6453	6424	},
6454	6425	}),
6455		.data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) },
	6426	.data = .{ .disp = -stack_dst.stack_offset },
6456	6427	});
6457	6428
6458	6429	return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
...	...	@@ -6551,7 +6522,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6551	6522	.reg2 = reg,
6552	6523	.flags = 0b01,
6553	6524	}),
6554		.data = .{ .imm = 0 },
	6525	.data = .{ .disp = 0 },
6555	6526	});
6556	6527	break :blk MCValue{ .register = reg };
6557	6528	},

src/arch/x86_64/Emit.zig+395-2244

...	...	@@ -1,3 +1,4 @@
	1	//!
1	2	//! This file contains the functionality for lowering x86_64 MIR into
2	3	//! machine code
3	4
...	...	@@ -7,6 +8,7 @@ const std = @import("std");
7	8	const assert = std.debug.assert;
8	9	const bits = @import("bits.zig");
9	10	const abi = @import("abi.zig");
	11	const encoder = @import("encoder.zig");
10	12	const link = @import("../../link.zig");
11	13	const log = std.log.scoped(.codegen);
12	14	const math = std.math;
...	...	@@ -19,12 +21,13 @@ const CodeGen = @import("CodeGen.zig");
19	21	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
20	22	const Encoder = bits.Encoder;
21	23	const ErrorMsg = Module.ErrorMsg;
	24	const Instruction = encoder.Instruction;
22	25	const MCValue = @import("CodeGen.zig").MCValue;
	26	const Memory = bits.Memory;
23	27	const Mir = @import("Mir.zig");
24	28	const Module = @import("../../Module.zig");
25		const Instruction = bits.Instruction;
26		const Type = @import("../../type.zig").Type;
27	29	const Register = bits.Register;
	30	const Type = @import("../../type.zig").Type;
28	31
29	32	mir: Mir,
30	33	bin_file: *link.File,
...	...	@@ -45,6 +48,8 @@ relocs: std.ArrayListUnmanaged(Reloc) = .{},
45	48	const InnerError = error{
46	49	OutOfMemory,
47	50	EmitFail,
	51	InvalidInstruction,
	52	CannotEncode,
48	53	};
49	54
50	55	const Reloc = struct {
...	...	@@ -153,8 +158,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
153	158	.push => try emit.mirPushPop(.push, inst),
154	159	.pop => try emit.mirPushPop(.pop, inst),
155	160
156		.jmp => try emit.mirJmpCall(.jmp_near, inst),
157		.call => try emit.mirJmpCall(.call_near, inst),
	161	.jmp => try emit.mirJmpCall(.jmp, inst),
	162	.call => try emit.mirJmpCall(.call, inst),
158	163
159	164	.cond_jmp => try emit.mirCondJmp(inst),
160	165	.cond_set_byte => try emit.mirCondSetByte(inst),
...	...	@@ -170,25 +175,15 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
170	175	.interrupt => try emit.mirInterrupt(inst),
171	176	.nop => {}, // just skip it
172	177
173		// SSE instructions
174		.mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),
175		.mov_f32_sse => try emit.mirMovFloatSse(.movss, inst),
176
177		.add_f64_sse => try emit.mirAddFloatSse(.addsd, inst),
178		.add_f32_sse => try emit.mirAddFloatSse(.addss, inst),
179
180		.cmp_f64_sse => try emit.mirCmpFloatSse(.ucomisd, inst),
181		.cmp_f32_sse => try emit.mirCmpFloatSse(.ucomiss, inst),
	178	// SSE/AVX instructions
	179	.mov_f64 => try emit.mirMovFloat(.movsd, inst),
	180	.mov_f32 => try emit.mirMovFloat(.movss, inst),
182	181
183		// AVX instructions
184		.mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst),
185		.mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst),
	182	.add_f64 => try emit.mirAddFloat(.addsd, inst),
	183	.add_f32 => try emit.mirAddFloat(.addss, inst),
186	184
187		.add_f64_avx => try emit.mirAddFloatAvx(.vaddsd, inst),
188		.add_f32_avx => try emit.mirAddFloatAvx(.vaddss, inst),
189
190		.cmp_f64_avx => try emit.mirCmpFloatAvx(.vucomisd, inst),
191		.cmp_f32_avx => try emit.mirCmpFloatAvx(.vucomiss, inst),
	185	.cmp_f64 => try emit.mirCmpFloat(.ucomisd, inst),
	186	.cmp_f32 => try emit.mirCmpFloat(.ucomiss, inst),
192	187
193	188	// Pseudo-instructions
194	189	.call_extern => try emit.mirCallExtern(inst),
...	...	@@ -235,8 +230,23 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
235	230	}
236	231	}
237	232
	233	fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct {
	234	op1: Instruction.Operand = .none,
	235	op2: Instruction.Operand = .none,
	236	op3: Instruction.Operand = .none,
	237	op4: Instruction.Operand = .none,
	238	}) InnerError!void {
	239	const inst = try Instruction.new(mnemonic, .{
	240	.op1 = ops.op1,
	241	.op2 = ops.op2,
	242	.op3 = ops.op3,
	243	.op4 = ops.op4,
	244	});
	245	return inst.encode(emit.code.writer());
	246	}
	247
238	248	fn mirUndefinedInstruction(emit: *Emit) InnerError!void {
239		return lowerToZoEnc(.ud2, emit.code);
	249	return emit.encode(.ud2, .{});
240	250	}
241	251
242	252	fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -244,45 +254,43 @@ fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
244	254	assert(tag == .interrupt);
245	255	const ops = emit.mir.instructions.items(.ops)[inst].decode();
246	256	switch (ops.flags) {
247		0b00 => return lowerToZoEnc(.int3, emit.code),
	257	0b00 => return emit.encode(.int3, .{}),
248	258	else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}),
249	259	}
250	260	}
251	261
252	262	fn mirSyscall(emit: *Emit) InnerError!void {
253		return lowerToZoEnc(.syscall, emit.code);
	263	return emit.encode(.syscall, .{});
254	264	}
255	265
256		fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	266	fn mirPushPop(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
257	267	const ops = emit.mir.instructions.items(.ops)[inst].decode();
258	268	switch (ops.flags) {
259	269	0b00 => {
260		// PUSH/POP reg
261		return lowerToOEnc(tag, ops.reg1, emit.code);
	270	return emit.encode(mnemonic, .{
	271	.op1 = .{ .reg = ops.reg1 },
	272	});
262	273	},
263	274	0b01 => {
264		// PUSH/POP r/m64
265		const imm = emit.mir.instructions.items(.data)[inst].imm;
266		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
267		16 => .word_ptr,
268		else => .qword_ptr,
269		};
270		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
271		.disp = imm,
272		.base = ops.reg1,
273		}), emit.code);
	275	const disp = emit.mir.instructions.items(.data)[inst].disp;
	276	return emit.encode(mnemonic, .{
	277	.op1 = .{ .mem = Memory.sib(.qword, .{
	278	.base = ops.reg1,
	279	.disp = disp,
	280	}) },
	281	});
274	282	},
275	283	0b10 => {
276		// PUSH imm32
277		assert(tag == .push);
278	284	const imm = emit.mir.instructions.items(.data)[inst].imm;
279		return lowerToIEnc(.push, imm, emit.code);
	285	return emit.encode(.push, .{
	286	.op1 = .{ .imm = imm },
	287	});
280	288	},
281	289	0b11 => unreachable,
282	290	}
283	291	}
284	292
285		fn mirPushPopRegisterList(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	293	fn mirPushPopRegisterList(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
286	294	const ops = emit.mir.instructions.items(.ops)[inst].decode();
287	295	const payload = emit.mir.instructions.items(.data)[inst].payload;
288	296	const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;
...	...	@@ -291,15 +299,20 @@ fn mirPushPopRegisterList(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerErro
291	299	const callee_preserved_regs = abi.getCalleePreservedRegs(emit.target.*);
292	300	for (callee_preserved_regs) \|reg\| {
293	301	if (reg_list.isSet(callee_preserved_regs, reg)) {
294		switch (tag) {
295		.push => try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
296		.disp = @bitCast(u32, disp),
297		.base = ops.reg1,
298		}), reg, emit.code),
299		.pop => try lowerToRmEnc(.mov, reg, RegisterOrMemory.mem(.qword_ptr, .{
300		.disp = @bitCast(u32, disp),
301		.base = ops.reg1,
302		}), emit.code),
	302	const op1: Instruction.Operand = .{ .mem = Memory.sib(.qword, .{
	303	.base = ops.reg1,
	304	.disp = disp,
	305	}) };
	306	const op2: Instruction.Operand = .{ .reg = reg };
	307	switch (mnemonic) {
	308	.push => try emit.encode(.mov, .{
	309	.op1 = op1,
	310	.op2 = op2,
	311	}),
	312	.pop => try emit.encode(.mov, .{
	313	.op1 = op2,
	314	.op2 = op1,
	315	}),
303	316	else => unreachable,
304	317	}
305	318	disp += 8;
...	...	@@ -307,13 +320,17 @@ fn mirPushPopRegisterList(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerErro
307	320	}
308	321	}
309	322
310		fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	323	fn mirJmpCall(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
311	324	const ops = emit.mir.instructions.items(.ops)[inst].decode();
312	325	switch (ops.flags) {
313	326	0b00 => {
314	327	const target = emit.mir.instructions.items(.data)[inst].inst;
315	328	const source = emit.code.items.len;
316		try lowerToDEnc(tag, 0, emit.code);
	329	try emit.encode(mnemonic, .{
	330	.op1 = .{
	331	.imm = 0,
	332	},
	333	});
317	334	try emit.relocs.append(emit.bin_file.allocator, .{
318	335	.source = source,
319	336	.target = target,
...	...	@@ -323,34 +340,33 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
323	340	},
324	341	0b01 => {
325	342	if (ops.reg1 == .none) {
326		// JMP/CALL [imm]
327	343	const imm = emit.mir.instructions.items(.data)[inst].imm;
328		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
329		16 => .word_ptr,
330		else => .qword_ptr,
331		};
332		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code);
	344	return emit.encode(mnemonic, .{
	345	.op1 = .{ .imm = imm },
	346	});
333	347	}
334		// JMP/CALL reg
335		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	348	return emit.encode(mnemonic, .{
	349	.op1 = .{ .reg = ops.reg1 },
	350	});
336	351	},
337	352	0b10 => {
338		// JMP/CALL r/m64
339		const imm = emit.mir.instructions.items(.data)[inst].imm;
340		return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
341		.disp = imm,
342		.base = ops.reg1,
343		}), emit.code);
	353	const disp = emit.mir.instructions.items(.data)[inst].disp;
	354	return emit.encode(mnemonic, .{
	355	.op1 = .{ .mem = Memory.sib(.qword, .{
	356	.base = ops.reg1,
	357	.disp = disp,
	358	}) },
	359	});
344	360	},
345	361	0b11 => return emit.fail("TODO unused variant jmp/call 0b11", .{}),
346	362	}
347	363	}
348	364
349	365	fn mirCondJmp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
350		const mir_tag = emit.mir.instructions.items(.tag)[inst];
351		assert(mir_tag == .cond_jmp);
	366	const tag = emit.mir.instructions.items(.tag)[inst];
	367	assert(tag == .cond_jmp);
352	368	const inst_cc = emit.mir.instructions.items(.data)[inst].inst_cc;
353		const tag: Tag = switch (inst_cc.cc) {
	369	const mnemonic: Instruction.Mnemonic = switch (inst_cc.cc) {
354	370	.a => .ja,
355	371	.ae => .jae,
356	372	.b => .jb,
...	...	@@ -383,7 +399,9 @@ fn mirCondJmp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
383	399	.z => .jz,
384	400	};
385	401	const source = emit.code.items.len;
386		try lowerToDEnc(tag, 0, emit.code);
	402	try emit.encode(mnemonic, .{
	403	.op1 = .{ .imm = 0 },
	404	});
387	405	try emit.relocs.append(emit.bin_file.allocator, .{
388	406	.source = source,
389	407	.target = inst_cc.inst,
...	...	@@ -393,11 +411,11 @@ fn mirCondJmp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
393	411	}
394	412
395	413	fn mirCondSetByte(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
396		const mir_tag = emit.mir.instructions.items(.tag)[inst];
397		assert(mir_tag == .cond_set_byte);
	414	const tag = emit.mir.instructions.items(.tag)[inst];
	415	assert(tag == .cond_set_byte);
398	416	const ops = emit.mir.instructions.items(.ops)[inst].decode();
399	417	const cc = emit.mir.instructions.items(.data)[inst].cc;
400		const tag: Tag = switch (cc) {
	418	const mnemonic: Instruction.Mnemonic = switch (cc) {
401	419	.a => .seta,
402	420	.ae => .setae,
403	421	.b => .setb,
...	...	@@ -429,15 +447,15 @@ fn mirCondSetByte(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
429	447	.s => .sets,
430	448	.z => .setz,
431	449	};
432		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code);
	450	return emit.encode(mnemonic, .{ .op1 = .{ .reg = ops.reg1 } });
433	451	}
434	452
435	453	fn mirCondMov(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
436		const mir_tag = emit.mir.instructions.items(.tag)[inst];
437		assert(mir_tag == .cond_mov);
	454	const tag = emit.mir.instructions.items(.tag)[inst];
	455	assert(tag == .cond_mov);
438	456	const ops = emit.mir.instructions.items(.ops)[inst].decode();
439	457	const cc = emit.mir.instructions.items(.data)[inst].cc;
440		const tag: Tag = switch (cc) {
	458	const mnemonic: Instruction.Mnemonic = switch (cc) {
441	459	.a => .cmova,
442	460	.ae => .cmovae,
443	461	.b => .cmovb,
...	...	@@ -469,21 +487,28 @@ fn mirCondMov(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
469	487	.s => .cmovs,
470	488	.z => .cmovz,
471	489	};
	490	const op1: Instruction.Operand = .{ .reg = ops.reg1 };
472	491
473	492	if (ops.flags == 0b00) {
474		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
	493	return emit.encode(mnemonic, .{
	494	.op1 = op1,
	495	.op2 = .{ .reg = ops.reg2 },
	496	});
475	497	}
476		const imm = emit.mir.instructions.items(.data)[inst].imm;
	498	const disp = emit.mir.instructions.items(.data)[inst].disp;
477	499	const ptr_size: Memory.PtrSize = switch (ops.flags) {
478	500	0b00 => unreachable,
479		0b01 => .word_ptr,
480		0b10 => .dword_ptr,
481		0b11 => .qword_ptr,
	501	0b01 => .word,
	502	0b10 => .dword,
	503	0b11 => .qword,
482	504	};
483		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ptr_size, .{
484		.disp = imm,
485		.base = ops.reg2,
486		}), emit.code);
	505	return emit.encode(mnemonic, .{
	506	.op1 = op1,
	507	.op2 = .{ .mem = Memory.sib(ptr_size, .{
	508	.base = ops.reg2,
	509	.disp = disp,
	510	}) },
	511	});
487	512	}
488	513
489	514	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -493,18 +518,16 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
493	518	switch (ops.flags) {
494	519	0b00 => {
495	520	if (ops.reg2 == .none) {
496		// TEST r/m64, imm32
497		// MI
498	521	const imm = emit.mir.instructions.items(.data)[inst].imm;
499		if (ops.reg1.to64() == .rax) {
500		// TEST rax, imm32
501		// I
502		return lowerToIEnc(.@"test", imm, emit.code);
503		}
504		return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
	522	return emit.encode(.@"test", .{
	523	.op1 = .{ .reg = ops.reg1 },
	524	.op2 = .{ .imm = imm },
	525	});
505	526	}
506		// TEST r/m64, r64
507		return lowerToMrEnc(.@"test", RegisterOrMemory.reg(ops.reg1), ops.reg2, emit.code);
	527	return emit.encode(.@"test", .{
	528	.op1 = .{ .reg = ops.reg1 },
	529	.op2 = .{ .reg = ops.reg2 },
	530	});
508	531	},
509	532	else => return emit.fail("TODO more TEST alternatives", .{}),
510	533	}
...	...	@@ -515,62 +538,59 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
515	538	assert(tag == .ret);
516	539	const ops = emit.mir.instructions.items(.ops)[inst].decode();
517	540	switch (ops.flags) {
518		0b00 => {
519		// RETF imm16
520		// I
521		const imm = emit.mir.instructions.items(.data)[inst].imm;
522		return lowerToIEnc(.ret_far, imm, emit.code);
523		},
524		0b01 => {
525		return lowerToZoEnc(.ret_far, emit.code);
526		},
	541	0b00 => unreachable,
	542	0b01 => unreachable,
527	543	0b10 => {
528		// RET imm16
529		// I
530	544	const imm = emit.mir.instructions.items(.data)[inst].imm;
531		return lowerToIEnc(.ret_near, imm, emit.code);
	545	return emit.encode(.ret, .{
	546	.op1 = .{ .imm = imm },
	547	});
532	548	},
533	549	0b11 => {
534		return lowerToZoEnc(.ret_near, emit.code);
	550	return emit.encode(.ret, .{});
535	551	},
536	552	}
537	553	}
538	554
539		fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	555	fn mirArith(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
540	556	const ops = emit.mir.instructions.items(.ops)[inst].decode();
541	557	switch (ops.flags) {
542	558	0b00 => {
543	559	if (ops.reg2 == .none) {
544		// mov reg1, imm32
545		// MI
546	560	const imm = emit.mir.instructions.items(.data)[inst].imm;
547		return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
	561	return emit.encode(mnemonic, .{
	562	.op1 = .{ .reg = ops.reg1 },
	563	.op2 = .{ .imm = imm },
	564	});
548	565	}
549		// mov reg1, reg2
550		// RM
551		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
	566	return emit.encode(mnemonic, .{
	567	.op1 = .{ .reg = ops.reg1 },
	568	.op2 = .{ .reg = ops.reg2 },
	569	});
552	570	},
553	571	0b01 => {
554		// mov reg1, [reg2 + imm32]
555		// RM
556		const imm = emit.mir.instructions.items(.data)[inst].imm;
557		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
558		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
559		.disp = imm,
560		.base = src_reg,
561		}), emit.code);
	572	const disp = emit.mir.instructions.items(.data)[inst].disp;
	573	const base: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
	574	return emit.encode(mnemonic, .{
	575	.op1 = .{ .reg = ops.reg1 },
	576	.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg1.size()), .{
	577	.base = base,
	578	.disp = disp,
	579	}) },
	580	});
562	581	},
563	582	0b10 => {
564	583	if (ops.reg2 == .none) {
565	584	return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
566	585	}
567		// mov [reg1 + imm32], reg2
568		// MR
569		const imm = emit.mir.instructions.items(.data)[inst].imm;
570		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
571		.disp = imm,
572		.base = ops.reg1,
573		}), ops.reg2, emit.code);
	586	const disp = emit.mir.instructions.items(.data)[inst].disp;
	587	return emit.encode(mnemonic, .{
	588	.op1 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg2.size()), .{
	589	.base = ops.reg1,
	590	.disp = disp,
	591	}) },
	592	.op2 = .{ .reg = ops.reg2 },
	593	});
574	594	},
575	595	0b11 => {
576	596	return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
...	...	@@ -578,169 +598,165 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
578	598	}
579	599	}
580	600
581		fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	601	fn mirArithMemImm(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
582	602	const ops = emit.mir.instructions.items(.ops)[inst].decode();
583	603	assert(ops.reg2 == .none);
584	604	const payload = emit.mir.instructions.items(.data)[inst].payload;
585	605	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
586	606	const ptr_size: Memory.PtrSize = switch (ops.flags) {
587		0b00 => .byte_ptr,
588		0b01 => .word_ptr,
589		0b10 => .dword_ptr,
590		0b11 => .qword_ptr,
591		};
592		return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
593		.disp = imm_pair.dest_off,
594		.base = ops.reg1,
595		}), imm_pair.operand, emit.code);
596		}
597
598		inline fn setRexWRegister(reg: Register) bool {
599		if (reg.size() > 64) return false;
600		if (reg.size() == 64) return true;
601		return switch (reg) {
602		.ah, .ch, .dh, .bh => true,
603		else => false,
	607	0b00 => .byte,
	608	0b01 => .word,
	609	0b10 => .dword,
	610	0b11 => .qword,
604	611	};
	612	return emit.encode(mnemonic, .{
	613	.op1 = .{ .mem = Memory.sib(ptr_size, .{
	614	.disp = imm_pair.dest_off,
	615	.base = ops.reg1,
	616	}) },
	617	.op2 = .{ .imm = imm_pair.operand },
	618	});
605	619	}
606	620
607		inline fn immOpSize(u_imm: u32) u6 {
608		const imm = @bitCast(i32, u_imm);
609		if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
610		return 8;
611		}
612		if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
613		return 16;
614		}
615		return 32;
616		}
617
618		fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	621	fn mirArithScaleSrc(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
619	622	const ops = emit.mir.instructions.items(.ops)[inst].decode();
620	623	const scale = ops.flags;
621	624	const payload = emit.mir.instructions.items(.data)[inst].payload;
622	625	const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
623		// OP reg1, [reg2 + scale*index + imm32]
624		const scale_index = ScaleIndex{
	626	const scale_index = Memory.ScaleIndex{
625	627	.scale = scale,
626	628	.index = index_reg_disp.index,
627	629	};
628		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
629		.disp = index_reg_disp.disp,
630		.base = ops.reg2,
631		.scale_index = scale_index,
632		}), emit.code);
	630	return emit.encode(mnemonic, .{
	631	.op1 = .{ .reg = ops.reg1 },
	632	.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg1.size()), .{
	633	.base = ops.reg2,
	634	.scale_index = scale_index,
	635	.disp = index_reg_disp.disp,
	636	}) },
	637	});
633	638	}
634	639
635		fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	640	fn mirArithScaleDst(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
636	641	const ops = emit.mir.instructions.items(.ops)[inst].decode();
637	642	const scale = ops.flags;
638	643	const payload = emit.mir.instructions.items(.data)[inst].payload;
639	644	const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
640		const scale_index = ScaleIndex{
	645	const scale_index = Memory.ScaleIndex{
641	646	.scale = scale,
642	647	.index = index_reg_disp.index,
643	648	};
644	649	assert(ops.reg2 != .none);
645		// OP [reg1 + scale*index + imm32], reg2
646		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
647		.disp = index_reg_disp.disp,
648		.base = ops.reg1,
649		.scale_index = scale_index,
650		}), ops.reg2, emit.code);
	650	return emit.encode(mnemonic, .{
	651	.op1 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg2.size()), .{
	652	.base = ops.reg1,
	653	.scale_index = scale_index,
	654	.disp = index_reg_disp.disp,
	655	}) },
	656	.op2 = .{ .reg = ops.reg2 },
	657	});
651	658	}
652	659
653		fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	660	fn mirArithScaleImm(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
654	661	const ops = emit.mir.instructions.items(.ops)[inst].decode();
655	662	const scale = ops.flags;
656	663	const payload = emit.mir.instructions.items(.data)[inst].payload;
657	664	const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();
658		const scale_index = ScaleIndex{
	665	const scale_index = Memory.ScaleIndex{
659	666	.scale = scale,
660	667	.index = index_reg_disp_imm.index,
661	668	};
662		// OP qword ptr [reg1 + scale*index + imm32], imm32
663		return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
664		.disp = index_reg_disp_imm.disp,
665		.base = ops.reg1,
666		.scale_index = scale_index,
667		}), index_reg_disp_imm.imm, emit.code);
	669	return emit.encode(mnemonic, .{
	670	.op1 = .{ .mem = Memory.sib(.qword, .{
	671	.base = ops.reg1,
	672	.disp = index_reg_disp_imm.disp,
	673	.scale_index = scale_index,
	674	}) },
	675	.op2 = .{ .imm = index_reg_disp_imm.imm },
	676	});
668	677	}
669	678
670		fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	679	fn mirArithMemIndexImm(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
671	680	const ops = emit.mir.instructions.items(.ops)[inst].decode();
672	681	assert(ops.reg2 == .none);
673	682	const payload = emit.mir.instructions.items(.data)[inst].payload;
674	683	const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();
675	684	const ptr_size: Memory.PtrSize = switch (ops.flags) {
676		0b00 => .byte_ptr,
677		0b01 => .word_ptr,
678		0b10 => .dword_ptr,
679		0b11 => .qword_ptr,
	685	0b00 => .byte,
	686	0b01 => .word,
	687	0b10 => .dword,
	688	0b11 => .qword,
680	689	};
681		const scale_index = ScaleIndex{
	690	const scale_index = Memory.ScaleIndex{
682	691	.scale = 0,
683	692	.index = index_reg_disp_imm.index,
684	693	};
685		// OP ptr [reg1 + index + imm32], imm32
686		return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
687		.disp = index_reg_disp_imm.disp,
688		.base = ops.reg1,
689		.scale_index = scale_index,
690		}), index_reg_disp_imm.imm, emit.code);
	694	return emit.encode(mnemonic, .{
	695	.op1 = .{ .mem = Memory.sib(ptr_size, .{
	696	.disp = index_reg_disp_imm.disp,
	697	.base = ops.reg1,
	698	.scale_index = scale_index,
	699	}) },
	700	.op2 = .{ .imm = index_reg_disp_imm.imm },
	701	});
691	702	}
692	703
693	704	fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
694		const mir_tag = emit.mir.instructions.items(.tag)[inst];
695		assert(mir_tag == .mov_sign_extend);
	705	const tag = emit.mir.instructions.items(.tag)[inst];
	706	assert(tag == .mov_sign_extend);
696	707	const ops = emit.mir.instructions.items(.ops)[inst].decode();
697		const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
	708	const disp = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].disp else undefined;
698	709	switch (ops.flags) {
699	710	0b00 => {
700		const tag: Tag = if (ops.reg2.size() == 32) .movsxd else .movsx;
701		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
702		},
703		0b01 => {
704		return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
705		.disp = imm,
706		.base = ops.reg2,
707		}), emit.code);
708		},
709		0b10 => {
710		return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
711		.disp = imm,
712		.base = ops.reg2,
713		}), emit.code);
	711	const mnemonic: Instruction.Mnemonic = if (ops.reg2.size() == 32) .movsxd else .movsx;
	712	return emit.encode(mnemonic, .{
	713	.op1 = .{ .reg = ops.reg1 },
	714	.op2 = .{ .reg = ops.reg2 },
	715	});
714	716	},
715		0b11 => {
716		return lowerToRmEnc(.movsxd, ops.reg1, RegisterOrMemory.mem(.dword_ptr, .{
717		.disp = imm,
718		.base = ops.reg2,
719		}), emit.code);
	717	else => {
	718	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	719	0b01 => .byte,
	720	0b10 => .word,
	721	0b11 => .qword,
	722	else => unreachable,
	723	};
	724	return emit.encode(.movsx, .{
	725	.op1 = .{ .reg = ops.reg1 },
	726	.op2 = .{ .mem = Memory.sib(ptr_size, .{
	727	.disp = disp,
	728	.base = ops.reg2,
	729	}) },
	730	});
720	731	},
721	732	}
722	733	}
723	734
724	735	fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
725		const mir_tag = emit.mir.instructions.items(.tag)[inst];
726		assert(mir_tag == .mov_zero_extend);
	736	const tag = emit.mir.instructions.items(.tag)[inst];
	737	assert(tag == .mov_zero_extend);
727	738	const ops = emit.mir.instructions.items(.ops)[inst].decode();
728		const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
	739	const disp = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].disp else undefined;
729	740	switch (ops.flags) {
730	741	0b00 => {
731		return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
732		},
733		0b01 => {
734		return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
735		.disp = imm,
736		.base = ops.reg2,
737		}), emit.code);
	742	return emit.encode(.movzx, .{
	743	.op1 = .{ .reg = ops.reg1 },
	744	.op2 = .{ .reg = ops.reg2 },
	745	});
738	746	},
739		0b10 => {
740		return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
741		.disp = imm,
742		.base = ops.reg2,
743		}), emit.code);
	747	0b01, 0b10 => {
	748	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	749	0b01 => .byte,
	750	0b10 => .word,
	751	else => unreachable,
	752	};
	753	return emit.encode(.movzx, .{
	754	.op1 = .{ .reg = ops.reg1 },
	755	.op2 = .{ .mem = Memory.sib(ptr_size, .{
	756	.disp = disp,
	757	.base = ops.reg2,
	758	}) },
	759	});
744	760	},
745	761	0b11 => {
746	762	return emit.fail("TODO unused variant: movzx 0b11", .{});
...	...	@@ -759,9 +775,10 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
759	775	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
760	776	break :blk imm.decode();
761	777	} else emit.mir.instructions.items(.data)[inst].imm;
762		// movabs reg, imm64
763		// OI
764		return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
	778	return emit.encode(.mov, .{
	779	.op1 = .{ .reg = ops.reg1 },
	780	.op2 = .{ .imm = @bitCast(i64, imm) },
	781	});
765	782	},
766	783	0b01 => {
767	784	if (ops.reg1 == .none) {
...	...	@@ -770,18 +787,20 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
770	787	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
771	788	break :blk imm.decode();
772	789	} else emit.mir.instructions.items(.data)[inst].imm;
773		// movabs moffs64, rax
774		// TD
775		return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
	790	return emit.encode(.mov, .{
	791	.op1 = .{ .mem = Memory.moffs(ops.reg2, imm) },
	792	.op2 = .{ .reg = .rax },
	793	});
776	794	}
777	795	const imm: u64 = if (ops.reg1.size() == 64) blk: {
778	796	const payload = emit.mir.instructions.items(.data)[inst].payload;
779	797	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
780	798	break :blk imm.decode();
781	799	} else emit.mir.instructions.items(.data)[inst].imm;
782		// movabs rax, moffs64
783		// FD
784		return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
	800	return emit.encode(.mov, .{
	801	.op1 = .{ .reg = .rax },
	802	.op2 = .{ .mem = Memory.moffs(ops.reg1, imm) },
	803	});
785	804	},
786	805	else => return emit.fail("TODO unused movabs variant", .{}),
787	806	}
...	...	@@ -791,63 +810,58 @@ fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
791	810	const tag = emit.mir.instructions.items(.tag)[inst];
792	811	assert(tag == .fisttp);
793	812	const ops = emit.mir.instructions.items(.ops)[inst].decode();
794
795		// the selecting between operand sizes for this particular `fisttp` instruction
796		// is done via opcode instead of the usual prefixes.
797
798		const opcode: Tag = switch (ops.flags) {
799		0b00 => .fisttp16,
800		0b01 => .fisttp32,
801		0b10 => .fisttp64,
	813	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	814	0b00 => .word,
	815	0b01 => .dword,
	816	0b10 => .qword,
802	817	else => unreachable,
803	818	};
804		const mem_or_reg = Memory{
805		.base = ops.reg1,
806		.disp = emit.mir.instructions.items(.data)[inst].imm,
807		.ptr_size = Memory.PtrSize.dword_ptr, // to prevent any prefix from being used
808		};
809		return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
	819	return emit.encode(.fisttp, .{
	820	.op1 = .{ .mem = Memory.sib(ptr_size, .{
	821	.base = ops.reg1,
	822	.disp = emit.mir.instructions.items(.data)[inst].disp,
	823	}) },
	824	});
810	825	}
811	826
812	827	fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
813	828	const tag = emit.mir.instructions.items(.tag)[inst];
814	829	assert(tag == .fld);
815	830	const ops = emit.mir.instructions.items(.ops)[inst].decode();
816
817		// the selecting between operand sizes for this particular `fisttp` instruction
818		// is done via opcode instead of the usual prefixes.
819
820		const opcode: Tag = switch (ops.flags) {
821		0b01 => .fld32,
822		0b10 => .fld64,
	831	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	832	0b01 => .dword,
	833	0b10 => .qword,
823	834	else => unreachable,
824	835	};
825		const mem_or_reg = Memory{
826		.base = ops.reg1,
827		.disp = emit.mir.instructions.items(.data)[inst].imm,
828		.ptr_size = Memory.PtrSize.dword_ptr, // to prevent any prefix from being used
829		};
830		return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
	836	return emit.encode(.fld, .{
	837	.op1 = .{ .mem = Memory.sib(ptr_size, .{
	838	.base = ops.reg1,
	839	.disp = emit.mir.instructions.items(.data)[inst].disp,
	840	}) },
	841	});
831	842	}
832	843
833		fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	844	fn mirShift(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
834	845	const ops = emit.mir.instructions.items(.ops)[inst].decode();
835	846	switch (ops.flags) {
836	847	0b00 => {
837		// sal reg1, 1
838		// M1
839		return lowerToM1Enc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	848	return emit.encode(mnemonic, .{
	849	.op1 = .{ .reg = ops.reg1 },
	850	.op2 = .{ .imm = 1 },
	851	});
840	852	},
841	853	0b01 => {
842		// sal reg1, .cl
843		// MC
844		return lowerToMcEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	854	return emit.encode(mnemonic, .{
	855	.op1 = .{ .reg = ops.reg1 },
	856	.op2 = .{ .reg = .cl },
	857	});
845	858	},
846	859	0b10 => {
847		// sal reg1, imm8
848		// MI
849	860	const imm = @truncate(u8, emit.mir.instructions.items(.data)[inst].imm);
850		return lowerToMiImm8Enc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
	861	return emit.encode(mnemonic, .{
	862	.op1 = .{ .reg = ops.reg1 },
	863	.op2 = .{ .imm = imm },
	864	});
851	865	},
852	866	0b11 => {
853	867	return emit.fail("TODO unused variant: SHIFT reg1, 0b11", .{});
...	...	@@ -855,24 +869,28 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
855	869	}
856	870	}
857	871
858		fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	872	fn mirMulDiv(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
859	873	const ops = emit.mir.instructions.items(.ops)[inst].decode();
860	874	if (ops.reg1 != .none) {
861	875	assert(ops.reg2 == .none);
862		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	876	return emit.encode(mnemonic, .{
	877	.op1 = .{ .reg = ops.reg1 },
	878	});
863	879	}
864	880	assert(ops.reg2 != .none);
865		const imm = emit.mir.instructions.items(.data)[inst].imm;
	881	const disp = emit.mir.instructions.items(.data)[inst].disp;
866	882	const ptr_size: Memory.PtrSize = switch (ops.flags) {
867		0b00 => .byte_ptr,
868		0b01 => .word_ptr,
869		0b10 => .dword_ptr,
870		0b11 => .qword_ptr,
	883	0b00 => .byte,
	884	0b01 => .word,
	885	0b10 => .dword,
	886	0b11 => .qword,
871	887	};
872		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
873		.disp = imm,
874		.base = ops.reg2,
875		}), emit.code);
	888	return emit.encode(mnemonic, .{
	889	.op1 = .{ .mem = Memory.sib(ptr_size, .{
	890	.base = ops.reg2,
	891	.disp = disp,
	892	}) },
	893	});
876	894	}
877	895
878	896	fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -881,40 +899,54 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
881	899	const ops = emit.mir.instructions.items(.ops)[inst].decode();
882	900	switch (ops.flags) {
883	901	0b00 => {
884		return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
	902	return emit.encode(.imul, .{
	903	.op1 = .{ .reg = ops.reg1 },
	904	.op2 = .{ .reg = ops.reg2 },
	905	});
885	906	},
886	907	0b01 => {
887		const imm = emit.mir.instructions.items(.data)[inst].imm;
	908	const disp = emit.mir.instructions.items(.data)[inst].disp;
888	909	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
889		return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
890		.disp = imm,
891		.base = src_reg,
892		}), emit.code);
	910	return emit.encode(.imul, .{
	911	.op1 = .{ .reg = ops.reg1 },
	912	.op2 = .{ .mem = Memory.sib(.qword, .{
	913	.base = src_reg,
	914	.disp = disp,
	915	}) },
	916	});
893	917	},
894	918	0b10 => {
895	919	const imm = emit.mir.instructions.items(.data)[inst].imm;
896		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code);
	920	return emit.encode(.imul, .{
	921	.op1 = .{ .reg = ops.reg1 },
	922	.op2 = .{ .reg = ops.reg2 },
	923	.op3 = .{ .imm = imm },
	924	});
897	925	},
898	926	0b11 => {
899	927	const payload = emit.mir.instructions.items(.data)[inst].payload;
900	928	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
901		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
902		.disp = imm_pair.dest_off,
903		.base = ops.reg2,
904		}), imm_pair.operand, emit.code);
	929	return emit.encode(.imul, .{
	930	.op1 = .{ .reg = ops.reg1 },
	931	.op2 = .{ .mem = Memory.sib(.qword, .{
	932	.base = ops.reg2,
	933	.disp = imm_pair.dest_off,
	934	}) },
	935	.op3 = .{ .imm = imm_pair.operand },
	936	});
905	937	},
906	938	}
907	939	}
908	940
909	941	fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
910	942	const ops = emit.mir.instructions.items(.ops)[inst].decode();
911		const tag: Tag = switch (ops.flags) {
	943	const mnemonic: Instruction.Mnemonic = switch (ops.flags) {
912	944	0b00 => .cbw,
913	945	0b01 => .cwd,
914	946	0b10 => .cdq,
915	947	0b11 => .cqo,
916	948	};
917		return lowerToZoEnc(tag, emit.code);
	949	return emit.encode(mnemonic, .{});
918	950	}
919	951
920	952	fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -923,30 +955,22 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
923	955	const ops = emit.mir.instructions.items(.ops)[inst].decode();
924	956	switch (ops.flags) {
925	957	0b00 => {
926		// lea reg1, [reg2 + imm32]
927		// RM
928		const imm = emit.mir.instructions.items(.data)[inst].imm;
	958	const disp = emit.mir.instructions.items(.data)[inst].disp;
929	959	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
930		return lowerToRmEnc(
931		.lea,
932		ops.reg1,
933		RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
934		.disp = imm,
	960	return emit.encode(.lea, .{
	961	.op1 = .{ .reg = ops.reg1 },
	962	.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg1.size()), .{
935	963	.base = src_reg,
936		}),
937		emit.code,
938		);
	964	.disp = disp,
	965	}) },
	966	});
939	967	},
940	968	0b01 => {
941		// lea reg1, [rip + imm32]
942		// RM
943	969	const start_offset = emit.code.items.len;
944		try lowerToRmEnc(
945		.lea,
946		ops.reg1,
947		RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
948		emit.code,
949		);
	970	try emit.encode(.lea, .{
	971	.op1 = .{ .reg = ops.reg1 },
	972	.op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromSize(ops.reg1.size()), 0) },
	973	});
950	974	const end_offset = emit.code.items.len;
951	975	// Backpatch the displacement
952	976	const payload = emit.mir.instructions.items(.data)[inst].payload;
...	...	@@ -955,24 +979,21 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
955	979	mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
956	980	},
957	981	0b10 => {
958		// lea reg, [rbp + index + imm32]
959	982	const payload = emit.mir.instructions.items(.data)[inst].payload;
960	983	const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
961	984	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
962		const scale_index = ScaleIndex{
	985	const scale_index = Memory.ScaleIndex{
963	986	.scale = 0,
964	987	.index = index_reg_disp.index,
965	988	};
966		return lowerToRmEnc(
967		.lea,
968		ops.reg1,
969		RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
970		.disp = index_reg_disp.disp,
	989	return emit.encode(.lea, .{
	990	.op1 = .{ .reg = ops.reg1 },
	991	.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg1.size()), .{
971	992	.base = src_reg,
972	993	.scale_index = scale_index,
973		}),
974		emit.code,
975		);
	994	.disp = index_reg_disp.disp,
	995	}) },
	996	});
976	997	},
977	998	0b11 => return emit.fail("TODO unused LEA variant 0b11", .{}),
978	999	}
...	...	@@ -989,14 +1010,10 @@ fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
989	1010	else => return emit.fail("TODO unused LEA PIC variant 0b11", .{}),
990	1011	}
991	1012
992		// lea reg1, [rip + reloc]
993		// RM
994		try lowerToRmEnc(
995		.lea,
996		ops.reg1,
997		RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
998		emit.code,
999		);
	1013	try emit.encode(.lea, .{
	1014	.op1 = .{ .reg = ops.reg1 },
	1015	.op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromSize(ops.reg1.size()), 0) },
	1016	});
1000	1017
1001	1018	const end_offset = emit.code.items.len;
1002	1019
...	...	@@ -1039,94 +1056,64 @@ fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1039	1056	}
1040	1057	}
1041	1058
1042		// SSE instructions
1043
1044		fn mirMovFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1045		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1046		switch (ops.flags) {
1047		0b00 => {
1048		const imm = emit.mir.instructions.items(.data)[inst].imm;
1049		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
1050		.disp = imm,
1051		.base = ops.reg2,
1052		}), emit.code);
1053		},
1054		0b01 => {
1055		const imm = emit.mir.instructions.items(.data)[inst].imm;
1056		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
1057		.disp = imm,
1058		.base = ops.reg1,
1059		}), ops.reg2, emit.code);
1060		},
1061		0b10 => {
1062		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1063		},
1064		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1065		}
1066		}
1067
1068		fn mirAddFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1069		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1070		switch (ops.flags) {
1071		0b00 => {
1072		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1073		},
1074		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1075		}
1076		}
1077
1078		fn mirCmpFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1079		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1080		switch (ops.flags) {
1081		0b00 => {
1082		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1083		},
1084		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1085		}
1086		}
1087		// AVX instructions
	1059	// SSE/AVX instructions
1088	1060
1089		fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	1061	fn mirMovFloat(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
1090	1062	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1091	1063	switch (ops.flags) {
1092	1064	0b00 => {
1093		const imm = emit.mir.instructions.items(.data)[inst].imm;
1094		return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
1095		.disp = imm,
1096		.base = ops.reg2,
1097		}), emit.code);
	1065	const disp = emit.mir.instructions.items(.data)[inst].disp;
	1066	return emit.encode(mnemonic, .{
	1067	.op1 = .{ .reg = ops.reg1 },
	1068	.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg2.size()), .{
	1069	.base = ops.reg2,
	1070	.disp = disp,
	1071	}) },
	1072	});
1098	1073	},
1099	1074	0b01 => {
1100		const imm = emit.mir.instructions.items(.data)[inst].imm;
1101		return lowerToMvEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
1102		.disp = imm,
1103		.base = ops.reg1,
1104		}), ops.reg2, emit.code);
	1075	const disp = emit.mir.instructions.items(.data)[inst].disp;
	1076	return emit.encode(mnemonic, .{
	1077	.op1 = .{ .mem = Memory.sib(Memory.PtrSize.fromSize(ops.reg1.size()), .{
	1078	.base = ops.reg1,
	1079	.disp = disp,
	1080	}) },
	1081	.op2 = .{ .reg = ops.reg2 },
	1082	});
1105	1083	},
1106	1084	0b10 => {
1107		return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
	1085	return emit.encode(mnemonic, .{
	1086	.op1 = .{ .reg = ops.reg1 },
	1087	.op2 = .{ .reg = ops.reg2 },
	1088	});
1108	1089	},
1109		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
	1090	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, mnemonic }),
1110	1091	}
1111	1092	}
1112	1093
1113		fn mirAddFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	1094	fn mirAddFloat(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
1114	1095	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1115	1096	switch (ops.flags) {
1116	1097	0b00 => {
1117		return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
	1098	return emit.encode(mnemonic, .{
	1099	.op1 = .{ .reg = ops.reg1 },
	1100	.op2 = .{ .reg = ops.reg2 },
	1101	});
1118	1102	},
1119		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
	1103	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, mnemonic }),
1120	1104	}
1121	1105	}
1122	1106
1123		fn mirCmpFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	1107	fn mirCmpFloat(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
1124	1108	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1125	1109	switch (ops.flags) {
1126	1110	0b00 => {
1127		return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
	1111	return emit.encode(mnemonic, .{
	1112	.op1 = .{ .reg = ops.reg1 },
	1113	.op2 = .{ .reg = ops.reg2 },
	1114	});
1128	1115	},
1129		else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
	1116	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, mnemonic }),
1130	1117	}
1131	1118	}
1132	1119
...	...	@@ -1139,7 +1126,9 @@ fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1139	1126
1140	1127	const offset = blk: {
1141	1128	// callq
1142		try lowerToDEnc(.call_near, 0, emit.code);
	1129	try emit.encode(.call, .{
	1130	.op1 = .{ .imm = 0 },
	1131	});
1143	1132	break :blk @intCast(u32, emit.code.items.len) - 4;
1144	1133	};
1145	1134
...	...	@@ -1264,1841 +1253,3 @@ fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1264	1253	.none => {},
1265	1254	}
1266	1255	}
1267
1268		const Tag = enum {
1269		adc,
1270		add,
1271		sub,
1272		xor,
1273		@"and",
1274		@"or",
1275		sbb,
1276		cmp,
1277		mov,
1278		movsx,
1279		movsxd,
1280		movzx,
1281		lea,
1282		jmp_near,
1283		call_near,
1284		push,
1285		pop,
1286		@"test",
1287		ud2,
1288		int3,
1289		nop,
1290		imul,
1291		mul,
1292		idiv,
1293		div,
1294		syscall,
1295		ret_near,
1296		ret_far,
1297		fisttp16,
1298		fisttp32,
1299		fisttp64,
1300		fld32,
1301		fld64,
1302		jo,
1303		jno,
1304		jb,
1305		jbe,
1306		jc,
1307		jnae,
1308		jnc,
1309		jae,
1310		je,
1311		jz,
1312		jne,
1313		jnz,
1314		jna,
1315		jnb,
1316		jnbe,
1317		ja,
1318		js,
1319		jns,
1320		jpe,
1321		jp,
1322		jpo,
1323		jnp,
1324		jnge,
1325		jl,
1326		jge,
1327		jnl,
1328		jle,
1329		jng,
1330		jg,
1331		jnle,
1332		seto,
1333		setno,
1334		setb,
1335		setc,
1336		setnae,
1337		setnb,
1338		setnc,
1339		setae,
1340		sete,
1341		setz,
1342		setne,
1343		setnz,
1344		setbe,
1345		setna,
1346		seta,
1347		setnbe,
1348		sets,
1349		setns,
1350		setp,
1351		setpe,
1352		setnp,
1353		setpo,
1354		setl,
1355		setnge,
1356		setnl,
1357		setge,
1358		setle,
1359		setng,
1360		setnle,
1361		setg,
1362		cmovo,
1363		cmovno,
1364		cmovb,
1365		cmovc,
1366		cmovnae,
1367		cmovnb,
1368		cmovnc,
1369		cmovae,
1370		cmove,
1371		cmovz,
1372		cmovne,
1373		cmovnz,
1374		cmovbe,
1375		cmovna,
1376		cmova,
1377		cmovnbe,
1378		cmovs,
1379		cmovns,
1380		cmovp,
1381		cmovpe,
1382		cmovnp,
1383		cmovpo,
1384		cmovl,
1385		cmovnge,
1386		cmovnl,
1387		cmovge,
1388		cmovle,
1389		cmovng,
1390		cmovnle,
1391		cmovg,
1392		shl,
1393		sal,
1394		shr,
1395		sar,
1396		cbw,
1397		cwd,
1398		cdq,
1399		cqo,
1400		movsd,
1401		movss,
1402		addsd,
1403		addss,
1404		cmpsd,
1405		cmpss,
1406		ucomisd,
1407		ucomiss,
1408		vmovsd,
1409		vmovss,
1410		vaddsd,
1411		vaddss,
1412		vcmpsd,
1413		vcmpss,
1414		vucomisd,
1415		vucomiss,
1416
1417		fn isSse(tag: Tag) bool {
1418		return switch (tag) {
1419		.movsd,
1420		.movss,
1421		.addsd,
1422		.addss,
1423		.cmpsd,
1424		.cmpss,
1425		.ucomisd,
1426		.ucomiss,
1427		=> true,
1428
1429		else => false,
1430		};
1431		}
1432
1433		fn isAvx(tag: Tag) bool {
1434		return switch (tag) {
1435		.vmovsd,
1436		.vmovss,
1437		.vaddsd,
1438		.vaddss,
1439		.vcmpsd,
1440		.vcmpss,
1441		.vucomisd,
1442		.vucomiss,
1443		=> true,
1444
1445		else => false,
1446		};
1447		}
1448
1449		fn isSetCC(tag: Tag) bool {
1450		return switch (tag) {
1451		.seto,
1452		.setno,
1453		.setb,
1454		.setc,
1455		.setnae,
1456		.setnb,
1457		.setnc,
1458		.setae,
1459		.sete,
1460		.setz,
1461		.setne,
1462		.setnz,
1463		.setbe,
1464		.setna,
1465		.seta,
1466		.setnbe,
1467		.sets,
1468		.setns,
1469		.setp,
1470		.setpe,
1471		.setnp,
1472		.setpo,
1473		.setl,
1474		.setnge,
1475		.setnl,
1476		.setge,
1477		.setle,
1478		.setng,
1479		.setnle,
1480		.setg,
1481		=> true,
1482		else => false,
1483		};
1484		}
1485		};
1486
1487		const Encoding = enum {
1488		/// OP
1489		zo,
1490
1491		/// OP rel32
1492		d,
1493
1494		/// OP r/m64
1495		m,
1496
1497		/// OP r64
1498		o,
1499
1500		/// OP imm32
1501		i,
1502
1503		/// OP r/m64, 1
1504		m1,
1505
1506		/// OP r/m64, .cl
1507		mc,
1508
1509		/// OP r/m64, imm32
1510		mi,
1511
1512		/// OP r/m64, imm8
1513		mi8,
1514
1515		/// OP r/m64, r64
1516		mr,
1517
1518		/// OP r64, r/m64
1519		rm,
1520
1521		/// OP r64, imm64
1522		oi,
1523
1524		/// OP al/ax/eax/rax, moffs
1525		fd,
1526
1527		/// OP moffs, al/ax/eax/rax
1528		td,
1529
1530		/// OP r64, r/m64, imm32
1531		rmi,
1532
1533		/// OP xmm1, xmm2/m64
1534		vm,
1535
1536		/// OP m64, xmm1
1537		mv,
1538
1539		/// OP xmm1, xmm2, xmm3/m64
1540		rvm,
1541
1542		/// OP xmm1, xmm2, xmm3/m64, imm8
1543		rvmi,
1544		};
1545
1546		const OpCode = struct {
1547		bytes: [3]u8,
1548		count: usize,
1549
1550		fn init(comptime in_bytes: []const u8) OpCode {
1551		comptime assert(in_bytes.len <= 3);
1552		comptime var bytes: [3]u8 = undefined;
1553		inline for (in_bytes, 0..) \|x, i\| {
1554		bytes[i] = x;
1555		}
1556		return .{ .bytes = bytes, .count = in_bytes.len };
1557		}
1558
1559		fn encode(opc: OpCode, encoder: Encoder) void {
1560		switch (opc.count) {
1561		1 => encoder.opcode_1byte(opc.bytes[0]),
1562		2 => encoder.opcode_2byte(opc.bytes[0], opc.bytes[1]),
1563		3 => encoder.opcode_3byte(opc.bytes[0], opc.bytes[1], opc.bytes[2]),
1564		else => unreachable,
1565		}
1566		}
1567
1568		fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1569		assert(opc.count == 1);
1570		encoder.opcode_withReg(opc.bytes[0], reg.lowEnc());
1571		}
1572		};
1573
1574		inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) OpCode {
1575		// zig fmt: off
1576		switch (enc) {
1577		.zo => return switch (tag) {
1578		.ret_near => OpCode.init(&.{0xc3}),
1579		.ret_far => OpCode.init(&.{0xcb}),
1580		.ud2 => OpCode.init(&.{ 0x0F, 0x0B }),
1581		.int3 => OpCode.init(&.{0xcc}),
1582		.nop => OpCode.init(&.{0x90}),
1583		.syscall => OpCode.init(&.{ 0x0f, 0x05 }),
1584		.cbw => OpCode.init(&.{0x98}),
1585		.cwd,
1586		.cdq,
1587		.cqo => OpCode.init(&.{0x99}),
1588		else => unreachable,
1589		},
1590		.d => return switch (tag) {
1591		.jmp_near => OpCode.init(&.{0xe9}),
1592		.call_near => OpCode.init(&.{0xe8}),
1593
1594		.jo => if (is_one_byte) OpCode.init(&.{0x70}) else OpCode.init(&.{0x0f,0x80}),
1595
1596		.jno => if (is_one_byte) OpCode.init(&.{0x71}) else OpCode.init(&.{0x0f,0x81}),
1597
1598		.jb,
1599		.jc,
1600		.jnae => if (is_one_byte) OpCode.init(&.{0x72}) else OpCode.init(&.{0x0f,0x82}),
1601
1602		.jnb,
1603		.jnc,
1604		.jae => if (is_one_byte) OpCode.init(&.{0x73}) else OpCode.init(&.{0x0f,0x83}),
1605
1606		.je,
1607		.jz => if (is_one_byte) OpCode.init(&.{0x74}) else OpCode.init(&.{0x0f,0x84}),
1608
1609		.jne,
1610		.jnz => if (is_one_byte) OpCode.init(&.{0x75}) else OpCode.init(&.{0x0f,0x85}),
1611
1612		.jna,
1613		.jbe => if (is_one_byte) OpCode.init(&.{0x76}) else OpCode.init(&.{0x0f,0x86}),
1614
1615		.jnbe,
1616		.ja => if (is_one_byte) OpCode.init(&.{0x77}) else OpCode.init(&.{0x0f,0x87}),
1617
1618		.js => if (is_one_byte) OpCode.init(&.{0x78}) else OpCode.init(&.{0x0f,0x88}),
1619
1620		.jns => if (is_one_byte) OpCode.init(&.{0x79}) else OpCode.init(&.{0x0f,0x89}),
1621
1622		.jpe,
1623		.jp => if (is_one_byte) OpCode.init(&.{0x7a}) else OpCode.init(&.{0x0f,0x8a}),
1624
1625		.jpo,
1626		.jnp => if (is_one_byte) OpCode.init(&.{0x7b}) else OpCode.init(&.{0x0f,0x8b}),
1627
1628		.jnge,
1629		.jl => if (is_one_byte) OpCode.init(&.{0x7c}) else OpCode.init(&.{0x0f,0x8c}),
1630
1631		.jge,
1632		.jnl => if (is_one_byte) OpCode.init(&.{0x7d}) else OpCode.init(&.{0x0f,0x8d}),
1633
1634		.jle,
1635		.jng => if (is_one_byte) OpCode.init(&.{0x7e}) else OpCode.init(&.{0x0f,0x8e}),
1636
1637		.jg,
1638		.jnle => if (is_one_byte) OpCode.init(&.{0x7f}) else OpCode.init(&.{0x0f,0x8f}),
1639
1640		else => unreachable,
1641		},
1642		.m => return switch (tag) {
1643		.jmp_near,
1644		.call_near,
1645		.push => OpCode.init(&.{0xff}),
1646
1647		.pop => OpCode.init(&.{0x8f}),
1648		.seto => OpCode.init(&.{0x0f,0x90}),
1649		.setno => OpCode.init(&.{0x0f,0x91}),
1650
1651		.setb,
1652		.setc,
1653		.setnae => OpCode.init(&.{0x0f,0x92}),
1654
1655		.setnb,
1656		.setnc,
1657		.setae => OpCode.init(&.{0x0f,0x93}),
1658
1659		.sete,
1660		.setz => OpCode.init(&.{0x0f,0x94}),
1661
1662		.setne,
1663		.setnz => OpCode.init(&.{0x0f,0x95}),
1664
1665		.setbe,
1666		.setna => OpCode.init(&.{0x0f,0x96}),
1667
1668		.seta,
1669		.setnbe => OpCode.init(&.{0x0f,0x97}),
1670
1671		.sets => OpCode.init(&.{0x0f,0x98}),
1672		.setns => OpCode.init(&.{0x0f,0x99}),
1673
1674		.setp,
1675		.setpe => OpCode.init(&.{0x0f,0x9a}),
1676
1677		.setnp,
1678		.setpo => OpCode.init(&.{0x0f,0x9b}),
1679
1680		.setl,
1681		.setnge => OpCode.init(&.{0x0f,0x9c}),
1682
1683		.setnl,
1684		.setge => OpCode.init(&.{0x0f,0x9d}),
1685
1686		.setle,
1687		.setng => OpCode.init(&.{0x0f,0x9e}),
1688
1689		.setnle,
1690		.setg => OpCode.init(&.{0x0f,0x9f}),
1691
1692		.idiv,
1693		.div,
1694		.imul,
1695		.mul => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1696
1697		.fisttp16 => OpCode.init(&.{0xdf}),
1698		.fisttp32 => OpCode.init(&.{0xdb}),
1699		.fisttp64 => OpCode.init(&.{0xdd}),
1700		.fld32 => OpCode.init(&.{0xd9}),
1701		.fld64 => OpCode.init(&.{0xdd}),
1702		else => unreachable,
1703		},
1704		.o => return switch (tag) {
1705		.push => OpCode.init(&.{0x50}),
1706		.pop => OpCode.init(&.{0x58}),
1707		else => unreachable,
1708		},
1709		.i => return switch (tag) {
1710		.push => if (is_one_byte) OpCode.init(&.{0x6a}) else OpCode.init(&.{0x68}),
1711		.@"test" => if (is_one_byte) OpCode.init(&.{0xa8}) else OpCode.init(&.{0xa9}),
1712		.ret_near => OpCode.init(&.{0xc2}),
1713		.ret_far => OpCode.init(&.{0xca}),
1714		else => unreachable,
1715		},
1716		.m1 => return switch (tag) {
1717		.shl, .sal,
1718		.shr, .sar => if (is_one_byte) OpCode.init(&.{0xd0}) else OpCode.init(&.{0xd1}),
1719		else => unreachable,
1720		},
1721		.mc => return switch (tag) {
1722		.shl, .sal,
1723		.shr, .sar => if (is_one_byte) OpCode.init(&.{0xd2}) else OpCode.init(&.{0xd3}),
1724		else => unreachable,
1725		},
1726		.mi => return switch (tag) {
1727		.adc, .add,
1728		.sub, .xor,
1729		.@"and", .@"or",
1730		.sbb, .cmp => if (is_one_byte) OpCode.init(&.{0x80}) else OpCode.init(&.{0x81}),
1731		.mov => if (is_one_byte) OpCode.init(&.{0xc6}) else OpCode.init(&.{0xc7}),
1732		.@"test" => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1733		else => unreachable,
1734		},
1735		.mi8 => return switch (tag) {
1736		.adc, .add,
1737		.sub, .xor,
1738		.@"and", .@"or",
1739		.sbb, .cmp => OpCode.init(&.{0x83}),
1740		.shl, .sal,
1741		.shr, .sar => if (is_one_byte) OpCode.init(&.{0xc0}) else OpCode.init(&.{0xc1}),
1742		else => unreachable,
1743		},
1744		.mr => return switch (tag) {
1745		.adc => if (is_one_byte) OpCode.init(&.{0x10}) else OpCode.init(&.{0x11}),
1746		.add => if (is_one_byte) OpCode.init(&.{0x00}) else OpCode.init(&.{0x01}),
1747		.sub => if (is_one_byte) OpCode.init(&.{0x28}) else OpCode.init(&.{0x29}),
1748		.xor => if (is_one_byte) OpCode.init(&.{0x30}) else OpCode.init(&.{0x31}),
1749		.@"and" => if (is_one_byte) OpCode.init(&.{0x20}) else OpCode.init(&.{0x21}),
1750		.@"or" => if (is_one_byte) OpCode.init(&.{0x08}) else OpCode.init(&.{0x09}),
1751		.sbb => if (is_one_byte) OpCode.init(&.{0x18}) else OpCode.init(&.{0x19}),
1752		.cmp => if (is_one_byte) OpCode.init(&.{0x38}) else OpCode.init(&.{0x39}),
1753		.mov => if (is_one_byte) OpCode.init(&.{0x88}) else OpCode.init(&.{0x89}),
1754		.@"test" => if (is_one_byte) OpCode.init(&.{0x84}) else OpCode.init(&.{0x85}),
1755		.movsd => OpCode.init(&.{0xf2,0x0f,0x11}),
1756		.movss => OpCode.init(&.{0xf3,0x0f,0x11}),
1757		else => unreachable,
1758		},
1759		.rm => return switch (tag) {
1760		.adc => if (is_one_byte) OpCode.init(&.{0x12}) else OpCode.init(&.{0x13}),
1761		.add => if (is_one_byte) OpCode.init(&.{0x02}) else OpCode.init(&.{0x03}),
1762		.sub => if (is_one_byte) OpCode.init(&.{0x2a}) else OpCode.init(&.{0x2b}),
1763		.xor => if (is_one_byte) OpCode.init(&.{0x32}) else OpCode.init(&.{0x33}),
1764		.@"and" => if (is_one_byte) OpCode.init(&.{0x22}) else OpCode.init(&.{0x23}),
1765		.@"or" => if (is_one_byte) OpCode.init(&.{0x0a}) else OpCode.init(&.{0x0b}),
1766		.sbb => if (is_one_byte) OpCode.init(&.{0x1a}) else OpCode.init(&.{0x1b}),
1767		.cmp => if (is_one_byte) OpCode.init(&.{0x3a}) else OpCode.init(&.{0x3b}),
1768		.mov => if (is_one_byte) OpCode.init(&.{0x8a}) else OpCode.init(&.{0x8b}),
1769		.movsx => if (is_one_byte) OpCode.init(&.{0x0f,0xbe}) else OpCode.init(&.{0x0f,0xbf}),
1770		.movsxd => OpCode.init(&.{0x63}),
1771		.movzx => if (is_one_byte) OpCode.init(&.{0x0f,0xb6}) else OpCode.init(&.{0x0f,0xb7}),
1772		.lea => if (is_one_byte) OpCode.init(&.{0x8c}) else OpCode.init(&.{0x8d}),
1773		.imul => OpCode.init(&.{0x0f,0xaf}),
1774
1775		.cmova,
1776		.cmovnbe, => OpCode.init(&.{0x0f,0x47}),
1777
1778		.cmovae,
1779		.cmovnb, => OpCode.init(&.{0x0f,0x43}),
1780
1781		.cmovb,
1782		.cmovc,
1783		.cmovnae => OpCode.init(&.{0x0f,0x42}),
1784
1785		.cmovbe,
1786		.cmovna, => OpCode.init(&.{0x0f,0x46}),
1787
1788		.cmove,
1789		.cmovz, => OpCode.init(&.{0x0f,0x44}),
1790
1791		.cmovg,
1792		.cmovnle, => OpCode.init(&.{0x0f,0x4f}),
1793
1794		.cmovge,
1795		.cmovnl, => OpCode.init(&.{0x0f,0x4d}),
1796
1797		.cmovl,
1798		.cmovnge, => OpCode.init(&.{0x0f,0x4c}),
1799
1800		.cmovle,
1801		.cmovng, => OpCode.init(&.{0x0f,0x4e}),
1802
1803		.cmovne,
1804		.cmovnz, => OpCode.init(&.{0x0f,0x45}),
1805
1806		.cmovno => OpCode.init(&.{0x0f,0x41}),
1807
1808		.cmovnp,
1809		.cmovpo, => OpCode.init(&.{0x0f,0x4b}),
1810
1811		.cmovns => OpCode.init(&.{0x0f,0x49}),
1812
1813		.cmovo => OpCode.init(&.{0x0f,0x40}),
1814
1815		.cmovp,
1816		.cmovpe, => OpCode.init(&.{0x0f,0x4a}),
1817
1818		.cmovs => OpCode.init(&.{0x0f,0x48}),
1819
1820		.movsd => OpCode.init(&.{0xf2,0x0f,0x10}),
1821		.movss => OpCode.init(&.{0xf3,0x0f,0x10}),
1822		.addsd => OpCode.init(&.{0xf2,0x0f,0x58}),
1823		.addss => OpCode.init(&.{0xf3,0x0f,0x58}),
1824		.ucomisd => OpCode.init(&.{0x66,0x0f,0x2e}),
1825		.ucomiss => OpCode.init(&.{0x0f,0x2e}),
1826		else => unreachable,
1827		},
1828		.oi => return switch (tag) {
1829		.mov => if (is_one_byte) OpCode.init(&.{0xb0}) else OpCode.init(&.{0xb8}),
1830		else => unreachable,
1831		},
1832		.fd => return switch (tag) {
1833		.mov => if (is_one_byte) OpCode.init(&.{0xa0}) else OpCode.init(&.{0xa1}),
1834		else => unreachable,
1835		},
1836		.td => return switch (tag) {
1837		.mov => if (is_one_byte) OpCode.init(&.{0xa2}) else OpCode.init(&.{0xa3}),
1838		else => unreachable,
1839		},
1840		.rmi => return switch (tag) {
1841		.imul => if (is_one_byte) OpCode.init(&.{0x6b}) else OpCode.init(&.{0x69}),
1842		else => unreachable,
1843		},
1844		.mv => return switch (tag) {
1845		.vmovsd,
1846		.vmovss => OpCode.init(&.{0x11}),
1847		else => unreachable,
1848		},
1849		.vm => return switch (tag) {
1850		.vmovsd,
1851		.vmovss => OpCode.init(&.{0x10}),
1852		.vucomisd,
1853		.vucomiss => OpCode.init(&.{0x2e}),
1854		else => unreachable,
1855		},
1856		.rvm => return switch (tag) {
1857		.vaddsd,
1858		.vaddss => OpCode.init(&.{0x58}),
1859		.vmovsd,
1860		.vmovss => OpCode.init(&.{0x10}),
1861		else => unreachable,
1862		},
1863		.rvmi => return switch (tag) {
1864		.vcmpsd,
1865		.vcmpss => OpCode.init(&.{0xc2}),
1866		else => unreachable,
1867		},
1868		}
1869		// zig fmt: on
1870		}
1871
1872		inline fn getModRmExt(tag: Tag) u3 {
1873		return switch (tag) {
1874		.adc => 0x2,
1875		.add => 0x0,
1876		.sub => 0x5,
1877		.xor => 0x6,
1878		.@"and" => 0x4,
1879		.@"or" => 0x1,
1880		.sbb => 0x3,
1881		.cmp => 0x7,
1882		.mov => 0x0,
1883		.jmp_near => 0x4,
1884		.call_near => 0x2,
1885		.push => 0x6,
1886		.pop => 0x0,
1887		.@"test" => 0x0,
1888		.seto,
1889		.setno,
1890		.setb,
1891		.setc,
1892		.setnae,
1893		.setnb,
1894		.setnc,
1895		.setae,
1896		.sete,
1897		.setz,
1898		.setne,
1899		.setnz,
1900		.setbe,
1901		.setna,
1902		.seta,
1903		.setnbe,
1904		.sets,
1905		.setns,
1906		.setp,
1907		.setpe,
1908		.setnp,
1909		.setpo,
1910		.setl,
1911		.setnge,
1912		.setnl,
1913		.setge,
1914		.setle,
1915		.setng,
1916		.setnle,
1917		.setg,
1918		=> 0x0,
1919		.shl,
1920		.sal,
1921		=> 0x4,
1922		.shr => 0x5,
1923		.sar => 0x7,
1924		.mul => 0x4,
1925		.imul => 0x5,
1926		.div => 0x6,
1927		.idiv => 0x7,
1928		.fisttp16 => 0x1,
1929		.fisttp32 => 0x1,
1930		.fisttp64 => 0x1,
1931		.fld32 => 0x0,
1932		.fld64 => 0x0,
1933		else => unreachable,
1934		};
1935		}
1936
1937		const VexEncoding = struct {
1938		prefix: Encoder.Vex,
1939		reg: ?enum {
1940		ndd,
1941		nds,
1942		dds,
1943		},
1944		};
1945
1946		inline fn getVexEncoding(tag: Tag, enc: Encoding) VexEncoding {
1947		const desc: struct {
1948		reg: enum {
1949		none,
1950		ndd,
1951		nds,
1952		dds,
1953		} = .none,
1954		len_256: bool = false,
1955		wig: bool = false,
1956		lig: bool = false,
1957		lz: bool = false,
1958		lead_opc: enum {
1959		l_0f,
1960		l_0f_3a,
1961		l_0f_38,
1962		} = .l_0f,
1963		simd_prefix: enum {
1964		none,
1965		p_66,
1966		p_f2,
1967		p_f3,
1968		} = .none,
1969		} = blk: {
1970		switch (enc) {
1971		.mv => switch (tag) {
1972		.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1973		.vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1974		else => unreachable,
1975		},
1976		.vm => switch (tag) {
1977		.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1978		.vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1979		.vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true },
1980		.vucomiss => break :blk .{ .lig = true, .wig = true },
1981		else => unreachable,
1982		},
1983		.rvm => switch (tag) {
1984		.vaddsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1985		.vaddss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1986		.vmovsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1987		.vmovss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1988		else => unreachable,
1989		},
1990		.rvmi => switch (tag) {
1991		.vcmpsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1992		.vcmpss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1993		else => unreachable,
1994		},
1995		else => unreachable,
1996		}
1997		};
1998
1999		var vex: Encoder.Vex = .{};
2000
2001		if (desc.len_256) vex.len_256();
2002		if (desc.wig) vex.wig();
2003		if (desc.lig) vex.lig();
2004		if (desc.lz) vex.lz();
2005
2006		switch (desc.lead_opc) {
2007		.l_0f => {},
2008		.l_0f_3a => vex.lead_opc_0f_3a(),
2009		.l_0f_38 => vex.lead_opc_0f_38(),
2010		}
2011
2012		switch (desc.simd_prefix) {
2013		.none => {},
2014		.p_66 => vex.simd_prefix_66(),
2015		.p_f2 => vex.simd_prefix_f2(),
2016		.p_f3 => vex.simd_prefix_f3(),
2017		}
2018
2019		return VexEncoding{ .prefix = vex, .reg = switch (desc.reg) {
2020		.none => null,
2021		.nds => .nds,
2022		.dds => .dds,
2023		.ndd => .ndd,
2024		} };
2025		}
2026
2027		const ScaleIndex = packed struct {
2028		scale: u2,
2029		index: Register,
2030		};
2031
2032		const Memory = struct {
2033		base: ?Register,
2034		rip: bool = false,
2035		disp: u32,
2036		ptr_size: PtrSize,
2037		scale_index: ?ScaleIndex = null,
2038
2039		const PtrSize = enum(u2) {
2040		byte_ptr = 0b00,
2041		word_ptr = 0b01,
2042		dword_ptr = 0b10,
2043		qword_ptr = 0b11,
2044
2045		fn new(bit_size: u64) PtrSize {
2046		return @intToEnum(PtrSize, math.log2_int(u4, @intCast(u4, @divExact(bit_size, 8))));
2047		}
2048
2049		/// Returns size in bits.
2050		fn size(ptr_size: PtrSize) u64 {
2051		return 8 * (math.powi(u8, 2, @enumToInt(ptr_size)) catch unreachable);
2052		}
2053		};
2054
2055		fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
2056		if (mem_op.base) \|base\| {
2057		const dst = base.lowEnc();
2058		const src = operand;
2059		if (dst == 4 or mem_op.scale_index != null) {
2060		if (mem_op.disp == 0 and dst != 5) {
2061		encoder.modRm_SIBDisp0(src);
2062		if (mem_op.scale_index) \|si\| {
2063		encoder.sib_scaleIndexBase(si.scale, si.index.lowEnc(), dst);
2064		} else {
2065		encoder.sib_base(dst);
2066		}
2067		} else if (immOpSize(mem_op.disp) == 8) {
2068		encoder.modRm_SIBDisp8(src);
2069		if (mem_op.scale_index) \|si\| {
2070		encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowEnc(), dst);
2071		} else {
2072		encoder.sib_baseDisp8(dst);
2073		}
2074		encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
2075		} else {
2076		encoder.modRm_SIBDisp32(src);
2077		if (mem_op.scale_index) \|si\| {
2078		encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowEnc(), dst);
2079		} else {
2080		encoder.sib_baseDisp32(dst);
2081		}
2082		encoder.disp32(@bitCast(i32, mem_op.disp));
2083		}
2084		} else {
2085		if (mem_op.disp == 0 and dst != 5) {
2086		encoder.modRm_indirectDisp0(src, dst);
2087		} else if (immOpSize(mem_op.disp) == 8) {
2088		encoder.modRm_indirectDisp8(src, dst);
2089		encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
2090		} else {
2091		encoder.modRm_indirectDisp32(src, dst);
2092		encoder.disp32(@bitCast(i32, mem_op.disp));
2093		}
2094		}
2095		} else {
2096		if (mem_op.rip) {
2097		encoder.modRm_RIPDisp32(operand);
2098		} else {
2099		encoder.modRm_SIBDisp0(operand);
2100		if (mem_op.scale_index) \|si\| {
2101		encoder.sib_scaleIndexDisp32(si.scale, si.index.lowEnc());
2102		} else {
2103		encoder.sib_disp32();
2104		}
2105		}
2106		encoder.disp32(@bitCast(i32, mem_op.disp));
2107		}
2108		}
2109
2110		/// Returns size in bits.
2111		fn size(memory: Memory) u64 {
2112		return memory.ptr_size.size();
2113		}
2114		};
2115
2116		fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
2117		switch (size) {
2118		8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
2119		16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
2120		32, 64 => encoder.imm32(@bitCast(i32, imm)),
2121		else => unreachable,
2122		}
2123		}
2124
2125		const RegisterOrMemory = union(enum) {
2126		register: Register,
2127		memory: Memory,
2128
2129		fn reg(register: Register) RegisterOrMemory {
2130		return .{ .register = register };
2131		}
2132
2133		fn mem(ptr_size: Memory.PtrSize, args: struct {
2134		disp: u32,
2135		base: ?Register = null,
2136		scale_index: ?ScaleIndex = null,
2137		}) RegisterOrMemory {
2138		return .{
2139		.memory = .{
2140		.base = args.base,
2141		.disp = args.disp,
2142		.ptr_size = ptr_size,
2143		.scale_index = args.scale_index,
2144		},
2145		};
2146		}
2147
2148		fn rip(ptr_size: Memory.PtrSize, disp: u32) RegisterOrMemory {
2149		return .{
2150		.memory = .{
2151		.base = null,
2152		.rip = true,
2153		.disp = disp,
2154		.ptr_size = ptr_size,
2155		},
2156		};
2157		}
2158
2159		/// Returns size in bits.
2160		fn size(reg_or_mem: RegisterOrMemory) u64 {
2161		return switch (reg_or_mem) {
2162		.register => \|register\| register.size(),
2163		.memory => \|memory\| memory.size(),
2164		};
2165		}
2166		};
2167
2168		fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
2169		assert(!tag.isAvx());
2170		const opc = getOpCode(tag, .zo, false);
2171		const encoder = try Encoder.init(code, 2);
2172		switch (tag) {
2173		.cqo => {
2174		encoder.rex(.{
2175		.w = true,
2176		});
2177		},
2178		else => {},
2179		}
2180		opc.encode(encoder);
2181		}
2182
2183		fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2184		assert(!tag.isAvx());
2185		if (tag == .ret_far or tag == .ret_near) {
2186		const encoder = try Encoder.init(code, 3);
2187		const opc = getOpCode(tag, .i, false);
2188		opc.encode(encoder);
2189		encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
2190		return;
2191		}
2192		const opc = getOpCode(tag, .i, immOpSize(imm) == 8);
2193		const encoder = try Encoder.init(code, 5);
2194		if (immOpSize(imm) == 16) {
2195		encoder.prefix16BitMode();
2196		}
2197		opc.encode(encoder);
2198		encodeImm(encoder, imm, immOpSize(imm));
2199		}
2200
2201		fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
2202		assert(!tag.isAvx());
2203		const opc = getOpCode(tag, .o, false);
2204		const encoder = try Encoder.init(code, 3);
2205		if (reg.size() == 16) {
2206		encoder.prefix16BitMode();
2207		}
2208		encoder.rex(.{
2209		.w = false,
2210		.b = reg.isExtended(),
2211		});
2212		opc.encodeWithReg(encoder, reg);
2213		}
2214
2215		fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2216		assert(!tag.isAvx());
2217		const opc = getOpCode(tag, .d, false);
2218		const encoder = try Encoder.init(code, 6);
2219		opc.encode(encoder);
2220		encoder.imm32(@bitCast(i32, imm));
2221		}
2222
2223		fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void {
2224		assert(!tag.isAvx());
2225		const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
2226		const modrm_ext = getModRmExt(tag);
2227		switch (reg_or_mem) {
2228		.register => \|reg\| {
2229		const encoder = try Encoder.init(code, 4);
2230		if (reg.size() == 16) {
2231		encoder.prefix16BitMode();
2232		}
2233		const wide = if (tag == .jmp_near) false else setRexWRegister(reg);
2234		encoder.rex(.{
2235		.w = wide,
2236		.b = reg.isExtended(),
2237		});
2238		opc.encode(encoder);
2239		encoder.modRm_direct(modrm_ext, reg.lowEnc());
2240		},
2241		.memory => \|mem_op\| {
2242		const encoder = try Encoder.init(code, 8);
2243		if (mem_op.ptr_size == .word_ptr) {
2244		encoder.prefix16BitMode();
2245		}
2246		if (mem_op.base) \|base\| {
2247		const wide = if (tag == .jmp_near) false else mem_op.ptr_size == .qword_ptr;
2248		encoder.rex(.{
2249		.w = wide,
2250		.b = base.isExtended(),
2251		.x = if (mem_op.scale_index) \|si\| si.index.isExtended() else false,
2252		});
2253		}
2254		opc.encode(encoder);
2255		mem_op.encode(encoder, modrm_ext);
2256		},
2257		}
2258		}
2259
2260		fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2261		return lowerToMxEnc(tag, reg_or_mem, .m, code);
2262		}
2263
2264		fn lowerToM1Enc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2265		return lowerToMxEnc(tag, reg_or_mem, .m1, code);
2266		}
2267
2268		fn lowerToMcEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2269		return lowerToMxEnc(tag, reg_or_mem, .mc, code);
2270		}
2271
2272		fn lowerToTdEnc(tag: Tag, moffs: u64, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
2273		return lowerToTdFdEnc(tag, reg, moffs, code, true);
2274		}
2275
2276		fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) InnerError!void {
2277		return lowerToTdFdEnc(tag, reg, moffs, code, false);
2278		}
2279
2280		fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void {
2281		assert(!tag.isAvx());
2282		const opc = if (td) getOpCode(tag, .td, reg.size() == 8) else getOpCode(tag, .fd, reg.size() == 8);
2283		const encoder = try Encoder.init(code, 10);
2284		if (reg.size() == 16) {
2285		encoder.prefix16BitMode();
2286		}
2287		encoder.rex(.{
2288		.w = setRexWRegister(reg),
2289		});
2290		opc.encode(encoder);
2291		switch (reg.size()) {
2292		8 => encoder.imm8(@bitCast(i8, @truncate(u8, moffs))),
2293		16 => encoder.imm16(@bitCast(i16, @truncate(u16, moffs))),
2294		32 => encoder.imm32(@bitCast(i32, @truncate(u32, moffs))),
2295		64 => encoder.imm64(moffs),
2296		else => unreachable,
2297		}
2298		}
2299
2300		fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void {
2301		assert(!tag.isAvx());
2302		const opc = getOpCode(tag, .oi, reg.size() == 8);
2303		const encoder = try Encoder.init(code, 10);
2304		if (reg.size() == 16) {
2305		encoder.prefix16BitMode();
2306		}
2307		encoder.rex(.{
2308		.w = setRexWRegister(reg),
2309		.b = reg.isExtended(),
2310		});
2311		opc.encodeWithReg(encoder, reg);
2312		switch (reg.size()) {
2313		8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
2314		16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
2315		32 => encoder.imm32(@bitCast(i32, @truncate(u32, imm))),
2316		64 => encoder.imm64(imm),
2317		else => unreachable,
2318		}
2319		}
2320
2321		fn lowerToMiXEnc(
2322		tag: Tag,
2323		reg_or_mem: RegisterOrMemory,
2324		imm: u32,
2325		enc: Encoding,
2326		code: *std.ArrayList(u8),
2327		) InnerError!void {
2328		assert(!tag.isAvx());
2329		const modrm_ext = getModRmExt(tag);
2330		const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
2331		switch (reg_or_mem) {
2332		.register => \|dst_reg\| {
2333		const encoder = try Encoder.init(code, 7);
2334		if (dst_reg.size() == 16) {
2335		// 0x66 prefix switches to the non-default size; here we assume a switch from
2336		// the default 32bits to 16bits operand-size.
2337		// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
2338		encoder.prefix16BitMode();
2339		}
2340		encoder.rex(.{
2341		.w = setRexWRegister(dst_reg),
2342		.b = dst_reg.isExtended(),
2343		});
2344		opc.encode(encoder);
2345		encoder.modRm_direct(modrm_ext, dst_reg.lowEnc());
2346		encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_reg.size());
2347		},
2348		.memory => \|dst_mem\| {
2349		const encoder = try Encoder.init(code, 12);
2350		if (dst_mem.ptr_size == .word_ptr) {
2351		encoder.prefix16BitMode();
2352		}
2353		if (dst_mem.base) \|base\| {
2354		encoder.rex(.{
2355		.w = dst_mem.ptr_size == .qword_ptr,
2356		.b = base.isExtended(),
2357		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2358		});
2359		} else {
2360		encoder.rex(.{
2361		.w = dst_mem.ptr_size == .qword_ptr,
2362		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2363		});
2364		}
2365		opc.encode(encoder);
2366		dst_mem.encode(encoder, modrm_ext);
2367		encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_mem.ptr_size.size());
2368		},
2369		}
2370		}
2371
2372		fn lowerToMiImm8Enc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u8, code: *std.ArrayList(u8)) InnerError!void {
2373		return lowerToMiXEnc(tag, reg_or_mem, imm, .mi8, code);
2374		}
2375
2376		fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2377		return lowerToMiXEnc(tag, reg_or_mem, imm, .mi, code);
2378		}
2379
2380		fn lowerToRmEnc(
2381		tag: Tag,
2382		reg: Register,
2383		reg_or_mem: RegisterOrMemory,
2384		code: *std.ArrayList(u8),
2385		) InnerError!void {
2386		assert(!tag.isAvx());
2387		const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8);
2388		switch (reg_or_mem) {
2389		.register => \|src_reg\| {
2390		const encoder = try Encoder.init(code, 5);
2391		if (reg.size() == 16) {
2392		encoder.prefix16BitMode();
2393		}
2394		encoder.rex(.{
2395		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
2396		.r = reg.isExtended(),
2397		.b = src_reg.isExtended(),
2398		});
2399		opc.encode(encoder);
2400		encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2401		},
2402		.memory => \|src_mem\| {
2403		const encoder = try Encoder.init(code, 9);
2404		if (reg.size() == 16) {
2405		encoder.prefix16BitMode();
2406		}
2407		if (src_mem.base) \|base\| {
2408		// TODO handle 32-bit base register - requires prefix 0x67
2409		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
2410		encoder.rex(.{
2411		.w = setRexWRegister(reg),
2412		.r = reg.isExtended(),
2413		.b = base.isExtended(),
2414		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2415		});
2416		} else {
2417		encoder.rex(.{
2418		.w = setRexWRegister(reg),
2419		.r = reg.isExtended(),
2420		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2421		});
2422		}
2423		opc.encode(encoder);
2424		src_mem.encode(encoder, reg.lowEnc());
2425		},
2426		}
2427		}
2428
2429		fn lowerToMrEnc(
2430		tag: Tag,
2431		reg_or_mem: RegisterOrMemory,
2432		reg: Register,
2433		code: *std.ArrayList(u8),
2434		) InnerError!void {
2435		assert(!tag.isAvx());
2436		const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8);
2437		switch (reg_or_mem) {
2438		.register => \|dst_reg\| {
2439		const encoder = try Encoder.init(code, 4);
2440		if (dst_reg.size() == 16) {
2441		encoder.prefix16BitMode();
2442		}
2443		encoder.rex(.{
2444		.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
2445		.r = reg.isExtended(),
2446		.b = dst_reg.isExtended(),
2447		});
2448		opc.encode(encoder);
2449		encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2450		},
2451		.memory => \|dst_mem\| {
2452		const encoder = try Encoder.init(code, 9);
2453		if (reg.size() == 16) {
2454		encoder.prefix16BitMode();
2455		}
2456		if (dst_mem.base) \|base\| {
2457		encoder.rex(.{
2458		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2459		.r = reg.isExtended(),
2460		.b = base.isExtended(),
2461		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2462		});
2463		} else {
2464		encoder.rex(.{
2465		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2466		.r = reg.isExtended(),
2467		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2468		});
2469		}
2470		opc.encode(encoder);
2471		dst_mem.encode(encoder, reg.lowEnc());
2472		},
2473		}
2474		}
2475
2476		fn lowerToRmiEnc(
2477		tag: Tag,
2478		reg: Register,
2479		reg_or_mem: RegisterOrMemory,
2480		imm: u32,
2481		code: *std.ArrayList(u8),
2482		) InnerError!void {
2483		assert(!tag.isAvx());
2484		const opc = getOpCode(tag, .rmi, false);
2485		const encoder = try Encoder.init(code, 13);
2486		if (reg.size() == 16) {
2487		encoder.prefix16BitMode();
2488		}
2489		switch (reg_or_mem) {
2490		.register => \|src_reg\| {
2491		encoder.rex(.{
2492		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
2493		.r = reg.isExtended(),
2494		.b = src_reg.isExtended(),
2495		});
2496		opc.encode(encoder);
2497		encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2498		},
2499		.memory => \|src_mem\| {
2500		if (src_mem.base) \|base\| {
2501		// TODO handle 32-bit base register - requires prefix 0x67
2502		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
2503		encoder.rex(.{
2504		.w = setRexWRegister(reg),
2505		.r = reg.isExtended(),
2506		.b = base.isExtended(),
2507		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2508		});
2509		} else {
2510		encoder.rex(.{
2511		.w = setRexWRegister(reg),
2512		.r = reg.isExtended(),
2513		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2514		});
2515		}
2516		opc.encode(encoder);
2517		src_mem.encode(encoder, reg.lowEnc());
2518		},
2519		}
2520		encodeImm(encoder, imm, reg.size());
2521		}
2522
2523		/// Also referred to as XM encoding in Intel manual.
2524		fn lowerToVmEnc(
2525		tag: Tag,
2526		reg: Register,
2527		reg_or_mem: RegisterOrMemory,
2528		code: *std.ArrayList(u8),
2529		) InnerError!void {
2530		const opc = getOpCode(tag, .vm, false);
2531		var enc = getVexEncoding(tag, .vm);
2532		const vex = &enc.prefix;
2533		switch (reg_or_mem) {
2534		.register => \|src_reg\| {
2535		const encoder = try Encoder.init(code, 5);
2536		vex.rex(.{
2537		.r = reg.isExtended(),
2538		.b = src_reg.isExtended(),
2539		});
2540		encoder.vex(enc.prefix);
2541		opc.encode(encoder);
2542		encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2543		},
2544		.memory => \|src_mem\| {
2545		const encoder = try Encoder.init(code, 10);
2546		if (src_mem.base) \|base\| {
2547		vex.rex(.{
2548		.r = reg.isExtended(),
2549		.b = base.isExtended(),
2550		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2551		});
2552		} else {
2553		vex.rex(.{
2554		.r = reg.isExtended(),
2555		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2556		});
2557		}
2558		encoder.vex(enc.prefix);
2559		opc.encode(encoder);
2560		src_mem.encode(encoder, reg.lowEnc());
2561		},
2562		}
2563		}
2564
2565		/// Usually referred to as MR encoding with V/V in Intel manual.
2566		fn lowerToMvEnc(
2567		tag: Tag,
2568		reg_or_mem: RegisterOrMemory,
2569		reg: Register,
2570		code: *std.ArrayList(u8),
2571		) InnerError!void {
2572		const opc = getOpCode(tag, .mv, false);
2573		var enc = getVexEncoding(tag, .mv);
2574		const vex = &enc.prefix;
2575		switch (reg_or_mem) {
2576		.register => \|dst_reg\| {
2577		const encoder = try Encoder.init(code, 4);
2578		vex.rex(.{
2579		.r = reg.isExtended(),
2580		.b = dst_reg.isExtended(),
2581		});
2582		encoder.vex(enc.prefix);
2583		opc.encode(encoder);
2584		encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2585		},
2586		.memory => \|dst_mem\| {
2587		const encoder = try Encoder.init(code, 10);
2588		if (dst_mem.base) \|base\| {
2589		vex.rex(.{
2590		.r = reg.isExtended(),
2591		.b = base.isExtended(),
2592		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2593		});
2594		} else {
2595		vex.rex(.{
2596		.r = reg.isExtended(),
2597		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2598		});
2599		}
2600		encoder.vex(enc.prefix);
2601		opc.encode(encoder);
2602		dst_mem.encode(encoder, reg.lowEnc());
2603		},
2604		}
2605		}
2606
2607		fn lowerToRvmEnc(
2608		tag: Tag,
2609		reg1: Register,
2610		reg2: Register,
2611		reg_or_mem: RegisterOrMemory,
2612		code: *std.ArrayList(u8),
2613		) InnerError!void {
2614		const opc = getOpCode(tag, .rvm, false);
2615		var enc = getVexEncoding(tag, .rvm);
2616		const vex = &enc.prefix;
2617		switch (reg_or_mem) {
2618		.register => \|reg3\| {
2619		if (enc.reg) \|vvvv\| {
2620		switch (vvvv) {
2621		.nds => vex.reg(reg2.enc()),
2622		else => unreachable, // TODO
2623		}
2624		}
2625		const encoder = try Encoder.init(code, 5);
2626		vex.rex(.{
2627		.r = reg1.isExtended(),
2628		.b = reg3.isExtended(),
2629		});
2630		encoder.vex(enc.prefix);
2631		opc.encode(encoder);
2632		encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2633		},
2634		.memory => \|dst_mem\| {
2635		_ = dst_mem;
2636		unreachable; // TODO
2637		},
2638		}
2639		}
2640
2641		fn lowerToRvmiEnc(
2642		tag: Tag,
2643		reg1: Register,
2644		reg2: Register,
2645		reg_or_mem: RegisterOrMemory,
2646		imm: u32,
2647		code: *std.ArrayList(u8),
2648		) InnerError!void {
2649		const opc = getOpCode(tag, .rvmi, false);
2650		var enc = getVexEncoding(tag, .rvmi);
2651		const vex = &enc.prefix;
2652		const encoder: Encoder = blk: {
2653		switch (reg_or_mem) {
2654		.register => \|reg3\| {
2655		if (enc.reg) \|vvvv\| {
2656		switch (vvvv) {
2657		.nds => vex.reg(reg2.enc()),
2658		else => unreachable, // TODO
2659		}
2660		}
2661		const encoder = try Encoder.init(code, 5);
2662		vex.rex(.{
2663		.r = reg1.isExtended(),
2664		.b = reg3.isExtended(),
2665		});
2666		encoder.vex(enc.prefix);
2667		opc.encode(encoder);
2668		encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2669		break :blk encoder;
2670		},
2671		.memory => \|dst_mem\| {
2672		_ = dst_mem;
2673		unreachable; // TODO
2674		},
2675		}
2676		};
2677		encodeImm(encoder, imm, 8); // TODO
2678		}
2679
2680		fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
2681		assert(expected.len > 0);
2682		if (mem.eql(u8, expected, given)) return;
2683		const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
2684		defer testing.allocator.free(expected_fmt);
2685		const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
2686		defer testing.allocator.free(given_fmt);
2687		const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
2688		var padding = try testing.allocator.alloc(u8, idx + 5);
2689		defer testing.allocator.free(padding);
2690		mem.set(u8, padding, ' ');
2691		std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
2692		assembly,
2693		expected_fmt,
2694		given_fmt,
2695		padding,
2696		});
2697		return error.TestFailed;
2698		}
2699
2700		const TestEmit = struct {
2701		code_buffer: std.ArrayList(u8),
2702		next: usize = 0,
2703
2704		fn init() TestEmit {
2705		return .{
2706		.code_buffer = std.ArrayList(u8).init(testing.allocator),
2707		};
2708		}
2709
2710		fn deinit(emit: *TestEmit) void {
2711		emit.code_buffer.deinit();
2712		emit.next = undefined;
2713		}
2714
2715		fn code(emit: TestEmit) std.ArrayList(u8) {
2716		emit.next = emit.code_buffer.items.len;
2717		return &emit.code_buffer;
2718		}
2719
2720		fn lowered(emit: TestEmit) []const u8 {
2721		return emit.code_buffer.items[emit.next..];
2722		}
2723		};
2724
2725		test "lower MI encoding" {
2726		var emit = TestEmit.init();
2727		defer emit.deinit();
2728		try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, emit.code());
2729		try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", emit.lowered(), "mov rax, 0x10");
2730		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, emit.code());
2731		try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", emit.lowered(), "mov dword ptr [r11 + 0], 0x10");
2732		try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{
2733		.disp = @bitCast(u32, @as(i32, -8)),
2734		.base = .rdx,
2735		}), 0x10, emit.code());
2736		try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", emit.lowered(), "add dword ptr [rdx - 8], 0x10");
2737		try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
2738		.disp = 0x10000000,
2739		.base = .r11,
2740		}), 0x10, emit.code());
2741		try expectEqualHexStrings(
2742		"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
2743		emit.lowered(),
2744		"sub dword ptr [r11 + 0x10000000], 0x10",
2745		);
2746		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, emit.code());
2747		try expectEqualHexStrings(
2748		"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
2749		emit.lowered(),
2750		"and dword ptr [ds:0x10000000], 0x10",
2751		);
2752		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
2753		.disp = 0x10000000,
2754		.base = .r12,
2755		}), 0x10, emit.code());
2756		try expectEqualHexStrings(
2757		"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
2758		emit.lowered(),
2759		"and dword ptr [r12 + 0x10000000], 0x10",
2760		);
2761		try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, emit.code());
2762		try expectEqualHexStrings(
2763		"\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
2764		emit.lowered(),
2765		"mov qword ptr [rip + 0x10], 0x10",
2766		);
2767		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2768		.disp = @bitCast(u32, @as(i32, -8)),
2769		.base = .rbp,
2770		}), 0x10, emit.code());
2771		try expectEqualHexStrings(
2772		"\x48\xc7\x45\xf8\x10\x00\x00\x00",
2773		emit.lowered(),
2774		"mov qword ptr [rbp - 8], 0x10",
2775		);
2776		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{
2777		.disp = @bitCast(u32, @as(i32, -2)),
2778		.base = .rbp,
2779		}), 0x10, emit.code());
2780		try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", emit.lowered(), "mov word ptr [rbp - 2], 0x10");
2781		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{
2782		.disp = @bitCast(u32, @as(i32, -1)),
2783		.base = .rbp,
2784		}), 0x10, emit.code());
2785		try expectEqualHexStrings("\xC6\x45\xFF\x10", emit.lowered(), "mov byte ptr [rbp - 1], 0x10");
2786		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2787		.disp = 0x10000000,
2788		.scale_index = .{
2789		.scale = 1,
2790		.index = .rcx,
2791		},
2792		}), 0x10, emit.code());
2793		try expectEqualHexStrings(
2794		"\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
2795		emit.lowered(),
2796		"mov qword ptr [rcx*2 + 0x10000000], 0x10",
2797		);
2798
2799		try lowerToMiImm8Enc(.add, RegisterOrMemory.reg(.rax), 0x10, emit.code());
2800		try expectEqualHexStrings("\x48\x83\xC0\x10", emit.lowered(), "add rax, 0x10");
2801		}
2802
2803		test "lower RM encoding" {
2804		var emit = TestEmit.init();
2805		defer emit.deinit();
2806		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code());
2807		try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx");
2808		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
2809		try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]");
2810		try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
2811		try expectEqualHexStrings(
2812		"\x4C\x03\x1C\x25\x00\x00\x00\x10",
2813		emit.lowered(),
2814		"add r11, qword ptr [ds:0x10000000]",
2815		);
2816		try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
2817		try expectEqualHexStrings(
2818		"\x44\x02\x24\x25\x00\x00\x00\x10",
2819		emit.lowered(),
2820		"add r11b, byte ptr [ds:0x10000000]",
2821		);
2822		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2823		.disp = 0x10000000,
2824		.base = .r13,
2825		}), emit.code());
2826		try expectEqualHexStrings(
2827		"\x4D\x2B\x9D\x00\x00\x00\x10",
2828		emit.lowered(),
2829		"sub r11, qword ptr [r13 + 0x10000000]",
2830		);
2831		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2832		.disp = 0x10000000,
2833		.base = .r12,
2834		}), emit.code());
2835		try expectEqualHexStrings(
2836		"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
2837		emit.lowered(),
2838		"sub r11, qword ptr [r12 + 0x10000000]",
2839		);
2840		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2841		.disp = @bitCast(u32, @as(i32, -4)),
2842		.base = .rbp,
2843		}), emit.code());
2844		try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]");
2845		try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2846		try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]");
2847		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2848		.disp = @bitCast(u32, @as(i32, -8)),
2849		.base = .rbp,
2850		.scale_index = .{
2851		.scale = 0,
2852		.index = .rcx,
2853		},
2854		}), emit.code());
2855		try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
2856		try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2857		.disp = @bitCast(u32, @as(i32, -4)),
2858		.base = .rbp,
2859		.scale_index = .{
2860		.scale = 2,
2861		.index = .rdx,
2862		},
2863		}), emit.code());
2864		try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
2865		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2866		.disp = @bitCast(u32, @as(i32, -8)),
2867		.base = .rbp,
2868		.scale_index = .{
2869		.scale = 3,
2870		.index = .rcx,
2871		},
2872		}), emit.code());
2873		try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
2874		try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
2875		.disp = @bitCast(u32, @as(i32, -24)),
2876		.base = .rsi,
2877		.scale_index = .{
2878		.scale = 0,
2879		.index = .rcx,
2880		},
2881		}), emit.code());
2882		try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
2883		try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
2884		.disp = 0,
2885		.base = .rbp,
2886		.scale_index = .{
2887		.scale = 0,
2888		.index = .rcx,
2889		},
2890		}), emit.code());
2891		try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
2892		}
2893
2894		test "lower MR encoding" {
2895		var emit = TestEmit.init();
2896		defer emit.deinit();
2897		try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code());
2898		try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx");
2899		try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2900		.disp = @bitCast(u32, @as(i32, -4)),
2901		.base = .rbp,
2902		}), .r11, emit.code());
2903		try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11");
2904		try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code());
2905		try expectEqualHexStrings(
2906		"\x44\x00\x24\x25\x00\x00\x00\x10",
2907		emit.lowered(),
2908		"add byte ptr [ds:0x10000000], r12b",
2909		);
2910		try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code());
2911		try expectEqualHexStrings(
2912		"\x44\x01\x24\x25\x00\x00\x00\x10",
2913		emit.lowered(),
2914		"add dword ptr [ds:0x10000000], r12d",
2915		);
2916		try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
2917		.disp = 0x10000000,
2918		.base = .r11,
2919		}), .r12, emit.code());
2920		try expectEqualHexStrings(
2921		"\x4D\x29\xA3\x00\x00\x00\x10",
2922		emit.lowered(),
2923		"sub qword ptr [r11 + 0x10000000], r12",
2924		);
2925		try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code());
2926		try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12");
2927		}
2928
2929		test "lower OI encoding" {
2930		var emit = TestEmit.init();
2931		defer emit.deinit();
2932		try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code());
2933		try expectEqualHexStrings(
2934		"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
2935		emit.lowered(),
2936		"movabs rax, 0x1000000000000000",
2937		);
2938		try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code());
2939		try expectEqualHexStrings(
2940		"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
2941		emit.lowered(),
2942		"movabs r11, 0x1000000000000000",
2943		);
2944		try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code());
2945		try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000");
2946		try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code());
2947		try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000");
2948		try lowerToOiEnc(.mov, .r11b, 0x10, emit.code());
2949		try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10");
2950		}
2951
2952		test "lower FD/TD encoding" {
2953		var emit = TestEmit.init();
2954		defer emit.deinit();
2955		try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code());
2956		try expectEqualHexStrings(
2957		"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2958		emit.lowered(),
2959		"mov rax, ds:0x1000000000000000",
2960		);
2961		try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code());
2962		try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000");
2963		try lowerToFdEnc(.mov, .ax, 0x1000, emit.code());
2964		try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000");
2965		try lowerToFdEnc(.mov, .al, 0x10, emit.code());
2966		try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10");
2967		}
2968
2969		test "lower M encoding" {
2970		var emit = TestEmit.init();
2971		defer emit.deinit();
2972		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), emit.code());
2973		try expectEqualHexStrings("\x41\xFF\xE4", emit.lowered(), "jmp r12");
2974		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), emit.code());
2975		try expectEqualHexStrings("\x66\x41\xFF\xE4", emit.lowered(), "jmp r12w");
2976		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2977		try expectEqualHexStrings("\x41\xFF\x24\x24", emit.lowered(), "jmp qword ptr [r12]");
2978		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2979		try expectEqualHexStrings("\x66\x41\xFF\x24\x24", emit.lowered(), "jmp word ptr [r12]");
2980		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), emit.code());
2981		try expectEqualHexStrings("\x41\xFF\x64\x24\x10", emit.lowered(), "jmp qword ptr [r12 + 0x10]");
2982		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2983		.disp = 0x1000,
2984		.base = .r12,
2985		}), emit.code());
2986		try expectEqualHexStrings(
2987		"\x41\xFF\xA4\x24\x00\x10\x00\x00",
2988		emit.lowered(),
2989		"jmp qword ptr [r12 + 0x1000]",
2990		);
2991		try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2992		try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [rip + 0x10]");
2993		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), emit.code());
2994		try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [ds:0x10]");
2995		try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), emit.code());
2996		try expectEqualHexStrings("\x41\x0F\x97\xC3", emit.lowered(), "seta r11b");
2997		try lowerToMEnc(.idiv, RegisterOrMemory.reg(.rax), emit.code());
2998		try expectEqualHexStrings("\x48\xF7\xF8", emit.lowered(), "idiv rax");
2999		try lowerToMEnc(.imul, RegisterOrMemory.reg(.al), emit.code());
3000		try expectEqualHexStrings("\xF6\xE8", emit.lowered(), "imul al");
3001		}
3002
3003		test "lower M1 and MC encodings" {
3004		var emit = TestEmit.init();
3005		defer emit.deinit();
3006		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12), emit.code());
3007		try expectEqualHexStrings("\x49\xD1\xE4", emit.lowered(), "sal r12, 1");
3008		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12d), emit.code());
3009		try expectEqualHexStrings("\x41\xD1\xE4", emit.lowered(), "sal r12d, 1");
3010		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12w), emit.code());
3011		try expectEqualHexStrings("\x66\x41\xD1\xE4", emit.lowered(), "sal r12w, 1");
3012		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12b), emit.code());
3013		try expectEqualHexStrings("\x41\xD0\xE4", emit.lowered(), "sal r12b, 1");
3014		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.rax), emit.code());
3015		try expectEqualHexStrings("\x48\xD1\xE0", emit.lowered(), "sal rax, 1");
3016		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.eax), emit.code());
3017		try expectEqualHexStrings("\xD1\xE0", emit.lowered(), "sal eax, 1");
3018		try lowerToM1Enc(.sal, RegisterOrMemory.mem(.qword_ptr, .{
3019		.disp = @bitCast(u32, @as(i32, -0x10)),
3020		.base = .rbp,
3021		}), emit.code());
3022		try expectEqualHexStrings("\x48\xD1\x65\xF0", emit.lowered(), "sal qword ptr [rbp - 0x10], 1");
3023		try lowerToM1Enc(.sal, RegisterOrMemory.mem(.dword_ptr, .{
3024		.disp = @bitCast(u32, @as(i32, -0x10)),
3025		.base = .rbp,
3026		}), emit.code());
3027		try expectEqualHexStrings("\xD1\x65\xF0", emit.lowered(), "sal dword ptr [rbp - 0x10], 1");
3028
3029		try lowerToMcEnc(.shr, RegisterOrMemory.reg(.r12), emit.code());
3030		try expectEqualHexStrings("\x49\xD3\xEC", emit.lowered(), "shr r12, cl");
3031		try lowerToMcEnc(.shr, RegisterOrMemory.reg(.rax), emit.code());
3032		try expectEqualHexStrings("\x48\xD3\xE8", emit.lowered(), "shr rax, cl");
3033
3034		try lowerToMcEnc(.sar, RegisterOrMemory.reg(.rsi), emit.code());
3035		try expectEqualHexStrings("\x48\xD3\xFE", emit.lowered(), "sar rsi, cl");
3036		}
3037
3038		test "lower O encoding" {
3039		var emit = TestEmit.init();
3040		defer emit.deinit();
3041		try lowerToOEnc(.pop, .r12, emit.code());
3042		try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12");
3043		try lowerToOEnc(.push, .r12w, emit.code());
3044		try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w");
3045		}
3046
3047		test "lower RMI encoding" {
3048		var emit = TestEmit.init();
3049		defer emit.deinit();
3050		try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
3051		.disp = @bitCast(u32, @as(i32, -8)),
3052		.base = .rbp,
3053		}), 0x10, emit.code());
3054		try expectEqualHexStrings(
3055		"\x48\x69\x45\xF8\x10\x00\x00\x00",
3056		emit.lowered(),
3057		"imul rax, qword ptr [rbp - 8], 0x10",
3058		);
3059		try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
3060		.disp = @bitCast(u32, @as(i32, -4)),
3061		.base = .rbp,
3062		}), 0x10, emit.code());
3063		try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
3064		try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
3065		.disp = @bitCast(u32, @as(i32, -2)),
3066		.base = .rbp,
3067		}), 0x10, emit.code());
3068		try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
3069		try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code());
3070		try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10");
3071		try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
3072		try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
3073		}
3074
3075		test "lower MV encoding" {
3076		var emit = TestEmit.init();
3077		defer emit.deinit();
3078		try lowerToMvEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), .xmm1, emit.code());
3079		try expectEqualHexStrings(
3080		"\xC5\xFB\x11\x0D\x10\x00\x00\x00",
3081		emit.lowered(),
3082		"vmovsd qword ptr [rip + 0x10], xmm1",
3083		);
3084		}
3085
3086		test "lower VM encoding" {
3087		var emit = TestEmit.init();
3088		defer emit.deinit();
3089		try lowerToVmEnc(.vmovsd, .xmm1, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
3090		try expectEqualHexStrings(
3091		"\xC5\xFB\x10\x0D\x10\x00\x00\x00",
3092		emit.lowered(),
3093		"vmovsd xmm1, qword ptr [rip + 0x10]",
3094		);
3095		}
3096
3097		test "lower to RVM encoding" {
3098		var emit = TestEmit.init();
3099		defer emit.deinit();
3100		try lowerToRvmEnc(.vaddsd, .xmm0, .xmm1, RegisterOrMemory.reg(.xmm2), emit.code());
3101		try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2");
3102		try lowerToRvmEnc(.vaddsd, .xmm0, .xmm0, RegisterOrMemory.reg(.xmm1), emit.code());
3103		try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1");
3104		}

src/arch/x86_64/Encoding.zig created+521

...	...	@@ -0,0 +1,521 @@
	1	const Encoding = @This();
	2
	3	const std = @import("std");
	4	const assert = std.debug.assert;
	5	const math = std.math;
	6
	7	const bits = @import("bits.zig");
	8	const encoder = @import("encoder.zig");
	9	const Instruction = encoder.Instruction;
	10	const Register = bits.Register;
	11	const Rex = encoder.Rex;
	12	const LegacyPrefixes = encoder.LegacyPrefixes;
	13
	14	const table = @import("encodings.zig").table;
	15
	16	mnemonic: Mnemonic,
	17	op_en: OpEn,
	18	op1: Op,
	19	op2: Op,
	20	op3: Op,
	21	op4: Op,
	22	opc_len: u2,
	23	opc: [3]u8,
	24	modrm_ext: u3,
	25	mode: Mode,
	26
	27	pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
	28	op1: Instruction.Operand,
	29	op2: Instruction.Operand,
	30	op3: Instruction.Operand,
	31	op4: Instruction.Operand,
	32	}) ?Encoding {
	33	const input_op1 = Op.fromOperand(args.op1);
	34	const input_op2 = Op.fromOperand(args.op2);
	35	const input_op3 = Op.fromOperand(args.op3);
	36	const input_op4 = Op.fromOperand(args.op4);
	37
	38	// TODO work out what is the maximum number of variants we can actually find in one swoop.
	39	var candidates: [10]Encoding = undefined;
	40	var count: usize = 0;
	41	inline for (table) \|entry\| {
	42	const enc = Encoding{
	43	.mnemonic = entry[0],
	44	.op_en = entry[1],
	45	.op1 = entry[2],
	46	.op2 = entry[3],
	47	.op3 = entry[4],
	48	.op4 = entry[5],
	49	.opc_len = entry[6],
	50	.opc = .{ entry[7], entry[8], entry[9] },
	51	.modrm_ext = entry[10],
	52	.mode = entry[11],
	53	};
	54	if (enc.mnemonic == mnemonic and
	55	input_op1.isSubset(enc.op1, enc.mode) and
	56	input_op2.isSubset(enc.op2, enc.mode) and
	57	input_op3.isSubset(enc.op3, enc.mode) and
	58	input_op4.isSubset(enc.op4, enc.mode))
	59	{
	60	candidates[count] = enc;
	61	count += 1;
	62	}
	63	}
	64
	65	if (count == 0) return null;
	66	if (count == 1) return candidates[0];
	67
	68	const EncodingLength = struct {
	69	fn estimate(encoding: Encoding, params: struct {
	70	op1: Instruction.Operand,
	71	op2: Instruction.Operand,
	72	op3: Instruction.Operand,
	73	op4: Instruction.Operand,
	74	}) usize {
	75	var inst = Instruction{
	76	.op1 = params.op1,
	77	.op2 = params.op2,
	78	.op3 = params.op3,
	79	.op4 = params.op4,
	80	.encoding = encoding,
	81	};
	82	var cwriter = std.io.countingWriter(std.io.null_writer);
	83	inst.encode(cwriter.writer()) catch unreachable;
	84	return cwriter.bytes_written;
	85	}
	86	};
	87
	88	var shortest_encoding: ?struct {
	89	index: usize,
	90	len: usize,
	91	} = null;
	92	var i: usize = 0;
	93	while (i < count) : (i += 1) {
	94	const len = EncodingLength.estimate(candidates[i], .{
	95	.op1 = args.op1,
	96	.op2 = args.op2,
	97	.op3 = args.op3,
	98	.op4 = args.op4,
	99	});
	100	const current = shortest_encoding orelse {
	101	shortest_encoding = .{ .index = i, .len = len };
	102	continue;
	103	};
	104	if (len < current.len) {
	105	shortest_encoding = .{ .index = i, .len = len };
	106	}
	107	}
	108
	109	return candidates[shortest_encoding.?.index];
	110	}
	111
	112	/// Returns first matching encoding by opcode.
	113	pub fn findByOpcode(opc: []const u8, prefixes: struct {
	114	legacy: LegacyPrefixes,
	115	rex: Rex,
	116	}, modrm_ext: ?u3) ?Encoding {
	117	inline for (table) \|entry\| {
	118	const enc = Encoding{
	119	.mnemonic = entry[0],
	120	.op_en = entry[1],
	121	.op1 = entry[2],
	122	.op2 = entry[3],
	123	.op3 = entry[4],
	124	.op4 = entry[5],
	125	.opc_len = entry[6],
	126	.opc = .{ entry[7], entry[8], entry[9] },
	127	.modrm_ext = entry[10],
	128	.mode = entry[11],
	129	};
	130	const match = match: {
	131	if (modrm_ext) \|ext\| {
	132	break :match ext == enc.modrm_ext and std.mem.eql(u8, enc.opcode(), opc);
	133	}
	134	break :match std.mem.eql(u8, enc.opcode(), opc);
	135	};
	136	if (match) {
	137	if (prefixes.rex.w) {
	138	switch (enc.mode) {
	139	.fpu, .sse, .sse2 => {},
	140	.long => return enc,
	141	.none => {
	142	// TODO this is a hack to allow parsing of instructions which contain
	143	// spurious prefix bytes such as
	144	// rex.W mov dil, 0x1
	145	// Here, rex.W is not needed.
	146	const rex_w_allowed = blk: {
	147	const bit_size = enc.operandSize();
	148	break :blk bit_size == 64 or bit_size == 8;
	149	};
	150	if (rex_w_allowed) return enc;
	151	},
	152	}
	153	} else if (prefixes.legacy.prefix_66) {
	154	switch (enc.operandSize()) {
	155	16 => return enc,
	156	else => {},
	157	}
	158	} else {
	159	if (enc.mode == .none) {
	160	switch (enc.operandSize()) {
	161	16 => {},
	162	else => return enc,
	163	}
	164	}
	165	}
	166	}
	167	}
	168	return null;
	169	}
	170
	171	pub fn opcode(encoding: *const Encoding) []const u8 {
	172	return encoding.opc[0..encoding.opc_len];
	173	}
	174
	175	pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 {
	176	const prefix = encoding.opc[0];
	177	return switch (prefix) {
	178	0x66, 0xf2, 0xf3 => prefix,
	179	else => null,
	180	};
	181	}
	182
	183	pub fn modRmExt(encoding: Encoding) u3 {
	184	return switch (encoding.op_en) {
	185	.m, .mi, .m1, .mc => encoding.modrm_ext,
	186	else => unreachable,
	187	};
	188	}
	189
	190	pub fn operandSize(encoding: Encoding) u32 {
	191	if (encoding.mode == .long) return 64;
	192	const bit_size: u32 = switch (encoding.op_en) {
	193	.np => switch (encoding.op1) {
	194	.o16 => 16,
	195	.o32 => 32,
	196	.o64 => 64,
	197	else => 32,
	198	},
	199	.td => encoding.op2.size(),
	200	else => encoding.op1.size(),
	201	};
	202	return bit_size;
	203	}
	204
	205	pub fn format(
	206	encoding: Encoding,
	207	comptime fmt: []const u8,
	208	options: std.fmt.FormatOptions,
	209	writer: anytype,
	210	) !void {
	211	_ = options;
	212	_ = fmt;
	213	switch (encoding.mode) {
	214	.long => try writer.writeAll("REX.W + "),
	215	else => {},
	216	}
	217
	218	for (encoding.opcode()) \|byte\| {
	219	try writer.print("{x:0>2} ", .{byte});
	220	}
	221
	222	switch (encoding.op_en) {
	223	.np, .fd, .td, .i, .zi, .d => {},
	224	.o, .oi => {
	225	const tag = switch (encoding.op1) {
	226	.r8 => "rb",
	227	.r16 => "rw",
	228	.r32 => "rd",
	229	.r64 => "rd",
	230	else => unreachable,
	231	};
	232	try writer.print("+{s} ", .{tag});
	233	},
	234	.m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
	235	.mr, .rm, .rmi => try writer.writeAll("/r "),
	236	}
	237
	238	switch (encoding.op_en) {
	239	.i, .d, .zi, .oi, .mi, .rmi => {
	240	const op = switch (encoding.op_en) {
	241	.i, .d => encoding.op1,
	242	.zi, .oi, .mi => encoding.op2,
	243	.rmi => encoding.op3,
	244	else => unreachable,
	245	};
	246	const tag = switch (op) {
	247	.imm8 => "ib",
	248	.imm16 => "iw",
	249	.imm32 => "id",
	250	.imm64 => "io",
	251	.rel8 => "cb",
	252	.rel16 => "cw",
	253	.rel32 => "cd",
	254	else => unreachable,
	255	};
	256	try writer.print("{s} ", .{tag});
	257	},
	258	.np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm => {},
	259	}
	260
	261	try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
	262
	263	const ops = &[_]Op{ encoding.op1, encoding.op2, encoding.op3, encoding.op4 };
	264	for (ops) \|op\| switch (op) {
	265	.none, .o16, .o32, .o64 => break,
	266	else => try writer.print("{s} ", .{@tagName(op)}),
	267	};
	268
	269	const op_en = switch (encoding.op_en) {
	270	.zi => .i,
	271	else => \|op_en\| op_en,
	272	};
	273	try writer.print("{s}", .{@tagName(op_en)});
	274	}
	275
	276	pub const Mnemonic = enum {
	277	// zig fmt: off
	278	// General-purpose
	279	adc, add, @"and",
	280	call, cbw, cwde, cdqe, cwd, cdq, cqo, cmp,
	281	cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
	282	cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
	283	cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
	284	div,
	285	fisttp, fld,
	286	idiv, imul, int3,
	287	ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe,
	288	jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
	289	jmp,
	290	lea,
	291	mov, movsx, movsxd, movzx, mul,
	292	nop,
	293	@"or",
	294	pop, push,
	295	ret,
	296	sal, sar, sbb, shl, shr, sub, syscall,
	297	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
	298	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
	299	setnz, seto, setp, setpe, setpo, sets, setz,
	300	@"test",
	301	ud2,
	302	xor,
	303	// SSE
	304	addss,
	305	cmpss,
	306	movss,
	307	ucomiss,
	308	// SSE2
	309	addsd,
	310	cmpsd,
	311	movq, movsd,
	312	ucomisd,
	313	// zig fmt: on
	314	};
	315
	316	pub const OpEn = enum {
	317	// zig fmt: off
	318	np,
	319	o, oi,
	320	i, zi,
	321	d, m,
	322	fd, td,
	323	m1, mc, mi, mr, rm, rmi,
	324	// zig fmt: on
	325	};
	326
	327	pub const Op = enum {
	328	// zig fmt: off
	329	none,
	330	o16, o32, o64,
	331	unity,
	332	imm8, imm16, imm32, imm64,
	333	al, ax, eax, rax,
	334	cl,
	335	r8, r16, r32, r64,
	336	rm8, rm16, rm32, rm64,
	337	m8, m16, m32, m64, m80,
	338	rel8, rel16, rel32,
	339	m,
	340	moffs,
	341	sreg,
	342	xmm, xmm_m32, xmm_m64,
	343	// zig fmt: on
	344
	345	pub fn fromOperand(operand: Instruction.Operand) Op {
	346	switch (operand) {
	347	.none => return .none,
	348
	349	.reg => \|reg\| {
	350	switch (reg.class()) {
	351	.segment => return .sreg,
	352	.floating_point => return switch (reg.size()) {
	353	128 => .xmm,
	354	else => unreachable,
	355	},
	356	.general_purpose => {
	357	if (reg.to64() == .rax) return switch (reg) {
	358	.al => .al,
	359	.ax => .ax,
	360	.eax => .eax,
	361	.rax => .rax,
	362	else => unreachable,
	363	};
	364	if (reg == .cl) return .cl;
	365	return switch (reg.size()) {
	366	8 => .r8,
	367	16 => .r16,
	368	32 => .r32,
	369	64 => .r64,
	370	else => unreachable,
	371	};
	372	},
	373	}
	374	},
	375
	376	.mem => \|mem\| switch (mem) {
	377	.moffs => return .moffs,
	378	.sib, .rip => {
	379	const bit_size = mem.size();
	380	return switch (bit_size) {
	381	8 => .m8,
	382	16 => .m16,
	383	32 => .m32,
	384	64 => .m64,
	385	80 => .m80,
	386	else => unreachable,
	387	};
	388	},
	389	},
	390
	391	.imm => \|imm\| {
	392	if (imm == 1) return .unity;
	393	if (math.cast(i8, imm)) \|_\| return .imm8;
	394	if (math.cast(i16, imm)) \|_\| return .imm16;
	395	if (math.cast(i32, imm)) \|_\| return .imm32;
	396	return .imm64;
	397	},
	398	}
	399	}
	400
	401	pub fn size(op: Op) u32 {
	402	return switch (op) {
	403	.none, .o16, .o32, .o64, .moffs, .m, .sreg, .unity => unreachable,
	404	.imm8, .al, .cl, .r8, .m8, .rm8, .rel8 => 8,
	405	.imm16, .ax, .r16, .m16, .rm16, .rel16 => 16,
	406	.imm32, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
	407	.imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
	408	.m80 => 80,
	409	.xmm => 128,
	410	};
	411	}
	412
	413	pub fn isRegister(op: Op) bool {
	414	// zig fmt: off
	415	return switch (op) {
	416	.cl,
	417	.al, .ax, .eax, .rax,
	418	.r8, .r16, .r32, .r64,
	419	.rm8, .rm16, .rm32, .rm64,
	420	.xmm, .xmm_m32, .xmm_m64,
	421	=> true,
	422	else => false,
	423	};
	424	// zig fmt: on
	425	}
	426
	427	pub fn isImmediate(op: Op) bool {
	428	// zig fmt: off
	429	return switch (op) {
	430	.imm8, .imm16, .imm32, .imm64,
	431	.rel8, .rel16, .rel32,
	432	.unity,
	433	=> true,
	434	else => false,
	435	};
	436	// zig fmt: on
	437	}
	438
	439	pub fn isMemory(op: Op) bool {
	440	// zig fmt: off
	441	return switch (op) {
	442	.rm8, .rm16, .rm32, .rm64,
	443	.m8, .m16, .m32, .m64, .m80,
	444	.m,
	445	.xmm_m32, .xmm_m64,
	446	=> true,
	447	else => false,
	448	};
	449	// zig fmt: on
	450	}
	451
	452	pub fn isSegmentRegister(op: Op) bool {
	453	return switch (op) {
	454	.moffs, .sreg => true,
	455	else => false,
	456	};
	457	}
	458
	459	pub fn isFloatingPointRegister(op: Op) bool {
	460	return switch (op) {
	461	.xmm, .xmm_m32, .xmm_m64 => true,
	462	else => false,
	463	};
	464	}
	465
	466	/// Given an operand `op` checks if `target` is a subset for the purposes
	467	/// of the encoding.
	468	pub fn isSubset(op: Op, target: Op, mode: Mode) bool {
	469	switch (op) {
	470	.m, .o16, .o32, .o64 => unreachable,
	471	.moffs, .sreg => return op == target,
	472	.none => switch (target) {
	473	.o16, .o32, .o64, .none => return true,
	474	else => return false,
	475	},
	476	else => {
	477	if (op.isRegister() and target.isRegister()) {
	478	switch (mode) {
	479	.sse, .sse2 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
	480	else => switch (target) {
	481	.cl, .al, .ax, .eax, .rax => return op == target,
	482	else => return op.size() == target.size(),
	483	},
	484	}
	485	}
	486	if (op.isMemory() and target.isMemory()) {
	487	switch (target) {
	488	.m => return true,
	489	else => return op.size() == target.size(),
	490	}
	491	}
	492	if (op.isImmediate() and target.isImmediate()) {
	493	switch (target) {
	494	.imm32, .rel32 => switch (op) {
	495	.unity, .imm8, .imm16, .imm32 => return true,
	496	else => return op == target,
	497	},
	498	.imm16, .rel16 => switch (op) {
	499	.unity, .imm8, .imm16 => return true,
	500	else => return op == target,
	501	},
	502	.imm8, .rel8 => switch (op) {
	503	.unity, .imm8 => return true,
	504	else => return op == target,
	505	},
	506	else => return op == target,
	507	}
	508	}
	509	return false;
	510	},
	511	}
	512	}
	513	};
	514
	515	pub const Mode = enum {
	516	none,
	517	fpu,
	518	long,
	519	sse,
	520	sse2,
	521	};

src/arch/x86_64/Mir.zig+16-32

...	...	@@ -339,41 +339,23 @@ pub const Inst = struct {
339	339	/// Nop
340	340	nop,
341	341
342		/// SSE instructions
	342	/// SSE/AVX instructions
343	343	/// ops flags: form:
344	344	/// 0b00 reg1, qword ptr [reg2 + imm32]
345	345	/// 0b01 qword ptr [reg1 + imm32], reg2
346	346	/// 0b10 reg1, reg2
347		mov_f64_sse,
348		mov_f32_sse,
	347	mov_f64,
	348	mov_f32,
349	349
350	350	/// ops flags: form:
351	351	/// 0b00 reg1, reg2
352		add_f64_sse,
353		add_f32_sse,
	352	add_f64,
	353	add_f32,
354	354
355	355	/// ops flags: form:
356	356	/// 0b00 reg1, reg2
357		cmp_f64_sse,
358		cmp_f32_sse,
359
360		/// AVX instructions
361		/// ops flags: form:
362		/// 0b00 reg1, qword ptr [reg2 + imm32]
363		/// 0b01 qword ptr [reg1 + imm32], reg2
364		/// 0b10 reg1, reg1, reg2
365		mov_f64_avx,
366		mov_f32_avx,
367
368		/// ops flags: form:
369		/// 0b00 reg1, reg1, reg2
370		add_f64_avx,
371		add_f32_avx,
372
373		/// ops flags: form:
374		/// 0b00 reg1, reg1, reg2
375		cmp_f64_avx,
376		cmp_f32_avx,
	357	cmp_f64,
	358	cmp_f32,
377	359
378	360	/// Pseudo-instructions
379	361	/// call extern function
...	...	@@ -439,6 +421,8 @@ pub const Inst = struct {
439	421	inst: Index,
440	422	/// A 32-bit immediate value.
441	423	imm: u32,
	424	/// A 32-bit signed displacement value.
	425	disp: i32,
442	426	/// A condition code for use with EFLAGS register.
443	427	cc: bits.Condition,
444	428	/// Another instruction with condition code.
...	...	@@ -476,9 +460,9 @@ pub const IndexRegisterDisp = struct {
476	460	index: u32,
477	461
478	462	/// Displacement value
479		disp: u32,
	463	disp: i32,
480	464
481		pub fn encode(index: Register, disp: u32) IndexRegisterDisp {
	465	pub fn encode(index: Register, disp: i32) IndexRegisterDisp {
482	466	return .{
483	467	.index = @enumToInt(index),
484	468	.disp = disp,
...	...	@@ -487,7 +471,7 @@ pub const IndexRegisterDisp = struct {
487	471
488	472	pub fn decode(this: IndexRegisterDisp) struct {
489	473	index: Register,
490		disp: u32,
	474	disp: i32,
491	475	} {
492	476	return .{
493	477	.index = @intToEnum(Register, this.index),
...	...	@@ -503,12 +487,12 @@ pub const IndexRegisterDispImm = struct {
503	487	index: u32,
504	488
505	489	/// Displacement value
506		disp: u32,
	490	disp: i32,
507	491
508	492	/// Immediate
509	493	imm: u32,
510	494
511		pub fn encode(index: Register, disp: u32, imm: u32) IndexRegisterDispImm {
	495	pub fn encode(index: Register, disp: i32, imm: u32) IndexRegisterDispImm {
512	496	return .{
513	497	.index = @enumToInt(index),
514	498	.disp = disp,
...	...	@@ -518,7 +502,7 @@ pub const IndexRegisterDispImm = struct {
518	502
519	503	pub fn decode(this: IndexRegisterDispImm) struct {
520	504	index: Register,
521		disp: u32,
	505	disp: i32,
522	506	imm: u32,
523	507	} {
524	508	return .{
...	...	@@ -576,7 +560,7 @@ pub const SaveRegisterList = struct {
576	560	};
577	561
578	562	pub const ImmPair = struct {
579		dest_off: u32,
	563	dest_off: i32,
580	564	operand: u32,
581	565	};
582	566

src/arch/x86_64/bits.zig+256-859

...	...	@@ -1,9 +1,9 @@
1	1	const std = @import("std");
2		const testing = std.testing;
3		const mem = std.mem;
4	2	const assert = std.debug.assert;
5		const ArrayList = std.ArrayList;
	3	const expect = std.testing.expect;
	4
6	5	const Allocator = std.mem.Allocator;
	6	const ArrayList = std.ArrayList;
7	7	const DW = std.dwarf;
8	8
9	9	/// EFLAGS condition codes
...	...	@@ -135,960 +135,357 @@ pub const Condition = enum(u5) {
135	135	}
136	136	};
137	137
138		/// Definitions of all of the general purpose x64 registers. The order is semantically meaningful.
139		/// The registers are defined such that IDs go in descending order of 64-bit,
140		/// 32-bit, 16-bit, and then 8-bit, and each set contains exactly sixteen
141		/// registers. This results in some useful properties:
142		///
143		/// Any 64-bit register can be turned into its 32-bit form by adding 16, and
144		/// vice versa. This also works between 32-bit and 16-bit forms. With 8-bit, it
145		/// works for all except for sp, bp, si, and di, which do not have an 8-bit
146		/// form.
147		///
148		/// If (register & 8) is set, the register is extended.
149		///
150		/// The ID can be easily determined by figuring out what range the register is
151		/// in, and then subtracting the base.
152	138	pub const Register = enum(u7) {
153	139	// zig fmt: off
154		// 0 through 15, 64-bit registers. 8-15 are extended.
155		// id is just the int value.
156	140	rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi,
157	141	r8, r9, r10, r11, r12, r13, r14, r15,
158	142
159		// 16 through 31, 32-bit registers. 24-31 are extended.
160		// id is int value - 16.
161	143	eax, ecx, edx, ebx, esp, ebp, esi, edi,
162	144	r8d, r9d, r10d, r11d, r12d, r13d, r14d, r15d,
163	145
164		// 32-47, 16-bit registers. 40-47 are extended.
165		// id is int value - 32.
166	146	ax, cx, dx, bx, sp, bp, si, di,
167	147	r8w, r9w, r10w, r11w, r12w, r13w, r14w, r15w,
168	148
169		// 48-63, 8-bit registers. 56-63 are extended.
170		// id is int value - 48.
171		al, cl, dl, bl, ah, ch, dh, bh,
	149	al, cl, dl, bl, spl, bpl, sil, dil,
172	150	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,
173	151
174		// 64-79, 256-bit registers.
175		// id is int value - 64.
	152	ah, ch, dh, bh,
	153
176	154	ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,
177	155	ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
178	156
179		// 80-95, 128-bit registers.
180		// id is int value - 80.
181	157	xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
182	158	xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
183	159
184		// Pseudo-value for MIR instructions.
	160	es, cs, ss, ds, fs, gs,
	161
185	162	none,
186	163	// zig fmt: on
187	164
188		pub fn id(self: Register) u7 {
189		return switch (@enumToInt(self)) {
190		0...63 => @as(u7, @truncate(u4, @enumToInt(self))),
191		64...79 => @enumToInt(self),
192		else => unreachable,
193		};
194		}
195
196		/// Returns the bit-width of the register.
197		pub fn size(self: Register) u9 {
198		return switch (@enumToInt(self)) {
199		0...15 => 64,
200		16...31 => 32,
201		32...47 => 16,
202		48...63 => 8,
203		64...79 => 256,
204		80...95 => 128,
205		else => unreachable,
206		};
207		}
208
209		/// Returns whether the register is extended. Extended registers are the
210		/// new registers added with amd64, r8 through r15. This also includes any
211		/// other variant of access to those registers, such as r8b, r15d, and so
212		/// on. This is needed because access to these registers requires special
213		/// handling via the REX prefix, via the B or R bits, depending on context.
214		pub fn isExtended(self: Register) bool {
215		return @enumToInt(self) & 0x08 != 0;
216		}
217
218		/// This returns the 4-bit register ID, which is used in practically every
219		/// opcode. Note that bit 3 (the highest bit) is never used directly in
220		/// an instruction (@see isExtended), and requires special handling. The
221		/// lower three bits are often embedded directly in instructions (such as
222		/// the B8 variant of moves), or used in R/M bytes.
223		pub fn enc(self: Register) u4 {
224		return @truncate(u4, @enumToInt(self));
225		}
226
227		/// Like enc, but only returns the lower 3 bits.
228		pub fn lowEnc(self: Register) u3 {
229		return @truncate(u3, @enumToInt(self));
230		}
231
232		pub fn to256(self: Register) Register {
233		return @intToEnum(Register, @as(u8, self.enc()) + 64);
234		}
	165	pub const Class = enum(u2) {
	166	general_purpose,
	167	floating_point,
	168	segment,
	169	};
235	170
236		pub fn to128(self: Register) Register {
237		return @intToEnum(Register, @as(u8, self.enc()) + 80);
238		}
	171	pub fn class(reg: Register) Class {
	172	return switch (@enumToInt(reg)) {
	173	// zig fmt: off
	174	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => .general_purpose,
	175	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => .general_purpose,
	176	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => .general_purpose,
	177	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => .general_purpose,
	178	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => .general_purpose,
239	179
240		/// Convert from any register to its 64 bit alias.
241		pub fn to64(self: Register) Register {
242		return @intToEnum(Register, self.enc());
243		}
	180	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => .floating_point,
	181	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => .floating_point,
244	182
245		/// Convert from any register to its 32 bit alias.
246		pub fn to32(self: Register) Register {
247		return @intToEnum(Register, @as(u8, self.enc()) + 16);
248		}
249
250		/// Convert from any register to its 16 bit alias.
251		pub fn to16(self: Register) Register {
252		return @intToEnum(Register, @as(u8, self.enc()) + 32);
253		}
	183	@enumToInt(Register.es) ... @enumToInt(Register.gs) => .segment,
254	184
255		/// Convert from any register to its 8 bit alias.
256		pub fn to8(self: Register) Register {
257		return @intToEnum(Register, @as(u8, self.enc()) + 48);
	185	else => unreachable,
	186	// zig fmt: on
	187	};
258	188	}
259	189
260		pub fn dwarfLocOp(self: Register) u8 {
261		switch (@enumToInt(self)) {
262		0...63 => return switch (self.to64()) {
263		.rax => DW.OP.reg0,
264		.rdx => DW.OP.reg1,
265		.rcx => DW.OP.reg2,
266		.rbx => DW.OP.reg3,
267		.rsi => DW.OP.reg4,
268		.rdi => DW.OP.reg5,
269		.rbp => DW.OP.reg6,
270		.rsp => DW.OP.reg7,
271
272		.r8 => DW.OP.reg8,
273		.r9 => DW.OP.reg9,
274		.r10 => DW.OP.reg10,
275		.r11 => DW.OP.reg11,
276		.r12 => DW.OP.reg12,
277		.r13 => DW.OP.reg13,
278		.r14 => DW.OP.reg14,
279		.r15 => DW.OP.reg15,
	190	pub fn id(reg: Register) u6 {
	191	const base = switch (@enumToInt(reg)) {
	192	// zig fmt: off
	193	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
	194	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
	195	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
	196	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
	197	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
280	198
281		else => unreachable,
282		},
	199	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0) - 16,
	200	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0) - 16,
283	201
284		64...79 => return @as(u8, self.enc()) + DW.OP.reg17,
	202	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es) - 32,
285	203
286	204	else => unreachable,
287		}
	205	// zig fmt: on
	206	};
	207	return @intCast(u6, @enumToInt(reg) - base);
288	208	}
289	209
290		/// DWARF encodings that push a value onto the DWARF stack that is either
291		/// the contents of a register or the result of adding the contents a given
292		/// register to a given signed offset.
293		pub fn dwarfLocOpDeref(self: Register) u8 {
294		switch (@enumToInt(self)) {
295		0...63 => return switch (self.to64()) {
296		.rax => DW.OP.breg0,
297		.rdx => DW.OP.breg1,
298		.rcx => DW.OP.breg2,
299		.rbx => DW.OP.breg3,
300		.rsi => DW.OP.breg4,
301		.rdi => DW.OP.breg5,
302		.rbp => DW.OP.breg6,
303		.rsp => DW.OP.fbreg,
304
305		.r8 => DW.OP.breg8,
306		.r9 => DW.OP.breg9,
307		.r10 => DW.OP.breg10,
308		.r11 => DW.OP.breg11,
309		.r12 => DW.OP.breg12,
310		.r13 => DW.OP.breg13,
311		.r14 => DW.OP.breg14,
312		.r15 => DW.OP.breg15,
	210	pub fn size(reg: Register) u32 {
	211	return switch (@enumToInt(reg)) {
	212	// zig fmt: off
	213	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => 64,
	214	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => 32,
	215	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => 16,
	216	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => 8,
	217	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => 8,
313	218
314		else => unreachable,
315		},
	219	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => 256,
	220	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => 128,
316	221
317		64...79 => return @as(u8, self.enc()) + DW.OP.breg17,
	222	@enumToInt(Register.es) ... @enumToInt(Register.gs) => 16,
318	223
319	224	else => unreachable,
320		}
321		}
322		};
323
324		// zig fmt: on
325
326		/// Encoding helper functions for x86_64 instructions
327		///
328		/// Many of these helpers do very little, but they can help make things
329		/// slightly more readable with more descriptive field names / function names.
330		///
331		/// Some of them also have asserts to ensure that we aren't doing dumb things.
332		/// For example, trying to use register 4 (esp) in an indirect modr/m byte is illegal,
333		/// you need to encode it with an SIB byte.
334		///
335		/// Note that ALL of these helper functions will assume capacity,
336		/// so ensure that the `code` has sufficient capacity before using them.
337		/// The `init` method is the recommended way to ensure capacity.
338		pub const Encoder = struct {
339		/// Non-owning reference to the code array
340		code: *ArrayList(u8),
341
342		const Self = @This();
343
344		/// Wrap `code` in Encoder to make it easier to call these helper functions
345		///
346		/// maximum_inst_size should contain the maximum number of bytes
347		/// that the encoded instruction will take.
348		/// This is because the helper functions will assume capacity
349		/// in order to avoid bounds checking.
350		pub fn init(code: *ArrayList(u8), maximum_inst_size: u8) !Self {
351		try code.ensureUnusedCapacity(maximum_inst_size);
352		return Self{ .code = code };
353		}
354
355		/// Directly write a number to the code array with big endianness
356		pub fn writeIntBig(self: Self, comptime T: type, value: T) void {
357		mem.writeIntBig(
358		T,
359		self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
360		value,
361		);
	225	// zig fmt: on
	226	};
362	227	}
363	228
364		/// Directly write a number to the code array with little endianness
365		pub fn writeIntLittle(self: Self, comptime T: type, value: T) void {
366		mem.writeIntLittle(
367		T,
368		self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
369		value,
370		);
371		}
	229	pub fn isExtended(reg: Register) bool {
	230	return switch (@enumToInt(reg)) {
	231	// zig fmt: off
	232	@enumToInt(Register.r8) ... @enumToInt(Register.r15) => true,
	233	@enumToInt(Register.r8d) ... @enumToInt(Register.r15d) => true,
	234	@enumToInt(Register.r8w) ... @enumToInt(Register.r15w) => true,
	235	@enumToInt(Register.r8b) ... @enumToInt(Register.r15b) => true,
372	236
373		// --------
374		// Prefixes
375		// --------
376
377		pub const LegacyPrefixes = packed struct {
378		/// LOCK
379		prefix_f0: bool = false,
380		/// REPNZ, REPNE, REP, Scalar Double-precision
381		prefix_f2: bool = false,
382		/// REPZ, REPE, REP, Scalar Single-precision
383		prefix_f3: bool = false,
384
385		/// CS segment override or Branch not taken
386		prefix_2e: bool = false,
387		/// DS segment override
388		prefix_36: bool = false,
389		/// ES segment override
390		prefix_26: bool = false,
391		/// FS segment override
392		prefix_64: bool = false,
393		/// GS segment override
394		prefix_65: bool = false,
395
396		/// Branch taken
397		prefix_3e: bool = false,
398
399		/// Operand size override (enables 16 bit operation)
400		prefix_66: bool = false,
401
402		/// Address size override (enables 16 bit address size)
403		prefix_67: bool = false,
404
405		padding: u5 = 0,
406		};
	237	@enumToInt(Register.ymm8) ... @enumToInt(Register.ymm15) => true,
	238	@enumToInt(Register.xmm8) ... @enumToInt(Register.xmm15) => true,
407	239
408		/// Encodes legacy prefixes
409		pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) void {
410		if (@bitCast(u16, prefixes) != 0) {
411		// Hopefully this path isn't taken very often, so we'll do it the slow way for now
412
413		// LOCK
414		if (prefixes.prefix_f0) self.code.appendAssumeCapacity(0xf0);
415		// REPNZ, REPNE, REP, Scalar Double-precision
416		if (prefixes.prefix_f2) self.code.appendAssumeCapacity(0xf2);
417		// REPZ, REPE, REP, Scalar Single-precision
418		if (prefixes.prefix_f3) self.code.appendAssumeCapacity(0xf3);
419
420		// CS segment override or Branch not taken
421		if (prefixes.prefix_2e) self.code.appendAssumeCapacity(0x2e);
422		// DS segment override
423		if (prefixes.prefix_36) self.code.appendAssumeCapacity(0x36);
424		// ES segment override
425		if (prefixes.prefix_26) self.code.appendAssumeCapacity(0x26);
426		// FS segment override
427		if (prefixes.prefix_64) self.code.appendAssumeCapacity(0x64);
428		// GS segment override
429		if (prefixes.prefix_65) self.code.appendAssumeCapacity(0x65);
430
431		// Branch taken
432		if (prefixes.prefix_3e) self.code.appendAssumeCapacity(0x3e);
433
434		// Operand size override
435		if (prefixes.prefix_66) self.code.appendAssumeCapacity(0x66);
436
437		// Address size override
438		if (prefixes.prefix_67) self.code.appendAssumeCapacity(0x67);
439		}
	240	else => false,
	241	// zig fmt: on
	242	};
440	243	}
441	244
442		/// Use 16 bit operand size
443		///
444		/// Note that this flag is overridden by REX.W, if both are present.
445		pub fn prefix16BitMode(self: Self) void {
446		self.code.appendAssumeCapacity(0x66);
	245	pub fn isRexInvalid(reg: Register) bool {
	246	return switch (@enumToInt(reg)) {
	247	@enumToInt(Register.ah)...@enumToInt(Register.bh) => true,
	248	else => false,
	249	};
447	250	}
448	251
449		pub const Vex = struct {
450		rex_prefix: Rex = .{},
451		lead_opc: u5 = 0b0_0001,
452		register: u4 = 0b1111,
453		length: u1 = 0b0,
454		simd_prefix: u2 = 0b00,
455		wig_desc: bool = false,
456		lig_desc: bool = false,
457		lz_desc: bool = false,
458
459		pub fn rex(self: *Vex, r: Rex) void {
460		self.rex_prefix = r;
461		}
462
463		pub fn lead_opc_0f(self: *Vex) void {
464		self.lead_opc = 0b0_0001;
465		}
466
467		pub fn lead_opc_0f_38(self: *Vex) void {
468		self.lead_opc = 0b0_0010;
469		}
470
471		pub fn lead_opc_0f_3a(self: *Vex) void {
472		self.lead_opc = 0b0_0011;
473		}
474
475		pub fn reg(self: *Vex, register: u4) void {
476		self.register = ~register;
477		}
478
479		pub fn len_128(self: *Vex) void {
480		self.length = 0;
481		}
482
483		pub fn len_256(self: *Vex) void {
484		assert(!self.lz_desc);
485		self.length = 1;
486		}
487
488		pub fn simd_prefix_66(self: *Vex) void {
489		self.simd_prefix = 0b01;
490		}
491
492		pub fn simd_prefix_f3(self: *Vex) void {
493		self.simd_prefix = 0b10;
494		}
495
496		pub fn simd_prefix_f2(self: *Vex) void {
497		self.simd_prefix = 0b11;
498		}
499
500		pub fn wig(self: *Vex) void {
501		self.wig_desc = true;
502		}
503
504		pub fn lig(self: *Vex) void {
505		self.lig_desc = true;
506		}
507
508		pub fn lz(self: *Vex) void {
509		self.lz_desc = true;
510		}
	252	pub fn enc(reg: Register) u4 {
	253	const base = switch (@enumToInt(reg)) {
	254	// zig fmt: off
	255	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
	256	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
	257	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
	258	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
	259	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
511	260
512		pub fn write(self: Vex, writer: anytype) usize {
513		var buf: [3]u8 = .{0} ** 3;
514		const form_3byte: bool = blk: {
515		if (self.rex_prefix.w and !self.wig_desc) break :blk true;
516		if (self.rex_prefix.x or self.rex_prefix.b) break :blk true;
517		break :blk self.lead_opc != 0b0_0001;
518		};
	261	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
	262	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
519	263
520		if (self.lz_desc) {
521		assert(self.length == 0);
522		}
523
524		if (form_3byte) {
525		// First byte
526		buf[0] = 0xc4;
527		// Second byte
528		const rxb_mask: u3 = @intCast(u3, @boolToInt(!self.rex_prefix.r)) << 2 \|
529		@intCast(u2, @boolToInt(!self.rex_prefix.x)) << 1 \|
530		@boolToInt(!self.rex_prefix.b);
531		buf[1] \|= @intCast(u8, rxb_mask) << 5;
532		buf[1] \|= self.lead_opc;
533		// Third byte
534		buf[2] \|= @intCast(u8, @boolToInt(!self.rex_prefix.w)) << 7;
535		buf[2] \|= @intCast(u7, self.register) << 3;
536		buf[2] \|= @intCast(u3, self.length) << 2;
537		buf[2] \|= self.simd_prefix;
538		} else {
539		// First byte
540		buf[0] = 0xc5;
541		// Second byte
542		buf[1] \|= @intCast(u8, @boolToInt(!self.rex_prefix.r)) << 7;
543		buf[1] \|= @intCast(u7, self.register) << 3;
544		buf[1] \|= @intCast(u3, self.length) << 2;
545		buf[1] \|= self.simd_prefix;
546		}
547
548		const count: usize = if (form_3byte) 3 else 2;
549		_ = writer.writeAll(buf[0..count]) catch unreachable;
550		return count;
551		}
552		};
	264	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es),
553	265
554		pub fn vex(self: Self, prefix: Vex) void {
555		_ = prefix.write(self.code.writer());
	266	else => unreachable,
	267	// zig fmt: on
	268	};
	269	return @truncate(u4, @enumToInt(reg) - base);
556	270	}
557	271
558		/// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB
559		pub const Rex = struct {
560		/// Wide, enables 64-bit operation
561		w: bool = false,
562		/// Extends the reg field in the ModR/M byte
563		r: bool = false,
564		/// Extends the index field in the SIB byte
565		x: bool = false,
566		/// Extends the r/m field in the ModR/M byte,
567		/// or the base field in the SIB byte,
568		/// or the reg field in the Opcode byte
569		b: bool = false,
570		};
571
572		/// Encodes a REX prefix byte given all the fields
573		///
574		/// Use this byte whenever you need 64 bit operation,
575		/// or one of reg, index, r/m, base, or opcode-reg might be extended.
576		///
577		/// See struct `Rex` for a description of each field.
578		///
579		/// Does not add a prefix byte if none of the fields are set!
580		pub fn rex(self: Self, byte: Rex) void {
581		var value: u8 = 0b0100_0000;
582
583		if (byte.w) value \|= 0b1000;
584		if (byte.r) value \|= 0b0100;
585		if (byte.x) value \|= 0b0010;
586		if (byte.b) value \|= 0b0001;
587
588		if (value != 0b0100_0000) {
589		self.code.appendAssumeCapacity(value);
590		}
	272	pub fn lowEnc(reg: Register) u3 {
	273	return @truncate(u3, reg.enc());
591	274	}
592	275
593		// ------
594		// Opcode
595		// ------
596
597		/// Encodes a 1 byte opcode
598		pub fn opcode_1byte(self: Self, opcode: u8) void {
599		self.code.appendAssumeCapacity(opcode);
	276	pub fn toSize(reg: Register, bit_size: u32) Register {
	277	return switch (bit_size) {
	278	8 => reg.to8(),
	279	16 => reg.to16(),
	280	32 => reg.to32(),
	281	64 => reg.to64(),
	282	128 => reg.to128(),
	283	256 => reg.to256(),
	284	else => unreachable,
	285	};
600	286	}
601	287
602		/// Encodes a 2 byte opcode
603		///
604		/// e.g. IMUL has the opcode 0x0f 0xaf, so you use
605		///
606		/// encoder.opcode_2byte(0x0f, 0xaf);
607		pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) void {
608		self.code.appendAssumeCapacity(prefix);
609		self.code.appendAssumeCapacity(opcode);
	288	fn gpBase(reg: Register) u7 {
	289	assert(reg.class() == .general_purpose);
	290	return switch (@enumToInt(reg)) {
	291	// zig fmt: off
	292	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
	293	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
	294	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
	295	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
	296	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
	297	else => unreachable,
	298	// zig fmt: on
	299	};
610	300	}
611	301
612		/// Encodes a 3 byte opcode
613		///
614		/// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
615		///
616		/// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
617		pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) void {
618		self.code.appendAssumeCapacity(prefix_1);
619		self.code.appendAssumeCapacity(prefix_2);
620		self.code.appendAssumeCapacity(opcode);
	302	pub fn to64(reg: Register) Register {
	303	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.rax));
621	304	}
622	305
623		/// Encodes a 1 byte opcode with a reg field
624		///
625		/// Remember to add a REX prefix byte if reg is extended!
626		pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) void {
627		assert(opcode & 0b111 == 0);
628		self.code.appendAssumeCapacity(opcode \| reg);
	306	pub fn to32(reg: Register) Register {
	307	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.eax));
629	308	}
630	309
631		// ------
632		// ModR/M
633		// ------
634
635		/// Construct a ModR/M byte given all the fields
636		///
637		/// Remember to add a REX prefix byte if reg or rm are extended!
638		pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) void {
639		self.code.appendAssumeCapacity(
640		@as(u8, mod) << 6 \| @as(u8, reg_or_opx) << 3 \| rm,
641		);
	310	pub fn to16(reg: Register) Register {
	311	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.ax));
642	312	}
643	313
644		/// Construct a ModR/M byte using direct r/m addressing
645		/// r/m effective address: r/m
646		///
647		/// Note reg's effective address is always just reg for the ModR/M byte.
648		/// Remember to add a REX prefix byte if reg or rm are extended!
649		pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) void {
650		self.modRm(0b11, reg_or_opx, rm);
	314	pub fn to8(reg: Register) Register {
	315	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.al));
651	316	}
652	317
653		/// Construct a ModR/M byte using indirect r/m addressing
654		/// r/m effective address: [r/m]
655		///
656		/// Note reg's effective address is always just reg for the ModR/M byte.
657		/// Remember to add a REX prefix byte if reg or rm are extended!
658		pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) void {
659		assert(rm != 4 and rm != 5);
660		self.modRm(0b00, reg_or_opx, rm);
	318	fn fpBase(reg: Register) u7 {
	319	assert(reg.class() == .floating_point);
	320	return switch (@enumToInt(reg)) {
	321	@enumToInt(Register.ymm0)...@enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
	322	@enumToInt(Register.xmm0)...@enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
	323	else => unreachable,
	324	};
661	325	}
662	326
663		/// Construct a ModR/M byte using indirect SIB addressing
664		/// r/m effective address: [SIB]
665		///
666		/// Note reg's effective address is always just reg for the ModR/M byte.
667		/// Remember to add a REX prefix byte if reg or rm are extended!
668		pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) void {
669		self.modRm(0b00, reg_or_opx, 0b100);
	327	pub fn to256(reg: Register) Register {
	328	return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.ymm0));
670	329	}
671	330
672		/// Construct a ModR/M byte using RIP-relative addressing
673		/// r/m effective address: [RIP + disp32]
674		///
675		/// Note reg's effective address is always just reg for the ModR/M byte.
676		/// Remember to add a REX prefix byte if reg or rm are extended!
677		pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) void {
678		self.modRm(0b00, reg_or_opx, 0b101);
	331	pub fn to128(reg: Register) Register {
	332	return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.xmm0));
679	333	}
680	334
681		/// Construct a ModR/M byte using indirect r/m with a 8bit displacement
682		/// r/m effective address: [r/m + disp8]
683		///
684		/// Note reg's effective address is always just reg for the ModR/M byte.
685		/// Remember to add a REX prefix byte if reg or rm are extended!
686		pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) void {
687		assert(rm != 4);
688		self.modRm(0b01, reg_or_opx, rm);
	335	pub fn dwarfLocOp(reg: Register) u8 {
	336	return switch (reg.class()) {
	337	.general_purpose => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.reg0,
	338	.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.reg17,
	339	else => unreachable,
	340	};
689	341	}
690	342
691		/// Construct a ModR/M byte using indirect SIB with a 8bit displacement
692		/// r/m effective address: [SIB + disp8]
693		///
694		/// Note reg's effective address is always just reg for the ModR/M byte.
695		/// Remember to add a REX prefix byte if reg or rm are extended!
696		pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) void {
697		self.modRm(0b01, reg_or_opx, 0b100);
	343	/// DWARF encodings that push a value onto the DWARF stack that is either
	344	/// the contents of a register or the result of adding the contents a given
	345	/// register to a given signed offset.
	346	pub fn dwarfLocOpDeref(reg: Register) u8 {
	347	return switch (reg.class()) {
	348	.general_purpose => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.breg0,
	349	.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.breg17,
	350	else => unreachable,
	351	};
698	352	}
	353	};
699	354
700		/// Construct a ModR/M byte using indirect r/m with a 32bit displacement
701		/// r/m effective address: [r/m + disp32]
702		///
703		/// Note reg's effective address is always just reg for the ModR/M byte.
704		/// Remember to add a REX prefix byte if reg or rm are extended!
705		pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) void {
706		assert(rm != 4);
707		self.modRm(0b10, reg_or_opx, rm);
708		}
	355	test "Register id - different classes" {
	356	try expect(Register.al.id() == Register.ax.id());
	357	try expect(Register.ah.id() == Register.spl.id());
	358	try expect(Register.ax.id() == Register.eax.id());
	359	try expect(Register.eax.id() == Register.rax.id());
709	360
710		/// Construct a ModR/M byte using indirect SIB with a 32bit displacement
711		/// r/m effective address: [SIB + disp32]
712		///
713		/// Note reg's effective address is always just reg for the ModR/M byte.
714		/// Remember to add a REX prefix byte if reg or rm are extended!
715		pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) void {
716		self.modRm(0b10, reg_or_opx, 0b100);
717		}
	361	try expect(Register.ymm0.id() == 0b10000);
	362	try expect(Register.ymm0.id() != Register.rax.id());
	363	try expect(Register.xmm0.id() == Register.ymm0.id());
718	364
719		// ---
720		// SIB
721		// ---
722
723		/// Construct a SIB byte given all the fields
724		///
725		/// Remember to add a REX prefix byte if index or base are extended!
726		pub fn sib(self: Self, scale: u2, index: u3, base: u3) void {
727		self.code.appendAssumeCapacity(
728		@as(u8, scale) << 6 \| @as(u8, index) << 3 \| base,
729		);
730		}
	365	try expect(Register.es.id() == 0b100000);
	366	}
731	367
732		/// Construct a SIB byte with scale * index + base, no frills.
733		/// r/m effective address: [base + scale * index]
734		///
735		/// Remember to add a REX prefix byte if index or base are extended!
736		pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) void {
737		assert(base != 5);
	368	test "Register enc - different classes" {
	369	try expect(Register.al.enc() == Register.ax.enc());
	370	try expect(Register.ax.enc() == Register.eax.enc());
	371	try expect(Register.eax.enc() == Register.rax.enc());
	372	try expect(Register.ymm0.enc() == Register.rax.enc());
	373	try expect(Register.xmm0.enc() == Register.ymm0.enc());
	374	try expect(Register.es.enc() == Register.rax.enc());
	375	}
738	376
739		self.sib(scale, index, base);
740		}
	377	test "Register classes" {
	378	try expect(Register.r11.class() == .general_purpose);
	379	try expect(Register.ymm11.class() == .floating_point);
	380	try expect(Register.fs.class() == .segment);
	381	}
741	382
742		/// Construct a SIB byte with scale * index + disp32
743		/// r/m effective address: [scale * index + disp32]
744		///
745		/// Remember to add a REX prefix byte if index or base are extended!
746		pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) void {
747		assert(index != 4);
748
749		// scale is actually ignored
750		// index = 4 means no index
751		// base = 5 means no base, if mod == 0.
752		self.sib(scale, index, 5);
753		}
	383	pub const Memory = union(enum) {
	384	sib: Sib,
	385	rip: Rip,
	386	moffs: Moffs,
754	387
755		/// Construct a SIB byte with just base
756		/// r/m effective address: [base]
757		///
758		/// Remember to add a REX prefix byte if index or base are extended!
759		pub fn sib_base(self: Self, base: u3) void {
760		assert(base != 5);
	388	pub const ScaleIndex = packed struct {
	389	scale: u4,
	390	index: Register,
	391	};
761	392
762		// scale is actually ignored
763		// index = 4 means no index
764		self.sib(0, 4, base);
765		}
	393	pub const PtrSize = enum {
	394	byte,
	395	word,
	396	dword,
	397	qword,
	398	tbyte,
	399
	400	pub fn fromSize(bit_size: u32) PtrSize {
	401	return switch (bit_size) {
	402	8 => .byte,
	403	16 => .word,
	404	32 => .dword,
	405	64 => .qword,
	406	80 => .tbyte,
	407	else => unreachable,
	408	};
	409	}
766	410
767		/// Construct a SIB byte with just disp32
768		/// r/m effective address: [disp32]
769		///
770		/// Remember to add a REX prefix byte if index or base are extended!
771		pub fn sib_disp32(self: Self) void {
772		// scale is actually ignored
773		// index = 4 means no index
774		// base = 5 means no base, if mod == 0.
775		self.sib(0, 4, 5);
776		}
	411	pub fn size(s: PtrSize) u32 {
	412	return switch (s) {
	413	.byte => 8,
	414	.word => 16,
	415	.dword => 32,
	416	.qword => 64,
	417	.tbyte => 80,
	418	};
	419	}
	420	};
777	421
778		/// Construct a SIB byte with scale * index + base + disp8
779		/// r/m effective address: [base + scale * index + disp8]
780		///
781		/// Remember to add a REX prefix byte if index or base are extended!
782		pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) void {
783		self.sib(scale, index, base);
784		}
	422	pub const Sib = struct {
	423	ptr_size: PtrSize,
	424	base: ?Register,
	425	scale_index: ?ScaleIndex,
	426	disp: i32,
	427	};
785	428
786		/// Construct a SIB byte with base + disp8, no index
787		/// r/m effective address: [base + disp8]
788		///
789		/// Remember to add a REX prefix byte if index or base are extended!
790		pub fn sib_baseDisp8(self: Self, base: u3) void {
791		// scale is ignored
792		// index = 4 means no index
793		self.sib(0, 4, base);
794		}
	429	pub const Rip = struct {
	430	ptr_size: PtrSize,
	431	disp: i32,
	432	};
795	433
796		/// Construct a SIB byte with scale * index + base + disp32
797		/// r/m effective address: [base + scale * index + disp32]
798		///
799		/// Remember to add a REX prefix byte if index or base are extended!
800		pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) void {
801		self.sib(scale, index, base);
802		}
	434	pub const Moffs = struct {
	435	seg: Register,
	436	offset: u64,
	437	};
803	438
804		/// Construct a SIB byte with base + disp32, no index
805		/// r/m effective address: [base + disp32]
806		///
807		/// Remember to add a REX prefix byte if index or base are extended!
808		pub fn sib_baseDisp32(self: Self, base: u3) void {
809		// scale is ignored
810		// index = 4 means no index
811		self.sib(0, 4, base);
	439	pub fn moffs(reg: Register, offset: u64) Memory {
	440	assert(reg.class() == .segment);
	441	return .{ .moffs = .{ .seg = reg, .offset = offset } };
812	442	}
813	443
814		// -------------------------
815		// Trivial (no bit fiddling)
816		// -------------------------
817
818		/// Encode an 8 bit immediate
819		///
820		/// It is sign-extended to 64 bits by the cpu.
821		pub fn imm8(self: Self, imm: i8) void {
822		self.code.appendAssumeCapacity(@bitCast(u8, imm));
	444	pub fn sib(ptr_size: PtrSize, args: struct {
	445	disp: i32,
	446	base: ?Register = null,
	447	scale_index: ?ScaleIndex = null,
	448	}) Memory {
	449	return .{ .sib = .{
	450	.base = args.base,
	451	.disp = args.disp,
	452	.ptr_size = ptr_size,
	453	.scale_index = args.scale_index,
	454	} };
823	455	}
824	456
825		/// Encode an 8 bit displacement
826		///
827		/// It is sign-extended to 64 bits by the cpu.
828		pub fn disp8(self: Self, disp: i8) void {
829		self.code.appendAssumeCapacity(@bitCast(u8, disp));
	457	pub fn rip(ptr_size: PtrSize, disp: i32) Memory {
	458	return .{ .rip = .{ .ptr_size = ptr_size, .disp = disp } };
830	459	}
831	460
832		/// Encode an 16 bit immediate
833		///
834		/// It is sign-extended to 64 bits by the cpu.
835		pub fn imm16(self: Self, imm: i16) void {
836		self.writeIntLittle(i16, imm);
	461	pub fn isSegmentRegister(mem: Memory) bool {
	462	return switch (mem) {
	463	.moffs => true,
	464	.rip => false,
	465	.sib => \|s\| if (s.base) \|r\| r.class() == .segment else false,
	466	};
837	467	}
838	468
839		/// Encode an 32 bit immediate
840		///
841		/// It is sign-extended to 64 bits by the cpu.
842		pub fn imm32(self: Self, imm: i32) void {
843		self.writeIntLittle(i32, imm);
	469	pub fn base(mem: Memory) ?Register {
	470	return switch (mem) {
	471	.moffs => \|m\| m.seg,
	472	.sib => \|s\| s.base,
	473	.rip => null,
	474	};
844	475	}
845	476
846		/// Encode an 32 bit displacement
847		///
848		/// It is sign-extended to 64 bits by the cpu.
849		pub fn disp32(self: Self, disp: i32) void {
850		self.writeIntLittle(i32, disp);
	477	pub fn scaleIndex(mem: Memory) ?ScaleIndex {
	478	return switch (mem) {
	479	.moffs, .rip => null,
	480	.sib => \|s\| s.scale_index,
	481	};
851	482	}
852	483
853		/// Encode an 64 bit immediate
854		///
855		/// It is sign-extended to 64 bits by the cpu.
856		pub fn imm64(self: Self, imm: u64) void {
857		self.writeIntLittle(u64, imm);
	484	pub fn size(mem: Memory) u32 {
	485	return switch (mem) {
	486	.rip => \|r\| r.ptr_size.size(),
	487	.sib => \|s\| s.ptr_size.size(),
	488	.moffs => unreachable,
	489	};
858	490	}
859	491	};
860
861		test "Encoder helpers - general purpose registers" {
862		var code = ArrayList(u8).init(testing.allocator);
863		defer code.deinit();
864
865		// simple integer multiplication
866
867		// imul eax,edi
868		// 0faf c7
869		{
870		try code.resize(0);
871		const encoder = try Encoder.init(&code, 4);
872		encoder.rex(.{
873		.r = Register.eax.isExtended(),
874		.b = Register.edi.isExtended(),
875		});
876		encoder.opcode_2byte(0x0f, 0xaf);
877		encoder.modRm_direct(
878		Register.eax.lowEnc(),
879		Register.edi.lowEnc(),
880		);
881
882		try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);
883		}
884
885		// simple mov
886
887		// mov eax,edi
888		// 89 f8
889		{
890		try code.resize(0);
891		const encoder = try Encoder.init(&code, 3);
892		encoder.rex(.{
893		.r = Register.edi.isExtended(),
894		.b = Register.eax.isExtended(),
895		});
896		encoder.opcode_1byte(0x89);
897		encoder.modRm_direct(
898		Register.edi.lowEnc(),
899		Register.eax.lowEnc(),
900		);
901
902		try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);
903		}
904
905		// signed integer addition of 32-bit sign extended immediate to 64 bit register
906
907		// add rcx, 2147483647
908		//
909		// Using the following opcode: REX.W + 81 /0 id, we expect the following encoding
910		//
911		// 48 : REX.W set for 64 bit operand (rcx)
912		// 81 : opcode for "<arithmetic> with immediate"
913		// c1 : id = rcx,
914		// : c1 = 11 <-- mod = 11 indicates r/m is register (rcx)
915		// : 000 <-- opcode_extension = 0 because opcode extension is /0. /0 specifies ADD
916		// : 001 <-- 001 is rcx
917		// ffffff7f : 2147483647
918		{
919		try code.resize(0);
920		const encoder = try Encoder.init(&code, 7);
921		encoder.rex(.{ .w = true }); // use 64 bit operation
922		encoder.opcode_1byte(0x81);
923		encoder.modRm_direct(
924		0,
925		Register.rcx.lowEnc(),
926		);
927		encoder.imm32(2147483647);
928
929		try testing.expectEqualSlices(u8, &[_]u8{ 0x48, 0x81, 0xc1, 0xff, 0xff, 0xff, 0x7f }, code.items);
930		}
931		}
932
933		test "Encoder helpers - Vex prefix" {
934		var buf: [3]u8 = undefined;
935		var stream = std.io.fixedBufferStream(&buf);
936		const writer = stream.writer();
937
938		{
939		var vex_prefix = Encoder.Vex{};
940		vex_prefix.rex(.{
941		.r = true,
942		});
943		const nwritten = vex_prefix.write(writer);
944		try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, buf[0..nwritten]);
945		}
946
947		{
948		stream.reset();
949		var vex_prefix = Encoder.Vex{};
950		vex_prefix.reg(Register.xmm15.enc());
951		const nwritten = vex_prefix.write(writer);
952		try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]);
953		}
954
955		{
956		stream.reset();
957		var vex_prefix = Encoder.Vex{};
958		vex_prefix.rex(.{
959		.w = true,
960		.x = true,
961		});
962		const nwritten = vex_prefix.write(writer);
963		try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, buf[0..nwritten]);
964		}
965
966		{
967		stream.reset();
968		var vex_prefix = Encoder.Vex{};
969		vex_prefix.rex(.{
970		.w = true,
971		.r = true,
972		});
973		vex_prefix.len_256();
974		vex_prefix.lead_opc_0f();
975		vex_prefix.simd_prefix_66();
976		const nwritten = vex_prefix.write(writer);
977		try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, buf[0..nwritten]);
978		}
979
980		var code = ArrayList(u8).init(testing.allocator);
981		defer code.deinit();
982
983		{
984		// vmovapd xmm1, xmm2
985		const encoder = try Encoder.init(&code, 4);
986		var vex = Encoder.Vex{};
987		vex.simd_prefix_66();
988		encoder.vex(vex); // use 64 bit operation
989		encoder.opcode_1byte(0x28);
990		encoder.modRm_direct(0, Register.xmm1.lowEnc());
991		try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items);
992		}
993
994		{
995		try code.resize(0);
996
997		// vmovhpd xmm13, xmm1, qword ptr [rip]
998		const encoder = try Encoder.init(&code, 9);
999		var vex = Encoder.Vex{};
1000		vex.len_128();
1001		vex.simd_prefix_66();
1002		vex.lead_opc_0f();
1003		vex.rex(.{ .r = true });
1004		vex.reg(Register.xmm1.enc());
1005		encoder.vex(vex);
1006		encoder.opcode_1byte(0x16);
1007		encoder.modRm_RIPDisp32(Register.xmm13.lowEnc());
1008		encoder.disp32(0);
1009		try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items);
1010		}
1011		}
1012
1013		// TODO add these registers to the enum and populate dwarfLocOp
1014		// // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register.
1015		// RA = (16, "RA"),
1016		//
1017		// XMM0 = (17, "xmm0"),
1018		// XMM1 = (18, "xmm1"),
1019		// XMM2 = (19, "xmm2"),
1020		// XMM3 = (20, "xmm3"),
1021		// XMM4 = (21, "xmm4"),
1022		// XMM5 = (22, "xmm5"),
1023		// XMM6 = (23, "xmm6"),
1024		// XMM7 = (24, "xmm7"),
1025		//
1026		// XMM8 = (25, "xmm8"),
1027		// XMM9 = (26, "xmm9"),
1028		// XMM10 = (27, "xmm10"),
1029		// XMM11 = (28, "xmm11"),
1030		// XMM12 = (29, "xmm12"),
1031		// XMM13 = (30, "xmm13"),
1032		// XMM14 = (31, "xmm14"),
1033		// XMM15 = (32, "xmm15"),
1034		//
1035		// ST0 = (33, "st0"),
1036		// ST1 = (34, "st1"),
1037		// ST2 = (35, "st2"),
1038		// ST3 = (36, "st3"),
1039		// ST4 = (37, "st4"),
1040		// ST5 = (38, "st5"),
1041		// ST6 = (39, "st6"),
1042		// ST7 = (40, "st7"),
1043		//
1044		// MM0 = (41, "mm0"),
1045		// MM1 = (42, "mm1"),
1046		// MM2 = (43, "mm2"),
1047		// MM3 = (44, "mm3"),
1048		// MM4 = (45, "mm4"),
1049		// MM5 = (46, "mm5"),
1050		// MM6 = (47, "mm6"),
1051		// MM7 = (48, "mm7"),
1052		//
1053		// RFLAGS = (49, "rFLAGS"),
1054		// ES = (50, "es"),
1055		// CS = (51, "cs"),
1056		// SS = (52, "ss"),
1057		// DS = (53, "ds"),
1058		// FS = (54, "fs"),
1059		// GS = (55, "gs"),
1060		//
1061		// FS_BASE = (58, "fs.base"),
1062		// GS_BASE = (59, "gs.base"),
1063		//
1064		// TR = (62, "tr"),
1065		// LDTR = (63, "ldtr"),
1066		// MXCSR = (64, "mxcsr"),
1067		// FCW = (65, "fcw"),
1068		// FSW = (66, "fsw"),
1069		//
1070		// XMM16 = (67, "xmm16"),
1071		// XMM17 = (68, "xmm17"),
1072		// XMM18 = (69, "xmm18"),
1073		// XMM19 = (70, "xmm19"),
1074		// XMM20 = (71, "xmm20"),
1075		// XMM21 = (72, "xmm21"),
1076		// XMM22 = (73, "xmm22"),
1077		// XMM23 = (74, "xmm23"),
1078		// XMM24 = (75, "xmm24"),
1079		// XMM25 = (76, "xmm25"),
1080		// XMM26 = (77, "xmm26"),
1081		// XMM27 = (78, "xmm27"),
1082		// XMM28 = (79, "xmm28"),
1083		// XMM29 = (80, "xmm29"),
1084		// XMM30 = (81, "xmm30"),
1085		// XMM31 = (82, "xmm31"),
1086		//
1087		// K0 = (118, "k0"),
1088		// K1 = (119, "k1"),
1089		// K2 = (120, "k2"),
1090		// K3 = (121, "k3"),
1091		// K4 = (122, "k4"),
1092		// K5 = (123, "k5"),
1093		// K6 = (124, "k6"),
1094		// K7 = (125, "k7"),

src/arch/x86_64/encoder.zig created+794

...	...	@@ -0,0 +1,794 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3	const math = std.math;
	4
	5	const bits = @import("bits.zig");
	6	const Encoding = @import("Encoding.zig");
	7	const Memory = bits.Memory;
	8	const Moffs = bits.Moffs;
	9	const PtrSize = bits.PtrSize;
	10	const Register = bits.Register;
	11
	12	pub const Instruction = struct {
	13	op1: Operand = .none,
	14	op2: Operand = .none,
	15	op3: Operand = .none,
	16	op4: Operand = .none,
	17	encoding: Encoding,
	18
	19	pub const Mnemonic = Encoding.Mnemonic;
	20
	21	pub const Operand = union(enum) {
	22	none,
	23	reg: Register,
	24	mem: Memory,
	25	imm: i64,
	26
	27	/// Returns the bitsize of the operand.
	28	/// Asserts the operand is either register or memory.
	29	pub fn size(op: Operand) u64 {
	30	return switch (op) {
	31	.none => unreachable,
	32	.reg => \|reg\| reg.size(),
	33	.mem => \|mem\| mem.size(),
	34	.imm => unreachable,
	35	};
	36	}
	37
	38	/// Returns true if the operand is a segment register.
	39	/// Asserts the operand is either register or memory.
	40	pub fn isSegmentRegister(op: Operand) bool {
	41	return switch (op) {
	42	.none => unreachable,
	43	.reg => \|reg\| reg.class() == .segment,
	44	.mem => \|mem\| mem.isSegmentRegister(),
	45	.imm => unreachable,
	46	};
	47	}
	48
	49	pub fn fmtPrint(op: Operand, enc_op: Encoding.Op, writer: anytype) !void {
	50	switch (op) {
	51	.none => {},
	52	.reg => \|reg\| try writer.writeAll(@tagName(reg)),
	53	.mem => \|mem\| switch (mem) {
	54	.rip => \|rip\| {
	55	try writer.print("{s} ptr [rip", .{@tagName(rip.ptr_size)});
	56	if (rip.disp != 0) {
	57	const sign_bit = if (sign(rip.disp) < 0) "-" else "+";
	58	const disp_abs = try std.math.absInt(rip.disp);
	59	try writer.print(" {s} 0x{x}", .{ sign_bit, disp_abs });
	60	}
	61	try writer.writeByte(']');
	62	},
	63	.sib => \|sib\| {
	64	try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});
	65
	66	if (mem.isSegmentRegister()) {
	67	return writer.print("{s}:0x{x}", .{ @tagName(sib.base.?), sib.disp });
	68	}
	69
	70	try writer.writeByte('[');
	71
	72	if (sib.base) \|base\| {
	73	try writer.print("{s}", .{@tagName(base)});
	74	}
	75	if (sib.scale_index) \|si\| {
	76	if (sib.base != null) {
	77	try writer.writeAll(" + ");
	78	}
	79	try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });
	80	}
	81	if (sib.disp != 0) {
	82	if (sib.base != null or sib.scale_index != null) {
	83	try writer.writeByte(' ');
	84	}
	85	try writer.writeByte(if (sign(sib.disp) < 0) '-' else '+');
	86	const disp_abs = try std.math.absInt(sib.disp);
	87	try writer.print(" 0x{x}", .{disp_abs});
	88	}
	89
	90	try writer.writeByte(']');
	91	},
	92	.moffs => \|moffs\| try writer.print("{s}:0x{x}", .{ @tagName(moffs.seg), moffs.offset }),
	93	},
	94	.imm => \|imm\| {
	95	if (enc_op == .imm64) {
	96	return writer.print("0x{x}", .{@bitCast(u64, imm)});
	97	}
	98	const imm_abs = try std.math.absInt(imm);
	99	if (sign(imm) < 0) {
	100	try writer.writeByte('-');
	101	}
	102	try writer.print("0x{x}", .{imm_abs});
	103	},
	104	}
	105	}
	106	};
	107
	108	pub fn new(mnemonic: Mnemonic, args: struct {
	109	op1: Operand = .none,
	110	op2: Operand = .none,
	111	op3: Operand = .none,
	112	op4: Operand = .none,
	113	}) !Instruction {
	114	const encoding = Encoding.findByMnemonic(mnemonic, .{
	115	.op1 = args.op1,
	116	.op2 = args.op2,
	117	.op3 = args.op3,
	118	.op4 = args.op4,
	119	}) orelse return error.InvalidInstruction;
	120	std.log.debug("{}", .{encoding});
	121	return .{
	122	.op1 = args.op1,
	123	.op2 = args.op2,
	124	.op3 = args.op3,
	125	.op4 = args.op4,
	126	.encoding = encoding,
	127	};
	128	}
	129
	130	pub fn fmtPrint(inst: Instruction, writer: anytype) !void {
	131	try writer.print("{s}", .{@tagName(inst.encoding.mnemonic)});
	132	const ops = [_]struct { Operand, Encoding.Op }{
	133	.{ inst.op1, inst.encoding.op1 },
	134	.{ inst.op2, inst.encoding.op2 },
	135	.{ inst.op3, inst.encoding.op3 },
	136	.{ inst.op4, inst.encoding.op4 },
	137	};
	138	for (&ops, 0..) \|op, i\| {
	139	if (op[0] == .none) break;
	140	if (i > 0) {
	141	try writer.writeByte(',');
	142	}
	143	try writer.writeByte(' ');
	144	try op[0].fmtPrint(op[1], writer);
	145	}
	146	}
	147
	148	pub fn encode(inst: Instruction, writer: anytype) !void {
	149	const encoder = Encoder(@TypeOf(writer)){ .writer = writer };
	150	const encoding = inst.encoding;
	151
	152	try inst.encodeLegacyPrefixes(encoder);
	153	try inst.encodeMandatoryPrefix(encoder);
	154	try inst.encodeRexPrefix(encoder);
	155	try inst.encodeOpcode(encoder);
	156
	157	switch (encoding.op_en) {
	158	.np, .o => {},
	159	.i, .d => try encodeImm(inst.op1.imm, encoding.op1, encoder),
	160	.zi, .oi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
	161	.fd => try encoder.imm64(inst.op2.mem.moffs.offset),
	162	.td => try encoder.imm64(inst.op1.mem.moffs.offset),
	163	else => {
	164	const mem_op = switch (encoding.op_en) {
	165	.m, .mi, .m1, .mc, .mr => inst.op1,
	166	.rm, .rmi => inst.op2,
	167	else => unreachable,
	168	};
	169	switch (mem_op) {
	170	.reg => \|reg\| {
	171	const rm = switch (encoding.op_en) {
	172	.m, .mi, .m1, .mc => encoding.modRmExt(),
	173	.mr => inst.op2.reg.lowEnc(),
	174	.rm, .rmi => inst.op1.reg.lowEnc(),
	175	else => unreachable,
	176	};
	177	try encoder.modRm_direct(rm, reg.lowEnc());
	178	},
	179	.mem => \|mem\| {
	180	const op = switch (encoding.op_en) {
	181	.m, .mi, .m1, .mc => .none,
	182	.mr => inst.op2,
	183	.rm, .rmi => inst.op1,
	184	else => unreachable,
	185	};
	186	try encodeMemory(encoding, mem, op, encoder);
	187	},
	188	else => unreachable,
	189	}
	190
	191	switch (encoding.op_en) {
	192	.mi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
	193	.rmi => try encodeImm(inst.op3.imm, encoding.op3, encoder),
	194	else => {},
	195	}
	196	},
	197	}
	198	}
	199
	200	fn encodeOpcode(inst: Instruction, encoder: anytype) !void {
	201	const opcode = inst.encoding.opcode();
	202	switch (inst.encoding.op_en) {
	203	.o, .oi => try encoder.opcode_withReg(opcode[0], inst.op1.reg.lowEnc()),
	204	else => {
	205	const index: usize = if (inst.encoding.mandatoryPrefix()) \|_\| 1 else 0;
	206	for (opcode[index..]) \|byte\| {
	207	try encoder.opcode_1byte(byte);
	208	}
	209	},
	210	}
	211	}
	212
	213	fn encodeLegacyPrefixes(inst: Instruction, encoder: anytype) !void {
	214	const enc = inst.encoding;
	215	const op_en = enc.op_en;
	216
	217	var legacy = LegacyPrefixes{};
	218	if (enc.mode == .none) {
	219	const bit_size = enc.operandSize();
	220	if (bit_size == 16) {
	221	legacy.set16BitOverride();
	222	}
	223	}
	224
	225	const segment_override: ?Register = switch (op_en) {
	226	.i, .zi, .o, .oi, .d, .np => null,
	227	.fd => inst.op2.mem.base().?,
	228	.td => inst.op1.mem.base().?,
	229	.rm, .rmi => if (inst.op2.isSegmentRegister()) blk: {
	230	break :blk switch (inst.op2) {
	231	.reg => \|r\| r,
	232	.mem => \|m\| m.base().?,
	233	else => unreachable,
	234	};
	235	} else null,
	236	.m, .mi, .m1, .mc, .mr => if (inst.op1.isSegmentRegister()) blk: {
	237	break :blk switch (inst.op1) {
	238	.reg => \|r\| r,
	239	.mem => \|m\| m.base().?,
	240	else => unreachable,
	241	};
	242	} else null,
	243	};
	244	if (segment_override) \|seg\| {
	245	legacy.setSegmentOverride(seg);
	246	}
	247
	248	try encoder.legacyPrefixes(legacy);
	249	}
	250
	251	fn encodeRexPrefix(inst: Instruction, encoder: anytype) !void {
	252	const op_en = inst.encoding.op_en;
	253
	254	// Check if we need REX and can actually encode it
	255	const is_rex_invalid = for (&[_]Operand{ inst.op1, inst.op2, inst.op3, inst.op4 }) \|op\| switch (op) {
	256	.reg => \|r\| if (r.isRexInvalid()) break true,
	257	else => {},
	258	} else false;
	259
	260	var rex = Rex{};
	261	rex.w = inst.encoding.mode == .long;
	262
	263	switch (op_en) {
	264	.np, .i, .zi, .fd, .td, .d => {},
	265	.o, .oi => {
	266	rex.b = inst.op1.reg.isExtended();
	267	},
	268	.m, .mi, .m1, .mc, .mr, .rm, .rmi => {
	269	const r_op = switch (op_en) {
	270	.rm, .rmi => inst.op1,
	271	.mr => inst.op2,
	272	else => null,
	273	};
	274	if (r_op) \|op\| {
	275	rex.r = op.reg.isExtended();
	276	}
	277
	278	const b_x_op = switch (op_en) {
	279	.rm, .rmi => inst.op2,
	280	.m, .mi, .m1, .mc, .mr => inst.op1,
	281	else => unreachable,
	282	};
	283	switch (b_x_op) {
	284	.reg => \|r\| {
	285	rex.b = r.isExtended();
	286	},
	287	.mem => \|mem\| {
	288	rex.b = if (mem.base()) \|base\| base.isExtended() else false;
	289	rex.x = if (mem.scaleIndex()) \|si\| si.index.isExtended() else false;
	290	},
	291	else => unreachable,
	292	}
	293	},
	294	}
	295
	296	if (rex.isSet() and is_rex_invalid) return error.CannotEncode;
	297
	298	try encoder.rex(rex);
	299	}
	300
	301	fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void {
	302	const prefix = inst.encoding.mandatoryPrefix() orelse return;
	303	try encoder.opcode_1byte(prefix);
	304	}
	305
	306	fn encodeMemory(encoding: Encoding, mem: Memory, operand: Operand, encoder: anytype) !void {
	307	const operand_enc = switch (operand) {
	308	.reg => \|reg\| reg.lowEnc(),
	309	.none => encoding.modRmExt(),
	310	else => unreachable,
	311	};
	312
	313	switch (mem) {
	314	.moffs => unreachable,
	315	.sib => \|sib\| {
	316	if (sib.base) \|base\| {
	317	if (base.class() == .segment) {
	318	// TODO audit this wrt SIB
	319	try encoder.modRm_SIBDisp0(operand_enc);
	320	if (sib.scale_index) \|si\| {
	321	const scale = math.log2_int(u4, si.scale);
	322	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
	323	} else {
	324	try encoder.sib_disp32();
	325	}
	326	try encoder.disp32(sib.disp);
	327	} else {
	328	assert(base.class() == .general_purpose);
	329	const dst = base.lowEnc();
	330	const src = operand_enc;
	331	if (dst == 4 or sib.scale_index != null) {
	332	if (sib.disp == 0 and dst != 5) {
	333	try encoder.modRm_SIBDisp0(src);
	334	if (sib.scale_index) \|si\| {
	335	const scale = math.log2_int(u4, si.scale);
	336	try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
	337	} else {
	338	try encoder.sib_base(dst);
	339	}
	340	} else if (math.cast(i8, sib.disp)) \|_\| {
	341	try encoder.modRm_SIBDisp8(src);
	342	if (sib.scale_index) \|si\| {
	343	const scale = math.log2_int(u4, si.scale);
	344	try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
	345	} else {
	346	try encoder.sib_baseDisp8(dst);
	347	}
	348	try encoder.disp8(@truncate(i8, sib.disp));
	349	} else {
	350	try encoder.modRm_SIBDisp32(src);
	351	if (sib.scale_index) \|si\| {
	352	const scale = math.log2_int(u4, si.scale);
	353	try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
	354	} else {
	355	try encoder.sib_baseDisp32(dst);
	356	}
	357	try encoder.disp32(sib.disp);
	358	}
	359	} else {
	360	if (sib.disp == 0 and dst != 5) {
	361	try encoder.modRm_indirectDisp0(src, dst);
	362	} else if (math.cast(i8, sib.disp)) \|_\| {
	363	try encoder.modRm_indirectDisp8(src, dst);
	364	try encoder.disp8(@truncate(i8, sib.disp));
	365	} else {
	366	try encoder.modRm_indirectDisp32(src, dst);
	367	try encoder.disp32(sib.disp);
	368	}
	369	}
	370	}
	371	} else {
	372	try encoder.modRm_SIBDisp0(operand_enc);
	373	if (sib.scale_index) \|si\| {
	374	const scale = math.log2_int(u4, si.scale);
	375	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
	376	} else {
	377	try encoder.sib_disp32();
	378	}
	379	try encoder.disp32(sib.disp);
	380	}
	381	},
	382	.rip => \|rip\| {
	383	try encoder.modRm_RIPDisp32(operand_enc);
	384	try encoder.disp32(rip.disp);
	385	},
	386	}
	387	}
	388
	389	fn encodeImm(imm: i64, kind: Encoding.Op, encoder: anytype) !void {
	390	switch (kind) {
	391	.imm8, .rel8 => try encoder.imm8(@truncate(i8, imm)),
	392	.imm16, .rel16 => try encoder.imm16(@truncate(i16, imm)),
	393	.imm32, .rel32 => try encoder.imm32(@truncate(i32, imm)),
	394	.imm64 => try encoder.imm64(@bitCast(u64, imm)),
	395	else => unreachable,
	396	}
	397	}
	398	};
	399
	400	inline fn sign(i: anytype) @TypeOf(i) {
	401	return @as(@TypeOf(i), @boolToInt(i > 0)) - @boolToInt(i < 0);
	402	}
	403
	404	pub const LegacyPrefixes = packed struct {
	405	/// LOCK
	406	prefix_f0: bool = false,
	407	/// REPNZ, REPNE, REP, Scalar Double-precision
	408	prefix_f2: bool = false,
	409	/// REPZ, REPE, REP, Scalar Single-precision
	410	prefix_f3: bool = false,
	411
	412	/// CS segment override or Branch not taken
	413	prefix_2e: bool = false,
	414	/// SS segment override
	415	prefix_36: bool = false,
	416	/// ES segment override
	417	prefix_26: bool = false,
	418	/// FS segment override
	419	prefix_64: bool = false,
	420	/// GS segment override
	421	prefix_65: bool = false,
	422
	423	/// Branch taken
	424	prefix_3e: bool = false,
	425
	426	/// Address size override (enables 16 bit address size)
	427	prefix_67: bool = false,
	428
	429	/// Operand size override (enables 16 bit operation)
	430	prefix_66: bool = false,
	431
	432	padding: u5 = 0,
	433
	434	pub fn setSegmentOverride(self: *LegacyPrefixes, reg: Register) void {
	435	assert(reg.class() == .segment);
	436	switch (reg) {
	437	.cs => self.prefix_2e = true,
	438	.ss => self.prefix_36 = true,
	439	.es => self.prefix_26 = true,
	440	.fs => self.prefix_64 = true,
	441	.gs => self.prefix_65 = true,
	442	.ds => {},
	443	else => unreachable,
	444	}
	445	}
	446
	447	pub fn set16BitOverride(self: *LegacyPrefixes) void {
	448	self.prefix_66 = true;
	449	}
	450	};
	451
	452	fn Encoder(comptime T: type) type {
	453	return struct {
	454	writer: T,
	455
	456	const Self = @This();
	457
	458	// --------
	459	// Prefixes
	460	// --------
	461
	462	/// Encodes legacy prefixes
	463	pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) !void {
	464	if (@bitCast(u16, prefixes) != 0) {
	465	// Hopefully this path isn't taken very often, so we'll do it the slow way for now
	466
	467	// LOCK
	468	if (prefixes.prefix_f0) try self.writer.writeByte(0xf0);
	469	// REPNZ, REPNE, REP, Scalar Double-precision
	470	if (prefixes.prefix_f2) try self.writer.writeByte(0xf2);
	471	// REPZ, REPE, REP, Scalar Single-precision
	472	if (prefixes.prefix_f3) try self.writer.writeByte(0xf3);
	473
	474	// CS segment override or Branch not taken
	475	if (prefixes.prefix_2e) try self.writer.writeByte(0x2e);
	476	// DS segment override
	477	if (prefixes.prefix_36) try self.writer.writeByte(0x36);
	478	// ES segment override
	479	if (prefixes.prefix_26) try self.writer.writeByte(0x26);
	480	// FS segment override
	481	if (prefixes.prefix_64) try self.writer.writeByte(0x64);
	482	// GS segment override
	483	if (prefixes.prefix_65) try self.writer.writeByte(0x65);
	484
	485	// Branch taken
	486	if (prefixes.prefix_3e) try self.writer.writeByte(0x3e);
	487
	488	// Operand size override
	489	if (prefixes.prefix_66) try self.writer.writeByte(0x66);
	490
	491	// Address size override
	492	if (prefixes.prefix_67) try self.writer.writeByte(0x67);
	493	}
	494	}
	495
	496	/// Use 16 bit operand size
	497	///
	498	/// Note that this flag is overridden by REX.W, if both are present.
	499	pub fn prefix16BitMode(self: Self) !void {
	500	try self.writer.writeByte(0x66);
	501	}
	502
	503	/// Encodes a REX prefix byte given all the fields
	504	///
	505	/// Use this byte whenever you need 64 bit operation,
	506	/// or one of reg, index, r/m, base, or opcode-reg might be extended.
	507	///
	508	/// See struct `Rex` for a description of each field.
	509	///
	510	/// Does not add a prefix byte if none of the fields are set!
	511	pub fn rex(self: Self, byte: Rex) !void {
	512	var value: u8 = 0b0100_0000;
	513
	514	if (byte.w) value \|= 0b1000;
	515	if (byte.r) value \|= 0b0100;
	516	if (byte.x) value \|= 0b0010;
	517	if (byte.b) value \|= 0b0001;
	518
	519	if (value != 0b0100_0000) {
	520	try self.writer.writeByte(value);
	521	}
	522	}
	523
	524	// ------
	525	// Opcode
	526	// ------
	527
	528	/// Encodes a 1 byte opcode
	529	pub fn opcode_1byte(self: Self, opcode: u8) !void {
	530	try self.writer.writeByte(opcode);
	531	}
	532
	533	/// Encodes a 2 byte opcode
	534	///
	535	/// e.g. IMUL has the opcode 0x0f 0xaf, so you use
	536	///
	537	/// encoder.opcode_2byte(0x0f, 0xaf);
	538	pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) !void {
	539	try self.writer.writeAll(&.{ prefix, opcode });
	540	}
	541
	542	/// Encodes a 3 byte opcode
	543	///
	544	/// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
	545	///
	546	/// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
	547	pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) !void {
	548	try self.writer.writeAll(&.{ prefix_1, prefix_2, opcode });
	549	}
	550
	551	/// Encodes a 1 byte opcode with a reg field
	552	///
	553	/// Remember to add a REX prefix byte if reg is extended!
	554	pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) !void {
	555	assert(opcode & 0b111 == 0);
	556	try self.writer.writeByte(opcode \| reg);
	557	}
	558
	559	// ------
	560	// ModR/M
	561	// ------
	562
	563	/// Construct a ModR/M byte given all the fields
	564	///
	565	/// Remember to add a REX prefix byte if reg or rm are extended!
	566	pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) !void {
	567	try self.writer.writeByte(@as(u8, mod) << 6 \| @as(u8, reg_or_opx) << 3 \| rm);
	568	}
	569
	570	/// Construct a ModR/M byte using direct r/m addressing
	571	/// r/m effective address: r/m
	572	///
	573	/// Note reg's effective address is always just reg for the ModR/M byte.
	574	/// Remember to add a REX prefix byte if reg or rm are extended!
	575	pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) !void {
	576	try self.modRm(0b11, reg_or_opx, rm);
	577	}
	578
	579	/// Construct a ModR/M byte using indirect r/m addressing
	580	/// r/m effective address: [r/m]
	581	///
	582	/// Note reg's effective address is always just reg for the ModR/M byte.
	583	/// Remember to add a REX prefix byte if reg or rm are extended!
	584	pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) !void {
	585	assert(rm != 4 and rm != 5);
	586	try self.modRm(0b00, reg_or_opx, rm);
	587	}
	588
	589	/// Construct a ModR/M byte using indirect SIB addressing
	590	/// r/m effective address: [SIB]
	591	///
	592	/// Note reg's effective address is always just reg for the ModR/M byte.
	593	/// Remember to add a REX prefix byte if reg or rm are extended!
	594	pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) !void {
	595	try self.modRm(0b00, reg_or_opx, 0b100);
	596	}
	597
	598	/// Construct a ModR/M byte using RIP-relative addressing
	599	/// r/m effective address: [RIP + disp32]
	600	///
	601	/// Note reg's effective address is always just reg for the ModR/M byte.
	602	/// Remember to add a REX prefix byte if reg or rm are extended!
	603	pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) !void {
	604	try self.modRm(0b00, reg_or_opx, 0b101);
	605	}
	606
	607	/// Construct a ModR/M byte using indirect r/m with a 8bit displacement
	608	/// r/m effective address: [r/m + disp8]
	609	///
	610	/// Note reg's effective address is always just reg for the ModR/M byte.
	611	/// Remember to add a REX prefix byte if reg or rm are extended!
	612	pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) !void {
	613	assert(rm != 4);
	614	try self.modRm(0b01, reg_or_opx, rm);
	615	}
	616
	617	/// Construct a ModR/M byte using indirect SIB with a 8bit displacement
	618	/// r/m effective address: [SIB + disp8]
	619	///
	620	/// Note reg's effective address is always just reg for the ModR/M byte.
	621	/// Remember to add a REX prefix byte if reg or rm are extended!
	622	pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) !void {
	623	try self.modRm(0b01, reg_or_opx, 0b100);
	624	}
	625
	626	/// Construct a ModR/M byte using indirect r/m with a 32bit displacement
	627	/// r/m effective address: [r/m + disp32]
	628	///
	629	/// Note reg's effective address is always just reg for the ModR/M byte.
	630	/// Remember to add a REX prefix byte if reg or rm are extended!
	631	pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) !void {
	632	assert(rm != 4);
	633	try self.modRm(0b10, reg_or_opx, rm);
	634	}
	635
	636	/// Construct a ModR/M byte using indirect SIB with a 32bit displacement
	637	/// r/m effective address: [SIB + disp32]
	638	///
	639	/// Note reg's effective address is always just reg for the ModR/M byte.
	640	/// Remember to add a REX prefix byte if reg or rm are extended!
	641	pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) !void {
	642	try self.modRm(0b10, reg_or_opx, 0b100);
	643	}
	644
	645	// ---
	646	// SIB
	647	// ---
	648
	649	/// Construct a SIB byte given all the fields
	650	///
	651	/// Remember to add a REX prefix byte if index or base are extended!
	652	pub fn sib(self: Self, scale: u2, index: u3, base: u3) !void {
	653	try self.writer.writeByte(@as(u8, scale) << 6 \| @as(u8, index) << 3 \| base);
	654	}
	655
	656	/// Construct a SIB byte with scale * index + base, no frills.
	657	/// r/m effective address: [base + scale * index]
	658	///
	659	/// Remember to add a REX prefix byte if index or base are extended!
	660	pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) !void {
	661	assert(base != 5);
	662
	663	try self.sib(scale, index, base);
	664	}
	665
	666	/// Construct a SIB byte with scale * index + disp32
	667	/// r/m effective address: [scale * index + disp32]
	668	///
	669	/// Remember to add a REX prefix byte if index or base are extended!
	670	pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) !void {
	671	// scale is actually ignored
	672	// index = 4 means no index if and only if we haven't extended the register
	673	// TODO enforce this
	674	// base = 5 means no base, if mod == 0.
	675	try self.sib(scale, index, 5);
	676	}
	677
	678	/// Construct a SIB byte with just base
	679	/// r/m effective address: [base]
	680	///
	681	/// Remember to add a REX prefix byte if index or base are extended!
	682	pub fn sib_base(self: Self, base: u3) !void {
	683	assert(base != 5);
	684
	685	// scale is actually ignored
	686	// index = 4 means no index
	687	try self.sib(0, 4, base);
	688	}
	689
	690	/// Construct a SIB byte with just disp32
	691	/// r/m effective address: [disp32]
	692	///
	693	/// Remember to add a REX prefix byte if index or base are extended!
	694	pub fn sib_disp32(self: Self) !void {
	695	// scale is actually ignored
	696	// index = 4 means no index
	697	// base = 5 means no base, if mod == 0.
	698	try self.sib(0, 4, 5);
	699	}
	700
	701	/// Construct a SIB byte with scale * index + base + disp8
	702	/// r/m effective address: [base + scale * index + disp8]
	703	///
	704	/// Remember to add a REX prefix byte if index or base are extended!
	705	pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) !void {
	706	try self.sib(scale, index, base);
	707	}
	708
	709	/// Construct a SIB byte with base + disp8, no index
	710	/// r/m effective address: [base + disp8]
	711	///
	712	/// Remember to add a REX prefix byte if index or base are extended!
	713	pub fn sib_baseDisp8(self: Self, base: u3) !void {
	714	// scale is ignored
	715	// index = 4 means no index
	716	try self.sib(0, 4, base);
	717	}
	718
	719	/// Construct a SIB byte with scale * index + base + disp32
	720	/// r/m effective address: [base + scale * index + disp32]
	721	///
	722	/// Remember to add a REX prefix byte if index or base are extended!
	723	pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) !void {
	724	try self.sib(scale, index, base);
	725	}
	726
	727	/// Construct a SIB byte with base + disp32, no index
	728	/// r/m effective address: [base + disp32]
	729	///
	730	/// Remember to add a REX prefix byte if index or base are extended!
	731	pub fn sib_baseDisp32(self: Self, base: u3) !void {
	732	// scale is ignored
	733	// index = 4 means no index
	734	try self.sib(0, 4, base);
	735	}
	736
	737	// -------------------------
	738	// Trivial (no bit fiddling)
	739	// -------------------------
	740
	741	/// Encode an 8 bit immediate
	742	///
	743	/// It is sign-extended to 64 bits by the cpu.
	744	pub fn imm8(self: Self, imm: i8) !void {
	745	try self.writer.writeByte(@bitCast(u8, imm));
	746	}
	747
	748	/// Encode an 8 bit displacement
	749	///
	750	/// It is sign-extended to 64 bits by the cpu.
	751	pub fn disp8(self: Self, disp: i8) !void {
	752	try self.writer.writeByte(@bitCast(u8, disp));
	753	}
	754
	755	/// Encode an 16 bit immediate
	756	///
	757	/// It is sign-extended to 64 bits by the cpu.
	758	pub fn imm16(self: Self, imm: i16) !void {
	759	try self.writer.writeIntLittle(i16, imm);
	760	}
	761
	762	/// Encode an 32 bit immediate
	763	///
	764	/// It is sign-extended to 64 bits by the cpu.
	765	pub fn imm32(self: Self, imm: i32) !void {
	766	try self.writer.writeIntLittle(i32, imm);
	767	}
	768
	769	/// Encode an 32 bit displacement
	770	///
	771	/// It is sign-extended to 64 bits by the cpu.
	772	pub fn disp32(self: Self, disp: i32) !void {
	773	try self.writer.writeIntLittle(i32, disp);
	774	}
	775
	776	/// Encode an 64 bit immediate
	777	///
	778	/// It is sign-extended to 64 bits by the cpu.
	779	pub fn imm64(self: Self, imm: u64) !void {
	780	try self.writer.writeIntLittle(u64, imm);
	781	}
	782	};
	783	}
	784
	785	pub const Rex = struct {
	786	w: bool = false,
	787	r: bool = false,
	788	x: bool = false,
	789	b: bool = false,
	790
	791	pub fn isSet(rex: Rex) bool {
	792	return rex.w or rex.r or rex.x or rex.b;
	793	}
	794	};

src/arch/x86_64/encodings.zig created+542

...	...	@@ -0,0 +1,542 @@
	1	const Encoding = @import("Encoding.zig");
	2	const Mnemonic = Encoding.Mnemonic;
	3	const OpEn = Encoding.OpEn;
	4	const Op = Encoding.Op;
	5	const Mode = Encoding.Mode;
	6
	7	const opcode_len = u2;
	8	const modrm_ext = u3;
	9
	10	const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, opcode_len, u8, u8, u8, modrm_ext, Mode };
	11
	12	// TODO move this into a .zon file when Zig is capable of importing .zon files
	13	// zig fmt: off
	14	pub const table = &[_]Entry{
	15	// General-purpose
	16	.{ .adc, .zi, .al, .imm8, .none, .none, 1, 0x14, 0x00, 0x00, 0, .none },
	17	.{ .adc, .zi, .ax, .imm16, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
	18	.{ .adc, .zi, .eax, .imm32, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
	19	.{ .adc, .zi, .rax, .imm32, .none, .none, 1, 0x15, 0x00, 0x00, 0, .long },
	20	.{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .none },
	21	.{ .adc, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
	22	.{ .adc, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
	23	.{ .adc, .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 2, .long },
	24	.{ .adc, .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
	25	.{ .adc, .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
	26	.{ .adc, .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 2, .long },
	27	.{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .none },
	28	.{ .adc, .mr, .rm16, .r16, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
	29	.{ .adc, .mr, .rm32, .r32, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
	30	.{ .adc, .mr, .rm64, .r64, .none, .none, 1, 0x11, 0x00, 0x00, 0, .long },
	31	.{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .none },
	32	.{ .adc, .rm, .r16, .rm16, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
	33	.{ .adc, .rm, .r32, .rm32, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
	34	.{ .adc, .rm, .r64, .rm64, .none, .none, 1, 0x13, 0x00, 0x00, 0, .long },
	35
	36	.{ .add, .zi, .al, .imm8, .none, .none, 1, 0x04, 0x00, 0x00, 0, .none },
	37	.{ .add, .zi, .ax, .imm16, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
	38	.{ .add, .zi, .eax, .imm32, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
	39	.{ .add, .zi, .rax, .imm32, .none, .none, 1, 0x05, 0x00, 0x00, 0, .long },
	40	.{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .none },
	41	.{ .add, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
	42	.{ .add, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
	43	.{ .add, .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 0, .long },
	44	.{ .add, .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
	45	.{ .add, .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
	46	.{ .add, .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 0, .long },
	47	.{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .none },
	48	.{ .add, .mr, .rm16, .r16, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
	49	.{ .add, .mr, .rm32, .r32, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
	50	.{ .add, .mr, .rm64, .r64, .none, .none, 1, 0x01, 0x00, 0x00, 0, .long },
	51	.{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .none },
	52	.{ .add, .rm, .r16, .rm16, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
	53	.{ .add, .rm, .r32, .rm32, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
	54	.{ .add, .rm, .r64, .rm64, .none, .none, 1, 0x03, 0x00, 0x00, 0, .long },
	55
	56	.{ .@"and", .zi, .al, .imm8, .none, .none, 1, 0x24, 0x00, 0x00, 0, .none },
	57	.{ .@"and", .zi, .ax, .imm16, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
	58	.{ .@"and", .zi, .eax, .imm32, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
	59	.{ .@"and", .zi, .rax, .imm32, .none, .none, 1, 0x25, 0x00, 0x00, 0, .long },
	60	.{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .none },
	61	.{ .@"and", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
	62	.{ .@"and", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
	63	.{ .@"and", .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 4, .long },
	64	.{ .@"and", .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
	65	.{ .@"and", .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
	66	.{ .@"and", .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 4, .long },
	67	.{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .none },
	68	.{ .@"and", .mr, .rm16, .r16, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
	69	.{ .@"and", .mr, .rm32, .r32, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
	70	.{ .@"and", .mr, .rm64, .r64, .none, .none, 1, 0x21, 0x00, 0x00, 0, .long },
	71	.{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .none },
	72	.{ .@"and", .rm, .r16, .rm16, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
	73	.{ .@"and", .rm, .r32, .rm32, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
	74	.{ .@"and", .rm, .r64, .rm64, .none, .none, 1, 0x23, 0x00, 0x00, 0, .long },
	75
	76	// This is M encoding according to Intel, but D makes more sense here.
	77	.{ .call, .d, .rel32, .none, .none, .none, 1, 0xe8, 0x00, 0x00, 0, .none },
	78	.{ .call, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 2, .none },
	79
	80	.{ .cbw, .np, .o16, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
	81	.{ .cwde, .np, .o32, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
	82	.{ .cdqe, .np, .o64, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .long },
	83
	84	.{ .cwd, .np, .o16, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
	85	.{ .cdq, .np, .o32, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
	86	.{ .cqo, .np, .o64, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .long },
	87
	88	.{ .cmova, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	89	.{ .cmova, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	90	.{ .cmova, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
	91	.{ .cmovae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	92	.{ .cmovae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	93	.{ .cmovae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
	94	.{ .cmovb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	95	.{ .cmovb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	96	.{ .cmovb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
	97	.{ .cmovbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	98	.{ .cmovbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	99	.{ .cmovbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
	100	.{ .cmovc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	101	.{ .cmovc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	102	.{ .cmovc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
	103	.{ .cmove, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	104	.{ .cmove, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	105	.{ .cmove, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
	106	.{ .cmovg, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	107	.{ .cmovg, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	108	.{ .cmovg, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
	109	.{ .cmovge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	110	.{ .cmovge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	111	.{ .cmovge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
	112	.{ .cmovl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	113	.{ .cmovl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	114	.{ .cmovl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
	115	.{ .cmovle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	116	.{ .cmovle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	117	.{ .cmovle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
	118	.{ .cmovna, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	119	.{ .cmovna, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	120	.{ .cmovna, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
	121	.{ .cmovnae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	122	.{ .cmovnae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	123	.{ .cmovnae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
	124	.{ .cmovnb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	125	.{ .cmovnb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	126	.{ .cmovnb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
	127	.{ .cmovnbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	128	.{ .cmovnbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	129	.{ .cmovnbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
	130	.{ .cmovnc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	131	.{ .cmovnc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	132	.{ .cmovnc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
	133	.{ .cmovne, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	134	.{ .cmovne, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	135	.{ .cmovne, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
	136	.{ .cmovng, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	137	.{ .cmovng, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	138	.{ .cmovng, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
	139	.{ .cmovnge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	140	.{ .cmovnge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	141	.{ .cmovnge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
	142	.{ .cmovnl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	143	.{ .cmovnl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	144	.{ .cmovnl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
	145	.{ .cmovnle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	146	.{ .cmovnle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	147	.{ .cmovnle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
	148	.{ .cmovno, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
	149	.{ .cmovno, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
	150	.{ .cmovno, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .long },
	151	.{ .cmovnp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	152	.{ .cmovnp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	153	.{ .cmovnp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
	154	.{ .cmovns, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
	155	.{ .cmovns, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
	156	.{ .cmovns, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .long },
	157	.{ .cmovnz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	158	.{ .cmovnz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	159	.{ .cmovnz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
	160	.{ .cmovo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
	161	.{ .cmovo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
	162	.{ .cmovo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .long },
	163	.{ .cmovp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	164	.{ .cmovp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	165	.{ .cmovp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
	166	.{ .cmovpe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	167	.{ .cmovpe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	168	.{ .cmovpe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
	169	.{ .cmovpo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	170	.{ .cmovpo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	171	.{ .cmovpo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
	172	.{ .cmovs, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
	173	.{ .cmovs, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
	174	.{ .cmovs, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .long },
	175	.{ .cmovz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	176	.{ .cmovz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	177	.{ .cmovz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
	178
	179	.{ .cmp, .zi, .al, .imm8, .none, .none, 1, 0x3c, 0x00, 0x00, 0, .none },
	180	.{ .cmp, .zi, .ax, .imm16, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
	181	.{ .cmp, .zi, .eax, .imm32, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
	182	.{ .cmp, .zi, .rax, .imm32, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .long },
	183	.{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .none },
	184	.{ .cmp, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
	185	.{ .cmp, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
	186	.{ .cmp, .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 7, .long },
	187	.{ .cmp, .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
	188	.{ .cmp, .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
	189	.{ .cmp, .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 7, .long },
	190	.{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .none },
	191	.{ .cmp, .mr, .rm16, .r16, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
	192	.{ .cmp, .mr, .rm32, .r32, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
	193	.{ .cmp, .mr, .rm64, .r64, .none, .none, 1, 0x39, 0x00, 0x00, 0, .long },
	194	.{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .none },
	195	.{ .cmp, .rm, .r16, .rm16, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
	196	.{ .cmp, .rm, .r32, .rm32, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
	197	.{ .cmp, .rm, .r64, .rm64, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .long },
	198
	199	.{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .none },
	200	.{ .div, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
	201	.{ .div, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
	202	.{ .div, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .long },
	203
	204	.{ .fisttp, .m, .m16, .none, .none, .none, 1, 0xdf, 0x00, 0x00, 1, .fpu },
	205	.{ .fisttp, .m, .m32, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 1, .fpu },
	206	.{ .fisttp, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 1, .fpu },
	207
	208	.{ .fld, .m, .m32, .none, .none, .none, 1, 0xd9, 0x00, 0x00, 0, .fpu },
	209	.{ .fld, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 0, .fpu },
	210	.{ .fld, .m, .m80, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 5, .fpu },
	211
	212	.{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .none },
	213	.{ .idiv, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
	214	.{ .idiv, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
	215	.{ .idiv, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .long },
	216
	217	.{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .none },
	218	.{ .imul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
	219	.{ .imul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
	220	.{ .imul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .long },
	221	.{ .imul, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
	222	.{ .imul, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
	223	.{ .imul, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .long },
	224	.{ .imul, .rmi, .r16, .rm16, .imm8, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
	225	.{ .imul, .rmi, .r32, .rm32, .imm8, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
	226	.{ .imul, .rmi, .r64, .rm64, .imm8, .none, 1, 0x6b, 0x00, 0x00, 0, .long },
	227	.{ .imul, .rmi, .r16, .rm16, .imm16, .none, 1, 0x69, 0x00, 0x00, 0, .none },
	228	.{ .imul, .rmi, .r32, .rm32, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .none },
	229	.{ .imul, .rmi, .r64, .rm64, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .long },
	230
	231	.{ .int3, .np, .none, .none, .none, .none, 1, 0xcc, 0x00, 0x00, 0, .none },
	232
	233	.{ .ja, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
	234	.{ .jae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
	235	.{ .jb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
	236	.{ .jbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
	237	.{ .jc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
	238	.{ .jrcxz, .d, .rel32, .none, .none, .none, 1, 0xe3, 0x00, 0x00, 0, .none },
	239	.{ .je, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
	240	.{ .jg, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
	241	.{ .jge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
	242	.{ .jl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
	243	.{ .jle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
	244	.{ .jna, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
	245	.{ .jnae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
	246	.{ .jnb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
	247	.{ .jnbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
	248	.{ .jnc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
	249	.{ .jne, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
	250	.{ .jng, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
	251	.{ .jnge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
	252	.{ .jnl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
	253	.{ .jnle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
	254	.{ .jno, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x81, 0x00, 0, .none },
	255	.{ .jnp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
	256	.{ .jns, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x89, 0x00, 0, .none },
	257	.{ .jnz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
	258	.{ .jo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x80, 0x00, 0, .none },
	259	.{ .jp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
	260	.{ .jpe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
	261	.{ .jpo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
	262	.{ .js, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x88, 0x00, 0, .none },
	263	.{ .jz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
	264
	265	.{ .jmp, .d, .rel32, .none, .none, .none, 1, 0xe9, 0x00, 0x00, 0, .none },
	266	.{ .jmp, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 4, .none },
	267
	268	.{ .lea, .rm, .r16, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
	269	.{ .lea, .rm, .r32, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
	270	.{ .lea, .rm, .r64, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .long },
	271
	272	.{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .none },
	273	.{ .mov, .mr, .rm16, .r16, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
	274	.{ .mov, .mr, .rm32, .r32, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
	275	.{ .mov, .mr, .rm64, .r64, .none, .none, 1, 0x89, 0x00, 0x00, 0, .long },
	276	.{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .none },
	277	.{ .mov, .rm, .r16, .rm16, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
	278	.{ .mov, .rm, .r32, .rm32, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
	279	.{ .mov, .rm, .r64, .rm64, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .long },
	280	.{ .mov, .mr, .rm16, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .none },
	281	.{ .mov, .mr, .rm64, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .long },
	282	.{ .mov, .rm, .sreg, .rm16, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .none },
	283	.{ .mov, .rm, .sreg, .rm64, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .long },
	284	.{ .mov, .fd, .al, .moffs, .none, .none, 1, 0xa0, 0x00, 0x00, 0, .none },
	285	.{ .mov, .fd, .ax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
	286	.{ .mov, .fd, .eax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
	287	.{ .mov, .fd, .rax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .long },
	288	.{ .mov, .td, .moffs, .al, .none, .none, 1, 0xa2, 0x00, 0x00, 0, .none },
	289	.{ .mov, .td, .moffs, .ax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
	290	.{ .mov, .td, .moffs, .eax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
	291	.{ .mov, .td, .moffs, .rax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .long },
	292	.{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .none },
	293	.{ .mov, .oi, .r16, .imm16, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
	294	.{ .mov, .oi, .r32, .imm32, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
	295	.{ .mov, .oi, .r64, .imm64, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .long },
	296	.{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .none },
	297	.{ .mov, .mi, .rm16, .imm16, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
	298	.{ .mov, .mi, .rm32, .imm32, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
	299	.{ .mov, .mi, .rm64, .imm32, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .long },
	300
	301	.{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
	302	.{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
	303	.{ .movsx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .long },
	304	.{ .movsx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .none },
	305	.{ .movsx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .long },
	306
	307	// This instruction is discouraged.
	308	.{ .movsxd, .rm, .r32, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .none },
	309	.{ .movsxd, .rm, .r64, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .long },
	310
	311	.{ .movzx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
	312	.{ .movzx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
	313	.{ .movzx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .long },
	314	.{ .movzx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .none },
	315	.{ .movzx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .long },
	316
	317	.{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .none },
	318	.{ .mul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
	319	.{ .mul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
	320	.{ .mul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .long },
	321
	322	.{ .nop, .np, .none, .none, .none, .none, 1, 0x90, 0x00, 0x00, 0, .none },
	323
	324	.{ .@"or", .zi, .al, .imm8, .none, .none, 1, 0x0c, 0x00, 0x00, 0, .none },
	325	.{ .@"or", .zi, .ax, .imm16, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
	326	.{ .@"or", .zi, .eax, .imm32, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
	327	.{ .@"or", .zi, .rax, .imm32, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .long },
	328	.{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .none },
	329	.{ .@"or", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
	330	.{ .@"or", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
	331	.{ .@"or", .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 1, .long },
	332	.{ .@"or", .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
	333	.{ .@"or", .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
	334	.{ .@"or", .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 1, .long },
	335	.{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .none },
	336	.{ .@"or", .mr, .rm16, .r16, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
	337	.{ .@"or", .mr, .rm32, .r32, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
	338	.{ .@"or", .mr, .rm64, .r64, .none, .none, 1, 0x09, 0x00, 0x00, 0, .long },
	339	.{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .none },
	340	.{ .@"or", .rm, .r16, .rm16, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
	341	.{ .@"or", .rm, .r32, .rm32, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
	342	.{ .@"or", .rm, .r64, .rm64, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .long },
	343
	344	.{ .pop, .o, .r16, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
	345	.{ .pop, .o, .r64, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
	346	.{ .pop, .m, .rm16, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
	347	.{ .pop, .m, .rm64, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
	348
	349	.{ .push, .o, .r16, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
	350	.{ .push, .o, .r64, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
	351	.{ .push, .m, .rm16, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
	352	.{ .push, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
	353	.{ .push, .i, .imm8, .none, .none, .none, 1, 0x6a, 0x00, 0x00, 0, .none },
	354	.{ .push, .i, .imm16, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
	355	.{ .push, .i, .imm32, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
	356
	357	.{ .ret, .np, .none, .none, .none, .none, 1, 0xc3, 0x00, 0x00, 0, .none },
	358
	359	.{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
	360	.{ .sal, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	361	.{ .sal, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	362	.{ .sal, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
	363	.{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
	364	.{ .sal, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	365	.{ .sal, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	366	.{ .sal, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
	367	.{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
	368	.{ .sal, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	369	.{ .sal, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	370	.{ .sal, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
	371
	372	.{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .none },
	373	.{ .sar, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
	374	.{ .sar, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
	375	.{ .sar, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .long },
	376	.{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .none },
	377	.{ .sar, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
	378	.{ .sar, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
	379	.{ .sar, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .long },
	380	.{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .none },
	381	.{ .sar, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
	382	.{ .sar, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
	383	.{ .sar, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .long },
	384
	385	.{ .sbb, .zi, .al, .imm8, .none, .none, 1, 0x1c, 0x00, 0x00, 0, .none },
	386	.{ .sbb, .zi, .ax, .imm16, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
	387	.{ .sbb, .zi, .eax, .imm32, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
	388	.{ .sbb, .zi, .rax, .imm32, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .long },
	389	.{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .none },
	390	.{ .sbb, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
	391	.{ .sbb, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
	392	.{ .sbb, .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 3, .long },
	393	.{ .sbb, .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
	394	.{ .sbb, .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
	395	.{ .sbb, .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 3, .long },
	396	.{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .none },
	397	.{ .sbb, .mr, .rm16, .r16, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
	398	.{ .sbb, .mr, .rm32, .r32, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
	399	.{ .sbb, .mr, .rm64, .r64, .none, .none, 1, 0x19, 0x00, 0x00, 0, .long },
	400	.{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .none },
	401	.{ .sbb, .rm, .r16, .rm16, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
	402	.{ .sbb, .rm, .r32, .rm32, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
	403	.{ .sbb, .rm, .r64, .rm64, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .long },
	404
	405	.{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
	406	.{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
	407	.{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
	408	.{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
	409	.{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
	410	.{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
	411	.{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
	412	.{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
	413	.{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
	414	.{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
	415	.{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
	416	.{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
	417	.{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
	418	.{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
	419	.{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
	420	.{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
	421	.{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
	422	.{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
	423	.{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
	424	.{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
	425	.{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .none },
	426	.{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
	427	.{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .none },
	428	.{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
	429	.{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .none },
	430	.{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
	431	.{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
	432	.{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
	433	.{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .none },
	434	.{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
	435
	436	.{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
	437	.{ .shl, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	438	.{ .shl, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	439	.{ .shl, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
	440	.{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
	441	.{ .shl, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	442	.{ .shl, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	443	.{ .shl, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
	444	.{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
	445	.{ .shl, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	446	.{ .shl, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	447	.{ .shl, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
	448
	449	.{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .none },
	450	.{ .shr, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
	451	.{ .shr, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
	452	.{ .shr, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .long },
	453	.{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .none },
	454	.{ .shr, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
	455	.{ .shr, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
	456	.{ .shr, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .long },
	457	.{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .none },
	458	.{ .shr, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
	459	.{ .shr, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
	460	.{ .shr, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .long },
	461
	462	.{ .sub, .zi, .al, .imm8, .none, .none, 1, 0x2c, 0x00, 0x00, 0, .none },
	463	.{ .sub, .zi, .ax, .imm16, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
	464	.{ .sub, .zi, .eax, .imm32, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
	465	.{ .sub, .zi, .rax, .imm32, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .long },
	466	.{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .none },
	467	.{ .sub, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
	468	.{ .sub, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
	469	.{ .sub, .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 5, .long },
	470	.{ .sub, .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
	471	.{ .sub, .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
	472	.{ .sub, .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 5, .long },
	473	.{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .none },
	474	.{ .sub, .mr, .rm16, .r16, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
	475	.{ .sub, .mr, .rm32, .r32, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
	476	.{ .sub, .mr, .rm64, .r64, .none, .none, 1, 0x29, 0x00, 0x00, 0, .long },
	477	.{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .none },
	478	.{ .sub, .rm, .r16, .rm16, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
	479	.{ .sub, .rm, .r32, .rm32, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
	480	.{ .sub, .rm, .r64, .rm64, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .long },
	481
	482	.{ .syscall, .np, .none, .none, .none, .none, 2, 0x0f, 0x05, 0x00, 0, .none },
	483
	484	.{ .@"test", .zi, .al, .imm8, .none, .none, 1, 0xa8, 0x00, 0x00, 0, .none },
	485	.{ .@"test", .zi, .ax, .imm16, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
	486	.{ .@"test", .zi, .eax, .imm32, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
	487	.{ .@"test", .zi, .rax, .imm32, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .long },
	488	.{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .none },
	489	.{ .@"test", .mi, .rm16, .imm16, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
	490	.{ .@"test", .mi, .rm32, .imm32, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
	491	.{ .@"test", .mi, .rm64, .imm32, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .long },
	492	.{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .none },
	493	.{ .@"test", .mr, .rm16, .r16, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
	494	.{ .@"test", .mr, .rm32, .r32, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
	495	.{ .@"test", .mr, .rm64, .r64, .none, .none, 1, 0x85, 0x00, 0x00, 0, .long },
	496
	497	.{ .ud2, .np, .none, .none, .none, .none, 2, 0x0f, 0x0b, 0x00, 0, .none },
	498
	499	.{ .xor, .zi, .al, .imm8, .none, .none, 1, 0x34, 0x00, 0x00, 0, .none },
	500	.{ .xor, .zi, .ax, .imm16, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
	501	.{ .xor, .zi, .eax, .imm32, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
	502	.{ .xor, .zi, .rax, .imm32, .none, .none, 1, 0x35, 0x00, 0x00, 0, .long },
	503	.{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .none },
	504	.{ .xor, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
	505	.{ .xor, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
	506	.{ .xor, .mi, .rm64, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 6, .long },
	507	.{ .xor, .mi, .rm16, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
	508	.{ .xor, .mi, .rm32, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
	509	.{ .xor, .mi, .rm64, .imm8, .none, .none, 1, 0x83, 0x00, 0x00, 6, .long },
	510	.{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .none },
	511	.{ .xor, .mr, .rm16, .r16, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
	512	.{ .xor, .mr, .rm32, .r32, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
	513	.{ .xor, .mr, .rm64, .r64, .none, .none, 1, 0x31, 0x00, 0x00, 0, .long },
	514	.{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .none },
	515	.{ .xor, .rm, .r16, .rm16, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
	516	.{ .xor, .rm, .r32, .rm32, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
	517	.{ .xor, .rm, .r64, .rm64, .none, .none, 1, 0x33, 0x00, 0x00, 0, .long },
	518
	519	// SSE
	520	.{ .addss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x58, 0, .sse },
	521
	522	.{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, 3, 0xf3, 0x0f, 0xc2, 0, .sse },
	523
	524	.{ .movss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x10, 0, .sse },
	525	.{ .movss, .mr, .xmm_m32, .xmm, .none, .none, 3, 0xf3, 0x0f, 0x11, 0, .sse },
	526
	527	.{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, 2, 0x0f, 0x2e, 0x00, 0, .sse },
	528
	529	// SSE2
	530	.{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x58, 0, .sse2 },
	531
	532	.{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, 3, 0xf2, 0x0f, 0xc2, 0, .sse2 },
	533
	534	.{ .movq, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf3, 0x0f, 0x7e, 0, .sse2 },
	535	.{ .movq, .mr, .xmm_m64, .xmm, .none, .none, 3, 0x66, 0x0f, 0xd6, 0, .sse2 },
	536
	537	.{ .movsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x10, 0, .sse2 },
	538	.{ .movsd, .mr, .xmm_m64, .xmm, .none, .none, 3, 0xf2, 0x0f, 0x11, 0, .sse2 },
	539
	540	.{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0x66, 0x0f, 0x2e, 0, .sse2 },
	541	};
	542	// zig fmt: on