Commit d525ecb523fed3c1496bf2f5315d06ea95716c8b | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-03-13 20:48:41+01:00
committer	GitHub <noreply@github.com> 2023-03-13 20:48:41+01:00
log	d525ecb523fed3c1496bf2f5315d06ea95716c8b
tree	751784e96af8e3932f7da7c4346888dc2ed91ab0
parent	4942e4e8701b9a137d7f30fcb8ac1bc2d46f2a99
parent	e9fc0aba4c3b0855e580ff3afe7251920ae3dcf9
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14886 from ziglang/x86_64-encoder

x86_64: add table-driven instruction encoder

8 files changed, 5333 insertions(+), 5340 deletions(-)

CMakeLists.txt
src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Mir.zig
    bits.zig
    encoder.zig
    encodings.zig

CMakeLists.txt+4-1

...	...	@@ -559,9 +559,12 @@ set(ZIG_STAGE2_SOURCES
559	559	"${CMAKE_SOURCE_DIR}/src/arch/wasm/Mir.zig"
560	560	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/CodeGen.zig"
561	561	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Emit.zig"
	562	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Encoding.zig"
562	563	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Mir.zig"
563		"${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
564	564	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/abi.zig"
	565	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
	566	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/encoder.zig"
	567	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/encodings.zig"
565	568	"${CMAKE_SOURCE_DIR}/src/clang.zig"
566	569	"${CMAKE_SOURCE_DIR}/src/clang_options.zig"
567	570	"${CMAKE_SOURCE_DIR}/src/clang_options_data.zig"

src/arch/x86_64/CodeGen.zig+803-1204

...	...	@@ -33,9 +33,11 @@ const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
33	33	const errUnionErrorOffset = codegen.errUnionErrorOffset;
34	34
35	35	const Condition = bits.Condition;
	36	const Immediate = bits.Immediate;
	37	const Memory = bits.Memory;
	38	const Register = bits.Register;
36	39	const RegisterManager = abi.RegisterManager;
37	40	const RegisterLock = RegisterManager.RegisterLock;
38		const Register = bits.Register;
39	41
40	42	const gp = abi.RegisterClass.gp;
41	43	const sse = abi.RegisterClass.sse;
...	...	@@ -303,7 +305,12 @@ pub fn generate(
303	305	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
304	306	error.CodegenFail => return Result{ .fail = function.err_msg.? },
305	307	error.OutOfRegisters => return Result{
306		.fail = try ErrorMsg.create(bin_file.allocator, src_loc, "CodeGen ran out of registers. This is a bug in the Zig compiler.", .{}),
	308	.fail = try ErrorMsg.create(
	309	bin_file.allocator,
	310	src_loc,
	311	"CodeGen ran out of registers. This is a bug in the Zig compiler.",
	312	.{},
	313	),
307	314	},
308	315	else => \|e\| return e,
309	316	};
...	...	@@ -342,6 +349,20 @@ pub fn generate(
342	349	defer emit.deinit();
343	350	emit.lowerMir() catch \|err\| switch (err) {
344	351	error.EmitFail => return Result{ .fail = emit.err_msg.? },
	352	error.InvalidInstruction, error.CannotEncode => \|e\| {
	353	const msg = switch (e) {
	354	error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
	355	error.CannotEncode => "CodeGen failed to encode the instruction.",
	356	};
	357	return Result{
	358	.fail = try ErrorMsg.create(
	359	bin_file.allocator,
	360	src_loc,
	361	"{s} This is a bug in the Zig compiler.",
	362	.{msg},
	363	),
	364	};
	365	},
345	366	else => \|e\| return e,
346	367	};
347	368
...	...	@@ -355,52 +376,263 @@ pub fn generate(
355	376	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
356	377	const gpa = self.gpa;
357	378	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
358		const result_index = @intCast(Air.Inst.Index, self.mir_instructions.len);
	379	const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
359	380	self.mir_instructions.appendAssumeCapacity(inst);
360	381	return result_index;
361	382	}
362	383
363		pub fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
	384	fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
364	385	const fields = std.meta.fields(@TypeOf(extra));
365	386	try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
366	387	return self.addExtraAssumeCapacity(extra);
367	388	}
368	389
369		pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
	390	fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
370	391	const fields = std.meta.fields(@TypeOf(extra));
371	392	const result = @intCast(u32, self.mir_extra.items.len);
372	393	inline for (fields) \|field\| {
373	394	self.mir_extra.appendAssumeCapacity(switch (field.type) {
374	395	u32 => @field(extra, field.name),
375	396	i32 => @bitCast(u32, @field(extra, field.name)),
376		else => @compileError("bad field type"),
	397	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
377	398	});
378	399	}
379	400	return result;
380	401	}
381	402
	403	fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
	404	_ = try self.addInst(.{
	405	.tag = .setcc,
	406	.ops = .r_c,
	407	.data = .{ .r_c = .{
	408	.r1 = reg,
	409	.cc = cc,
	410	} },
	411	});
	412	}
	413
	414	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
	415	_ = try self.addInst(.{
	416	.tag = .cmovcc,
	417	.ops = .rr_c,
	418	.data = .{ .rr_c = .{
	419	.r1 = reg1,
	420	.r2 = reg2,
	421	.cc = cc,
	422	} },
	423	});
	424	}
	425
	426	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
	427	return self.addInst(.{
	428	.tag = .jmp_reloc,
	429	.ops = undefined,
	430	.data = .{ .inst = target },
	431	});
	432	}
	433
	434	fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {
	435	return self.addInst(.{
	436	.tag = .jcc,
	437	.ops = .inst_cc,
	438	.data = .{ .inst_cc = .{
	439	.inst = target,
	440	.cc = cc,
	441	} },
	442	});
	443	}
	444
	445	fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
	446	_ = try self.addInst(.{
	447	.tag = tag,
	448	.ops = .none,
	449	.data = undefined,
	450	});
	451	}
	452
	453	fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
	454	_ = try self.addInst(.{
	455	.tag = tag,
	456	.ops = .r,
	457	.data = .{ .r = reg },
	458	});
	459	}
	460
	461	fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
	462	const ops: Mir.Inst.Ops = if (imm == .signed) .imm_s else .imm_u;
	463	_ = try self.addInst(.{
	464	.tag = tag,
	465	.ops = ops,
	466	.data = .{ .imm = switch (imm) {
	467	.signed => \|x\| @bitCast(u32, x),
	468	.unsigned => \|x\| @intCast(u32, x),
	469	} },
	470	});
	471	}
	472
	473	fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Register) !void {
	474	_ = try self.addInst(.{
	475	.tag = tag,
	476	.ops = .rr,
	477	.data = .{ .rr = .{
	478	.r1 = reg1,
	479	.r2 = reg2,
	480	} },
	481	});
	482	}
	483
	484	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {
	485	const ops: Mir.Inst.Ops = switch (imm) {
	486	.signed => .ri_s,
	487	.unsigned => \|x\| if (x <= math.maxInt(u32)) .ri_u else .ri64,
	488	};
	489	const data: Mir.Inst.Data = switch (ops) {
	490	.ri_s => .{ .ri = .{
	491	.r1 = reg,
	492	.imm = @bitCast(u32, imm.signed),
	493	} },
	494	.ri_u => .{ .ri = .{
	495	.r1 = reg,
	496	.imm = @intCast(u32, imm.unsigned),
	497	} },
	498	.ri64 => .{ .rx = .{
	499	.r1 = reg,
	500	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
	501	} },
	502	else => unreachable,
	503	};
	504	_ = try self.addInst(.{
	505	.tag = tag,
	506	.ops = ops,
	507	.data = data,
	508	});
	509	}
	510
	511	fn asmRegisterRegisterImmediate(
	512	self: *Self,
	513	tag: Mir.Inst.Tag,
	514	reg1: Register,
	515	reg2: Register,
	516	imm: Immediate,
	517	) !void {
	518	const ops: Mir.Inst.Ops = switch (imm) {
	519	.signed => .rri_s,
	520	.unsigned => .rri_u,
	521	};
	522	const data: Mir.Inst.Data = switch (ops) {
	523	.rri_s => .{ .rri = .{
	524	.r1 = reg1,
	525	.r2 = reg2,
	526	.imm = @bitCast(u32, imm.signed),
	527	} },
	528	.rri_u => .{ .rri = .{
	529	.r1 = reg1,
	530	.r2 = reg2,
	531	.imm = @intCast(u32, imm.unsigned),
	532	} },
	533	else => unreachable,
	534	};
	535	_ = try self.addInst(.{
	536	.tag = tag,
	537	.ops = ops,
	538	.data = data,
	539	});
	540	}
	541
	542	fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
	543	const ops: Mir.Inst.Ops = switch (m) {
	544	.sib => .m_sib,
	545	.rip => .m_rip,
	546	else => unreachable,
	547	};
	548	const data: Mir.Inst.Data = .{ .payload = switch (ops) {
	549	.m_sib => try self.addExtra(Mir.MemorySib.encode(m)),
	550	.m_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	551	else => unreachable,
	552	} };
	553	_ = try self.addInst(.{
	554	.tag = tag,
	555	.ops = ops,
	556	.data = data,
	557	});
	558	}
	559
	560	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
	561	const ops: Mir.Inst.Ops = switch (m) {
	562	.sib => if (imm == .signed) .mi_s_sib else .mi_u_sib,
	563	.rip => if (imm == .signed) .mi_s_rip else .mi_u_rip,
	564	else => unreachable,
	565	};
	566	const payload: u32 = switch (ops) {
	567	.mi_s_sib, .mi_u_sib => try self.addExtra(Mir.MemorySib.encode(m)),
	568	.mi_s_rip, .mi_u_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	569	else => unreachable,
	570	};
	571	const data: Mir.Inst.Data = .{
	572	.xi = .{ .payload = payload, .imm = switch (imm) {
	573	.signed => \|x\| @bitCast(u32, x),
	574	.unsigned => \|x\| @intCast(u32, x),
	575	} },
	576	};
	577	_ = try self.addInst(.{
	578	.tag = tag,
	579	.ops = ops,
	580	.data = data,
	581	});
	582	}
	583
	584	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
	585	const ops: Mir.Inst.Ops = switch (m) {
	586	.sib => .rm_sib,
	587	.rip => .rm_rip,
	588	else => unreachable,
	589	};
	590	const data: Mir.Inst.Data = .{
	591	.rx = .{ .r1 = reg, .payload = switch (ops) {
	592	.rm_sib => try self.addExtra(Mir.MemorySib.encode(m)),
	593	.rm_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	594	else => unreachable,
	595	} },
	596	};
	597	_ = try self.addInst(.{
	598	.tag = tag,
	599	.ops = ops,
	600	.data = data,
	601	});
	602	}
	603
	604	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
	605	const ops: Mir.Inst.Ops = switch (m) {
	606	.sib => .mr_sib,
	607	.rip => .mr_rip,
	608	else => unreachable,
	609	};
	610	const data: Mir.Inst.Data = .{
	611	.rx = .{ .r1 = reg, .payload = switch (ops) {
	612	.mr_sib => try self.addExtra(Mir.MemorySib.encode(m)),
	613	.mr_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	614	else => unreachable,
	615	} },
	616	};
	617	_ = try self.addInst(.{
	618	.tag = tag,
	619	.ops = ops,
	620	.data = data,
	621	});
	622	}
	623
382	624	fn gen(self: *Self) InnerError!void {
383	625	const cc = self.fn_type.fnCallingConvention();
384	626	if (cc != .Naked) {
385		_ = try self.addInst(.{
386		.tag = .push,
387		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
388		.data = undefined, // unused for push reg,
389		});
390		_ = try self.addInst(.{
391		.tag = .mov,
392		.ops = Mir.Inst.Ops.encode(.{
393		.reg1 = .rbp,
394		.reg2 = .rsp,
395		}),
396		.data = undefined,
397		});
	627	try self.asmRegister(.push, .rbp);
	628	try self.asmRegisterRegister(.mov, .rbp, .rsp);
	629
398	630	// We want to subtract the aligned stack frame size from rsp here, but we don't
399	631	// yet know how big it will be, so we leave room for a 4-byte stack size.
400	632	// TODO During semantic analysis, check if there are no function calls. If there
401	633	// are none, here we can omit the part where we subtract and then add rsp.
402	634	const backpatch_stack_sub = try self.addInst(.{
403		.tag = .nop,
	635	.tag = .dead,
404	636	.ops = undefined,
405	637	.data = undefined,
406	638	});
...	...	@@ -427,7 +659,7 @@ fn gen(self: *Self) InnerError!void {
427	659
428	660	// Push callee-preserved regs that were used actually in use.
429	661	const backpatch_push_callee_preserved_regs = try self.addInst(.{
430		.tag = .nop,
	662	.tag = .dead,
431	663	.ops = undefined,
432	664	.data = undefined,
433	665	});
...	...	@@ -458,7 +690,7 @@ fn gen(self: *Self) InnerError!void {
458	690
459	691	// Pop saved callee-preserved regs.
460	692	const backpatch_pop_callee_preserved_regs = try self.addInst(.{
461		.tag = .nop,
	693	.tag = .dead,
462	694	.ops = undefined,
463	695	.data = undefined,
464	696	});
...	...	@@ -471,22 +703,13 @@ fn gen(self: *Self) InnerError!void {
471	703
472	704	// Maybe add rsp, x if required. This is backpatched later.
473	705	const backpatch_stack_add = try self.addInst(.{
474		.tag = .nop,
	706	.tag = .dead,
475	707	.ops = undefined,
476	708	.data = undefined,
477	709	});
478	710
479		_ = try self.addInst(.{
480		.tag = .pop,
481		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
482		.data = undefined,
483		});
484
485		_ = try self.addInst(.{
486		.tag = .ret,
487		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
488		.data = undefined,
489		});
	711	try self.asmRegister(.pop, .rbp);
	712	try self.asmOpOnly(.ret);
490	713
491	714	// Adjust the stack
492	715	if (self.max_end_stack > math.maxInt(i32)) {
...	...	@@ -500,27 +723,34 @@ fn gen(self: *Self) InnerError!void {
500	723	if (aligned_stack_end > 0) {
501	724	self.mir_instructions.set(backpatch_stack_sub, .{
502	725	.tag = .sub,
503		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
504		.data = .{ .imm = aligned_stack_end },
	726	.ops = .ri_u,
	727	.data = .{ .ri = .{
	728	.r1 = .rsp,
	729	.imm = aligned_stack_end,
	730	} },
505	731	});
506	732	self.mir_instructions.set(backpatch_stack_add, .{
507	733	.tag = .add,
508		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
509		.data = .{ .imm = aligned_stack_end },
	734	.ops = .ri_u,
	735	.data = .{ .ri = .{
	736	.r1 = .rsp,
	737	.imm = aligned_stack_end,
	738	} },
510	739	});
511	740
512	741	const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
	742	.base_reg = @enumToInt(Register.rbp),
513	743	.register_list = reg_list.asInt(),
514	744	.stack_end = aligned_stack_end,
515	745	});
516	746	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
517	747	.tag = .push_regs,
518		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
	748	.ops = undefined,
519	749	.data = .{ .payload = save_reg_list },
520	750	});
521	751	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
522	752	.tag = .pop_regs,
523		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
	753	.ops = undefined,
524	754	.data = .{ .payload = save_reg_list },
525	755	});
526	756	}
...	...	@@ -815,10 +1045,10 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
815	1045	branch.inst_table.putAssumeCapacity(inst, .dead);
816	1046	switch (prev_value) {
817	1047	.register => \|reg\| {
818		self.register_manager.freeReg(reg.to64());
	1048	self.register_manager.freeReg(reg);
819	1049	},
820	1050	.register_overflow => \|ro\| {
821		self.register_manager.freeReg(ro.reg.to64());
	1051	self.register_manager.freeReg(ro.reg);
822	1052	self.eflags_inst = null;
823	1053	},
824	1054	.eflags => {
...	...	@@ -1216,7 +1446,13 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1216	1446	};
1217	1447	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1218	1448
1219		const mask = ~@as(u64, 0);
	1449	const mask = switch (operand_ty.abiSize(self.target.*)) {
	1450	1 => ~@as(u8, 0),
	1451	2 => ~@as(u16, 0),
	1452	4 => ~@as(u32, 0),
	1453	8 => ~@as(u64, 0),
	1454	else => unreachable,
	1455	};
1220	1456	try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = mask });
1221	1457
1222	1458	break :result dst_mcv;
...	...	@@ -1263,14 +1499,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1263	1499	.unsigned => .b,
1264	1500	.signed => .l,
1265	1501	};
1266		_ = try self.addInst(.{
1267		.tag = .cond_mov,
1268		.ops = Mir.Inst.Ops.encode(.{
1269		.reg1 = dst_mcv.register,
1270		.reg2 = lhs_reg,
1271		}),
1272		.data = .{ .cc = cc },
1273		});
	1502	try self.asmCmovccRegisterRegister(dst_mcv.register, lhs_reg, cc);
1274	1503
1275	1504	break :result dst_mcv;
1276	1505	};
...	...	@@ -1470,13 +1699,7 @@ fn genSetStackTruncatedOverflowCompare(
1470	1699	.signed => .o,
1471	1700	.unsigned => .c,
1472	1701	};
1473		_ = try self.addInst(.{
1474		.tag = .cond_set_byte,
1475		.ops = Mir.Inst.Ops.encode(.{
1476		.reg1 = overflow_reg.to8(),
1477		}),
1478		.data = .{ .cc = cc },
1479		});
	1702	try self.asmSetccRegister(overflow_reg.to8(), cc);
1480	1703
1481	1704	const scratch_reg = temp_regs[1];
1482	1705	try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });
...	...	@@ -1489,12 +1712,7 @@ fn genSetStackTruncatedOverflowCompare(
1489	1712	);
1490	1713
1491	1714	const eq_reg = temp_regs[2];
1492		_ = try self.addInst(.{
1493		.tag = .cond_set_byte,
1494		.ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }),
1495		.data = .{ .cc = .ne },
1496		});
1497
	1715	try self.asmSetccRegister(eq_reg.to8(), .ne);
1498	1716	try self.genBinOpMir(
1499	1717	.@"or",
1500	1718	Type.u8,
...	...	@@ -1638,23 +1856,8 @@ fn genIntMulDivOpMir(
1638	1856	}
1639	1857
1640	1858	switch (signedness) {
1641		.signed => {
1642		_ = try self.addInst(.{
1643		.tag = .cwd,
1644		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
1645		.data = undefined,
1646		});
1647		},
1648		.unsigned => {
1649		_ = try self.addInst(.{
1650		.tag = .xor,
1651		.ops = Mir.Inst.Ops.encode(.{
1652		.reg1 = .rdx,
1653		.reg2 = .rdx,
1654		}),
1655		.data = undefined,
1656		});
1657		},
	1859	.signed => try self.asmOpOnly(.cqo),
	1860	.unsigned => try self.asmRegisterRegister(.xor, .rdx, .rdx),
1658	1861	}
1659	1862
1660	1863	const factor = switch (rhs) {
...	...	@@ -1667,29 +1870,11 @@ fn genIntMulDivOpMir(
1667	1870	};
1668	1871
1669	1872	switch (factor) {
1670		.register => \|reg\| {
1671		_ = try self.addInst(.{
1672		.tag = tag,
1673		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
1674		.data = undefined,
1675		});
1676		},
1677		.stack_offset => \|off\| {
1678		_ = try self.addInst(.{
1679		.tag = tag,
1680		.ops = Mir.Inst.Ops.encode(.{
1681		.reg2 = .rbp,
1682		.flags = switch (abi_size) {
1683		1 => 0b00,
1684		2 => 0b01,
1685		4 => 0b10,
1686		8 => 0b11,
1687		else => unreachable,
1688		},
1689		}),
1690		.data = .{ .imm = @bitCast(u32, -off) },
1691		});
1692		},
	1873	.register => \|reg\| try self.asmRegister(tag, reg),
	1874	.stack_offset => \|off\| try self.asmMemory(tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	1875	.base = .rbp,
	1876	.disp = -off,
	1877	})),
1693	1878	else => unreachable,
1694	1879	}
1695	1880	}
...	...	@@ -1717,38 +1902,10 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
1717	1902	.unsigned => .div,
1718	1903	}, Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor });
1719	1904
1720		_ = try self.addInst(.{
1721		.tag = .xor,
1722		.ops = Mir.Inst.Ops.encode(.{
1723		.reg1 = divisor.to64(),
1724		.reg2 = dividend.to64(),
1725		}),
1726		.data = undefined,
1727		});
1728		_ = try self.addInst(.{
1729		.tag = .sar,
1730		.ops = Mir.Inst.Ops.encode(.{
1731		.reg1 = divisor.to64(),
1732		.flags = 0b10,
1733		}),
1734		.data = .{ .imm = 63 },
1735		});
1736		_ = try self.addInst(.{
1737		.tag = .@"test",
1738		.ops = Mir.Inst.Ops.encode(.{
1739		.reg1 = .rdx,
1740		.reg2 = .rdx,
1741		}),
1742		.data = undefined,
1743		});
1744		_ = try self.addInst(.{
1745		.tag = .cond_mov,
1746		.ops = Mir.Inst.Ops.encode(.{
1747		.reg1 = divisor.to64(),
1748		.reg2 = .rdx,
1749		}),
1750		.data = .{ .cc = .e },
1751		});
	1905	try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64());
	1906	try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63));
	1907	try self.asmRegisterRegister(.@"test", .rdx, .rdx);
	1908	try self.asmCmovccRegisterRegister(divisor.to64(), .rdx, .e);
1752	1909	try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
1753	1910	return MCValue{ .register = divisor };
1754	1911	}
...	...	@@ -2182,18 +2339,10 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2182	2339
2183	2340	const addr_reg = try self.register_manager.allocReg(null, gp);
2184	2341	switch (slice_mcv) {
2185		.stack_offset => \|off\| {
2186		// mov reg, [rbp - 8]
2187		_ = try self.addInst(.{
2188		.tag = .mov,
2189		.ops = Mir.Inst.Ops.encode(.{
2190		.reg1 = addr_reg.to64(),
2191		.reg2 = .rbp,
2192		.flags = 0b01,
2193		}),
2194		.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
2195		});
2196		},
	2342	.stack_offset => \|off\| try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
	2343	.base = .rbp,
	2344	.disp = -off,
	2345	})),
2197	2346	else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
2198	2347	}
2199	2348	// TODO we could allocate register here, but need to expect addr register and potentially
...	...	@@ -2268,26 +2417,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2268	2417	array_ty.abiAlignment(self.target.*),
2269	2418	));
2270	2419	try self.genSetStack(array_ty, off, array, .{});
2271		// lea reg, [rbp]
2272		_ = try self.addInst(.{
2273		.tag = .lea,
2274		.ops = Mir.Inst.Ops.encode(.{
2275		.reg1 = addr_reg.to64(),
2276		.reg2 = .rbp,
2277		}),
2278		.data = .{ .imm = @bitCast(u32, -off) },
2279		});
	2420	try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
	2421	.base = .rbp,
	2422	.disp = -off,
	2423	}));
2280	2424	},
2281	2425	.stack_offset => \|off\| {
2282		// lea reg, [rbp]
2283		_ = try self.addInst(.{
2284		.tag = .lea,
2285		.ops = Mir.Inst.Ops.encode(.{
2286		.reg1 = addr_reg.to64(),
2287		.reg2 = .rbp,
2288		}),
2289		.data = .{ .imm = @bitCast(u32, -off) },
2290		});
	2426	try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
	2427	.base = .rbp,
	2428	.disp = -off,
	2429	}));
2291	2430	},
2292	2431	.memory, .linker_load => {
2293	2432	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
...	...	@@ -2324,7 +2463,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2324	2463	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
2325	2464
2326	2465	const elem_ty = ptr_ty.elemType2();
2327		const elem_abi_size = elem_ty.abiSize(self.target.*);
	2466	const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
2328	2467	const index_ty = self.air.typeOf(bin_op.rhs);
2329	2468	const index = try self.resolveInst(bin_op.rhs);
2330	2469	const index_lock: ?RegisterLock = switch (index) {
...	...	@@ -2344,16 +2483,14 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2344	2483	if (elem_abi_size > 8) {
2345	2484	return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});
2346	2485	} else {
2347		// mov dst_mcv, [dst_mcv]
2348		_ = try self.addInst(.{
2349		.tag = .mov,
2350		.ops = Mir.Inst.Ops.encode(.{
2351		.reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),
2352		.reg2 = dst_mcv.register,
2353		.flags = 0b01,
	2486	try self.asmRegisterMemory(
	2487	.mov,
	2488	registerAlias(dst_mcv.register, elem_abi_size),
	2489	Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{
	2490	.base = dst_mcv.register,
	2491	.disp = 0,
2354	2492	}),
2355		.data = .{ .imm = 0 },
2356		});
	2493	);
2357	2494	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
2358	2495	}
2359	2496	};
...	...	@@ -2580,7 +2717,7 @@ fn reuseOperand(
2580	2717
2581	2718	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
2582	2719	const elem_ty = ptr_ty.elemType();
2583		const abi_size = elem_ty.abiSize(self.target.*);
	2720	const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
2584	2721	switch (ptr) {
2585	2722	.none => unreachable,
2586	2723	.undef => unreachable,
...	...	@@ -2607,16 +2744,11 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2607	2744	.undef => unreachable,
2608	2745	.eflags => unreachable,
2609	2746	.register => \|dst_reg\| {
2610		// mov dst_reg, [reg]
2611		_ = try self.addInst(.{
2612		.tag = .mov,
2613		.ops = Mir.Inst.Ops.encode(.{
2614		.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
2615		.reg2 = reg,
2616		.flags = 0b01,
2617		}),
2618		.data = .{ .imm = 0 },
2619		});
	2747	try self.asmRegisterMemory(
	2748	.mov,
	2749	registerAlias(dst_reg, abi_size),
	2750	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg, .disp = 0 }),
	2751	);
2620	2752	},
2621	2753	.stack_offset => \|off\| {
2622	2754	if (abi_size <= 8) {
...	...	@@ -2676,23 +2808,19 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
2676	2808	const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl);
2677	2809	break :blk coff_file.getAtom(atom).getSymbolIndex().?;
2678	2810	} else unreachable;
2679		const flags: u2 = switch (load_struct.type) {
2680		.got => 0b00,
2681		.direct => 0b01,
2682		.import => 0b10,
	2811	const ops: Mir.Inst.Ops = switch (load_struct.type) {
	2812	.got => .got_reloc,
	2813	.direct => .direct_reloc,
	2814	.import => .import_reloc,
2683	2815	};
2684	2816	_ = try self.addInst(.{
2685		.tag = .lea_pic,
2686		.ops = Mir.Inst.Ops.encode(.{
2687		.reg1 = registerAlias(reg, abi_size),
2688		.flags = flags,
2689		}),
2690		.data = .{
2691		.relocation = .{
2692		.atom_index = atom_index,
2693		.sym_index = load_struct.sym_index,
2694		},
2695		},
	2817	.tag = .lea_linker,
	2818	.ops = ops,
	2819	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
	2820	.reg = @enumToInt(registerAlias(reg, abi_size)),
	2821	.atom_index = atom_index,
	2822	.sym_index = load_struct.sym_index,
	2823	}) },
2696	2824	});
2697	2825	},
2698	2826	.memory => \|addr\| {
...	...	@@ -2705,7 +2833,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
2705	2833	}
2706	2834
2707	2835	fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
2708		const abi_size = value_ty.abiSize(self.target.*);
	2836	const abi_size = @intCast(u32, value_ty.abiSize(self.target.*));
2709	2837	switch (ptr) {
2710	2838	.none => unreachable,
2711	2839	.undef => unreachable,
...	...	@@ -2738,25 +2866,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2738	2866	.immediate => \|imm\| {
2739	2867	switch (abi_size) {
2740	2868	1, 2, 4 => {
2741		// TODO this is wasteful!
2742		// introduce new MIR tag specifically for mov [reg + 0], imm
2743		const payload = try self.addExtra(Mir.ImmPair{
2744		.dest_off = 0,
2745		.operand = @truncate(u32, imm),
2746		});
2747		_ = try self.addInst(.{
2748		.tag = .mov_mem_imm,
2749		.ops = Mir.Inst.Ops.encode(.{
2750		.reg1 = reg.to64(),
2751		.flags = switch (abi_size) {
2752		1 => 0b00,
2753		2 => 0b01,
2754		4 => 0b10,
2755		else => unreachable,
2756		},
2757		}),
2758		.data = .{ .payload = payload },
2759		});
	2869	const immediate = if (value_ty.isSignedInt())
	2870	Immediate.s(@intCast(i32, @bitCast(i64, imm)))
	2871	else
	2872	Immediate.u(@truncate(u32, imm));
	2873	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	2874	.base = reg.to64(),
	2875	.disp = 0,
	2876	}), immediate);
2760	2877	},
2761	2878	8 => {
2762	2879	// TODO: optimization: if the imm is only using the lower
...	...	@@ -2786,13 +2903,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2786	2903	const overflow_bit_ty = value_ty.structFieldType(1);
2787	2904	const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);
2788	2905	const tmp_reg = try self.register_manager.allocReg(null, gp);
2789		_ = try self.addInst(.{
2790		.tag = .cond_set_byte,
2791		.ops = Mir.Inst.Ops.encode(.{
2792		.reg1 = tmp_reg.to8(),
2793		}),
2794		.data = .{ .cc = ro.eflags },
2795		});
	2906	try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
2796	2907	try self.genInlineMemcpyRegisterRegister(
2797	2908	overflow_bit_ty,
2798	2909	reg,
...	...	@@ -2833,17 +2944,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2833	2944
2834	2945	try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
2835	2946
2836		// to get the actual address of the value we want to modify we have to go through the GOT
2837		// mov reg, [reg]
2838		_ = try self.addInst(.{
2839		.tag = .mov,
2840		.ops = Mir.Inst.Ops.encode(.{
2841		.reg1 = addr_reg.to64(),
2842		.reg2 = addr_reg.to64(),
2843		.flags = 0b01,
2844		}),
2845		.data = .{ .imm = 0 },
2846		});
	2947	// To get the actual address of the value we want to modify we have to go through the GOT
	2948	try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
	2949	.base = addr_reg.to64(),
	2950	.disp = 0,
	2951	}));
2847	2952
2848	2953	const new_ptr = MCValue{ .register = addr_reg.to64() };
2849	2954
...	...	@@ -2853,19 +2958,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2853	2958	return self.fail("TODO saving imm to memory for abi_size {}", .{abi_size});
2854	2959	}
2855	2960
2856		const payload = try self.addExtra(Mir.ImmPair{
2857		.dest_off = 0,
2858		// TODO check if this logic is correct
2859		.operand = @truncate(u32, imm),
2860		});
2861		const flags: u2 = switch (abi_size) {
2862		1 => 0b00,
2863		2 => 0b01,
2864		4 => 0b10,
2865		8 => 0b11,
2866		else => unreachable,
2867		};
2868		if (flags == 0b11) {
	2961	if (abi_size == 8) {
	2962	// TODO
2869	2963	const top_bits: u32 = @intCast(u32, imm >> 32);
2870	2964	const can_extend = if (value_ty.isUnsignedInt())
2871	2965	(top_bits == 0) and (imm & 0x8000_0000) == 0
...	...	@@ -2876,14 +2970,10 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2876	2970	return self.fail("TODO imm64 would get incorrectly sign extended", .{});
2877	2971	}
2878	2972	}
2879		_ = try self.addInst(.{
2880		.tag = .mov_mem_imm,
2881		.ops = Mir.Inst.Ops.encode(.{
2882		.reg1 = addr_reg.to64(),
2883		.flags = flags,
2884		}),
2885		.data = .{ .payload = payload },
2886		});
	2973	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	2974	.base = addr_reg.to64(),
	2975	.disp = 0,
	2976	}), Immediate.u(@intCast(u32, imm)));
2887	2977	},
2888	2978	.register => {
2889	2979	return self.store(new_ptr, value, ptr_ty, value_ty);
...	...	@@ -2895,16 +2985,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2895	2985	defer self.register_manager.unlockReg(tmp_reg_lock);
2896	2986
2897	2987	try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
	2988	try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(.qword, .{
	2989	.base = tmp_reg,
	2990	.disp = 0,
	2991	}));
2898	2992
2899		_ = try self.addInst(.{
2900		.tag = .mov,
2901		.ops = Mir.Inst.Ops.encode(.{
2902		.reg1 = tmp_reg,
2903		.reg2 = tmp_reg,
2904		.flags = 0b01,
2905		}),
2906		.data = .{ .imm = 0 },
2907		});
2908	2993	return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2909	2994	}
2910	2995
...	...	@@ -3052,8 +3137,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3052	3137	const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
3053	3138	try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv.register, .{ .immediate = shift });
3054	3139
3055		// Mask with reg.size() - struct_field_size
3056		const max_reg_bit_width = Register.rax.size();
	3140	// Mask with reg.bitSize() - struct_field_size
	3141	const max_reg_bit_width = Register.rax.bitSize();
3057	3142	const mask_shift = @intCast(u6, (max_reg_bit_width - struct_field_ty.bitSize(self.target.*)));
3058	3143	const mask = (~@as(u64, 0)) >> mask_shift;
3059	3144
...	...	@@ -3066,14 +3151,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3066	3151	};
3067	3152	const field_size = @intCast(u32, struct_field_ty.abiSize(self.target.*));
3068	3153	if (signedness == .signed and field_size < 8) {
3069		_ = try self.addInst(.{
3070		.tag = .mov_sign_extend,
3071		.ops = Mir.Inst.Ops.encode(.{
3072		.reg1 = dst_mcv.register,
3073		.reg2 = registerAlias(dst_mcv.register, field_size),
3074		}),
3075		.data = undefined,
3076		});
	3154	try self.asmRegisterRegister(
	3155	.movsx,
	3156	dst_mcv.register,
	3157	registerAlias(dst_mcv.register, field_size),
	3158	);
3077	3159	}
3078	3160
3079	3161	break :result dst_mcv;
...	...	@@ -3090,13 +3172,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3090	3172	defer self.register_manager.unlockReg(reg_lock);
3091	3173
3092	3174	const dst_reg = try self.register_manager.allocReg(inst, gp);
3093		_ = try self.addInst(.{
3094		.tag = .cond_set_byte,
3095		.ops = Mir.Inst.Ops.encode(.{
3096		.reg1 = dst_reg.to8(),
3097		}),
3098		.data = .{ .cc = ro.eflags },
3099		});
	3175	try self.asmSetccRegister(dst_reg.to8(), ro.eflags);
3100	3176	break :result MCValue{ .register = dst_reg.to8() };
3101	3177	},
3102	3178	else => unreachable,
...	...	@@ -3132,25 +3208,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3132	3208	switch (shift) {
3133	3209	.immediate => \|imm\| switch (imm) {
3134	3210	0 => return,
3135		1 => {
3136		_ = try self.addInst(.{
3137		.tag = tag,
3138		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
3139		.data = undefined,
3140		});
3141		return;
3142		},
3143		else => {
3144		_ = try self.addInst(.{
3145		.tag = tag,
3146		.ops = Mir.Inst.Ops.encode(.{
3147		.reg1 = registerAlias(reg, abi_size),
3148		.flags = 0b10,
3149		}),
3150		.data = .{ .imm = @intCast(u8, imm) },
3151		});
3152		return;
3153		},
	3211	else => return self.asmRegisterImmediate(tag, registerAlias(reg, abi_size), Immediate.u(imm)),
3154	3212	},
3155	3213	.register => \|shift_reg\| {
3156	3214	if (shift_reg == .rcx) break :blk;
...	...	@@ -3162,14 +3220,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3162	3220	try self.genSetReg(Type.u8, .rcx, shift);
3163	3221	}
3164	3222
3165		_ = try self.addInst(.{
3166		.tag = tag,
3167		.ops = Mir.Inst.Ops.encode(.{
3168		.reg1 = registerAlias(reg, abi_size),
3169		.flags = 0b01,
3170		}),
3171		.data = undefined,
3172		});
	3223	try self.asmRegisterRegister(tag, registerAlias(reg, abi_size), .cl);
3173	3224	}
3174	3225
3175	3226	/// Result is always a register.
...	...	@@ -3542,59 +3593,44 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3542	3593	if (intrinsicsAllowed(self.target.*, dst_ty)) {
3543	3594	const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3544	3595	.f32 => switch (mir_tag) {
3545		.add => if (hasAvxSupport(self.target.*))
3546		Mir.Inst.Tag.add_f32_avx
3547		else
3548		Mir.Inst.Tag.add_f32_sse,
3549		.cmp => if (hasAvxSupport(self.target.*))
3550		Mir.Inst.Tag.cmp_f32_avx
3551		else
3552		Mir.Inst.Tag.cmp_f32_sse,
3553		else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
	3596	.add => .addss,
	3597	.cmp => .ucomiss,
	3598	else => return self.fail(
	3599	"TODO genBinOpMir for f32 register-register with MIR tag {}",
	3600	.{mir_tag},
	3601	),
3554	3602	},
3555	3603	.f64 => switch (mir_tag) {
3556		.add => if (hasAvxSupport(self.target.*))
3557		Mir.Inst.Tag.add_f64_avx
3558		else
3559		Mir.Inst.Tag.add_f64_sse,
3560		.cmp => if (hasAvxSupport(self.target.*))
3561		Mir.Inst.Tag.cmp_f64_avx
3562		else
3563		Mir.Inst.Tag.cmp_f64_sse,
3564		else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
	3604	.add => .addsd,
	3605	.cmp => .ucomisd,
	3606	else => return self.fail(
	3607	"TODO genBinOpMir for f64 register-register with MIR tag {}",
	3608	.{mir_tag},
	3609	),
3565	3610	},
3566		else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
	3611	else => return self.fail(
	3612	"TODO genBinOpMir for float register-register and type {}",
	3613	.{dst_ty.fmtDebug()},
	3614	),
3567	3615	};
3568		_ = try self.addInst(.{
3569		.tag = actual_tag,
3570		.ops = Mir.Inst.Ops.encode(.{
3571		.reg1 = dst_reg.to128(),
3572		.reg2 = src_reg.to128(),
3573		}),
3574		.data = undefined,
3575		});
3576		return;
	3616	return self.asmRegisterRegister(actual_tag, dst_reg.to128(), src_reg.to128());
3577	3617	}
3578	3618
3579	3619	return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
3580	3620	},
3581		else => {
3582		_ = try self.addInst(.{
3583		.tag = mir_tag,
3584		.ops = Mir.Inst.Ops.encode(.{
3585		.reg1 = registerAlias(dst_reg, abi_size),
3586		.reg2 = registerAlias(src_reg, abi_size),
3587		}),
3588		.data = undefined,
3589		});
3590		},
	3621	else => try self.asmRegisterRegister(
	3622	mir_tag,
	3623	registerAlias(dst_reg, abi_size),
	3624	registerAlias(src_reg, abi_size),
	3625	),
3591	3626	},
3592	3627	.immediate => \|imm\| {
3593		_ = try self.addInst(.{
3594		.tag = mir_tag,
3595		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }),
3596		.data = .{ .imm = @truncate(u32, imm) },
3597		});
	3628	// TODO
	3629	try self.asmRegisterImmediate(
	3630	mir_tag,
	3631	registerAlias(dst_reg, abi_size),
	3632	Immediate.u(@intCast(u32, imm)),
	3633	);
3598	3634	},
3599	3635	.memory,
3600	3636	.linker_load,
...	...	@@ -3611,15 +3647,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3611	3647	if (off > math.maxInt(i32)) {
3612	3648	return self.fail("stack offset too large", .{});
3613	3649	}
3614		_ = try self.addInst(.{
3615		.tag = mir_tag,
3616		.ops = Mir.Inst.Ops.encode(.{
3617		.reg1 = registerAlias(dst_reg, abi_size),
3618		.reg2 = .rbp,
3619		.flags = 0b01,
3620		}),
3621		.data = .{ .imm = @bitCast(u32, -off) },
3622		});
	3650	try self.asmRegisterMemory(
	3651	mir_tag,
	3652	registerAlias(dst_reg, abi_size),
	3653	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
	3654	);
3623	3655	},
3624	3656	}
3625	3657	},
...	...	@@ -3637,45 +3669,17 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3637	3669	.dead, .unreach => unreachable,
3638	3670	.register_overflow => unreachable,
3639	3671	.register => \|src_reg\| {
3640		_ = try self.addInst(.{
3641		.tag = mir_tag,
3642		.ops = Mir.Inst.Ops.encode(.{
3643		.reg1 = .rbp,
3644		.reg2 = registerAlias(src_reg, abi_size),
3645		.flags = 0b10,
3646		}),
3647		.data = .{ .imm = @bitCast(u32, -off) },
3648		});
	3672	try self.asmMemoryRegister(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	3673	.base = .rbp,
	3674	.disp = -off,
	3675	}), registerAlias(src_reg, abi_size));
3649	3676	},
3650	3677	.immediate => \|imm\| {
3651		const tag: Mir.Inst.Tag = switch (mir_tag) {
3652		.add => .add_mem_imm,
3653		.@"or" => .or_mem_imm,
3654		.@"and" => .and_mem_imm,
3655		.sub => .sub_mem_imm,
3656		.xor => .xor_mem_imm,
3657		.cmp => .cmp_mem_imm,
3658		else => unreachable,
3659		};
3660		const flags: u2 = switch (abi_size) {
3661		1 => 0b00,
3662		2 => 0b01,
3663		4 => 0b10,
3664		8 => 0b11,
3665		else => unreachable,
3666		};
3667		const payload = try self.addExtra(Mir.ImmPair{
3668		.dest_off = @bitCast(u32, -off),
3669		.operand = @truncate(u32, imm),
3670		});
3671		_ = try self.addInst(.{
3672		.tag = tag,
3673		.ops = Mir.Inst.Ops.encode(.{
3674		.reg1 = .rbp,
3675		.flags = flags,
3676		}),
3677		.data = .{ .payload = payload },
3678		});
	3678	// TODO
	3679	try self.asmMemoryImmediate(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	3680	.base = .rbp,
	3681	.disp = -off,
	3682	}), Immediate.u(@intCast(u32, imm)));
3679	3683	},
3680	3684	.memory,
3681	3685	.stack_offset,
...	...	@@ -3718,30 +3722,21 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3718	3722	.dead, .unreach => unreachable,
3719	3723	.ptr_stack_offset => unreachable,
3720	3724	.register_overflow => unreachable,
3721		.register => \|src_reg\| {
3722		// register, register
3723		_ = try self.addInst(.{
3724		.tag = .imul_complex,
3725		.ops = Mir.Inst.Ops.encode(.{
3726		.reg1 = registerAlias(dst_reg, abi_size),
3727		.reg2 = registerAlias(src_reg, abi_size),
3728		}),
3729		.data = undefined,
3730		});
3731		},
	3725	.register => \|src_reg\| try self.asmRegisterRegister(
	3726	.imul,
	3727	registerAlias(dst_reg, abi_size),
	3728	registerAlias(src_reg, abi_size),
	3729	),
3732	3730	.immediate => \|imm\| {
3733		// TODO take into account the type's ABI size when selecting the register alias
3734		// register, immediate
3735	3731	if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
3736		_ = try self.addInst(.{
3737		.tag = .imul_complex,
3738		.ops = Mir.Inst.Ops.encode(.{
3739		.reg1 = dst_reg.to32(),
3740		.reg2 = dst_reg.to32(),
3741		.flags = 0b10,
3742		}),
3743		.data = .{ .imm = @truncate(u32, imm) },
3744		});
	3732	// TODO take into account the type's ABI size when selecting the register alias
	3733	// register, immediate
	3734	try self.asmRegisterRegisterImmediate(
	3735	.imul,
	3736	dst_reg.to32(),
	3737	dst_reg.to32(),
	3738	Immediate.u(@intCast(u32, imm)),
	3739	);
3745	3740	} else {
3746	3741	// TODO verify we don't spill and assign to the same register as dst_mcv
3747	3742	const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);
...	...	@@ -3749,15 +3744,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3749	3744	}
3750	3745	},
3751	3746	.stack_offset => \|off\| {
3752		_ = try self.addInst(.{
3753		.tag = .imul_complex,
3754		.ops = Mir.Inst.Ops.encode(.{
3755		.reg1 = registerAlias(dst_reg, abi_size),
3756		.reg2 = .rbp,
3757		.flags = 0b01,
3758		}),
3759		.data = .{ .imm = @bitCast(u32, -off) },
3760		});
	3747	try self.asmRegisterMemory(
	3748	.imul,
	3749	registerAlias(dst_reg, abi_size),
	3750	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
	3751	);
3761	3752	},
3762	3753	.memory => {
3763	3754	return self.fail("TODO implement x86 multiply source memory", .{});
...	...	@@ -3780,16 +3771,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3780	3771	.register => \|src_reg\| {
3781	3772	// copy dst to a register
3782	3773	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
3783		// multiply into dst_reg
3784		// register, register
3785		_ = try self.addInst(.{
3786		.tag = .imul_complex,
3787		.ops = Mir.Inst.Ops.encode(.{
3788		.reg1 = registerAlias(dst_reg, abi_size),
3789		.reg2 = registerAlias(src_reg, abi_size),
3790		}),
3791		.data = undefined,
3792		});
	3774	try self.asmRegisterRegister(
	3775	.imul,
	3776	registerAlias(dst_reg, abi_size),
	3777	registerAlias(src_reg, abi_size),
	3778	);
3793	3779	// copy dst_reg back out
3794	3780	return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
3795	3781	},
...	...	@@ -3915,20 +3901,12 @@ fn genVarDbgInfo(
3915	3901	}
3916	3902
3917	3903	fn airTrap(self: *Self) !void {
3918		_ = try self.addInst(.{
3919		.tag = .ud,
3920		.ops = Mir.Inst.Ops.encode(.{}),
3921		.data = undefined,
3922		});
	3904	try self.asmOpOnly(.ud2);
3923	3905	return self.finishAirBookkeeping();
3924	3906	}
3925	3907
3926	3908	fn airBreakpoint(self: *Self) !void {
3927		_ = try self.addInst(.{
3928		.tag = .interrupt,
3929		.ops = Mir.Inst.Ops.encode(.{}),
3930		.data = undefined,
3931		});
	3909	try self.asmOpOnly(.int3);
3932	3910	return self.finishAirBookkeeping();
3933	3911	}
3934	3912
...	...	@@ -4023,11 +4001,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4023	4001
4024	4002	if (info.stack_byte_count > 0) {
4025	4003	// Adjust the stack
4026		_ = try self.addInst(.{
4027		.tag = .sub,
4028		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4029		.data = .{ .imm = info.stack_byte_count },
4030		});
	4004	try self.asmRegisterImmediate(.sub, .rsp, Immediate.u(info.stack_byte_count));
4031	4005	}
4032	4006
4033	4007	// Due to incremental compilation, how function calls are generated depends
...	...	@@ -4040,12 +4014,11 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4040	4014	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
4041	4015	const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
4042	4016	const atom = elf_file.getAtom(atom_index);
4043		const got_addr = @intCast(u32, atom.getOffsetTableAddress(elf_file));
4044		_ = try self.addInst(.{
4045		.tag = .call,
4046		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4047		.data = .{ .imm = got_addr },
4048		});
	4017	const got_addr = atom.getOffsetTableAddress(elf_file);
	4018	try self.asmMemory(.call, Memory.sib(.qword, .{
	4019	.base = .ds,
	4020	.disp = @intCast(i32, got_addr),
	4021	}));
4049	4022	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
4050	4023	const atom_index = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
4051	4024	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
...	...	@@ -4055,14 +4028,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4055	4028	.sym_index = sym_index,
4056	4029	},
4057	4030	});
4058		_ = try self.addInst(.{
4059		.tag = .call,
4060		.ops = Mir.Inst.Ops.encode(.{
4061		.reg1 = .rax,
4062		.flags = 0b01,
4063		}),
4064		.data = undefined,
4065		});
	4031	try self.asmRegister(.call, .rax);
4066	4032	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4067	4033	const atom_index = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
4068	4034	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
...	...	@@ -4072,14 +4038,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4072	4038	.sym_index = sym_index,
4073	4039	},
4074	4040	});
4075		_ = try self.addInst(.{
4076		.tag = .call,
4077		.ops = Mir.Inst.Ops.encode(.{
4078		.reg1 = .rax,
4079		.flags = 0b01,
4080		}),
4081		.data = undefined,
4082		});
	4041	try self.asmRegister(.call, .rax);
4083	4042	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
4084	4043	const decl_block_index = try p9.seeDecl(func.owner_decl);
4085	4044	const decl_block = p9.getDeclBlock(decl_block_index);
...	...	@@ -4088,11 +4047,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4088	4047	const got_addr = p9.bases.data;
4089	4048	const got_index = decl_block.got_index.?;
4090	4049	const fn_got_addr = got_addr + got_index * ptr_bytes;
4091		_ = try self.addInst(.{
4092		.tag = .call,
4093		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4094		.data = .{ .imm = @intCast(u32, fn_got_addr) },
4095		});
	4050	try self.asmMemory(.call, Memory.sib(.qword, .{
	4051	.base = .ds,
	4052	.disp = @intCast(i32, fn_got_addr),
	4053	}));
4096	4054	} else unreachable;
4097	4055	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
4098	4056	const extern_fn = func_payload.data;
...	...	@@ -4112,14 +4070,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4112	4070	.sym_index = sym_index,
4113	4071	},
4114	4072	});
4115		_ = try self.addInst(.{
4116		.tag = .call,
4117		.ops = Mir.Inst.Ops.encode(.{
4118		.reg1 = .rax,
4119		.flags = 0b01,
4120		}),
4121		.data = undefined,
4122		});
	4073	try self.asmRegister(.call, .rax);
4123	4074	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4124	4075	const sym_index = try macho_file.getGlobalSymbol(mem.sliceTo(decl_name, 0));
4125	4076	const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl);
...	...	@@ -4142,23 +4093,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4142	4093	assert(ty.zigTypeTag() == .Pointer);
4143	4094	const mcv = try self.resolveInst(callee);
4144	4095	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
4145		_ = try self.addInst(.{
4146		.tag = .call,
4147		.ops = Mir.Inst.Ops.encode(.{
4148		.reg1 = .rax,
4149		.flags = 0b01,
4150		}),
4151		.data = undefined,
4152		});
	4096	try self.asmRegister(.call, .rax);
4153	4097	}
4154	4098
4155	4099	if (info.stack_byte_count > 0) {
4156	4100	// Readjust the stack
4157		_ = try self.addInst(.{
4158		.tag = .add,
4159		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4160		.data = .{ .imm = info.stack_byte_count },
4161		});
	4101	try self.asmRegisterImmediate(.add, .rsp, Immediate.u(info.stack_byte_count));
4162	4102	}
4163	4103
4164	4104	const result: MCValue = result: {
...	...	@@ -4215,11 +4155,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
4215	4155	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
4216	4156	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
4217	4157	// which is available if the jump is 127 bytes or less forward.
4218		const jmp_reloc = try self.addInst(.{
4219		.tag = .jmp,
4220		.ops = Mir.Inst.Ops.encode(.{}),
4221		.data = .{ .inst = undefined },
4222		});
	4158	const jmp_reloc = try self.asmJmpReloc(undefined);
4223	4159	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
4224	4160	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4225	4161	}
...	...	@@ -4251,11 +4187,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
4251	4187	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
4252	4188	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
4253	4189	// which is available if the jump is 127 bytes or less forward.
4254		const jmp_reloc = try self.addInst(.{
4255		.tag = .jmp,
4256		.ops = Mir.Inst.Ops.encode(.{}),
4257		.data = .{ .inst = undefined },
4258		});
	4190	const jmp_reloc = try self.asmJmpReloc(undefined);
4259	4191	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
4260	4192	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4261	4193	}
...	...	@@ -4430,33 +4362,13 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4430	4362	const abi_size = ty.abiSize(self.target.*);
4431	4363	switch (mcv) {
4432	4364	.eflags => \|cc\| {
4433		return self.addInst(.{
4434		.tag = .cond_jmp,
4435		.ops = Mir.Inst.Ops.encode(.{}),
4436		.data = .{
4437		.inst_cc = .{
4438		.inst = undefined,
4439		// Here we map the opposites since the jump is to the false branch.
4440		.cc = cc.negate(),
4441		},
4442		},
4443		});
	4365	// Here we map the opposites since the jump is to the false branch.
	4366	return self.asmJccReloc(undefined, cc.negate());
4444	4367	},
4445	4368	.register => \|reg\| {
4446	4369	try self.spillEflagsIfOccupied();
4447		_ = try self.addInst(.{
4448		.tag = .@"test",
4449		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
4450		.data = .{ .imm = 1 },
4451		});
4452		return self.addInst(.{
4453		.tag = .cond_jmp,
4454		.ops = Mir.Inst.Ops.encode(.{}),
4455		.data = .{ .inst_cc = .{
4456		.inst = undefined,
4457		.cc = .e,
4458		} },
4459		});
	4370	try self.asmRegisterImmediate(.@"test", reg, Immediate.u(1));
	4371	return self.asmJccReloc(undefined, .e);
4460	4372	},
4461	4373	.immediate,
4462	4374	.stack_offset,
...	...	@@ -4470,6 +4382,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4470	4382	},
4471	4383	else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}),
4472	4384	}
	4385	return 0; // TODO
4473	4386	}
4474	4387
4475	4388	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4795,11 +4708,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
4795	4708	const body = self.air.extra[loop.end..][0..loop.data.body_len];
4796	4709	const jmp_target = @intCast(u32, self.mir_instructions.len);
4797	4710	try self.genBody(body);
4798		_ = try self.addInst(.{
4799		.tag = .jmp,
4800		.ops = Mir.Inst.Ops.encode(.{}),
4801		.data = .{ .inst = jmp_target },
4802		});
	4711	_ = try self.asmJmpReloc(jmp_target);
4803	4712	return self.finishAirBookkeeping();
4804	4713	}
4805	4714
...	...	@@ -4844,23 +4753,16 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4844	4753	.none => unreachable,
4845	4754	.undef => unreachable,
4846	4755	.dead, .unreach => unreachable,
4847		.immediate => \|imm\| {
4848		_ = try self.addInst(.{
4849		.tag = .xor,
4850		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }),
4851		.data = .{ .imm = @intCast(u32, imm) },
4852		});
4853		},
4854		.register => \|reg\| {
4855		_ = try self.addInst(.{
4856		.tag = .xor,
4857		.ops = Mir.Inst.Ops.encode(.{
4858		.reg1 = registerAlias(cond_reg, abi_size),
4859		.reg2 = registerAlias(reg, abi_size),
4860		}),
4861		.data = undefined,
4862		});
4863		},
	4756	.immediate => \|imm\| try self.asmRegisterImmediate(
	4757	.xor,
	4758	registerAlias(cond_reg, abi_size),
	4759	Immediate.u(imm),
	4760	),
	4761	.register => \|reg\| try self.asmRegisterRegister(
	4762	.xor,
	4763	registerAlias(cond_reg, abi_size),
	4764	registerAlias(reg, abi_size),
	4765	),
4864	4766	.stack_offset => {
4865	4767	if (abi_size <= 8) {
4866	4768	const reg = try self.copyToTmpRegister(ty, case);
...	...	@@ -4874,22 +4776,9 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4874	4776	},
4875	4777	}
4876	4778
4877		_ = try self.addInst(.{
4878		.tag = .@"test",
4879		.ops = Mir.Inst.Ops.encode(.{
4880		.reg1 = registerAlias(cond_reg, abi_size),
4881		.reg2 = registerAlias(cond_reg, abi_size),
4882		}),
4883		.data = undefined,
4884		});
4885		return self.addInst(.{
4886		.tag = .cond_jmp,
4887		.ops = Mir.Inst.Ops.encode(.{}),
4888		.data = .{ .inst_cc = .{
4889		.inst = undefined,
4890		.cc = .ne,
4891		} },
4892		});
	4779	const aliased_reg = registerAlias(cond_reg, abi_size);
	4780	try self.asmRegisterRegister(.@"test", aliased_reg, aliased_reg);
	4781	return self.asmJccReloc(undefined, .ne);
4893	4782	},
4894	4783	.stack_offset => {
4895	4784	try self.spillEflagsIfOccupied();
...	...	@@ -4907,6 +4796,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4907	4796	return self.fail("TODO implemenent switch mir when condition is {}", .{condition});
4908	4797	},
4909	4798	}
	4799	return 0; // TODO
4910	4800	}
4911	4801
4912	4802	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5101,10 +4991,10 @@ fn canonicaliseBranches(self: Self, parent_branch: Branch, canon_branch: *Bran
5101	4991	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
5102	4992	const next_inst = @intCast(u32, self.mir_instructions.len);
5103	4993	switch (self.mir_instructions.items(.tag)[reloc]) {
5104		.cond_jmp => {
	4994	.jcc => {
5105	4995	self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;
5106	4996	},
5107		.jmp => {
	4997	.jmp_reloc => {
5108	4998	self.mir_instructions.items(.data)[reloc].inst = next_inst;
5109	4999	},
5110	5000	else => unreachable,
...	...	@@ -5146,11 +5036,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
5146	5036	// Emit a jump with a relocation. It will be patched up after the block ends.
5147	5037	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
5148	5038	// Leave the jump offset undefined
5149		const jmp_reloc = try self.addInst(.{
5150		.tag = .jmp,
5151		.ops = Mir.Inst.Ops.encode(.{}),
5152		.data = .{ .inst = undefined },
5153		});
	5039	const jmp_reloc = try self.asmJmpReloc(undefined);
5154	5040	block_data.relocs.appendAssumeCapacity(jmp_reloc);
5155	5041	}
5156	5042
...	...	@@ -5223,31 +5109,19 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5223	5109	var iter = std.mem.tokenize(u8, asm_source, "\n\r");
5224	5110	while (iter.next()) \|ins\| {
5225	5111	if (mem.eql(u8, ins, "syscall")) {
5226		_ = try self.addInst(.{
5227		.tag = .syscall,
5228		.ops = undefined,
5229		.data = undefined,
5230		});
	5112	try self.asmOpOnly(.syscall);
5231	5113	} else if (mem.indexOf(u8, ins, "push")) \|_\| {
5232	5114	const arg = ins[4..];
5233	5115	if (mem.indexOf(u8, arg, "$")) \|l\| {
5234	5116	const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch {
5235	5117	return self.fail("TODO implement more inline asm int parsing", .{});
5236	5118	};
5237		_ = try self.addInst(.{
5238		.tag = .push,
5239		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b10 }),
5240		.data = .{ .imm = n },
5241		});
	5119	try self.asmImmediate(.push, Immediate.u(n));
5242	5120	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
5243	5121	const reg_name = ins[4 + l + 2 ..];
5244	5122	const reg = parseRegName(reg_name) orelse
5245	5123	return self.fail("unrecognized register: '{s}'", .{reg_name});
5246		_ = try self.addInst(.{
5247		.tag = .push,
5248		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5249		.data = undefined,
5250		});
	5124	try self.asmRegister(.push, reg);
5251	5125	} else return self.fail("TODO more push operands", .{});
5252	5126	} else if (mem.indexOf(u8, ins, "pop")) \|_\| {
5253	5127	const arg = ins[3..];
...	...	@@ -5255,11 +5129,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5255	5129	const reg_name = ins[3 + l + 2 ..];
5256	5130	const reg = parseRegName(reg_name) orelse
5257	5131	return self.fail("unrecognized register: '{s}'", .{reg_name});
5258		_ = try self.addInst(.{
5259		.tag = .pop,
5260		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5261		.data = undefined,
5262		});
	5132	try self.asmRegister(.pop, reg);
5263	5133	} else return self.fail("TODO more pop operands", .{});
5264	5134	} else {
5265	5135	return self.fail("TODO implement support for more x86 assembly instructions", .{});
...	...	@@ -5332,7 +5202,7 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
5332	5202	}
5333	5203
5334	5204	fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {
5335		const abi_size = ty.abiSize(self.target.*);
	5205	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5336	5206	switch (mcv) {
5337	5207	.dead => unreachable,
5338	5208	.unreach, .none => return,
...	...	@@ -5356,26 +5226,17 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5356	5226	.immediate => \|imm\| {
5357	5227	switch (abi_size) {
5358	5228	1, 2, 4 => {
	5229	// TODO
5359	5230	// We have a positive stack offset value but we want a twos complement negative
5360	5231	// offset from rbp, which is at the top of the stack frame.
5361		// mov [rbp+offset], immediate
5362		const payload = try self.addExtra(Mir.ImmPair{
5363		.dest_off = @bitCast(u32, -stack_offset),
5364		.operand = @truncate(u32, imm),
5365		});
5366		_ = try self.addInst(.{
5367		.tag = .mov_mem_imm,
5368		.ops = Mir.Inst.Ops.encode(.{
5369		.reg1 = .rsp,
5370		.flags = switch (abi_size) {
5371		1 => 0b00,
5372		2 => 0b01,
5373		4 => 0b10,
5374		else => unreachable,
5375		},
5376		}),
5377		.data = .{ .payload = payload },
5378		});
	5232	const immediate = if (ty.isSignedInt())
	5233	Immediate.s(@intCast(i32, @bitCast(i64, imm)))
	5234	else
	5235	Immediate.u(@intCast(u32, imm));
	5236	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5237	.base = .rsp,
	5238	.disp = -stack_offset,
	5239	}), immediate);
5379	5240	},
5380	5241	8 => {
5381	5242	const reg = try self.copyToTmpRegister(ty, mcv);
...	...	@@ -5400,44 +5261,32 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5400	5261	.Float => {
5401	5262	if (intrinsicsAllowed(self.target.*, ty)) {
5402	5263	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5403		.f32 => if (hasAvxSupport(self.target.*))
5404		Mir.Inst.Tag.mov_f32_avx
5405		else
5406		Mir.Inst.Tag.mov_f32_sse,
5407		.f64 => if (hasAvxSupport(self.target.*))
5408		Mir.Inst.Tag.mov_f64_avx
5409		else
5410		Mir.Inst.Tag.mov_f64_sse,
5411		else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
	5264	.f32 => .movss,
	5265	.f64 => .movsd,
	5266	else => return self.fail(
	5267	"TODO genSetStackArg for register for type {}",
	5268	.{ty.fmtDebug()},
	5269	),
5412	5270	};
5413		_ = try self.addInst(.{
5414		.tag = tag,
5415		.ops = Mir.Inst.Ops.encode(.{
5416		.reg1 = switch (ty.tag()) {
5417		.f32 => .esp,
5418		.f64 => .rsp,
5419		else => unreachable,
5420		},
5421		.reg2 = reg.to128(),
5422		.flags = 0b01,
5423		}),
5424		.data = .{ .imm = @bitCast(u32, -stack_offset) },
5425		});
5426		return;
	5271	// TODO verify this
	5272	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
	5273	.f32 => .dword,
	5274	.f64 => .qword,
	5275	else => unreachable,
	5276	};
	5277	return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
	5278	.base = .rsp,
	5279	.disp = -stack_offset,
	5280	}), reg.to128());
5427	5281	}
5428	5282
5429	5283	return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
5430	5284	},
5431	5285	else => {
5432		_ = try self.addInst(.{
5433		.tag = .mov,
5434		.ops = Mir.Inst.Ops.encode(.{
5435		.reg1 = .rsp,
5436		.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5437		.flags = 0b10,
5438		}),
5439		.data = .{ .imm = @bitCast(u32, -stack_offset) },
5440		});
	5286	try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5287	.base = .rsp,
	5288	.disp = -stack_offset,
	5289	}), registerAlias(reg, abi_size));
5441	5290	},
5442	5291	}
5443	5292	},
...	...	@@ -5460,7 +5309,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5460	5309	}
5461	5310
5462	5311	fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {
5463		const abi_size = ty.abiSize(self.target.*);
	5312	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5464	5313	switch (mcv) {
5465	5314	.dead => unreachable,
5466	5315	.unreach, .none => return, // Nothing to do.
...	...	@@ -5468,10 +5317,19 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5468	5317	if (!self.wantSafety())
5469	5318	return; // The already existing value will do just fine.
5470	5319	// TODO Upgrade this to a memset call when we have that available.
5471		switch (ty.abiSize(self.target.*)) {
5472		1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }, opts),
5473		2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }, opts),
5474		4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }, opts),
	5320	switch (abi_size) {
	5321	1, 2, 4 => {
	5322	const value: u64 = switch (abi_size) {
	5323	1 => 0xaa,
	5324	2 => 0xaaaa,
	5325	4 => 0xaaaaaaaa,
	5326	else => unreachable,
	5327	};
	5328	return self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5329	.base = opts.dest_stack_base orelse .rbp,
	5330	.disp = -stack_offset,
	5331	}), Immediate.u(value));
	5332	},
5475	5333	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts),
5476	5334	else => \|x\| return self.genInlineMemset(
5477	5335	.{ .stack_offset = stack_offset },
...	...	@@ -5491,13 +5349,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5491	5349	const overflow_bit_ty = ty.structFieldType(1);
5492	5350	const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
5493	5351	const tmp_reg = try self.register_manager.allocReg(null, gp);
5494		_ = try self.addInst(.{
5495		.tag = .cond_set_byte,
5496		.ops = Mir.Inst.Ops.encode(.{
5497		.reg1 = tmp_reg.to8(),
5498		}),
5499		.data = .{ .cc = ro.eflags },
5500		});
	5352	try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
5501	5353
5502	5354	return self.genSetStack(
5503	5355	overflow_bit_ty,
...	...	@@ -5512,72 +5364,36 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5512	5364	},
5513	5365	.immediate => \|x_big\| {
5514	5366	const base_reg = opts.dest_stack_base orelse .rbp;
	5367	// TODO
5515	5368	switch (abi_size) {
5516	5369	0 => {
5517	5370	assert(ty.isError());
5518		const payload = try self.addExtra(Mir.ImmPair{
5519		.dest_off = @bitCast(u32, -stack_offset),
5520		.operand = @truncate(u32, x_big),
5521		});
5522		_ = try self.addInst(.{
5523		.tag = .mov_mem_imm,
5524		.ops = Mir.Inst.Ops.encode(.{
5525		.reg1 = base_reg,
5526		.flags = 0b00,
5527		}),
5528		.data = .{ .payload = payload },
5529		});
	5371	try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
	5372	.base = base_reg,
	5373	.disp = -stack_offset,
	5374	}), Immediate.u(@truncate(u8, x_big)));
5530	5375	},
5531	5376	1, 2, 4 => {
5532		const payload = try self.addExtra(Mir.ImmPair{
5533		.dest_off = @bitCast(u32, -stack_offset),
5534		.operand = @truncate(u32, x_big),
5535		});
5536		_ = try self.addInst(.{
5537		.tag = .mov_mem_imm,
5538		.ops = Mir.Inst.Ops.encode(.{
5539		.reg1 = base_reg,
5540		.flags = switch (abi_size) {
5541		1 => 0b00,
5542		2 => 0b01,
5543		4 => 0b10,
5544		else => unreachable,
5545		},
5546		}),
5547		.data = .{ .payload = payload },
5548		});
	5377	const immediate = if (ty.isSignedInt())
	5378	Immediate.s(@truncate(i32, @bitCast(i64, x_big)))
	5379	else
	5380	Immediate.u(@intCast(u32, x_big));
	5381	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5382	.base = base_reg,
	5383	.disp = -stack_offset,
	5384	}), immediate);
5549	5385	},
5550	5386	8 => {
5551	5387	// 64 bit write to memory would take two mov's anyways so we
5552	5388	// insted just use two 32 bit writes to avoid register allocation
5553		{
5554		const payload = try self.addExtra(Mir.ImmPair{
5555		.dest_off = @bitCast(u32, -stack_offset + 4),
5556		.operand = @truncate(u32, x_big >> 32),
5557		});
5558		_ = try self.addInst(.{
5559		.tag = .mov_mem_imm,
5560		.ops = Mir.Inst.Ops.encode(.{
5561		.reg1 = base_reg,
5562		.flags = 0b10,
5563		}),
5564		.data = .{ .payload = payload },
5565		});
5566		}
5567		{
5568		const payload = try self.addExtra(Mir.ImmPair{
5569		.dest_off = @bitCast(u32, -stack_offset),
5570		.operand = @truncate(u32, x_big),
5571		});
5572		_ = try self.addInst(.{
5573		.tag = .mov_mem_imm,
5574		.ops = Mir.Inst.Ops.encode(.{
5575		.reg1 = base_reg,
5576		.flags = 0b10,
5577		}),
5578		.data = .{ .payload = payload },
5579		});
5580		}
	5389	try self.asmMemoryImmediate(.mov, Memory.sib(.dword, .{
	5390	.base = base_reg,
	5391	.disp = -stack_offset + 4,
	5392	}), Immediate.u(@truncate(u32, x_big >> 32)));
	5393	try self.asmMemoryImmediate(.mov, Memory.sib(.dword, .{
	5394	.base = base_reg,
	5395	.disp = -stack_offset,
	5396	}), Immediate.u(@truncate(u32, x_big)));
5581	5397	},
5582	5398	else => {
5583	5399	return self.fail("TODO implement set abi_size=large stack variable with immediate", .{});
...	...	@@ -5595,30 +5411,22 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5595	5411	.Float => {
5596	5412	if (intrinsicsAllowed(self.target.*, ty)) {
5597	5413	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5598		.f32 => if (hasAvxSupport(self.target.*))
5599		Mir.Inst.Tag.mov_f32_avx
5600		else
5601		Mir.Inst.Tag.mov_f32_sse,
5602		.f64 => if (hasAvxSupport(self.target.*))
5603		Mir.Inst.Tag.mov_f64_avx
5604		else
5605		Mir.Inst.Tag.mov_f64_sse,
5606		else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
	5414	.f32 => .movss,
	5415	.f64 => .movsd,
	5416	else => return self.fail(
	5417	"TODO genSetStack for register for type {}",
	5418	.{ty.fmtDebug()},
	5419	),
5607	5420	};
5608		_ = try self.addInst(.{
5609		.tag = tag,
5610		.ops = Mir.Inst.Ops.encode(.{
5611		.reg1 = switch (ty.tag()) {
5612		.f32 => base_reg.to32(),
5613		.f64 => base_reg.to64(),
5614		else => unreachable,
5615		},
5616		.reg2 = reg.to128(),
5617		.flags = 0b01,
5618		}),
5619		.data = .{ .imm = @bitCast(u32, -stack_offset) },
5620		});
5621		return;
	5421	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
	5422	.f32 => .dword,
	5423	.f64 => .qword,
	5424	else => unreachable,
	5425	};
	5426	return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
	5427	.base = base_reg.to64(),
	5428	.disp = -stack_offset,
	5429	}), reg.to128());
5622	5430	}
5623	5431
5624	5432	return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
...	...	@@ -5666,7 +5474,7 @@ fn genInlineMemcpyRegisterRegister(
5666	5474	src_reg: Register,
5667	5475	offset: i32,
5668	5476	) InnerError!void {
5669		assert(dst_reg.size() == 64);
	5477	assert(dst_reg.bitSize() == 64);
5670	5478
5671	5479	const dst_reg_lock = self.register_manager.lockReg(dst_reg);
5672	5480	defer if (dst_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -5683,16 +5491,10 @@ fn genInlineMemcpyRegisterRegister(
5683	5491	var remainder = abi_size;
5684	5492	while (remainder > 0) {
5685	5493	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5686
5687		_ = try self.addInst(.{
5688		.tag = .mov,
5689		.ops = Mir.Inst.Ops.encode(.{
5690		.reg1 = dst_reg,
5691		.reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5692		.flags = 0b10,
5693		}),
5694		.data = .{ .imm = @bitCast(u32, -next_offset) },
5695		});
	5494	try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
	5495	.base = dst_reg,
	5496	.disp = -next_offset,
	5497	}), registerAlias(tmp_reg, nearest_power_of_two));
5696	5498
5697	5499	if (nearest_power_of_two > 1) {
5698	5500	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{
...	...	@@ -5704,15 +5506,10 @@ fn genInlineMemcpyRegisterRegister(
5704	5506	next_offset -= nearest_power_of_two;
5705	5507	}
5706	5508	} else {
5707		_ = try self.addInst(.{
5708		.tag = .mov,
5709		.ops = Mir.Inst.Ops.encode(.{
5710		.reg1 = dst_reg,
5711		.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
5712		.flags = 0b10,
5713		}),
5714		.data = .{ .imm = @bitCast(u32, -offset) },
5715		});
	5509	try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5510	.base = dst_reg,
	5511	.disp = -offset,
	5512	}), registerAlias(src_reg, abi_size));
5716	5513	}
5717	5514	}
5718	5515
...	...	@@ -5752,24 +5549,17 @@ fn genInlineMemcpy(
5752	5549	try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
5753	5550	},
5754	5551	.ptr_stack_offset, .stack_offset => \|off\| {
5755		_ = try self.addInst(.{
5756		.tag = .lea,
5757		.ops = Mir.Inst.Ops.encode(.{
5758		.reg1 = dst_addr_reg.to64(),
5759		.reg2 = opts.dest_stack_base orelse .rbp,
5760		}),
5761		.data = .{ .imm = @bitCast(u32, -off) },
5762		});
	5552	try self.asmRegisterMemory(.lea, dst_addr_reg.to64(), Memory.sib(.qword, .{
	5553	.base = opts.dest_stack_base orelse .rbp,
	5554	.disp = -off,
	5555	}));
5763	5556	},
5764	5557	.register => \|reg\| {
5765		_ = try self.addInst(.{
5766		.tag = .mov,
5767		.ops = Mir.Inst.Ops.encode(.{
5768		.reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),
5769		.reg2 = reg,
5770		}),
5771		.data = undefined,
5772		});
	5558	try self.asmRegisterRegister(
	5559	.mov,
	5560	registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	5561	reg,
	5562	);
5773	5563	},
5774	5564	else => {
5775	5565	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
...	...	@@ -5781,24 +5571,17 @@ fn genInlineMemcpy(
5781	5571	try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
5782	5572	},
5783	5573	.ptr_stack_offset, .stack_offset => \|off\| {
5784		_ = try self.addInst(.{
5785		.tag = .lea,
5786		.ops = Mir.Inst.Ops.encode(.{
5787		.reg1 = src_addr_reg.to64(),
5788		.reg2 = opts.source_stack_base orelse .rbp,
5789		}),
5790		.data = .{ .imm = @bitCast(u32, -off) },
5791		});
	5574	try self.asmRegisterMemory(.lea, src_addr_reg.to64(), Memory.sib(.qword, .{
	5575	.base = opts.source_stack_base orelse .rbp,
	5576	.disp = -off,
	5577	}));
5792	5578	},
5793	5579	.register => \|reg\| {
5794		_ = try self.addInst(.{
5795		.tag = .mov,
5796		.ops = Mir.Inst.Ops.encode(.{
5797		.reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),
5798		.reg2 = reg,
5799		}),
5800		.data = undefined,
5801		});
	5580	try self.asmRegisterRegister(
	5581	.mov,
	5582	registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	5583	reg,
	5584	);
5802	5585	},
5803	5586	else => {
5804	5587	return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
...	...	@@ -5806,74 +5589,35 @@ fn genInlineMemcpy(
5806	5589	}
5807	5590
5808	5591	try self.genSetReg(Type.usize, count_reg, len);
5809
5810		// mov index_reg, 0
5811		_ = try self.addInst(.{
5812		.tag = .mov,
5813		.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5814		.data = .{ .imm = 0 },
5815		});
5816
5817		// loop:
5818		// cmp count, 0
	5592	try self.asmRegisterImmediate(.mov, index_reg, Immediate.u(0));
5819	5593	const loop_start = try self.addInst(.{
5820	5594	.tag = .cmp,
5821		.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5822		.data = .{ .imm = 0 },
5823		});
5824
5825		// je end
5826		const loop_reloc = try self.addInst(.{
5827		.tag = .cond_jmp,
5828		.ops = Mir.Inst.Ops.encode(.{}),
5829		.data = .{ .inst_cc = .{
5830		.inst = undefined,
5831		.cc = .e,
	5595	.ops = .ri_u,
	5596	.data = .{ .ri = .{
	5597	.r1 = count_reg,
	5598	.imm = 0,
5832	5599	} },
5833	5600	});
5834
5835		// mov tmp, [addr + index_reg]
5836		_ = try self.addInst(.{
5837		.tag = .mov_scale_src,
5838		.ops = Mir.Inst.Ops.encode(.{
5839		.reg1 = tmp_reg.to8(),
5840		.reg2 = src_addr_reg,
5841		}),
5842		.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },
5843		});
5844
5845		// mov [stack_offset + index_reg], tmp
5846		_ = try self.addInst(.{
5847		.tag = .mov_scale_dst,
5848		.ops = Mir.Inst.Ops.encode(.{
5849		.reg1 = dst_addr_reg,
5850		.reg2 = tmp_reg.to8(),
5851		}),
5852		.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },
5853		});
5854
5855		// add index_reg, 1
5856		_ = try self.addInst(.{
5857		.tag = .add,
5858		.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5859		.data = .{ .imm = 1 },
5860		});
5861
5862		// sub count, 1
5863		_ = try self.addInst(.{
5864		.tag = .sub,
5865		.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5866		.data = .{ .imm = 1 },
5867		});
5868
5869		// jmp loop
5870		_ = try self.addInst(.{
5871		.tag = .jmp,
5872		.ops = Mir.Inst.Ops.encode(.{}),
5873		.data = .{ .inst = loop_start },
5874		});
5875
5876		// end:
	5601	const loop_reloc = try self.asmJccReloc(undefined, .e);
	5602	try self.asmRegisterMemory(.mov, tmp_reg.to8(), Memory.sib(.byte, .{
	5603	.base = src_addr_reg,
	5604	.scale_index = .{
	5605	.scale = 1,
	5606	.index = index_reg,
	5607	},
	5608	.disp = 0,
	5609	}));
	5610	try self.asmMemoryRegister(.mov, Memory.sib(.byte, .{
	5611	.base = dst_addr_reg,
	5612	.scale_index = .{
	5613	.scale = 1,
	5614	.index = index_reg,
	5615	},
	5616	.disp = 0,
	5617	}), tmp_reg.to8());
	5618	try self.asmRegisterImmediate(.add, index_reg, Immediate.u(1));
	5619	try self.asmRegisterImmediate(.sub, count_reg, Immediate.u(1));
	5620	_ = try self.asmJmpReloc(loop_start);
5877	5621	try self.performReloc(loop_reloc);
5878	5622	}
5879	5623
...	...	@@ -5905,24 +5649,17 @@ fn genInlineMemset(
5905	5649	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
5906	5650	},
5907	5651	.ptr_stack_offset, .stack_offset => \|off\| {
5908		_ = try self.addInst(.{
5909		.tag = .lea,
5910		.ops = Mir.Inst.Ops.encode(.{
5911		.reg1 = addr_reg.to64(),
5912		.reg2 = opts.dest_stack_base orelse .rbp,
5913		}),
5914		.data = .{ .imm = @bitCast(u32, -off) },
5915		});
	5652	try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
	5653	.base = opts.dest_stack_base orelse .rbp,
	5654	.disp = -off,
	5655	}));
5916	5656	},
5917	5657	.register => \|reg\| {
5918		_ = try self.addInst(.{
5919		.tag = .mov,
5920		.ops = Mir.Inst.Ops.encode(.{
5921		.reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
5922		.reg2 = reg,
5923		}),
5924		.data = undefined,
5925		});
	5658	try self.asmRegisterRegister(
	5659	.mov,
	5660	registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	5661	reg,
	5662	);
5926	5663	},
5927	5664	else => {
5928	5665	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
...	...	@@ -5932,54 +5669,35 @@ fn genInlineMemset(
5932	5669	try self.genSetReg(Type.usize, index_reg, len);
5933	5670	try self.genBinOpMir(.sub, Type.usize, .{ .register = index_reg }, .{ .immediate = 1 });
5934	5671
5935		// loop:
5936		// cmp index_reg, -1
5937	5672	const loop_start = try self.addInst(.{
5938	5673	.tag = .cmp,
5939		.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5940		.data = .{ .imm = @bitCast(u32, @as(i32, -1)) },
5941		});
5942
5943		// je end
5944		const loop_reloc = try self.addInst(.{
5945		.tag = .cond_jmp,
5946		.ops = Mir.Inst.Ops.encode(.{}),
5947		.data = .{ .inst_cc = .{
5948		.inst = undefined,
5949		.cc = .e,
	5674	.ops = .ri_s,
	5675	.data = .{ .ri = .{
	5676	.r1 = index_reg,
	5677	.imm = @bitCast(u32, @as(i32, -1)),
5950	5678	} },
5951	5679	});
	5680	const loop_reloc = try self.asmJccReloc(undefined, .e);
5952	5681
5953	5682	switch (value) {
5954	5683	.immediate => \|x\| {
5955	5684	if (x > math.maxInt(i32)) {
5956	5685	return self.fail("TODO inline memset for value immediate larger than 32bits", .{});
5957	5686	}
5958		// mov byte ptr [rbp + index_reg + stack_offset], imm
5959		_ = try self.addInst(.{
5960		.tag = .mov_mem_index_imm,
5961		.ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg }),
5962		.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDispImm.encode(index_reg, 0, @truncate(u32, x))) },
5963		});
	5687	try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
	5688	.base = addr_reg,
	5689	.scale_index = .{
	5690	.scale = 1,
	5691	.index = index_reg,
	5692	},
	5693	.disp = 0,
	5694	}), Immediate.u(@intCast(u8, x)));
5964	5695	},
5965	5696	else => return self.fail("TODO inline memset for value of type {}", .{value}),
5966	5697	}
5967	5698
5968		// sub index_reg, 1
5969		_ = try self.addInst(.{
5970		.tag = .sub,
5971		.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5972		.data = .{ .imm = 1 },
5973		});
5974
5975		// jmp loop
5976		_ = try self.addInst(.{
5977		.tag = .jmp,
5978		.ops = Mir.Inst.Ops.encode(.{}),
5979		.data = .{ .inst = loop_start },
5980		});
5981
5982		// end:
	5699	try self.asmRegisterImmediate(.sub, index_reg, Immediate.u(1));
	5700	_ = try self.asmJmpReloc(loop_start);
5983	5701	try self.performReloc(loop_reloc);
5984	5702	}
5985	5703
...	...	@@ -5992,21 +5710,18 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5992	5710	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
5993	5711	return self.fail("stack offset too large", .{});
5994	5712	}
5995		_ = try self.addInst(.{
5996		.tag = .lea,
5997		.ops = Mir.Inst.Ops.encode(.{
5998		.reg1 = registerAlias(reg, abi_size),
5999		.reg2 = .rbp,
6000		}),
6001		.data = .{ .imm = @bitCast(u32, -off) },
6002		});
	5713	try self.asmRegisterMemory(
	5714	.lea,
	5715	registerAlias(reg, abi_size),
	5716	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
	5717	);
6003	5718	},
6004	5719	.unreach, .none => return, // Nothing to do.
6005	5720	.undef => {
6006	5721	if (!self.wantSafety())
6007	5722	return; // The already existing value will do just fine.
6008	5723	// Write the debug undefined value.
6009		switch (registerAlias(reg, abi_size).size()) {
	5724	switch (registerAlias(reg, abi_size).bitSize()) {
6010	5725	8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
6011	5726	16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
6012	5727	32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
...	...	@@ -6015,50 +5730,29 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6015	5730	}
6016	5731	},
6017	5732	.eflags => \|cc\| {
6018		_ = try self.addInst(.{
6019		.tag = .cond_set_byte,
6020		.ops = Mir.Inst.Ops.encode(.{
6021		.reg1 = reg.to8(),
6022		}),
6023		.data = .{ .cc = cc },
6024		});
	5733	return self.asmSetccRegister(reg.to8(), cc);
6025	5734	},
6026	5735	.immediate => \|x\| {
6027		// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
6028		// register is the fastest way to zero a register.
6029	5736	if (x == 0) {
6030		_ = try self.addInst(.{
6031		.tag = .xor,
6032		.ops = Mir.Inst.Ops.encode(.{
6033		.reg1 = reg.to32(),
6034		.reg2 = reg.to32(),
6035		}),
6036		.data = undefined,
6037		});
6038		return;
	5737	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
	5738	// register is the fastest way to zero a register.
	5739	return self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
6039	5740	}
6040		if (x <= math.maxInt(i32)) {
6041		// Next best case: if we set the lower four bytes, the upper four will be zeroed.
6042		_ = try self.addInst(.{
6043		.tag = .mov,
6044		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
6045		.data = .{ .imm = @truncate(u32, x) },
6046		});
6047		return;
	5741	if (ty.isSignedInt()) {
	5742	const signed_x = @bitCast(i64, x);
	5743	if (math.minInt(i32) <= signed_x and signed_x <= math.maxInt(i32)) {
	5744	return self.asmRegisterImmediate(
	5745	.mov,
	5746	registerAlias(reg, abi_size),
	5747	Immediate.s(@intCast(i32, signed_x)),
	5748	);
	5749	}
6048	5750	}
6049		// Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls
6050		// this `movabs`, though this is officially just a different variant of the plain `mov`
6051		// instruction.
6052		//
6053		// This encoding is, in fact, the same as the one used for 32-bit loads. The only
6054		// difference is that we set REX.W before the instruction, which extends the load to
6055		// 64-bit and uses the full bit-width of the register.
6056		const payload = try self.addExtra(Mir.Imm64.encode(x));
6057		_ = try self.addInst(.{
6058		.tag = .movabs,
6059		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }),
6060		.data = .{ .payload = payload },
6061		});
	5751	return self.asmRegisterImmediate(
	5752	.mov,
	5753	registerAlias(reg, abi_size),
	5754	Immediate.u(x),
	5755	);
6062	5756	},
6063	5757	.register => \|src_reg\| {
6064	5758	// If the registers are the same, nothing to do.
...	...	@@ -6069,69 +5763,38 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6069	5763	.Int => switch (ty.intInfo(self.target.*).signedness) {
6070	5764	.signed => {
6071	5765	if (abi_size <= 4) {
6072		_ = try self.addInst(.{
6073		.tag = .mov_sign_extend,
6074		.ops = Mir.Inst.Ops.encode(.{
6075		.reg1 = reg.to64(),
6076		.reg2 = registerAlias(src_reg, abi_size),
6077		}),
6078		.data = undefined,
6079		});
6080		return;
	5766	return self.asmRegisterRegister(
	5767	.movsx,
	5768	reg.to64(),
	5769	registerAlias(src_reg, abi_size),
	5770	);
6081	5771	}
6082	5772	},
6083	5773	.unsigned => {
6084	5774	if (abi_size <= 2) {
6085		_ = try self.addInst(.{
6086		.tag = .mov_zero_extend,
6087		.ops = Mir.Inst.Ops.encode(.{
6088		.reg1 = reg.to64(),
6089		.reg2 = registerAlias(src_reg, abi_size),
6090		}),
6091		.data = undefined,
6092		});
6093		return;
	5775	return self.asmRegisterRegister(
	5776	.movzx,
	5777	reg.to64(),
	5778	registerAlias(src_reg, abi_size),
	5779	);
6094	5780	}
6095	5781	},
6096	5782	},
6097	5783	.Float => {
6098	5784	if (intrinsicsAllowed(self.target.*, ty)) {
6099	5785	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6100		.f32 => if (hasAvxSupport(self.target.*))
6101		Mir.Inst.Tag.mov_f32_avx
6102		else
6103		Mir.Inst.Tag.mov_f32_sse,
6104		.f64 => if (hasAvxSupport(self.target.*))
6105		Mir.Inst.Tag.mov_f64_avx
6106		else
6107		Mir.Inst.Tag.mov_f64_sse,
	5786	.f32 => .movss,
	5787	.f64 => .movsd,
6108	5788	else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
6109	5789	};
6110		_ = try self.addInst(.{
6111		.tag = tag,
6112		.ops = Mir.Inst.Ops.encode(.{
6113		.reg1 = reg.to128(),
6114		.reg2 = src_reg.to128(),
6115		.flags = 0b10,
6116		}),
6117		.data = undefined,
6118		});
6119		return;
	5790	return self.asmRegisterRegister(tag, reg.to128(), src_reg.to128());
6120	5791	}
6121
6122	5792	return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
6123	5793	},
6124	5794	else => {},
6125	5795	}
6126	5796
6127		_ = try self.addInst(.{
6128		.tag = .mov,
6129		.ops = Mir.Inst.Ops.encode(.{
6130		.reg1 = registerAlias(reg, abi_size),
6131		.reg2 = registerAlias(src_reg, abi_size),
6132		}),
6133		.data = undefined,
6134		});
	5797	try self.asmRegisterRegister(.mov, registerAlias(reg, abi_size), registerAlias(src_reg, abi_size));
6135	5798	},
6136	5799	.linker_load => {
6137	5800	switch (ty.zigTypeTag()) {
...	...	@@ -6141,45 +5804,30 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6141	5804
6142	5805	if (intrinsicsAllowed(self.target.*, ty)) {
6143	5806	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6144		.f32 => if (hasAvxSupport(self.target.*))
6145		Mir.Inst.Tag.mov_f32_avx
6146		else
6147		Mir.Inst.Tag.mov_f32_sse,
6148		.f64 => if (hasAvxSupport(self.target.*))
6149		Mir.Inst.Tag.mov_f64_avx
6150		else
6151		Mir.Inst.Tag.mov_f64_sse,
	5807	.f32 => .movss,
	5808	.f64 => .movsd,
6152	5809	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6153	5810	};
6154
6155		_ = try self.addInst(.{
6156		.tag = tag,
6157		.ops = Mir.Inst.Ops.encode(.{
6158		.reg1 = reg.to128(),
6159		.reg2 = switch (ty.tag()) {
6160		.f32 => base_reg.to32(),
6161		.f64 => base_reg.to64(),
6162		else => unreachable,
6163		},
6164		}),
6165		.data = .{ .imm = 0 },
6166		});
6167		return;
	5811	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
	5812	.f32 => .dword,
	5813	.f64 => .qword,
	5814	else => unreachable,
	5815	};
	5816	return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
	5817	.base = base_reg.to64(),
	5818	.disp = 0,
	5819	}));
6168	5820	}
6169	5821
6170	5822	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
6171	5823	},
6172	5824	else => {
6173	5825	try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
6174		_ = try self.addInst(.{
6175		.tag = .mov,
6176		.ops = Mir.Inst.Ops.encode(.{
6177		.reg1 = registerAlias(reg, abi_size),
6178		.reg2 = reg.to64(),
6179		.flags = 0b01,
6180		}),
6181		.data = .{ .imm = 0 },
6182		});
	5826	try self.asmRegisterMemory(
	5827	.mov,
	5828	registerAlias(reg, abi_size),
	5829	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
	5830	);
6183	5831	},
6184	5832	}
6185	5833	},
...	...	@@ -6190,73 +5838,56 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6190	5838
6191	5839	if (intrinsicsAllowed(self.target.*, ty)) {
6192	5840	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6193		.f32 => if (hasAvxSupport(self.target.*))
6194		Mir.Inst.Tag.mov_f32_avx
6195		else
6196		Mir.Inst.Tag.mov_f32_sse,
6197		.f64 => if (hasAvxSupport(self.target.*))
6198		Mir.Inst.Tag.mov_f64_avx
6199		else
6200		Mir.Inst.Tag.mov_f64_sse,
	5841	.f32 => .movss,
	5842	.f64 => .movsd,
6201	5843	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6202	5844	};
6203
6204		_ = try self.addInst(.{
6205		.tag = tag,
6206		.ops = Mir.Inst.Ops.encode(.{
6207		.reg1 = reg.to128(),
6208		.reg2 = switch (ty.tag()) {
6209		.f32 => base_reg.to32(),
6210		.f64 => base_reg.to64(),
6211		else => unreachable,
6212		},
6213		}),
6214		.data = .{ .imm = 0 },
6215		});
6216		return;
	5845	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
	5846	.f32 => .dword,
	5847	.f64 => .qword,
	5848	else => unreachable,
	5849	};
	5850	return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
	5851	.base = base_reg.to64(),
	5852	.disp = 0,
	5853	}));
6217	5854	}
6218	5855
6219	5856	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
6220	5857	},
6221	5858	else => {
6222	5859	if (x <= math.maxInt(i32)) {
6223		// mov reg, [ds:imm32]
6224		_ = try self.addInst(.{
6225		.tag = .mov,
6226		.ops = Mir.Inst.Ops.encode(.{
6227		.reg1 = registerAlias(reg, abi_size),
6228		.flags = 0b01,
	5860	try self.asmRegisterMemory(
	5861	.mov,
	5862	registerAlias(reg, abi_size),
	5863	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5864	.base = .ds,
	5865	.disp = @intCast(i32, x),
6229	5866	}),
6230		.data = .{ .imm = @truncate(u32, x) },
6231		});
	5867	);
6232	5868	} else {
6233		// If this is RAX, we can use a direct load.
6234		// Otherwise, we need to load the address, then indirectly load the value.
6235		if (reg.id() == 0) {
6236		// movabs rax, ds:moffs64
6237		const payload = try self.addExtra(Mir.Imm64.encode(x));
	5869	if (reg.to64() == .rax) {
	5870	// If this is RAX, we can use a direct load.
	5871	// Otherwise, we need to load the address, then indirectly load the value.
	5872	var moffs: Mir.MemoryMoffs = .{
	5873	.seg = @enumToInt(Register.ds),
	5874	.msb = undefined,
	5875	.lsb = undefined,
	5876	};
	5877	moffs.encodeOffset(x);
6238	5878	_ = try self.addInst(.{
6239		.tag = .movabs,
6240		.ops = Mir.Inst.Ops.encode(.{
6241		.reg1 = .rax,
6242		.flags = 0b01, // imm64 will become moffs64
6243		}),
6244		.data = .{ .payload = payload },
	5879	.tag = .mov_moffs,
	5880	.ops = .rax_moffs,
	5881	.data = .{ .payload = try self.addExtra(moffs) },
6245	5882	});
6246	5883	} else {
6247	5884	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
6248	5885	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
6249
6250		// mov reg, [reg + 0x0]
6251		_ = try self.addInst(.{
6252		.tag = .mov,
6253		.ops = Mir.Inst.Ops.encode(.{
6254		.reg1 = registerAlias(reg, abi_size),
6255		.reg2 = reg.to64(),
6256		.flags = 0b01,
6257		}),
6258		.data = .{ .imm = 0 },
6259		});
	5886	try self.asmRegisterMemory(
	5887	.mov,
	5888	registerAlias(reg, abi_size),
	5889	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
	5890	);
6260	5891	}
6261	5892	}
6262	5893	},
...	...	@@ -6270,85 +5901,59 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6270	5901	.Int => switch (ty.intInfo(self.target.*).signedness) {
6271	5902	.signed => {
6272	5903	if (abi_size <= 4) {
6273		const flags: u2 = switch (abi_size) {
6274		1 => 0b01,
6275		2 => 0b10,
6276		4 => 0b11,
6277		else => unreachable,
6278		};
6279		_ = try self.addInst(.{
6280		.tag = .mov_sign_extend,
6281		.ops = Mir.Inst.Ops.encode(.{
6282		.reg1 = reg.to64(),
6283		.reg2 = .rbp,
6284		.flags = flags,
	5904	return self.asmRegisterMemory(
	5905	.movsx,
	5906	reg.to64(),
	5907	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5908	.base = .rbp,
	5909	.disp = -off,
6285	5910	}),
6286		.data = .{ .imm = @bitCast(u32, -off) },
6287		});
6288		return;
	5911	);
6289	5912	}
6290	5913	},
6291	5914	.unsigned => {
6292	5915	if (abi_size <= 2) {
6293		const flags: u2 = switch (abi_size) {
6294		1 => 0b01,
6295		2 => 0b10,
6296		else => unreachable,
6297		};
6298		_ = try self.addInst(.{
6299		.tag = .mov_zero_extend,
6300		.ops = Mir.Inst.Ops.encode(.{
6301		.reg1 = reg.to64(),
6302		.reg2 = .rbp,
6303		.flags = flags,
	5916	return self.asmRegisterMemory(
	5917	.movzx,
	5918	reg.to64(),
	5919	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5920	.base = .rbp,
	5921	.disp = -off,
6304	5922	}),
6305		.data = .{ .imm = @bitCast(u32, -off) },
6306		});
6307		return;
	5923	);
6308	5924	}
6309	5925	},
6310	5926	},
6311	5927	.Float => {
6312	5928	if (intrinsicsAllowed(self.target.*, ty)) {
6313	5929	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6314		.f32 => if (hasAvxSupport(self.target.*))
6315		Mir.Inst.Tag.mov_f32_avx
6316		else
6317		Mir.Inst.Tag.mov_f32_sse,
6318		.f64 => if (hasAvxSupport(self.target.*))
6319		Mir.Inst.Tag.mov_f64_avx
6320		else
6321		Mir.Inst.Tag.mov_f64_sse,
6322		else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
	5930	.f32 => .movss,
	5931	.f64 => .movsd,
	5932	else => return self.fail(
	5933	"TODO genSetReg from stack offset for {}",
	5934	.{ty.fmtDebug()},
	5935	),
6323	5936	};
6324		_ = try self.addInst(.{
6325		.tag = tag,
6326		.ops = Mir.Inst.Ops.encode(.{
6327		.reg1 = reg.to128(),
6328		.reg2 = switch (ty.tag()) {
6329		.f32 => .ebp,
6330		.f64 => .rbp,
6331		else => unreachable,
6332		},
6333		}),
6334		.data = .{ .imm = @bitCast(u32, -off) },
6335		});
6336		return;
	5937	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
	5938	.f32 => .dword,
	5939	.f64 => .qword,
	5940	else => unreachable,
	5941	};
	5942	return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
	5943	.base = .rbp,
	5944	.disp = -off,
	5945	}));
6337	5946	}
6338	5947	return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
6339	5948	},
6340	5949	else => {},
6341	5950	}
6342	5951
6343		_ = try self.addInst(.{
6344		.tag = .mov,
6345		.ops = Mir.Inst.Ops.encode(.{
6346		.reg1 = registerAlias(reg, abi_size),
6347		.reg2 = .rbp,
6348		.flags = 0b01,
6349		}),
6350		.data = .{ .imm = @bitCast(u32, -off) },
6351		});
	5952	try self.asmRegisterMemory(
	5953	.mov,
	5954	registerAlias(reg, abi_size),
	5955	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
	5956	);
6352	5957	},
6353	5958	}
6354	5959	}
...	...	@@ -6412,6 +6017,16 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6412	6017	const src_ty = self.air.typeOf(ty_op.operand);
6413	6018	const dst_ty = self.air.typeOfIndex(inst);
6414	6019	const operand = try self.resolveInst(ty_op.operand);
	6020	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
	6021	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	6022
	6023	switch (src_abi_size) {
	6024	4, 8 => {},
	6025	else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
	6026	}
	6027	if (dst_abi_size > 8) {
	6028	return self.fail("TODO convert float with abiSize={}", .{dst_abi_size});
	6029	}
6415	6030
6416	6031	// move float src to ST(0)
6417	6032	const stack_offset = switch (operand) {
...	...	@@ -6419,41 +6034,24 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6419	6034	else => blk: {
6420	6035	const offset = @intCast(i32, try self.allocMem(
6421	6036	inst,
6422		@intCast(u32, src_ty.abiSize(self.target.*)),
	6037	src_abi_size,
6423	6038	src_ty.abiAlignment(self.target.*),
6424	6039	));
6425	6040	try self.genSetStack(src_ty, offset, operand, .{});
6426	6041	break :blk offset;
6427	6042	},
6428	6043	};
6429		_ = try self.addInst(.{
6430		.tag = .fld,
6431		.ops = Mir.Inst.Ops.encode(.{
6432		.reg1 = .rbp,
6433		.flags = switch (src_ty.abiSize(self.target.*)) {
6434		4 => 0b01,
6435		8 => 0b10,
6436		else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
6437		},
6438		}),
6439		.data = .{ .imm = @bitCast(u32, -stack_offset) },
6440		});
	6044	try self.asmMemory(.fld, Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
	6045	.base = .rbp,
	6046	.disp = -stack_offset,
	6047	}));
6441	6048
6442	6049	// convert
6443	6050	const stack_dst = try self.allocRegOrMem(inst, false);
6444		_ = try self.addInst(.{
6445		.tag = .fisttp,
6446		.ops = Mir.Inst.Ops.encode(.{
6447		.reg1 = .rbp,
6448		.flags = switch (dst_ty.abiSize(self.target.*)) {
6449		1...2 => 0b00,
6450		3...4 => 0b01,
6451		5...8 => 0b10,
6452		else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),
6453		},
6454		}),
6455		.data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) },
6456		});
	6051	try self.asmMemory(.fisttp, Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
	6052	.base = .rbp,
	6053	.disp = -stack_dst.stack_offset,
	6054	}));
6457	6055
6458	6056	return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
6459	6057	}
...	...	@@ -6544,15 +6142,10 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6544	6142	.linker_load, .memory => {
6545	6143	const reg = try self.register_manager.allocReg(null, gp);
6546	6144	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
6547		_ = try self.addInst(.{
6548		.tag = .mov,
6549		.ops = Mir.Inst.Ops.encode(.{
6550		.reg1 = reg,
6551		.reg2 = reg,
6552		.flags = 0b01,
6553		}),
6554		.data = .{ .imm = 0 },
6555		});
	6145	try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{
	6146	.base = reg,
	6147	.disp = 0,
	6148	}));
6556	6149	break :blk MCValue{ .register = reg };
6557	6150	},
6558	6151	else => break :blk src_ptr,
...	...	@@ -6903,7 +6496,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
6903	6496	if (ret_ty_size == 0) {
6904	6497	assert(ret_ty.isError());
6905	6498	result.return_value = .{ .immediate = 0 };
6906		} else if (ret_ty_size <= 8) {
	6499	} else if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
6907	6500	const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
6908	6501	result.return_value = .{ .register = aliased_reg };
6909	6502	} else {
...	...	@@ -6978,28 +6571,34 @@ fn parseRegName(name: []const u8) ?Register {
6978	6571
6979	6572	/// Returns register wide enough to hold at least `size_bytes`.
6980	6573	fn registerAlias(reg: Register, size_bytes: u32) Register {
6981		if (size_bytes == 0) {
6982		unreachable; // should be comptime-known
6983		} else if (size_bytes <= 1) {
6984		return reg.to8();
6985		} else if (size_bytes <= 2) {
6986		return reg.to16();
6987		} else if (size_bytes <= 4) {
6988		return reg.to32();
6989		} else if (size_bytes <= 8) {
6990		return reg.to64();
6991		} else if (size_bytes <= 16) {
6992		return reg.to128();
6993		} else if (size_bytes <= 32) {
6994		return reg.to256();
6995		} else unreachable;
	6574	return switch (reg.class()) {
	6575	.general_purpose => if (size_bytes == 0)
	6576	unreachable // should be comptime-known
	6577	else if (size_bytes <= 1)
	6578	reg.to8()
	6579	else if (size_bytes <= 2)
	6580	reg.to16()
	6581	else if (size_bytes <= 4)
	6582	reg.to32()
	6583	else if (size_bytes <= 8)
	6584	reg.to64()
	6585	else
	6586	unreachable,
	6587	.floating_point => if (size_bytes <= 16)
	6588	reg.to128()
	6589	else if (size_bytes <= 32)
	6590	reg.to256()
	6591	else
	6592	unreachable,
	6593	.segment => unreachable,
	6594	};
6996	6595	}
6997	6596
6998	6597	/// Truncates the value in the register in place.
6999	6598	/// Clobbers any remaining bits.
7000	6599	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
7001	6600	const int_info = ty.intInfo(self.target.*);
7002		const max_reg_bit_width = Register.rax.size();
	6601	const max_reg_bit_width = Register.rax.bitSize();
7003	6602	switch (int_info.signedness) {
7004	6603	.signed => {
7005	6604	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
...	...	@@ -7009,7 +6608,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
7009	6608	.unsigned => {
7010	6609	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
7011	6610	const mask = (~@as(u64, 0)) >> shift;
7012		if (int_info.bits <= 32) {
	6611	if (int_info.bits < 32) {
7013	6612	try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .immediate = mask });
7014	6613	} else {
7015	6614	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });

src/arch/x86_64/Emit.zig+384-2836

...	...	@@ -7,6 +7,7 @@ const std = @import("std");
7	7	const assert = std.debug.assert;
8	8	const bits = @import("bits.zig");
9	9	const abi = @import("abi.zig");
	10	const encoder = @import("encoder.zig");
10	11	const link = @import("../../link.zig");
11	12	const log = std.log.scoped(.codegen);
12	13	const math = std.math;
...	...	@@ -19,12 +20,14 @@ const CodeGen = @import("CodeGen.zig");
19	20	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
20	21	const Encoder = bits.Encoder;
21	22	const ErrorMsg = Module.ErrorMsg;
	23	const Immediate = bits.Immediate;
	24	const Instruction = encoder.Instruction;
22	25	const MCValue = @import("CodeGen.zig").MCValue;
	26	const Memory = bits.Memory;
23	27	const Mir = @import("Mir.zig");
24	28	const Module = @import("../../Module.zig");
25		const Instruction = bits.Instruction;
26		const Type = @import("../../type.zig").Type;
27	29	const Register = bits.Register;
	30	const Type = @import("../../type.zig").Type;
28	31
29	32	mir: Mir,
30	33	bin_file: *link.File,
...	...	@@ -45,6 +48,8 @@ relocs: std.ArrayListUnmanaged(Reloc) = .{},
45	48	const InnerError = error{
46	49	OutOfMemory,
47	50	EmitFail,
	51	InvalidInstruction,
	52	CannotEncode,
48	53	};
49	54
50	55	const Reloc = struct {
...	...	@@ -65,133 +70,66 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
65	70	const inst = @intCast(u32, index);
66	71	try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
67	72	switch (tag) {
68		// GPR instructions
69		.adc => try emit.mirArith(.adc, inst),
70		.add => try emit.mirArith(.add, inst),
71		.sub => try emit.mirArith(.sub, inst),
72		.xor => try emit.mirArith(.xor, inst),
73		.@"and" => try emit.mirArith(.@"and", inst),
74		.@"or" => try emit.mirArith(.@"or", inst),
75		.sbb => try emit.mirArith(.sbb, inst),
76		.cmp => try emit.mirArith(.cmp, inst),
77		.mov => try emit.mirArith(.mov, inst),
78
79		.adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
80		.add_mem_imm => try emit.mirArithMemImm(.add, inst),
81		.sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
82		.xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
83		.and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
84		.or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
85		.sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
86		.cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
87		.mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
88
89		.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
90		.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
91		.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
92		.xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),
93		.and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),
94		.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
95		.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
96		.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
97		.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
98
99		.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
100		.add_scale_dst => try emit.mirArithScaleDst(.add, inst),
101		.sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),
102		.xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),
103		.and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),
104		.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
105		.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
106		.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
107		.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
108
109		.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
110		.add_scale_imm => try emit.mirArithScaleImm(.add, inst),
111		.sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
112		.xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
113		.and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
114		.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
115		.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
116		.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
117		.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
118
119		.adc_mem_index_imm => try emit.mirArithMemIndexImm(.adc, inst),
120		.add_mem_index_imm => try emit.mirArithMemIndexImm(.add, inst),
121		.sub_mem_index_imm => try emit.mirArithMemIndexImm(.sub, inst),
122		.xor_mem_index_imm => try emit.mirArithMemIndexImm(.xor, inst),
123		.and_mem_index_imm => try emit.mirArithMemIndexImm(.@"and", inst),
124		.or_mem_index_imm => try emit.mirArithMemIndexImm(.@"or", inst),
125		.sbb_mem_index_imm => try emit.mirArithMemIndexImm(.sbb, inst),
126		.cmp_mem_index_imm => try emit.mirArithMemIndexImm(.cmp, inst),
127		.mov_mem_index_imm => try emit.mirArithMemIndexImm(.mov, inst),
128
129		.mov_sign_extend => try emit.mirMovSignExtend(inst),
130		.mov_zero_extend => try emit.mirMovZeroExtend(inst),
131
132		.movabs => try emit.mirMovabs(inst),
133
134		.fisttp => try emit.mirFisttp(inst),
135		.fld => try emit.mirFld(inst),
136
137		.lea => try emit.mirLea(inst),
138		.lea_pic => try emit.mirLeaPic(inst),
139
140		.shl => try emit.mirShift(.shl, inst),
141		.sal => try emit.mirShift(.sal, inst),
142		.shr => try emit.mirShift(.shr, inst),
143		.sar => try emit.mirShift(.sar, inst),
144
145		.imul => try emit.mirMulDiv(.imul, inst),
146		.mul => try emit.mirMulDiv(.mul, inst),
147		.idiv => try emit.mirMulDiv(.idiv, inst),
148		.div => try emit.mirMulDiv(.div, inst),
149		.imul_complex => try emit.mirIMulComplex(inst),
150
151		.cwd => try emit.mirCwd(inst),
152
153		.push => try emit.mirPushPop(.push, inst),
154		.pop => try emit.mirPushPop(.pop, inst),
155
156		.jmp => try emit.mirJmpCall(.jmp_near, inst),
157		.call => try emit.mirJmpCall(.call_near, inst),
158
159		.cond_jmp => try emit.mirCondJmp(inst),
160		.cond_set_byte => try emit.mirCondSetByte(inst),
161		.cond_mov => try emit.mirCondMov(inst),
162
163		.ret => try emit.mirRet(inst),
164
165		.syscall => try emit.mirSyscall(),
166
167		.@"test" => try emit.mirTest(inst),
168
169		.ud => try emit.mirUndefinedInstruction(),
170		.interrupt => try emit.mirInterrupt(inst),
171		.nop => {}, // just skip it
172
173		// SSE instructions
174		.mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),
175		.mov_f32_sse => try emit.mirMovFloatSse(.movss, inst),
	73	.adc,
	74	.add,
	75	.@"and",
	76	.call,
	77	.cbw,
	78	.cwde,
	79	.cdqe,
	80	.cwd,
	81	.cdq,
	82	.cqo,
	83	.cmp,
	84	.div,
	85	.fisttp,
	86	.fld,
	87	.idiv,
	88	.imul,
	89	.int3,
	90	.jmp,
	91	.lea,
	92	.mov,
	93	.movzx,
	94	.mul,
	95	.nop,
	96	.@"or",
	97	.pop,
	98	.push,
	99	.ret,
	100	.sal,
	101	.sar,
	102	.sbb,
	103	.shl,
	104	.shr,
	105	.sub,
	106	.syscall,
	107	.@"test",
	108	.ud2,
	109	.xor,
176	110
177		.add_f64_sse => try emit.mirAddFloatSse(.addsd, inst),
178		.add_f32_sse => try emit.mirAddFloatSse(.addss, inst),
	111	.addss,
	112	.cmpss,
	113	.movss,
	114	.ucomiss,
	115	.addsd,
	116	.cmpsd,
	117	.movsd,
	118	.ucomisd,
	119	=> try emit.mirEncodeGeneric(tag, inst),
179	120
180		.cmp_f64_sse => try emit.mirCmpFloatSse(.ucomisd, inst),
181		.cmp_f32_sse => try emit.mirCmpFloatSse(.ucomiss, inst),
	121	.jmp_reloc => try emit.mirJmpReloc(inst),
182	122
183		// AVX instructions
184		.mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst),
185		.mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst),
	123	.call_extern => try emit.mirCallExtern(inst),
186	124
187		.add_f64_avx => try emit.mirAddFloatAvx(.vaddsd, inst),
188		.add_f32_avx => try emit.mirAddFloatAvx(.vaddss, inst),
	125	.lea_linker => try emit.mirLeaLinker(inst),
189	126
190		.cmp_f64_avx => try emit.mirCmpFloatAvx(.vucomisd, inst),
191		.cmp_f32_avx => try emit.mirCmpFloatAvx(.vucomiss, inst),
	127	.mov_moffs => try emit.mirMovMoffs(inst),
192	128
193		// Pseudo-instructions
194		.call_extern => try emit.mirCallExtern(inst),
	129	.movsx => try emit.mirMovsx(inst),
	130	.cmovcc => try emit.mirCmovcc(inst),
	131	.setcc => try emit.mirSetcc(inst),
	132	.jcc => try emit.mirJcc(inst),
195	133
196	134	.dbg_line => try emit.mirDbgLine(inst),
197	135	.dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
...	...	@@ -200,9 +138,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
200	138	.push_regs => try emit.mirPushPopRegisterList(.push, inst),
201	139	.pop_regs => try emit.mirPushPopRegisterList(.pop, inst),
202	140
203		else => {
204		return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {}", .{tag});
205		},
	141	.dead => {},
206	142	}
207	143	}
208	144
...	...	@@ -235,798 +171,385 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
235	171	}
236	172	}
237	173
238		fn mirUndefinedInstruction(emit: *Emit) InnerError!void {
239		return lowerToZoEnc(.ud2, emit.code);
	174	fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct {
	175	op1: Instruction.Operand = .none,
	176	op2: Instruction.Operand = .none,
	177	op3: Instruction.Operand = .none,
	178	op4: Instruction.Operand = .none,
	179	}) InnerError!void {
	180	const inst = try Instruction.new(mnemonic, .{
	181	.op1 = ops.op1,
	182	.op2 = ops.op2,
	183	.op3 = ops.op3,
	184	.op4 = ops.op4,
	185	});
	186	return inst.encode(emit.code.writer());
240	187	}
241	188
242		fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
243		const tag = emit.mir.instructions.items(.tag)[inst];
244		assert(tag == .interrupt);
245		const ops = emit.mir.instructions.items(.ops)[inst].decode();
246		switch (ops.flags) {
247		0b00 => return lowerToZoEnc(.int3, emit.code),
248		else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}),
249		}
250		}
	189	fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	190	const mnemonic = inline for (@typeInfo(Instruction.Mnemonic).Enum.fields) \|field\| {
	191	if (mem.eql(u8, field.name, @tagName(tag))) break @field(Instruction.Mnemonic, field.name);
	192	} else unreachable;
251	193
252		fn mirSyscall(emit: *Emit) InnerError!void {
253		return lowerToZoEnc(.syscall, emit.code);
254		}
	194	const ops = emit.mir.instructions.items(.ops)[inst];
	195	const data = emit.mir.instructions.items(.data)[inst];
255	196
256		fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
257		const ops = emit.mir.instructions.items(.ops)[inst].decode();
258		switch (ops.flags) {
259		0b00 => {
260		// PUSH/POP reg
261		return lowerToOEnc(tag, ops.reg1, emit.code);
262		},
263		0b01 => {
264		// PUSH/POP r/m64
265		const imm = emit.mir.instructions.items(.data)[inst].imm;
266		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
267		16 => .word_ptr,
268		else => .qword_ptr,
269		};
270		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
271		.disp = imm,
272		.base = ops.reg1,
273		}), emit.code);
274		},
275		0b10 => {
276		// PUSH imm32
277		assert(tag == .push);
278		const imm = emit.mir.instructions.items(.data)[inst].imm;
279		return lowerToIEnc(.push, imm, emit.code);
280		},
281		0b11 => unreachable,
282		}
283		}
	197	var op1: Instruction.Operand = .none;
	198	var op2: Instruction.Operand = .none;
	199	var op3: Instruction.Operand = .none;
	200	var op4: Instruction.Operand = .none;
284	201
285		fn mirPushPopRegisterList(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
286		const ops = emit.mir.instructions.items(.ops)[inst].decode();
287		const payload = emit.mir.instructions.items(.data)[inst].payload;
288		const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;
289		var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
290		const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
291		const callee_preserved_regs = abi.getCalleePreservedRegs(emit.target.*);
292		for (callee_preserved_regs) \|reg\| {
293		if (reg_list.isSet(callee_preserved_regs, reg)) {
294		switch (tag) {
295		.push => try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
296		.disp = @bitCast(u32, disp),
297		.base = ops.reg1,
298		}), reg, emit.code),
299		.pop => try lowerToRmEnc(.mov, reg, RegisterOrMemory.mem(.qword_ptr, .{
300		.disp = @bitCast(u32, disp),
301		.base = ops.reg1,
302		}), emit.code),
	202	switch (ops) {
	203	.none => {},
	204	.imm_s => op1 = .{ .imm = Immediate.s(@bitCast(i32, data.imm)) },
	205	.imm_u => op1 = .{ .imm = Immediate.u(data.imm) },
	206	.r => op1 = .{ .reg = data.r },
	207	.rr => {
	208	op1 = .{ .reg = data.rr.r1 };
	209	op2 = .{ .reg = data.rr.r2 };
	210	},
	211	.ri_s, .ri_u => {
	212	const imm = switch (ops) {
	213	.ri_s => Immediate.s(@bitCast(i32, data.ri.imm)),
	214	.ri_u => Immediate.u(data.ri.imm),
	215	else => unreachable,
	216	};
	217	op1 = .{ .reg = data.ri.r1 };
	218	op2 = .{ .imm = imm };
	219	},
	220	.ri64 => {
	221	const imm64 = emit.mir.extraData(Mir.Imm64, data.rx.payload).data;
	222	op1 = .{ .reg = data.rx.r1 };
	223	op2 = .{ .imm = Immediate.u(Mir.Imm64.decode(imm64)) };
	224	},
	225	.rri_s, .rri_u => {
	226	const imm = switch (ops) {
	227	.rri_s => Immediate.s(@bitCast(i32, data.rri.imm)),
	228	.rri_u => Immediate.u(data.rri.imm),
	229	else => unreachable,
	230	};
	231	op1 = .{ .reg = data.rri.r1 };
	232	op2 = .{ .reg = data.rri.r2 };
	233	op3 = .{ .imm = imm };
	234	},
	235	.m_sib => {
	236	const msib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
	237	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
	238	},
	239	.m_rip => {
	240	const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
	241	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
	242	},
	243	.mi_s_sib, .mi_u_sib => {
	244	const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data;
	245	const imm = switch (ops) {
	246	.mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
	247	.mi_u_sib => Immediate.u(data.xi.imm),
	248	else => unreachable,
	249	};
	250	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
	251	op2 = .{ .imm = imm };
	252	},
	253	.mi_u_rip, .mi_s_rip => {
	254	const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data;
	255	const imm = switch (ops) {
	256	.mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
	257	.mi_u_rip => Immediate.u(data.xi.imm),
	258	else => unreachable,
	259	};
	260	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
	261	op2 = .{ .imm = imm };
	262	},
	263	.rm_sib, .mr_sib => {
	264	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
	265	const op_r = .{ .reg = data.rx.r1 };
	266	const op_m = .{ .mem = Mir.MemorySib.decode(msib) };
	267	switch (ops) {
	268	.rm_sib => {
	269	op1 = op_r;
	270	op2 = op_m;
	271	},
	272	.mr_sib => {
	273	op1 = op_m;
	274	op2 = op_r;
	275	},
303	276	else => unreachable,
304	277	}
305		disp += 8;
306		}
307		}
308		}
309
310		fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
311		const ops = emit.mir.instructions.items(.ops)[inst].decode();
312		switch (ops.flags) {
313		0b00 => {
314		const target = emit.mir.instructions.items(.data)[inst].inst;
315		const source = emit.code.items.len;
316		try lowerToDEnc(tag, 0, emit.code);
317		try emit.relocs.append(emit.bin_file.allocator, .{
318		.source = source,
319		.target = target,
320		.offset = emit.code.items.len - 4,
321		.length = 5,
322		});
323	278	},
324		0b01 => {
325		if (ops.reg1 == .none) {
326		// JMP/CALL [imm]
327		const imm = emit.mir.instructions.items(.data)[inst].imm;
328		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
329		16 => .word_ptr,
330		else => .qword_ptr,
331		};
332		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code);
	279	.rm_rip, .mr_rip => {
	280	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
	281	const op_r = .{ .reg = data.rx.r1 };
	282	const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) };
	283	switch (ops) {
	284	.rm_sib => {
	285	op1 = op_r;
	286	op2 = op_m;
	287	},
	288	.mr_sib => {
	289	op1 = op_m;
	290	op2 = op_r;
	291	},
	292	else => unreachable,
333	293	}
334		// JMP/CALL reg
335		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
336	294	},
337		0b10 => {
338		// JMP/CALL r/m64
339		const imm = emit.mir.instructions.items(.data)[inst].imm;
340		return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
341		.disp = imm,
342		.base = ops.reg1,
343		}), emit.code);
344		},
345		0b11 => return emit.fail("TODO unused variant jmp/call 0b11", .{}),
	295	else => return emit.fail("TODO handle generic encoding: {s}, {s}", .{
	296	@tagName(mnemonic),
	297	@tagName(ops),
	298	}),
346	299	}
347		}
348	300
349		fn mirCondJmp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
350		const mir_tag = emit.mir.instructions.items(.tag)[inst];
351		assert(mir_tag == .cond_jmp);
352		const inst_cc = emit.mir.instructions.items(.data)[inst].inst_cc;
353		const tag: Tag = switch (inst_cc.cc) {
354		.a => .ja,
355		.ae => .jae,
356		.b => .jb,
357		.be => .jbe,
358		.c => .jc,
359		.e => .je,
360		.g => .jg,
361		.ge => .jge,
362		.l => .jl,
363		.le => .jle,
364		.na => .jna,
365		.nae => .jnae,
366		.nb => .jnb,
367		.nbe => .jnbe,
368		.nc => .jnc,
369		.ne => .jne,
370		.ng => .jng,
371		.nge => .jnge,
372		.nl => .jnl,
373		.nle => .jnle,
374		.no => .jno,
375		.np => .jnp,
376		.ns => .jns,
377		.nz => .jnz,
378		.o => .jo,
379		.p => .jp,
380		.pe => .jpe,
381		.po => .jpo,
382		.s => .js,
383		.z => .jz,
384		};
385		const source = emit.code.items.len;
386		try lowerToDEnc(tag, 0, emit.code);
387		try emit.relocs.append(emit.bin_file.allocator, .{
388		.source = source,
389		.target = inst_cc.inst,
390		.offset = emit.code.items.len - 4,
391		.length = 6,
	301	return emit.encode(mnemonic, .{
	302	.op1 = op1,
	303	.op2 = op2,
	304	.op3 = op3,
	305	.op4 = op4,
392	306	});
393	307	}
394	308
395		fn mirCondSetByte(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
396		const mir_tag = emit.mir.instructions.items(.tag)[inst];
397		assert(mir_tag == .cond_set_byte);
398		const ops = emit.mir.instructions.items(.ops)[inst].decode();
399		const cc = emit.mir.instructions.items(.data)[inst].cc;
400		const tag: Tag = switch (cc) {
401		.a => .seta,
402		.ae => .setae,
403		.b => .setb,
404		.be => .setbe,
405		.c => .setc,
406		.e => .sete,
407		.g => .setg,
408		.ge => .setge,
409		.l => .setl,
410		.le => .setle,
411		.na => .setna,
412		.nae => .setnae,
413		.nb => .setnb,
414		.nbe => .setnbe,
415		.nc => .setnc,
416		.ne => .setne,
417		.ng => .setng,
418		.nge => .setnge,
419		.nl => .setnl,
420		.nle => .setnle,
421		.no => .setno,
422		.np => .setnp,
423		.ns => .setns,
424		.nz => .setnz,
425		.o => .seto,
426		.p => .setp,
427		.pe => .setpe,
428		.po => .setpo,
429		.s => .sets,
430		.z => .setz,
431		};
432		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code);
433		}
434
435		fn mirCondMov(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
436		const mir_tag = emit.mir.instructions.items(.tag)[inst];
437		assert(mir_tag == .cond_mov);
438		const ops = emit.mir.instructions.items(.ops)[inst].decode();
439		const cc = emit.mir.instructions.items(.data)[inst].cc;
440		const tag: Tag = switch (cc) {
441		.a => .cmova,
442		.ae => .cmovae,
443		.b => .cmovb,
444		.be => .cmovbe,
445		.c => .cmovc,
446		.e => .cmove,
447		.g => .cmovg,
448		.ge => .cmovge,
449		.l => .cmovl,
450		.le => .cmovle,
451		.na => .cmovna,
452		.nae => .cmovnae,
453		.nb => .cmovnb,
454		.nbe => .cmovnbe,
455		.nc => .cmovnc,
456		.ne => .cmovne,
457		.ng => .cmovng,
458		.nge => .cmovnge,
459		.nl => .cmovnl,
460		.nle => .cmovnle,
461		.no => .cmovno,
462		.np => .cmovnp,
463		.ns => .cmovns,
464		.nz => .cmovnz,
465		.o => .cmovo,
466		.p => .cmovp,
467		.pe => .cmovpe,
468		.po => .cmovpo,
469		.s => .cmovs,
470		.z => .cmovz,
471		};
472
473		if (ops.flags == 0b00) {
474		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
475		}
476		const imm = emit.mir.instructions.items(.data)[inst].imm;
477		const ptr_size: Memory.PtrSize = switch (ops.flags) {
478		0b00 => unreachable,
479		0b01 => .word_ptr,
480		0b10 => .dword_ptr,
481		0b11 => .qword_ptr,
482		};
483		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ptr_size, .{
484		.disp = imm,
485		.base = ops.reg2,
486		}), emit.code);
487		}
488
489		fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
490		const tag = emit.mir.instructions.items(.tag)[inst];
491		assert(tag == .@"test");
492		const ops = emit.mir.instructions.items(.ops)[inst].decode();
493		switch (ops.flags) {
494		0b00 => {
495		if (ops.reg2 == .none) {
496		// TEST r/m64, imm32
497		// MI
498		const imm = emit.mir.instructions.items(.data)[inst].imm;
499		if (ops.reg1.to64() == .rax) {
500		// TEST rax, imm32
501		// I
502		return lowerToIEnc(.@"test", imm, emit.code);
503		}
504		return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
505		}
506		// TEST r/m64, r64
507		return lowerToMrEnc(.@"test", RegisterOrMemory.reg(ops.reg1), ops.reg2, emit.code);
508		},
509		else => return emit.fail("TODO more TEST alternatives", .{}),
510		}
511		}
512
513		fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
514		const tag = emit.mir.instructions.items(.tag)[inst];
515		assert(tag == .ret);
516		const ops = emit.mir.instructions.items(.ops)[inst].decode();
517		switch (ops.flags) {
518		0b00 => {
519		// RETF imm16
520		// I
521		const imm = emit.mir.instructions.items(.data)[inst].imm;
522		return lowerToIEnc(.ret_far, imm, emit.code);
523		},
524		0b01 => {
525		return lowerToZoEnc(.ret_far, emit.code);
526		},
527		0b10 => {
528		// RET imm16
529		// I
530		const imm = emit.mir.instructions.items(.data)[inst].imm;
531		return lowerToIEnc(.ret_near, imm, emit.code);
	309	fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	310	const ops = emit.mir.instructions.items(.ops)[inst];
	311	const payload = emit.mir.instructions.items(.data)[inst].payload;
	312	const moffs = emit.mir.extraData(Mir.MemoryMoffs, payload).data;
	313	const seg = @intToEnum(Register, moffs.seg);
	314	const offset = moffs.decodeOffset();
	315	switch (ops) {
	316	.rax_moffs => {
	317	try emit.encode(.mov, .{
	318	.op1 = .{ .reg = .rax },
	319	.op2 = .{ .mem = Memory.moffs(seg, offset) },
	320	});
532	321	},
533		0b11 => {
534		return lowerToZoEnc(.ret_near, emit.code);
	322	.moffs_rax => {
	323	try emit.encode(.mov, .{
	324	.op1 = .{ .mem = Memory.moffs(seg, offset) },
	325	.op2 = .{ .reg = .rax },
	326	});
535	327	},
	328	else => unreachable,
536	329	}
537	330	}
538	331
539		fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
540		const ops = emit.mir.instructions.items(.ops)[inst].decode();
541		switch (ops.flags) {
542		0b00 => {
543		if (ops.reg2 == .none) {
544		// mov reg1, imm32
545		// MI
546		const imm = emit.mir.instructions.items(.data)[inst].imm;
547		return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
548		}
549		// mov reg1, reg2
550		// RM
551		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
552		},
553		0b01 => {
554		// mov reg1, [reg2 + imm32]
555		// RM
556		const imm = emit.mir.instructions.items(.data)[inst].imm;
557		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
558		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
559		.disp = imm,
560		.base = src_reg,
561		}), emit.code);
	332	fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	333	const ops = emit.mir.instructions.items(.ops)[inst];
	334	const data = emit.mir.instructions.items(.data)[inst];
	335
	336	var op1: Instruction.Operand = .none;
	337	var op2: Instruction.Operand = .none;
	338	switch (ops) {
	339	.rr => {
	340	op1 = .{ .reg = data.rr.r1 };
	341	op2 = .{ .reg = data.rr.r2 };
562	342	},
563		0b10 => {
564		if (ops.reg2 == .none) {
565		return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
566		}
567		// mov [reg1 + imm32], reg2
568		// MR
569		const imm = emit.mir.instructions.items(.data)[inst].imm;
570		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
571		.disp = imm,
572		.base = ops.reg1,
573		}), ops.reg2, emit.code);
	343	.rm_sib => {
	344	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
	345	op1 = .{ .reg = data.rx.r1 };
	346	op2 = .{ .mem = Mir.MemorySib.decode(msib) };
574	347	},
575		0b11 => {
576		return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
	348	.rm_rip => {
	349	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
	350	op1 = .{ .reg = data.rx.r1 };
	351	op2 = .{ .mem = Mir.MemoryRip.decode(mrip) };
577	352	},
	353	else => unreachable, // TODO
578	354	}
579		}
580
581		fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
582		const ops = emit.mir.instructions.items(.ops)[inst].decode();
583		assert(ops.reg2 == .none);
584		const payload = emit.mir.instructions.items(.data)[inst].payload;
585		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
586		const ptr_size: Memory.PtrSize = switch (ops.flags) {
587		0b00 => .byte_ptr,
588		0b01 => .word_ptr,
589		0b10 => .dword_ptr,
590		0b11 => .qword_ptr,
591		};
592		return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
593		.disp = imm_pair.dest_off,
594		.base = ops.reg1,
595		}), imm_pair.operand, emit.code);
596		}
597
598		inline fn setRexWRegister(reg: Register) bool {
599		if (reg.size() > 64) return false;
600		if (reg.size() == 64) return true;
601		return switch (reg) {
602		.ah, .ch, .dh, .bh => true,
603		else => false,
604		};
605		}
606
607		inline fn immOpSize(u_imm: u32) u6 {
608		const imm = @bitCast(i32, u_imm);
609		if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
610		return 8;
611		}
612		if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
613		return 16;
614		}
615		return 32;
616		}
617	355
618		fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
619		const ops = emit.mir.instructions.items(.ops)[inst].decode();
620		const scale = ops.flags;
621		const payload = emit.mir.instructions.items(.data)[inst].payload;
622		const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
623		// OP reg1, [reg2 + scale*index + imm32]
624		const scale_index = ScaleIndex{
625		.scale = scale,
626		.index = index_reg_disp.index,
627		};
628		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
629		.disp = index_reg_disp.disp,
630		.base = ops.reg2,
631		.scale_index = scale_index,
632		}), emit.code);
633		}
634
635		fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
636		const ops = emit.mir.instructions.items(.ops)[inst].decode();
637		const scale = ops.flags;
638		const payload = emit.mir.instructions.items(.data)[inst].payload;
639		const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
640		const scale_index = ScaleIndex{
641		.scale = scale,
642		.index = index_reg_disp.index,
	356	const mnemonic: Instruction.Mnemonic = switch (op1.bitSize()) {
	357	32, 64 => if (op2.bitSize() == 32) .movsxd else .movsx,
	358	else => .movsx,
643	359	};
644		assert(ops.reg2 != .none);
645		// OP [reg1 + scale*index + imm32], reg2
646		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
647		.disp = index_reg_disp.disp,
648		.base = ops.reg1,
649		.scale_index = scale_index,
650		}), ops.reg2, emit.code);
651		}
652	360
653		fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
654		const ops = emit.mir.instructions.items(.ops)[inst].decode();
655		const scale = ops.flags;
656		const payload = emit.mir.instructions.items(.data)[inst].payload;
657		const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();
658		const scale_index = ScaleIndex{
659		.scale = scale,
660		.index = index_reg_disp_imm.index,
661		};
662		// OP qword ptr [reg1 + scale*index + imm32], imm32
663		return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
664		.disp = index_reg_disp_imm.disp,
665		.base = ops.reg1,
666		.scale_index = scale_index,
667		}), index_reg_disp_imm.imm, emit.code);
	361	return emit.encode(mnemonic, .{
	362	.op1 = op1,
	363	.op2 = op2,
	364	});
668	365	}
669	366
670		fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
671		const ops = emit.mir.instructions.items(.ops)[inst].decode();
672		assert(ops.reg2 == .none);
673		const payload = emit.mir.instructions.items(.data)[inst].payload;
674		const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();
675		const ptr_size: Memory.PtrSize = switch (ops.flags) {
676		0b00 => .byte_ptr,
677		0b01 => .word_ptr,
678		0b10 => .dword_ptr,
679		0b11 => .qword_ptr,
680		};
681		const scale_index = ScaleIndex{
682		.scale = 0,
683		.index = index_reg_disp_imm.index,
684		};
685		// OP ptr [reg1 + index + imm32], imm32
686		return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
687		.disp = index_reg_disp_imm.disp,
688		.base = ops.reg1,
689		.scale_index = scale_index,
690		}), index_reg_disp_imm.imm, emit.code);
	367	fn mnemonicFromConditionCode(comptime basename: []const u8, cc: bits.Condition) Instruction.Mnemonic {
	368	inline for (@typeInfo(bits.Condition).Enum.fields) \|field\| {
	369	if (mem.eql(u8, field.name, @tagName(cc)))
	370	return @field(Instruction.Mnemonic, basename ++ field.name);
	371	} else unreachable;
691	372	}
692	373
693		fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
694		const mir_tag = emit.mir.instructions.items(.tag)[inst];
695		assert(mir_tag == .mov_sign_extend);
696		const ops = emit.mir.instructions.items(.ops)[inst].decode();
697		const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
698		switch (ops.flags) {
699		0b00 => {
700		const tag: Tag = if (ops.reg2.size() == 32) .movsxd else .movsx;
701		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
702		},
703		0b01 => {
704		return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
705		.disp = imm,
706		.base = ops.reg2,
707		}), emit.code);
708		},
709		0b10 => {
710		return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
711		.disp = imm,
712		.base = ops.reg2,
713		}), emit.code);
714		},
715		0b11 => {
716		return lowerToRmEnc(.movsxd, ops.reg1, RegisterOrMemory.mem(.dword_ptr, .{
717		.disp = imm,
718		.base = ops.reg2,
719		}), emit.code);
	374	fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	375	const ops = emit.mir.instructions.items(.ops)[inst];
	376	switch (ops) {
	377	.rr_c => {
	378	const data = emit.mir.instructions.items(.data)[inst].rr_c;
	379	const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
	380	return emit.encode(mnemonic, .{
	381	.op1 = .{ .reg = data.r1 },
	382	.op2 = .{ .reg = data.r2 },
	383	});
720	384	},
	385	else => unreachable, // TODO
721	386	}
722	387	}
723	388
724		fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
725		const mir_tag = emit.mir.instructions.items(.tag)[inst];
726		assert(mir_tag == .mov_zero_extend);
727		const ops = emit.mir.instructions.items(.ops)[inst].decode();
728		const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
729		switch (ops.flags) {
730		0b00 => {
731		return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
732		},
733		0b01 => {
734		return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
735		.disp = imm,
736		.base = ops.reg2,
737		}), emit.code);
738		},
739		0b10 => {
740		return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
741		.disp = imm,
742		.base = ops.reg2,
743		}), emit.code);
744		},
745		0b11 => {
746		return emit.fail("TODO unused variant: movzx 0b11", .{});
	389	fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	390	const ops = emit.mir.instructions.items(.ops)[inst];
	391	switch (ops) {
	392	.r_c => {
	393	const data = emit.mir.instructions.items(.data)[inst].r_c;
	394	const mnemonic = mnemonicFromConditionCode("set", data.cc);
	395	return emit.encode(mnemonic, .{
	396	.op1 = .{ .reg = data.r1 },
	397	});
747	398	},
	399	else => unreachable, // TODO
748	400	}
749	401	}
750	402
751		fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
752		const tag = emit.mir.instructions.items(.tag)[inst];
753		assert(tag == .movabs);
754		const ops = emit.mir.instructions.items(.ops)[inst].decode();
755		switch (ops.flags) {
756		0b00 => {
757		const imm: u64 = if (ops.reg1.size() == 64) blk: {
758		const payload = emit.mir.instructions.items(.data)[inst].payload;
759		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
760		break :blk imm.decode();
761		} else emit.mir.instructions.items(.data)[inst].imm;
762		// movabs reg, imm64
763		// OI
764		return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
765		},
766		0b01 => {
767		if (ops.reg1 == .none) {
768		const imm: u64 = if (ops.reg2.size() == 64) blk: {
769		const payload = emit.mir.instructions.items(.data)[inst].payload;
770		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
771		break :blk imm.decode();
772		} else emit.mir.instructions.items(.data)[inst].imm;
773		// movabs moffs64, rax
774		// TD
775		return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
776		}
777		const imm: u64 = if (ops.reg1.size() == 64) blk: {
778		const payload = emit.mir.instructions.items(.data)[inst].payload;
779		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
780		break :blk imm.decode();
781		} else emit.mir.instructions.items(.data)[inst].imm;
782		// movabs rax, moffs64
783		// FD
784		return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
	403	fn mirJcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	404	const ops = emit.mir.instructions.items(.ops)[inst];
	405	switch (ops) {
	406	.inst_cc => {
	407	const data = emit.mir.instructions.items(.data)[inst].inst_cc;
	408	const mnemonic = mnemonicFromConditionCode("j", data.cc);
	409	const source = emit.code.items.len;
	410	try emit.encode(mnemonic, .{
	411	.op1 = .{ .imm = Immediate.s(0) },
	412	});
	413	try emit.relocs.append(emit.bin_file.allocator, .{
	414	.source = source,
	415	.target = data.inst,
	416	.offset = emit.code.items.len - 4,
	417	.length = 6,
	418	});
785	419	},
786		else => return emit.fail("TODO unused movabs variant", .{}),
	420	else => unreachable, // TODO
787	421	}
788	422	}
789	423
790		fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
791		const tag = emit.mir.instructions.items(.tag)[inst];
792		assert(tag == .fisttp);
793		const ops = emit.mir.instructions.items(.ops)[inst].decode();
794
795		// the selecting between operand sizes for this particular `fisttp` instruction
796		// is done via opcode instead of the usual prefixes.
797
798		const opcode: Tag = switch (ops.flags) {
799		0b00 => .fisttp16,
800		0b01 => .fisttp32,
801		0b10 => .fisttp64,
802		else => unreachable,
803		};
804		const mem_or_reg = Memory{
805		.base = ops.reg1,
806		.disp = emit.mir.instructions.items(.data)[inst].imm,
807		.ptr_size = Memory.PtrSize.dword_ptr, // to prevent any prefix from being used
808		};
809		return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
810		}
811
812		fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
813		const tag = emit.mir.instructions.items(.tag)[inst];
814		assert(tag == .fld);
815		const ops = emit.mir.instructions.items(.ops)[inst].decode();
816
817		// the selecting between operand sizes for this particular `fisttp` instruction
818		// is done via opcode instead of the usual prefixes.
819
820		const opcode: Tag = switch (ops.flags) {
821		0b01 => .fld32,
822		0b10 => .fld64,
823		else => unreachable,
824		};
825		const mem_or_reg = Memory{
826		.base = ops.reg1,
827		.disp = emit.mir.instructions.items(.data)[inst].imm,
828		.ptr_size = Memory.PtrSize.dword_ptr, // to prevent any prefix from being used
829		};
830		return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
	424	fn mirJmpReloc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	425	const target = emit.mir.instructions.items(.data)[inst].inst;
	426	const source = emit.code.items.len;
	427	try emit.encode(.jmp, .{
	428	.op1 = .{ .imm = Immediate.s(0) },
	429	});
	430	try emit.relocs.append(emit.bin_file.allocator, .{
	431	.source = source,
	432	.target = target,
	433	.offset = emit.code.items.len - 4,
	434	.length = 5,
	435	});
831	436	}
832	437
833		fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
834		const ops = emit.mir.instructions.items(.ops)[inst].decode();
835		switch (ops.flags) {
836		0b00 => {
837		// sal reg1, 1
838		// M1
839		return lowerToM1Enc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
840		},
841		0b01 => {
842		// sal reg1, .cl
843		// MC
844		return lowerToMcEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
845		},
846		0b10 => {
847		// sal reg1, imm8
848		// MI
849		const imm = @truncate(u8, emit.mir.instructions.items(.data)[inst].imm);
850		return lowerToMiImm8Enc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
851		},
852		0b11 => {
853		return emit.fail("TODO unused variant: SHIFT reg1, 0b11", .{});
854		},
855		}
856		}
	438	fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	439	const relocation = emit.mir.instructions.items(.data)[inst].relocation;
857	440
858		fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
859		const ops = emit.mir.instructions.items(.ops)[inst].decode();
860		if (ops.reg1 != .none) {
861		assert(ops.reg2 == .none);
862		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
863		}
864		assert(ops.reg2 != .none);
865		const imm = emit.mir.instructions.items(.data)[inst].imm;
866		const ptr_size: Memory.PtrSize = switch (ops.flags) {
867		0b00 => .byte_ptr,
868		0b01 => .word_ptr,
869		0b10 => .dword_ptr,
870		0b11 => .qword_ptr,
	441	const offset = blk: {
	442	try emit.encode(.call, .{
	443	.op1 = .{ .imm = Immediate.s(0) },
	444	});
	445	break :blk @intCast(u32, emit.code.items.len) - 4;
871	446	};
872		return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
873		.disp = imm,
874		.base = ops.reg2,
875		}), emit.code);
876		}
877	447
878		fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
879		const tag = emit.mir.instructions.items(.tag)[inst];
880		assert(tag == .imul_complex);
881		const ops = emit.mir.instructions.items(.ops)[inst].decode();
882		switch (ops.flags) {
883		0b00 => {
884		return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
885		},
886		0b01 => {
887		const imm = emit.mir.instructions.items(.data)[inst].imm;
888		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
889		return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
890		.disp = imm,
891		.base = src_reg,
892		}), emit.code);
893		},
894		0b10 => {
895		const imm = emit.mir.instructions.items(.data)[inst].imm;
896		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code);
897		},
898		0b11 => {
899		const payload = emit.mir.instructions.items(.data)[inst].payload;
900		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
901		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
902		.disp = imm_pair.dest_off,
903		.base = ops.reg2,
904		}), imm_pair.operand, emit.code);
905		},
	448	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	449	// Add relocation to the decl.
	450	const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	451	const target = macho_file.getGlobalByIndex(relocation.sym_index);
	452	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
	453	.type = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	454	.target = target,
	455	.offset = offset,
	456	.addend = 0,
	457	.pcrel = true,
	458	.length = 2,
	459	});
	460	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	461	// Add relocation to the decl.
	462	const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	463	const target = coff_file.getGlobalByIndex(relocation.sym_index);
	464	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
	465	.type = .direct,
	466	.target = target,
	467	.offset = offset,
	468	.addend = 0,
	469	.pcrel = true,
	470	.length = 2,
	471	});
	472	} else {
	473	return emit.fail("TODO implement call_extern for linking backends different than MachO and COFF", .{});
906	474	}
907	475	}
908	476
909		fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
910		const ops = emit.mir.instructions.items(.ops)[inst].decode();
911		const tag: Tag = switch (ops.flags) {
912		0b00 => .cbw,
913		0b01 => .cwd,
914		0b10 => .cdq,
915		0b11 => .cqo,
916		};
917		return lowerToZoEnc(tag, emit.code);
918		}
919
920		fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
921		const tag = emit.mir.instructions.items(.tag)[inst];
922		assert(tag == .lea);
923		const ops = emit.mir.instructions.items(.ops)[inst].decode();
924		switch (ops.flags) {
925		0b00 => {
926		// lea reg1, [reg2 + imm32]
927		// RM
928		const imm = emit.mir.instructions.items(.data)[inst].imm;
929		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
930		return lowerToRmEnc(
931		.lea,
932		ops.reg1,
933		RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
934		.disp = imm,
935		.base = src_reg,
	477	fn mirPushPopRegisterList(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	478	const payload = emit.mir.instructions.items(.data)[inst].payload;
	479	const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;
	480	const base = @intToEnum(Register, save_reg_list.base_reg);
	481	var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
	482	const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
	483	const callee_preserved_regs = abi.getCalleePreservedRegs(emit.target.*);
	484	for (callee_preserved_regs) \|reg\| {
	485	if (reg_list.isSet(callee_preserved_regs, reg)) {
	486	const op1: Instruction.Operand = .{ .mem = Memory.sib(.qword, .{
	487	.base = base,
	488	.disp = disp,
	489	}) };
	490	const op2: Instruction.Operand = .{ .reg = reg };
	491	switch (tag) {
	492	.push => try emit.encode(.mov, .{
	493	.op1 = op1,
	494	.op2 = op2,
936	495	}),
937		emit.code,
938		);
939		},
940		0b01 => {
941		// lea reg1, [rip + imm32]
942		// RM
943		const start_offset = emit.code.items.len;
944		try lowerToRmEnc(
945		.lea,
946		ops.reg1,
947		RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
948		emit.code,
949		);
950		const end_offset = emit.code.items.len;
951		// Backpatch the displacement
952		const payload = emit.mir.instructions.items(.data)[inst].payload;
953		const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
954		const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
955		mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
956		},
957		0b10 => {
958		// lea reg, [rbp + index + imm32]
959		const payload = emit.mir.instructions.items(.data)[inst].payload;
960		const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
961		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
962		const scale_index = ScaleIndex{
963		.scale = 0,
964		.index = index_reg_disp.index,
965		};
966		return lowerToRmEnc(
967		.lea,
968		ops.reg1,
969		RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
970		.disp = index_reg_disp.disp,
971		.base = src_reg,
972		.scale_index = scale_index,
	496	.pop => try emit.encode(.mov, .{
	497	.op1 = op2,
	498	.op2 = op1,
973	499	}),
974		emit.code,
975		);
976		},
977		0b11 => return emit.fail("TODO unused LEA variant 0b11", .{}),
	500	else => unreachable,
	501	}
	502	disp += 8;
	503	}
978	504	}
979	505	}
980	506
981		fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
982		const tag = emit.mir.instructions.items(.tag)[inst];
983		assert(tag == .lea_pic);
984		const ops = emit.mir.instructions.items(.ops)[inst].decode();
985		const relocation = emit.mir.instructions.items(.data)[inst].relocation;
986
987		switch (ops.flags) {
988		0b00, 0b01, 0b10 => {},
989		else => return emit.fail("TODO unused LEA PIC variant 0b11", .{}),
990		}
	507	fn mirLeaLinker(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	508	const ops = emit.mir.instructions.items(.ops)[inst];
	509	const payload = emit.mir.instructions.items(.data)[inst].payload;
	510	const metadata = emit.mir.extraData(Mir.LeaRegisterReloc, payload).data;
	511	const reg = @intToEnum(Register, metadata.reg);
991	512
992		// lea reg1, [rip + reloc]
993		// RM
994		try lowerToRmEnc(
995		.lea,
996		ops.reg1,
997		RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
998		emit.code,
999		);
	513	try emit.encode(.lea, .{
	514	.op1 = .{ .reg = reg },
	515	.op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
	516	});
1000	517
1001	518	const end_offset = emit.code.items.len;
1002	519
1003	520	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1004		const reloc_type = switch (ops.flags) {
1005		0b00 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
1006		0b01 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_SIGNED),
	521	const reloc_type = switch (ops) {
	522	.got_reloc => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	523	.direct_reloc => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_SIGNED),
1007	524	else => unreachable,
1008	525	};
1009		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	526	const atom_index = macho_file.getAtomIndexForSymbol(.{
	527	.sym_index = metadata.atom_index,
	528	.file = null,
	529	}).?;
1010	530	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
1011	531	.type = reloc_type,
1012		.target = .{ .sym_index = relocation.sym_index, .file = null },
	532	.target = .{ .sym_index = metadata.sym_index, .file = null },
1013	533	.offset = @intCast(u32, end_offset - 4),
1014	534	.addend = 0,
1015	535	.pcrel = true,
1016	536	.length = 2,
1017	537	});
1018	538	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
1019		const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	539	const atom_index = coff_file.getAtomIndexForSymbol(.{
	540	.sym_index = metadata.atom_index,
	541	.file = null,
	542	}).?;
1020	543	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
1021		.type = switch (ops.flags) {
1022		0b00 => .got,
1023		0b01 => .direct,
1024		0b10 => .import,
	544	.type = switch (ops) {
	545	.got_reloc => .got,
	546	.direct_reloc => .direct,
	547	.import_reloc => .import,
1025	548	else => unreachable,
1026	549	},
1027		.target = switch (ops.flags) {
1028		0b00, 0b01 => .{ .sym_index = relocation.sym_index, .file = null },
1029		0b10 => coff_file.getGlobalByIndex(relocation.sym_index),
	550	.target = switch (ops) {
	551	.got_reloc, .direct_reloc => .{ .sym_index = metadata.sym_index, .file = null },
	552	.import_reloc => coff_file.getGlobalByIndex(metadata.sym_index),
1030	553	else => unreachable,
1031	554	},
1032	555	.offset = @intCast(u32, end_offset - 4),
...	...	@@ -1039,158 +562,23 @@ fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1039	562	}
1040	563	}
1041	564
1042		// SSE instructions
1043
1044		fn mirMovFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1045		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1046		switch (ops.flags) {
1047		0b00 => {
1048		const imm = emit.mir.instructions.items(.data)[inst].imm;
1049		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
1050		.disp = imm,
1051		.base = ops.reg2,
1052		}), emit.code);
1053		},
1054		0b01 => {
1055		const imm = emit.mir.instructions.items(.data)[inst].imm;
1056		return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
1057		.disp = imm,
1058		.base = ops.reg1,
1059		}), ops.reg2, emit.code);
1060		},
1061		0b10 => {
1062		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1063		},
1064		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1065		}
	565	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	566	const payload = emit.mir.instructions.items(.data)[inst].payload;
	567	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
	568	log.debug("mirDbgLine", .{});
	569	try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
1066	570	}
1067	571
1068		fn mirAddFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1069		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1070		switch (ops.flags) {
1071		0b00 => {
1072		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1073		},
1074		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1075		}
1076		}
1077
1078		fn mirCmpFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1079		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1080		switch (ops.flags) {
1081		0b00 => {
1082		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1083		},
1084		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1085		}
1086		}
1087		// AVX instructions
1088
1089		fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1090		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1091		switch (ops.flags) {
1092		0b00 => {
1093		const imm = emit.mir.instructions.items(.data)[inst].imm;
1094		return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
1095		.disp = imm,
1096		.base = ops.reg2,
1097		}), emit.code);
1098		},
1099		0b01 => {
1100		const imm = emit.mir.instructions.items(.data)[inst].imm;
1101		return lowerToMvEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
1102		.disp = imm,
1103		.base = ops.reg1,
1104		}), ops.reg2, emit.code);
1105		},
1106		0b10 => {
1107		return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1108		},
1109		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1110		}
1111		}
1112
1113		fn mirAddFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1114		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1115		switch (ops.flags) {
1116		0b00 => {
1117		return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1118		},
1119		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1120		}
1121		}
1122
1123		fn mirCmpFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1124		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1125		switch (ops.flags) {
1126		0b00 => {
1127		return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1128		},
1129		else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
1130		}
1131		}
1132
1133		// Pseudo-instructions
1134
1135		fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1136		const tag = emit.mir.instructions.items(.tag)[inst];
1137		assert(tag == .call_extern);
1138		const relocation = emit.mir.instructions.items(.data)[inst].relocation;
1139
1140		const offset = blk: {
1141		// callq
1142		try lowerToDEnc(.call_near, 0, emit.code);
1143		break :blk @intCast(u32, emit.code.items.len) - 4;
1144		};
1145
1146		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1147		// Add relocation to the decl.
1148		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1149		const target = macho_file.getGlobalByIndex(relocation.sym_index);
1150		try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
1151		.type = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
1152		.target = target,
1153		.offset = offset,
1154		.addend = 0,
1155		.pcrel = true,
1156		.length = 2,
1157		});
1158		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
1159		// Add relocation to the decl.
1160		const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1161		const target = coff_file.getGlobalByIndex(relocation.sym_index);
1162		try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
1163		.type = .direct,
1164		.target = target,
1165		.offset = offset,
1166		.addend = 0,
1167		.pcrel = true,
1168		.length = 2,
1169		});
1170		} else {
1171		return emit.fail("TODO implement call_extern for linking backends different than MachO and COFF", .{});
1172		}
1173		}
1174
1175		fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1176		const tag = emit.mir.instructions.items(.tag)[inst];
1177		assert(tag == .dbg_line);
1178		const payload = emit.mir.instructions.items(.data)[inst].payload;
1179		const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
1180		log.debug("mirDbgLine", .{});
1181		try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
1182		}
1183
1184		fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1185		const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
1186		const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
1187		log.debug(" (advance pc={d} and line={d})", .{ delta_line, delta_pc });
1188		switch (emit.debug_output) {
1189		.dwarf => \|dw\| {
1190		try dw.advancePCAndLine(delta_line, delta_pc);
1191		emit.prev_di_line = line;
1192		emit.prev_di_column = column;
1193		emit.prev_di_pc = emit.code.items.len;
	572	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
	573	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
	574	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
	575	log.debug(" (advance pc={d} and line={d})", .{ delta_line, delta_pc });
	576	switch (emit.debug_output) {
	577	.dwarf => \|dw\| {
	578	try dw.advancePCAndLine(delta_line, delta_pc);
	579	emit.prev_di_line = line;
	580	emit.prev_di_column = column;
	581	emit.prev_di_pc = emit.code.items.len;
1194	582	},
1195	583	.plan9 => \|dbg_out\| {
1196	584	if (delta_pc <= 0) return; // only do this when the pc changes
...	...	@@ -1232,8 +620,7 @@ fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1232	620	}
1233	621
1234	622	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1235		const tag = emit.mir.instructions.items(.tag)[inst];
1236		assert(tag == .dbg_prologue_end);
	623	_ = inst;
1237	624	switch (emit.debug_output) {
1238	625	.dwarf => \|dw\| {
1239	626	try dw.setPrologueEnd();
...	...	@@ -1249,8 +636,7 @@ fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1249	636	}
1250	637
1251	638	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1252		const tag = emit.mir.instructions.items(.tag)[inst];
1253		assert(tag == .dbg_epilogue_begin);
	639	_ = inst;
1254	640	switch (emit.debug_output) {
1255	641	.dwarf => \|dw\| {
1256	642	try dw.setEpilogueBegin();
...	...	@@ -1264,1841 +650,3 @@ fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1264	650	.none => {},
1265	651	}
1266	652	}
1267
1268		const Tag = enum {
1269		adc,
1270		add,
1271		sub,
1272		xor,
1273		@"and",
1274		@"or",
1275		sbb,
1276		cmp,
1277		mov,
1278		movsx,
1279		movsxd,
1280		movzx,
1281		lea,
1282		jmp_near,
1283		call_near,
1284		push,
1285		pop,
1286		@"test",
1287		ud2,
1288		int3,
1289		nop,
1290		imul,
1291		mul,
1292		idiv,
1293		div,
1294		syscall,
1295		ret_near,
1296		ret_far,
1297		fisttp16,
1298		fisttp32,
1299		fisttp64,
1300		fld32,
1301		fld64,
1302		jo,
1303		jno,
1304		jb,
1305		jbe,
1306		jc,
1307		jnae,
1308		jnc,
1309		jae,
1310		je,
1311		jz,
1312		jne,
1313		jnz,
1314		jna,
1315		jnb,
1316		jnbe,
1317		ja,
1318		js,
1319		jns,
1320		jpe,
1321		jp,
1322		jpo,
1323		jnp,
1324		jnge,
1325		jl,
1326		jge,
1327		jnl,
1328		jle,
1329		jng,
1330		jg,
1331		jnle,
1332		seto,
1333		setno,
1334		setb,
1335		setc,
1336		setnae,
1337		setnb,
1338		setnc,
1339		setae,
1340		sete,
1341		setz,
1342		setne,
1343		setnz,
1344		setbe,
1345		setna,
1346		seta,
1347		setnbe,
1348		sets,
1349		setns,
1350		setp,
1351		setpe,
1352		setnp,
1353		setpo,
1354		setl,
1355		setnge,
1356		setnl,
1357		setge,
1358		setle,
1359		setng,
1360		setnle,
1361		setg,
1362		cmovo,
1363		cmovno,
1364		cmovb,
1365		cmovc,
1366		cmovnae,
1367		cmovnb,
1368		cmovnc,
1369		cmovae,
1370		cmove,
1371		cmovz,
1372		cmovne,
1373		cmovnz,
1374		cmovbe,
1375		cmovna,
1376		cmova,
1377		cmovnbe,
1378		cmovs,
1379		cmovns,
1380		cmovp,
1381		cmovpe,
1382		cmovnp,
1383		cmovpo,
1384		cmovl,
1385		cmovnge,
1386		cmovnl,
1387		cmovge,
1388		cmovle,
1389		cmovng,
1390		cmovnle,
1391		cmovg,
1392		shl,
1393		sal,
1394		shr,
1395		sar,
1396		cbw,
1397		cwd,
1398		cdq,
1399		cqo,
1400		movsd,
1401		movss,
1402		addsd,
1403		addss,
1404		cmpsd,
1405		cmpss,
1406		ucomisd,
1407		ucomiss,
1408		vmovsd,
1409		vmovss,
1410		vaddsd,
1411		vaddss,
1412		vcmpsd,
1413		vcmpss,
1414		vucomisd,
1415		vucomiss,
1416
1417		fn isSse(tag: Tag) bool {
1418		return switch (tag) {
1419		.movsd,
1420		.movss,
1421		.addsd,
1422		.addss,
1423		.cmpsd,
1424		.cmpss,
1425		.ucomisd,
1426		.ucomiss,
1427		=> true,
1428
1429		else => false,
1430		};
1431		}
1432
1433		fn isAvx(tag: Tag) bool {
1434		return switch (tag) {
1435		.vmovsd,
1436		.vmovss,
1437		.vaddsd,
1438		.vaddss,
1439		.vcmpsd,
1440		.vcmpss,
1441		.vucomisd,
1442		.vucomiss,
1443		=> true,
1444
1445		else => false,
1446		};
1447		}
1448
1449		fn isSetCC(tag: Tag) bool {
1450		return switch (tag) {
1451		.seto,
1452		.setno,
1453		.setb,
1454		.setc,
1455		.setnae,
1456		.setnb,
1457		.setnc,
1458		.setae,
1459		.sete,
1460		.setz,
1461		.setne,
1462		.setnz,
1463		.setbe,
1464		.setna,
1465		.seta,
1466		.setnbe,
1467		.sets,
1468		.setns,
1469		.setp,
1470		.setpe,
1471		.setnp,
1472		.setpo,
1473		.setl,
1474		.setnge,
1475		.setnl,
1476		.setge,
1477		.setle,
1478		.setng,
1479		.setnle,
1480		.setg,
1481		=> true,
1482		else => false,
1483		};
1484		}
1485		};
1486
1487		const Encoding = enum {
1488		/// OP
1489		zo,
1490
1491		/// OP rel32
1492		d,
1493
1494		/// OP r/m64
1495		m,
1496
1497		/// OP r64
1498		o,
1499
1500		/// OP imm32
1501		i,
1502
1503		/// OP r/m64, 1
1504		m1,
1505
1506		/// OP r/m64, .cl
1507		mc,
1508
1509		/// OP r/m64, imm32
1510		mi,
1511
1512		/// OP r/m64, imm8
1513		mi8,
1514
1515		/// OP r/m64, r64
1516		mr,
1517
1518		/// OP r64, r/m64
1519		rm,
1520
1521		/// OP r64, imm64
1522		oi,
1523
1524		/// OP al/ax/eax/rax, moffs
1525		fd,
1526
1527		/// OP moffs, al/ax/eax/rax
1528		td,
1529
1530		/// OP r64, r/m64, imm32
1531		rmi,
1532
1533		/// OP xmm1, xmm2/m64
1534		vm,
1535
1536		/// OP m64, xmm1
1537		mv,
1538
1539		/// OP xmm1, xmm2, xmm3/m64
1540		rvm,
1541
1542		/// OP xmm1, xmm2, xmm3/m64, imm8
1543		rvmi,
1544		};
1545
1546		const OpCode = struct {
1547		bytes: [3]u8,
1548		count: usize,
1549
1550		fn init(comptime in_bytes: []const u8) OpCode {
1551		comptime assert(in_bytes.len <= 3);
1552		comptime var bytes: [3]u8 = undefined;
1553		inline for (in_bytes, 0..) \|x, i\| {
1554		bytes[i] = x;
1555		}
1556		return .{ .bytes = bytes, .count = in_bytes.len };
1557		}
1558
1559		fn encode(opc: OpCode, encoder: Encoder) void {
1560		switch (opc.count) {
1561		1 => encoder.opcode_1byte(opc.bytes[0]),
1562		2 => encoder.opcode_2byte(opc.bytes[0], opc.bytes[1]),
1563		3 => encoder.opcode_3byte(opc.bytes[0], opc.bytes[1], opc.bytes[2]),
1564		else => unreachable,
1565		}
1566		}
1567
1568		fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1569		assert(opc.count == 1);
1570		encoder.opcode_withReg(opc.bytes[0], reg.lowEnc());
1571		}
1572		};
1573
1574		inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) OpCode {
1575		// zig fmt: off
1576		switch (enc) {
1577		.zo => return switch (tag) {
1578		.ret_near => OpCode.init(&.{0xc3}),
1579		.ret_far => OpCode.init(&.{0xcb}),
1580		.ud2 => OpCode.init(&.{ 0x0F, 0x0B }),
1581		.int3 => OpCode.init(&.{0xcc}),
1582		.nop => OpCode.init(&.{0x90}),
1583		.syscall => OpCode.init(&.{ 0x0f, 0x05 }),
1584		.cbw => OpCode.init(&.{0x98}),
1585		.cwd,
1586		.cdq,
1587		.cqo => OpCode.init(&.{0x99}),
1588		else => unreachable,
1589		},
1590		.d => return switch (tag) {
1591		.jmp_near => OpCode.init(&.{0xe9}),
1592		.call_near => OpCode.init(&.{0xe8}),
1593
1594		.jo => if (is_one_byte) OpCode.init(&.{0x70}) else OpCode.init(&.{0x0f,0x80}),
1595
1596		.jno => if (is_one_byte) OpCode.init(&.{0x71}) else OpCode.init(&.{0x0f,0x81}),
1597
1598		.jb,
1599		.jc,
1600		.jnae => if (is_one_byte) OpCode.init(&.{0x72}) else OpCode.init(&.{0x0f,0x82}),
1601
1602		.jnb,
1603		.jnc,
1604		.jae => if (is_one_byte) OpCode.init(&.{0x73}) else OpCode.init(&.{0x0f,0x83}),
1605
1606		.je,
1607		.jz => if (is_one_byte) OpCode.init(&.{0x74}) else OpCode.init(&.{0x0f,0x84}),
1608
1609		.jne,
1610		.jnz => if (is_one_byte) OpCode.init(&.{0x75}) else OpCode.init(&.{0x0f,0x85}),
1611
1612		.jna,
1613		.jbe => if (is_one_byte) OpCode.init(&.{0x76}) else OpCode.init(&.{0x0f,0x86}),
1614
1615		.jnbe,
1616		.ja => if (is_one_byte) OpCode.init(&.{0x77}) else OpCode.init(&.{0x0f,0x87}),
1617
1618		.js => if (is_one_byte) OpCode.init(&.{0x78}) else OpCode.init(&.{0x0f,0x88}),
1619
1620		.jns => if (is_one_byte) OpCode.init(&.{0x79}) else OpCode.init(&.{0x0f,0x89}),
1621
1622		.jpe,
1623		.jp => if (is_one_byte) OpCode.init(&.{0x7a}) else OpCode.init(&.{0x0f,0x8a}),
1624
1625		.jpo,
1626		.jnp => if (is_one_byte) OpCode.init(&.{0x7b}) else OpCode.init(&.{0x0f,0x8b}),
1627
1628		.jnge,
1629		.jl => if (is_one_byte) OpCode.init(&.{0x7c}) else OpCode.init(&.{0x0f,0x8c}),
1630
1631		.jge,
1632		.jnl => if (is_one_byte) OpCode.init(&.{0x7d}) else OpCode.init(&.{0x0f,0x8d}),
1633
1634		.jle,
1635		.jng => if (is_one_byte) OpCode.init(&.{0x7e}) else OpCode.init(&.{0x0f,0x8e}),
1636
1637		.jg,
1638		.jnle => if (is_one_byte) OpCode.init(&.{0x7f}) else OpCode.init(&.{0x0f,0x8f}),
1639
1640		else => unreachable,
1641		},
1642		.m => return switch (tag) {
1643		.jmp_near,
1644		.call_near,
1645		.push => OpCode.init(&.{0xff}),
1646
1647		.pop => OpCode.init(&.{0x8f}),
1648		.seto => OpCode.init(&.{0x0f,0x90}),
1649		.setno => OpCode.init(&.{0x0f,0x91}),
1650
1651		.setb,
1652		.setc,
1653		.setnae => OpCode.init(&.{0x0f,0x92}),
1654
1655		.setnb,
1656		.setnc,
1657		.setae => OpCode.init(&.{0x0f,0x93}),
1658
1659		.sete,
1660		.setz => OpCode.init(&.{0x0f,0x94}),
1661
1662		.setne,
1663		.setnz => OpCode.init(&.{0x0f,0x95}),
1664
1665		.setbe,
1666		.setna => OpCode.init(&.{0x0f,0x96}),
1667
1668		.seta,
1669		.setnbe => OpCode.init(&.{0x0f,0x97}),
1670
1671		.sets => OpCode.init(&.{0x0f,0x98}),
1672		.setns => OpCode.init(&.{0x0f,0x99}),
1673
1674		.setp,
1675		.setpe => OpCode.init(&.{0x0f,0x9a}),
1676
1677		.setnp,
1678		.setpo => OpCode.init(&.{0x0f,0x9b}),
1679
1680		.setl,
1681		.setnge => OpCode.init(&.{0x0f,0x9c}),
1682
1683		.setnl,
1684		.setge => OpCode.init(&.{0x0f,0x9d}),
1685
1686		.setle,
1687		.setng => OpCode.init(&.{0x0f,0x9e}),
1688
1689		.setnle,
1690		.setg => OpCode.init(&.{0x0f,0x9f}),
1691
1692		.idiv,
1693		.div,
1694		.imul,
1695		.mul => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1696
1697		.fisttp16 => OpCode.init(&.{0xdf}),
1698		.fisttp32 => OpCode.init(&.{0xdb}),
1699		.fisttp64 => OpCode.init(&.{0xdd}),
1700		.fld32 => OpCode.init(&.{0xd9}),
1701		.fld64 => OpCode.init(&.{0xdd}),
1702		else => unreachable,
1703		},
1704		.o => return switch (tag) {
1705		.push => OpCode.init(&.{0x50}),
1706		.pop => OpCode.init(&.{0x58}),
1707		else => unreachable,
1708		},
1709		.i => return switch (tag) {
1710		.push => if (is_one_byte) OpCode.init(&.{0x6a}) else OpCode.init(&.{0x68}),
1711		.@"test" => if (is_one_byte) OpCode.init(&.{0xa8}) else OpCode.init(&.{0xa9}),
1712		.ret_near => OpCode.init(&.{0xc2}),
1713		.ret_far => OpCode.init(&.{0xca}),
1714		else => unreachable,
1715		},
1716		.m1 => return switch (tag) {
1717		.shl, .sal,
1718		.shr, .sar => if (is_one_byte) OpCode.init(&.{0xd0}) else OpCode.init(&.{0xd1}),
1719		else => unreachable,
1720		},
1721		.mc => return switch (tag) {
1722		.shl, .sal,
1723		.shr, .sar => if (is_one_byte) OpCode.init(&.{0xd2}) else OpCode.init(&.{0xd3}),
1724		else => unreachable,
1725		},
1726		.mi => return switch (tag) {
1727		.adc, .add,
1728		.sub, .xor,
1729		.@"and", .@"or",
1730		.sbb, .cmp => if (is_one_byte) OpCode.init(&.{0x80}) else OpCode.init(&.{0x81}),
1731		.mov => if (is_one_byte) OpCode.init(&.{0xc6}) else OpCode.init(&.{0xc7}),
1732		.@"test" => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1733		else => unreachable,
1734		},
1735		.mi8 => return switch (tag) {
1736		.adc, .add,
1737		.sub, .xor,
1738		.@"and", .@"or",
1739		.sbb, .cmp => OpCode.init(&.{0x83}),
1740		.shl, .sal,
1741		.shr, .sar => if (is_one_byte) OpCode.init(&.{0xc0}) else OpCode.init(&.{0xc1}),
1742		else => unreachable,
1743		},
1744		.mr => return switch (tag) {
1745		.adc => if (is_one_byte) OpCode.init(&.{0x10}) else OpCode.init(&.{0x11}),
1746		.add => if (is_one_byte) OpCode.init(&.{0x00}) else OpCode.init(&.{0x01}),
1747		.sub => if (is_one_byte) OpCode.init(&.{0x28}) else OpCode.init(&.{0x29}),
1748		.xor => if (is_one_byte) OpCode.init(&.{0x30}) else OpCode.init(&.{0x31}),
1749		.@"and" => if (is_one_byte) OpCode.init(&.{0x20}) else OpCode.init(&.{0x21}),
1750		.@"or" => if (is_one_byte) OpCode.init(&.{0x08}) else OpCode.init(&.{0x09}),
1751		.sbb => if (is_one_byte) OpCode.init(&.{0x18}) else OpCode.init(&.{0x19}),
1752		.cmp => if (is_one_byte) OpCode.init(&.{0x38}) else OpCode.init(&.{0x39}),
1753		.mov => if (is_one_byte) OpCode.init(&.{0x88}) else OpCode.init(&.{0x89}),
1754		.@"test" => if (is_one_byte) OpCode.init(&.{0x84}) else OpCode.init(&.{0x85}),
1755		.movsd => OpCode.init(&.{0xf2,0x0f,0x11}),
1756		.movss => OpCode.init(&.{0xf3,0x0f,0x11}),
1757		else => unreachable,
1758		},
1759		.rm => return switch (tag) {
1760		.adc => if (is_one_byte) OpCode.init(&.{0x12}) else OpCode.init(&.{0x13}),
1761		.add => if (is_one_byte) OpCode.init(&.{0x02}) else OpCode.init(&.{0x03}),
1762		.sub => if (is_one_byte) OpCode.init(&.{0x2a}) else OpCode.init(&.{0x2b}),
1763		.xor => if (is_one_byte) OpCode.init(&.{0x32}) else OpCode.init(&.{0x33}),
1764		.@"and" => if (is_one_byte) OpCode.init(&.{0x22}) else OpCode.init(&.{0x23}),
1765		.@"or" => if (is_one_byte) OpCode.init(&.{0x0a}) else OpCode.init(&.{0x0b}),
1766		.sbb => if (is_one_byte) OpCode.init(&.{0x1a}) else OpCode.init(&.{0x1b}),
1767		.cmp => if (is_one_byte) OpCode.init(&.{0x3a}) else OpCode.init(&.{0x3b}),
1768		.mov => if (is_one_byte) OpCode.init(&.{0x8a}) else OpCode.init(&.{0x8b}),
1769		.movsx => if (is_one_byte) OpCode.init(&.{0x0f,0xbe}) else OpCode.init(&.{0x0f,0xbf}),
1770		.movsxd => OpCode.init(&.{0x63}),
1771		.movzx => if (is_one_byte) OpCode.init(&.{0x0f,0xb6}) else OpCode.init(&.{0x0f,0xb7}),
1772		.lea => if (is_one_byte) OpCode.init(&.{0x8c}) else OpCode.init(&.{0x8d}),
1773		.imul => OpCode.init(&.{0x0f,0xaf}),
1774
1775		.cmova,
1776		.cmovnbe, => OpCode.init(&.{0x0f,0x47}),
1777
1778		.cmovae,
1779		.cmovnb, => OpCode.init(&.{0x0f,0x43}),
1780
1781		.cmovb,
1782		.cmovc,
1783		.cmovnae => OpCode.init(&.{0x0f,0x42}),
1784
1785		.cmovbe,
1786		.cmovna, => OpCode.init(&.{0x0f,0x46}),
1787
1788		.cmove,
1789		.cmovz, => OpCode.init(&.{0x0f,0x44}),
1790
1791		.cmovg,
1792		.cmovnle, => OpCode.init(&.{0x0f,0x4f}),
1793
1794		.cmovge,
1795		.cmovnl, => OpCode.init(&.{0x0f,0x4d}),
1796
1797		.cmovl,
1798		.cmovnge, => OpCode.init(&.{0x0f,0x4c}),
1799
1800		.cmovle,
1801		.cmovng, => OpCode.init(&.{0x0f,0x4e}),
1802
1803		.cmovne,
1804		.cmovnz, => OpCode.init(&.{0x0f,0x45}),
1805
1806		.cmovno => OpCode.init(&.{0x0f,0x41}),
1807
1808		.cmovnp,
1809		.cmovpo, => OpCode.init(&.{0x0f,0x4b}),
1810
1811		.cmovns => OpCode.init(&.{0x0f,0x49}),
1812
1813		.cmovo => OpCode.init(&.{0x0f,0x40}),
1814
1815		.cmovp,
1816		.cmovpe, => OpCode.init(&.{0x0f,0x4a}),
1817
1818		.cmovs => OpCode.init(&.{0x0f,0x48}),
1819
1820		.movsd => OpCode.init(&.{0xf2,0x0f,0x10}),
1821		.movss => OpCode.init(&.{0xf3,0x0f,0x10}),
1822		.addsd => OpCode.init(&.{0xf2,0x0f,0x58}),
1823		.addss => OpCode.init(&.{0xf3,0x0f,0x58}),
1824		.ucomisd => OpCode.init(&.{0x66,0x0f,0x2e}),
1825		.ucomiss => OpCode.init(&.{0x0f,0x2e}),
1826		else => unreachable,
1827		},
1828		.oi => return switch (tag) {
1829		.mov => if (is_one_byte) OpCode.init(&.{0xb0}) else OpCode.init(&.{0xb8}),
1830		else => unreachable,
1831		},
1832		.fd => return switch (tag) {
1833		.mov => if (is_one_byte) OpCode.init(&.{0xa0}) else OpCode.init(&.{0xa1}),
1834		else => unreachable,
1835		},
1836		.td => return switch (tag) {
1837		.mov => if (is_one_byte) OpCode.init(&.{0xa2}) else OpCode.init(&.{0xa3}),
1838		else => unreachable,
1839		},
1840		.rmi => return switch (tag) {
1841		.imul => if (is_one_byte) OpCode.init(&.{0x6b}) else OpCode.init(&.{0x69}),
1842		else => unreachable,
1843		},
1844		.mv => return switch (tag) {
1845		.vmovsd,
1846		.vmovss => OpCode.init(&.{0x11}),
1847		else => unreachable,
1848		},
1849		.vm => return switch (tag) {
1850		.vmovsd,
1851		.vmovss => OpCode.init(&.{0x10}),
1852		.vucomisd,
1853		.vucomiss => OpCode.init(&.{0x2e}),
1854		else => unreachable,
1855		},
1856		.rvm => return switch (tag) {
1857		.vaddsd,
1858		.vaddss => OpCode.init(&.{0x58}),
1859		.vmovsd,
1860		.vmovss => OpCode.init(&.{0x10}),
1861		else => unreachable,
1862		},
1863		.rvmi => return switch (tag) {
1864		.vcmpsd,
1865		.vcmpss => OpCode.init(&.{0xc2}),
1866		else => unreachable,
1867		},
1868		}
1869		// zig fmt: on
1870		}
1871
1872		inline fn getModRmExt(tag: Tag) u3 {
1873		return switch (tag) {
1874		.adc => 0x2,
1875		.add => 0x0,
1876		.sub => 0x5,
1877		.xor => 0x6,
1878		.@"and" => 0x4,
1879		.@"or" => 0x1,
1880		.sbb => 0x3,
1881		.cmp => 0x7,
1882		.mov => 0x0,
1883		.jmp_near => 0x4,
1884		.call_near => 0x2,
1885		.push => 0x6,
1886		.pop => 0x0,
1887		.@"test" => 0x0,
1888		.seto,
1889		.setno,
1890		.setb,
1891		.setc,
1892		.setnae,
1893		.setnb,
1894		.setnc,
1895		.setae,
1896		.sete,
1897		.setz,
1898		.setne,
1899		.setnz,
1900		.setbe,
1901		.setna,
1902		.seta,
1903		.setnbe,
1904		.sets,
1905		.setns,
1906		.setp,
1907		.setpe,
1908		.setnp,
1909		.setpo,
1910		.setl,
1911		.setnge,
1912		.setnl,
1913		.setge,
1914		.setle,
1915		.setng,
1916		.setnle,
1917		.setg,
1918		=> 0x0,
1919		.shl,
1920		.sal,
1921		=> 0x4,
1922		.shr => 0x5,
1923		.sar => 0x7,
1924		.mul => 0x4,
1925		.imul => 0x5,
1926		.div => 0x6,
1927		.idiv => 0x7,
1928		.fisttp16 => 0x1,
1929		.fisttp32 => 0x1,
1930		.fisttp64 => 0x1,
1931		.fld32 => 0x0,
1932		.fld64 => 0x0,
1933		else => unreachable,
1934		};
1935		}
1936
1937		const VexEncoding = struct {
1938		prefix: Encoder.Vex,
1939		reg: ?enum {
1940		ndd,
1941		nds,
1942		dds,
1943		},
1944		};
1945
1946		inline fn getVexEncoding(tag: Tag, enc: Encoding) VexEncoding {
1947		const desc: struct {
1948		reg: enum {
1949		none,
1950		ndd,
1951		nds,
1952		dds,
1953		} = .none,
1954		len_256: bool = false,
1955		wig: bool = false,
1956		lig: bool = false,
1957		lz: bool = false,
1958		lead_opc: enum {
1959		l_0f,
1960		l_0f_3a,
1961		l_0f_38,
1962		} = .l_0f,
1963		simd_prefix: enum {
1964		none,
1965		p_66,
1966		p_f2,
1967		p_f3,
1968		} = .none,
1969		} = blk: {
1970		switch (enc) {
1971		.mv => switch (tag) {
1972		.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1973		.vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1974		else => unreachable,
1975		},
1976		.vm => switch (tag) {
1977		.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1978		.vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1979		.vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true },
1980		.vucomiss => break :blk .{ .lig = true, .wig = true },
1981		else => unreachable,
1982		},
1983		.rvm => switch (tag) {
1984		.vaddsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1985		.vaddss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1986		.vmovsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1987		.vmovss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1988		else => unreachable,
1989		},
1990		.rvmi => switch (tag) {
1991		.vcmpsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1992		.vcmpss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1993		else => unreachable,
1994		},
1995		else => unreachable,
1996		}
1997		};
1998
1999		var vex: Encoder.Vex = .{};
2000
2001		if (desc.len_256) vex.len_256();
2002		if (desc.wig) vex.wig();
2003		if (desc.lig) vex.lig();
2004		if (desc.lz) vex.lz();
2005
2006		switch (desc.lead_opc) {
2007		.l_0f => {},
2008		.l_0f_3a => vex.lead_opc_0f_3a(),
2009		.l_0f_38 => vex.lead_opc_0f_38(),
2010		}
2011
2012		switch (desc.simd_prefix) {
2013		.none => {},
2014		.p_66 => vex.simd_prefix_66(),
2015		.p_f2 => vex.simd_prefix_f2(),
2016		.p_f3 => vex.simd_prefix_f3(),
2017		}
2018
2019		return VexEncoding{ .prefix = vex, .reg = switch (desc.reg) {
2020		.none => null,
2021		.nds => .nds,
2022		.dds => .dds,
2023		.ndd => .ndd,
2024		} };
2025		}
2026
2027		const ScaleIndex = packed struct {
2028		scale: u2,
2029		index: Register,
2030		};
2031
2032		const Memory = struct {
2033		base: ?Register,
2034		rip: bool = false,
2035		disp: u32,
2036		ptr_size: PtrSize,
2037		scale_index: ?ScaleIndex = null,
2038
2039		const PtrSize = enum(u2) {
2040		byte_ptr = 0b00,
2041		word_ptr = 0b01,
2042		dword_ptr = 0b10,
2043		qword_ptr = 0b11,
2044
2045		fn new(bit_size: u64) PtrSize {
2046		return @intToEnum(PtrSize, math.log2_int(u4, @intCast(u4, @divExact(bit_size, 8))));
2047		}
2048
2049		/// Returns size in bits.
2050		fn size(ptr_size: PtrSize) u64 {
2051		return 8 * (math.powi(u8, 2, @enumToInt(ptr_size)) catch unreachable);
2052		}
2053		};
2054
2055		fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
2056		if (mem_op.base) \|base\| {
2057		const dst = base.lowEnc();
2058		const src = operand;
2059		if (dst == 4 or mem_op.scale_index != null) {
2060		if (mem_op.disp == 0 and dst != 5) {
2061		encoder.modRm_SIBDisp0(src);
2062		if (mem_op.scale_index) \|si\| {
2063		encoder.sib_scaleIndexBase(si.scale, si.index.lowEnc(), dst);
2064		} else {
2065		encoder.sib_base(dst);
2066		}
2067		} else if (immOpSize(mem_op.disp) == 8) {
2068		encoder.modRm_SIBDisp8(src);
2069		if (mem_op.scale_index) \|si\| {
2070		encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowEnc(), dst);
2071		} else {
2072		encoder.sib_baseDisp8(dst);
2073		}
2074		encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
2075		} else {
2076		encoder.modRm_SIBDisp32(src);
2077		if (mem_op.scale_index) \|si\| {
2078		encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowEnc(), dst);
2079		} else {
2080		encoder.sib_baseDisp32(dst);
2081		}
2082		encoder.disp32(@bitCast(i32, mem_op.disp));
2083		}
2084		} else {
2085		if (mem_op.disp == 0 and dst != 5) {
2086		encoder.modRm_indirectDisp0(src, dst);
2087		} else if (immOpSize(mem_op.disp) == 8) {
2088		encoder.modRm_indirectDisp8(src, dst);
2089		encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
2090		} else {
2091		encoder.modRm_indirectDisp32(src, dst);
2092		encoder.disp32(@bitCast(i32, mem_op.disp));
2093		}
2094		}
2095		} else {
2096		if (mem_op.rip) {
2097		encoder.modRm_RIPDisp32(operand);
2098		} else {
2099		encoder.modRm_SIBDisp0(operand);
2100		if (mem_op.scale_index) \|si\| {
2101		encoder.sib_scaleIndexDisp32(si.scale, si.index.lowEnc());
2102		} else {
2103		encoder.sib_disp32();
2104		}
2105		}
2106		encoder.disp32(@bitCast(i32, mem_op.disp));
2107		}
2108		}
2109
2110		/// Returns size in bits.
2111		fn size(memory: Memory) u64 {
2112		return memory.ptr_size.size();
2113		}
2114		};
2115
2116		fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
2117		switch (size) {
2118		8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
2119		16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
2120		32, 64 => encoder.imm32(@bitCast(i32, imm)),
2121		else => unreachable,
2122		}
2123		}
2124
2125		const RegisterOrMemory = union(enum) {
2126		register: Register,
2127		memory: Memory,
2128
2129		fn reg(register: Register) RegisterOrMemory {
2130		return .{ .register = register };
2131		}
2132
2133		fn mem(ptr_size: Memory.PtrSize, args: struct {
2134		disp: u32,
2135		base: ?Register = null,
2136		scale_index: ?ScaleIndex = null,
2137		}) RegisterOrMemory {
2138		return .{
2139		.memory = .{
2140		.base = args.base,
2141		.disp = args.disp,
2142		.ptr_size = ptr_size,
2143		.scale_index = args.scale_index,
2144		},
2145		};
2146		}
2147
2148		fn rip(ptr_size: Memory.PtrSize, disp: u32) RegisterOrMemory {
2149		return .{
2150		.memory = .{
2151		.base = null,
2152		.rip = true,
2153		.disp = disp,
2154		.ptr_size = ptr_size,
2155		},
2156		};
2157		}
2158
2159		/// Returns size in bits.
2160		fn size(reg_or_mem: RegisterOrMemory) u64 {
2161		return switch (reg_or_mem) {
2162		.register => \|register\| register.size(),
2163		.memory => \|memory\| memory.size(),
2164		};
2165		}
2166		};
2167
2168		fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
2169		assert(!tag.isAvx());
2170		const opc = getOpCode(tag, .zo, false);
2171		const encoder = try Encoder.init(code, 2);
2172		switch (tag) {
2173		.cqo => {
2174		encoder.rex(.{
2175		.w = true,
2176		});
2177		},
2178		else => {},
2179		}
2180		opc.encode(encoder);
2181		}
2182
2183		fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2184		assert(!tag.isAvx());
2185		if (tag == .ret_far or tag == .ret_near) {
2186		const encoder = try Encoder.init(code, 3);
2187		const opc = getOpCode(tag, .i, false);
2188		opc.encode(encoder);
2189		encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
2190		return;
2191		}
2192		const opc = getOpCode(tag, .i, immOpSize(imm) == 8);
2193		const encoder = try Encoder.init(code, 5);
2194		if (immOpSize(imm) == 16) {
2195		encoder.prefix16BitMode();
2196		}
2197		opc.encode(encoder);
2198		encodeImm(encoder, imm, immOpSize(imm));
2199		}
2200
2201		fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
2202		assert(!tag.isAvx());
2203		const opc = getOpCode(tag, .o, false);
2204		const encoder = try Encoder.init(code, 3);
2205		if (reg.size() == 16) {
2206		encoder.prefix16BitMode();
2207		}
2208		encoder.rex(.{
2209		.w = false,
2210		.b = reg.isExtended(),
2211		});
2212		opc.encodeWithReg(encoder, reg);
2213		}
2214
2215		fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2216		assert(!tag.isAvx());
2217		const opc = getOpCode(tag, .d, false);
2218		const encoder = try Encoder.init(code, 6);
2219		opc.encode(encoder);
2220		encoder.imm32(@bitCast(i32, imm));
2221		}
2222
2223		fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void {
2224		assert(!tag.isAvx());
2225		const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
2226		const modrm_ext = getModRmExt(tag);
2227		switch (reg_or_mem) {
2228		.register => \|reg\| {
2229		const encoder = try Encoder.init(code, 4);
2230		if (reg.size() == 16) {
2231		encoder.prefix16BitMode();
2232		}
2233		const wide = if (tag == .jmp_near) false else setRexWRegister(reg);
2234		encoder.rex(.{
2235		.w = wide,
2236		.b = reg.isExtended(),
2237		});
2238		opc.encode(encoder);
2239		encoder.modRm_direct(modrm_ext, reg.lowEnc());
2240		},
2241		.memory => \|mem_op\| {
2242		const encoder = try Encoder.init(code, 8);
2243		if (mem_op.ptr_size == .word_ptr) {
2244		encoder.prefix16BitMode();
2245		}
2246		if (mem_op.base) \|base\| {
2247		const wide = if (tag == .jmp_near) false else mem_op.ptr_size == .qword_ptr;
2248		encoder.rex(.{
2249		.w = wide,
2250		.b = base.isExtended(),
2251		.x = if (mem_op.scale_index) \|si\| si.index.isExtended() else false,
2252		});
2253		}
2254		opc.encode(encoder);
2255		mem_op.encode(encoder, modrm_ext);
2256		},
2257		}
2258		}
2259
2260		fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2261		return lowerToMxEnc(tag, reg_or_mem, .m, code);
2262		}
2263
2264		fn lowerToM1Enc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2265		return lowerToMxEnc(tag, reg_or_mem, .m1, code);
2266		}
2267
2268		fn lowerToMcEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2269		return lowerToMxEnc(tag, reg_or_mem, .mc, code);
2270		}
2271
2272		fn lowerToTdEnc(tag: Tag, moffs: u64, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
2273		return lowerToTdFdEnc(tag, reg, moffs, code, true);
2274		}
2275
2276		fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) InnerError!void {
2277		return lowerToTdFdEnc(tag, reg, moffs, code, false);
2278		}
2279
2280		fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void {
2281		assert(!tag.isAvx());
2282		const opc = if (td) getOpCode(tag, .td, reg.size() == 8) else getOpCode(tag, .fd, reg.size() == 8);
2283		const encoder = try Encoder.init(code, 10);
2284		if (reg.size() == 16) {
2285		encoder.prefix16BitMode();
2286		}
2287		encoder.rex(.{
2288		.w = setRexWRegister(reg),
2289		});
2290		opc.encode(encoder);
2291		switch (reg.size()) {
2292		8 => encoder.imm8(@bitCast(i8, @truncate(u8, moffs))),
2293		16 => encoder.imm16(@bitCast(i16, @truncate(u16, moffs))),
2294		32 => encoder.imm32(@bitCast(i32, @truncate(u32, moffs))),
2295		64 => encoder.imm64(moffs),
2296		else => unreachable,
2297		}
2298		}
2299
2300		fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void {
2301		assert(!tag.isAvx());
2302		const opc = getOpCode(tag, .oi, reg.size() == 8);
2303		const encoder = try Encoder.init(code, 10);
2304		if (reg.size() == 16) {
2305		encoder.prefix16BitMode();
2306		}
2307		encoder.rex(.{
2308		.w = setRexWRegister(reg),
2309		.b = reg.isExtended(),
2310		});
2311		opc.encodeWithReg(encoder, reg);
2312		switch (reg.size()) {
2313		8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
2314		16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
2315		32 => encoder.imm32(@bitCast(i32, @truncate(u32, imm))),
2316		64 => encoder.imm64(imm),
2317		else => unreachable,
2318		}
2319		}
2320
2321		fn lowerToMiXEnc(
2322		tag: Tag,
2323		reg_or_mem: RegisterOrMemory,
2324		imm: u32,
2325		enc: Encoding,
2326		code: *std.ArrayList(u8),
2327		) InnerError!void {
2328		assert(!tag.isAvx());
2329		const modrm_ext = getModRmExt(tag);
2330		const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
2331		switch (reg_or_mem) {
2332		.register => \|dst_reg\| {
2333		const encoder = try Encoder.init(code, 7);
2334		if (dst_reg.size() == 16) {
2335		// 0x66 prefix switches to the non-default size; here we assume a switch from
2336		// the default 32bits to 16bits operand-size.
2337		// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
2338		encoder.prefix16BitMode();
2339		}
2340		encoder.rex(.{
2341		.w = setRexWRegister(dst_reg),
2342		.b = dst_reg.isExtended(),
2343		});
2344		opc.encode(encoder);
2345		encoder.modRm_direct(modrm_ext, dst_reg.lowEnc());
2346		encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_reg.size());
2347		},
2348		.memory => \|dst_mem\| {
2349		const encoder = try Encoder.init(code, 12);
2350		if (dst_mem.ptr_size == .word_ptr) {
2351		encoder.prefix16BitMode();
2352		}
2353		if (dst_mem.base) \|base\| {
2354		encoder.rex(.{
2355		.w = dst_mem.ptr_size == .qword_ptr,
2356		.b = base.isExtended(),
2357		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2358		});
2359		} else {
2360		encoder.rex(.{
2361		.w = dst_mem.ptr_size == .qword_ptr,
2362		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2363		});
2364		}
2365		opc.encode(encoder);
2366		dst_mem.encode(encoder, modrm_ext);
2367		encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_mem.ptr_size.size());
2368		},
2369		}
2370		}
2371
2372		fn lowerToMiImm8Enc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u8, code: *std.ArrayList(u8)) InnerError!void {
2373		return lowerToMiXEnc(tag, reg_or_mem, imm, .mi8, code);
2374		}
2375
2376		fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2377		return lowerToMiXEnc(tag, reg_or_mem, imm, .mi, code);
2378		}
2379
2380		fn lowerToRmEnc(
2381		tag: Tag,
2382		reg: Register,
2383		reg_or_mem: RegisterOrMemory,
2384		code: *std.ArrayList(u8),
2385		) InnerError!void {
2386		assert(!tag.isAvx());
2387		const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8);
2388		switch (reg_or_mem) {
2389		.register => \|src_reg\| {
2390		const encoder = try Encoder.init(code, 5);
2391		if (reg.size() == 16) {
2392		encoder.prefix16BitMode();
2393		}
2394		encoder.rex(.{
2395		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
2396		.r = reg.isExtended(),
2397		.b = src_reg.isExtended(),
2398		});
2399		opc.encode(encoder);
2400		encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2401		},
2402		.memory => \|src_mem\| {
2403		const encoder = try Encoder.init(code, 9);
2404		if (reg.size() == 16) {
2405		encoder.prefix16BitMode();
2406		}
2407		if (src_mem.base) \|base\| {
2408		// TODO handle 32-bit base register - requires prefix 0x67
2409		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
2410		encoder.rex(.{
2411		.w = setRexWRegister(reg),
2412		.r = reg.isExtended(),
2413		.b = base.isExtended(),
2414		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2415		});
2416		} else {
2417		encoder.rex(.{
2418		.w = setRexWRegister(reg),
2419		.r = reg.isExtended(),
2420		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2421		});
2422		}
2423		opc.encode(encoder);
2424		src_mem.encode(encoder, reg.lowEnc());
2425		},
2426		}
2427		}
2428
2429		fn lowerToMrEnc(
2430		tag: Tag,
2431		reg_or_mem: RegisterOrMemory,
2432		reg: Register,
2433		code: *std.ArrayList(u8),
2434		) InnerError!void {
2435		assert(!tag.isAvx());
2436		const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8);
2437		switch (reg_or_mem) {
2438		.register => \|dst_reg\| {
2439		const encoder = try Encoder.init(code, 4);
2440		if (dst_reg.size() == 16) {
2441		encoder.prefix16BitMode();
2442		}
2443		encoder.rex(.{
2444		.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
2445		.r = reg.isExtended(),
2446		.b = dst_reg.isExtended(),
2447		});
2448		opc.encode(encoder);
2449		encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2450		},
2451		.memory => \|dst_mem\| {
2452		const encoder = try Encoder.init(code, 9);
2453		if (reg.size() == 16) {
2454		encoder.prefix16BitMode();
2455		}
2456		if (dst_mem.base) \|base\| {
2457		encoder.rex(.{
2458		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2459		.r = reg.isExtended(),
2460		.b = base.isExtended(),
2461		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2462		});
2463		} else {
2464		encoder.rex(.{
2465		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2466		.r = reg.isExtended(),
2467		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2468		});
2469		}
2470		opc.encode(encoder);
2471		dst_mem.encode(encoder, reg.lowEnc());
2472		},
2473		}
2474		}
2475
2476		fn lowerToRmiEnc(
2477		tag: Tag,
2478		reg: Register,
2479		reg_or_mem: RegisterOrMemory,
2480		imm: u32,
2481		code: *std.ArrayList(u8),
2482		) InnerError!void {
2483		assert(!tag.isAvx());
2484		const opc = getOpCode(tag, .rmi, false);
2485		const encoder = try Encoder.init(code, 13);
2486		if (reg.size() == 16) {
2487		encoder.prefix16BitMode();
2488		}
2489		switch (reg_or_mem) {
2490		.register => \|src_reg\| {
2491		encoder.rex(.{
2492		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
2493		.r = reg.isExtended(),
2494		.b = src_reg.isExtended(),
2495		});
2496		opc.encode(encoder);
2497		encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2498		},
2499		.memory => \|src_mem\| {
2500		if (src_mem.base) \|base\| {
2501		// TODO handle 32-bit base register - requires prefix 0x67
2502		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
2503		encoder.rex(.{
2504		.w = setRexWRegister(reg),
2505		.r = reg.isExtended(),
2506		.b = base.isExtended(),
2507		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2508		});
2509		} else {
2510		encoder.rex(.{
2511		.w = setRexWRegister(reg),
2512		.r = reg.isExtended(),
2513		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2514		});
2515		}
2516		opc.encode(encoder);
2517		src_mem.encode(encoder, reg.lowEnc());
2518		},
2519		}
2520		encodeImm(encoder, imm, reg.size());
2521		}
2522
2523		/// Also referred to as XM encoding in Intel manual.
2524		fn lowerToVmEnc(
2525		tag: Tag,
2526		reg: Register,
2527		reg_or_mem: RegisterOrMemory,
2528		code: *std.ArrayList(u8),
2529		) InnerError!void {
2530		const opc = getOpCode(tag, .vm, false);
2531		var enc = getVexEncoding(tag, .vm);
2532		const vex = &enc.prefix;
2533		switch (reg_or_mem) {
2534		.register => \|src_reg\| {
2535		const encoder = try Encoder.init(code, 5);
2536		vex.rex(.{
2537		.r = reg.isExtended(),
2538		.b = src_reg.isExtended(),
2539		});
2540		encoder.vex(enc.prefix);
2541		opc.encode(encoder);
2542		encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2543		},
2544		.memory => \|src_mem\| {
2545		const encoder = try Encoder.init(code, 10);
2546		if (src_mem.base) \|base\| {
2547		vex.rex(.{
2548		.r = reg.isExtended(),
2549		.b = base.isExtended(),
2550		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2551		});
2552		} else {
2553		vex.rex(.{
2554		.r = reg.isExtended(),
2555		.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2556		});
2557		}
2558		encoder.vex(enc.prefix);
2559		opc.encode(encoder);
2560		src_mem.encode(encoder, reg.lowEnc());
2561		},
2562		}
2563		}
2564
2565		/// Usually referred to as MR encoding with V/V in Intel manual.
2566		fn lowerToMvEnc(
2567		tag: Tag,
2568		reg_or_mem: RegisterOrMemory,
2569		reg: Register,
2570		code: *std.ArrayList(u8),
2571		) InnerError!void {
2572		const opc = getOpCode(tag, .mv, false);
2573		var enc = getVexEncoding(tag, .mv);
2574		const vex = &enc.prefix;
2575		switch (reg_or_mem) {
2576		.register => \|dst_reg\| {
2577		const encoder = try Encoder.init(code, 4);
2578		vex.rex(.{
2579		.r = reg.isExtended(),
2580		.b = dst_reg.isExtended(),
2581		});
2582		encoder.vex(enc.prefix);
2583		opc.encode(encoder);
2584		encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2585		},
2586		.memory => \|dst_mem\| {
2587		const encoder = try Encoder.init(code, 10);
2588		if (dst_mem.base) \|base\| {
2589		vex.rex(.{
2590		.r = reg.isExtended(),
2591		.b = base.isExtended(),
2592		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2593		});
2594		} else {
2595		vex.rex(.{
2596		.r = reg.isExtended(),
2597		.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2598		});
2599		}
2600		encoder.vex(enc.prefix);
2601		opc.encode(encoder);
2602		dst_mem.encode(encoder, reg.lowEnc());
2603		},
2604		}
2605		}
2606
2607		fn lowerToRvmEnc(
2608		tag: Tag,
2609		reg1: Register,
2610		reg2: Register,
2611		reg_or_mem: RegisterOrMemory,
2612		code: *std.ArrayList(u8),
2613		) InnerError!void {
2614		const opc = getOpCode(tag, .rvm, false);
2615		var enc = getVexEncoding(tag, .rvm);
2616		const vex = &enc.prefix;
2617		switch (reg_or_mem) {
2618		.register => \|reg3\| {
2619		if (enc.reg) \|vvvv\| {
2620		switch (vvvv) {
2621		.nds => vex.reg(reg2.enc()),
2622		else => unreachable, // TODO
2623		}
2624		}
2625		const encoder = try Encoder.init(code, 5);
2626		vex.rex(.{
2627		.r = reg1.isExtended(),
2628		.b = reg3.isExtended(),
2629		});
2630		encoder.vex(enc.prefix);
2631		opc.encode(encoder);
2632		encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2633		},
2634		.memory => \|dst_mem\| {
2635		_ = dst_mem;
2636		unreachable; // TODO
2637		},
2638		}
2639		}
2640
2641		fn lowerToRvmiEnc(
2642		tag: Tag,
2643		reg1: Register,
2644		reg2: Register,
2645		reg_or_mem: RegisterOrMemory,
2646		imm: u32,
2647		code: *std.ArrayList(u8),
2648		) InnerError!void {
2649		const opc = getOpCode(tag, .rvmi, false);
2650		var enc = getVexEncoding(tag, .rvmi);
2651		const vex = &enc.prefix;
2652		const encoder: Encoder = blk: {
2653		switch (reg_or_mem) {
2654		.register => \|reg3\| {
2655		if (enc.reg) \|vvvv\| {
2656		switch (vvvv) {
2657		.nds => vex.reg(reg2.enc()),
2658		else => unreachable, // TODO
2659		}
2660		}
2661		const encoder = try Encoder.init(code, 5);
2662		vex.rex(.{
2663		.r = reg1.isExtended(),
2664		.b = reg3.isExtended(),
2665		});
2666		encoder.vex(enc.prefix);
2667		opc.encode(encoder);
2668		encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2669		break :blk encoder;
2670		},
2671		.memory => \|dst_mem\| {
2672		_ = dst_mem;
2673		unreachable; // TODO
2674		},
2675		}
2676		};
2677		encodeImm(encoder, imm, 8); // TODO
2678		}
2679
2680		fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
2681		assert(expected.len > 0);
2682		if (mem.eql(u8, expected, given)) return;
2683		const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
2684		defer testing.allocator.free(expected_fmt);
2685		const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
2686		defer testing.allocator.free(given_fmt);
2687		const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
2688		var padding = try testing.allocator.alloc(u8, idx + 5);
2689		defer testing.allocator.free(padding);
2690		mem.set(u8, padding, ' ');
2691		std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
2692		assembly,
2693		expected_fmt,
2694		given_fmt,
2695		padding,
2696		});
2697		return error.TestFailed;
2698		}
2699
2700		const TestEmit = struct {
2701		code_buffer: std.ArrayList(u8),
2702		next: usize = 0,
2703
2704		fn init() TestEmit {
2705		return .{
2706		.code_buffer = std.ArrayList(u8).init(testing.allocator),
2707		};
2708		}
2709
2710		fn deinit(emit: *TestEmit) void {
2711		emit.code_buffer.deinit();
2712		emit.next = undefined;
2713		}
2714
2715		fn code(emit: TestEmit) std.ArrayList(u8) {
2716		emit.next = emit.code_buffer.items.len;
2717		return &emit.code_buffer;
2718		}
2719
2720		fn lowered(emit: TestEmit) []const u8 {
2721		return emit.code_buffer.items[emit.next..];
2722		}
2723		};
2724
2725		test "lower MI encoding" {
2726		var emit = TestEmit.init();
2727		defer emit.deinit();
2728		try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, emit.code());
2729		try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", emit.lowered(), "mov rax, 0x10");
2730		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, emit.code());
2731		try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", emit.lowered(), "mov dword ptr [r11 + 0], 0x10");
2732		try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{
2733		.disp = @bitCast(u32, @as(i32, -8)),
2734		.base = .rdx,
2735		}), 0x10, emit.code());
2736		try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", emit.lowered(), "add dword ptr [rdx - 8], 0x10");
2737		try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
2738		.disp = 0x10000000,
2739		.base = .r11,
2740		}), 0x10, emit.code());
2741		try expectEqualHexStrings(
2742		"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
2743		emit.lowered(),
2744		"sub dword ptr [r11 + 0x10000000], 0x10",
2745		);
2746		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, emit.code());
2747		try expectEqualHexStrings(
2748		"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
2749		emit.lowered(),
2750		"and dword ptr [ds:0x10000000], 0x10",
2751		);
2752		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
2753		.disp = 0x10000000,
2754		.base = .r12,
2755		}), 0x10, emit.code());
2756		try expectEqualHexStrings(
2757		"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
2758		emit.lowered(),
2759		"and dword ptr [r12 + 0x10000000], 0x10",
2760		);
2761		try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, emit.code());
2762		try expectEqualHexStrings(
2763		"\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
2764		emit.lowered(),
2765		"mov qword ptr [rip + 0x10], 0x10",
2766		);
2767		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2768		.disp = @bitCast(u32, @as(i32, -8)),
2769		.base = .rbp,
2770		}), 0x10, emit.code());
2771		try expectEqualHexStrings(
2772		"\x48\xc7\x45\xf8\x10\x00\x00\x00",
2773		emit.lowered(),
2774		"mov qword ptr [rbp - 8], 0x10",
2775		);
2776		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{
2777		.disp = @bitCast(u32, @as(i32, -2)),
2778		.base = .rbp,
2779		}), 0x10, emit.code());
2780		try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", emit.lowered(), "mov word ptr [rbp - 2], 0x10");
2781		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{
2782		.disp = @bitCast(u32, @as(i32, -1)),
2783		.base = .rbp,
2784		}), 0x10, emit.code());
2785		try expectEqualHexStrings("\xC6\x45\xFF\x10", emit.lowered(), "mov byte ptr [rbp - 1], 0x10");
2786		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2787		.disp = 0x10000000,
2788		.scale_index = .{
2789		.scale = 1,
2790		.index = .rcx,
2791		},
2792		}), 0x10, emit.code());
2793		try expectEqualHexStrings(
2794		"\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
2795		emit.lowered(),
2796		"mov qword ptr [rcx*2 + 0x10000000], 0x10",
2797		);
2798
2799		try lowerToMiImm8Enc(.add, RegisterOrMemory.reg(.rax), 0x10, emit.code());
2800		try expectEqualHexStrings("\x48\x83\xC0\x10", emit.lowered(), "add rax, 0x10");
2801		}
2802
2803		test "lower RM encoding" {
2804		var emit = TestEmit.init();
2805		defer emit.deinit();
2806		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code());
2807		try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx");
2808		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
2809		try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]");
2810		try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
2811		try expectEqualHexStrings(
2812		"\x4C\x03\x1C\x25\x00\x00\x00\x10",
2813		emit.lowered(),
2814		"add r11, qword ptr [ds:0x10000000]",
2815		);
2816		try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
2817		try expectEqualHexStrings(
2818		"\x44\x02\x24\x25\x00\x00\x00\x10",
2819		emit.lowered(),
2820		"add r11b, byte ptr [ds:0x10000000]",
2821		);
2822		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2823		.disp = 0x10000000,
2824		.base = .r13,
2825		}), emit.code());
2826		try expectEqualHexStrings(
2827		"\x4D\x2B\x9D\x00\x00\x00\x10",
2828		emit.lowered(),
2829		"sub r11, qword ptr [r13 + 0x10000000]",
2830		);
2831		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2832		.disp = 0x10000000,
2833		.base = .r12,
2834		}), emit.code());
2835		try expectEqualHexStrings(
2836		"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
2837		emit.lowered(),
2838		"sub r11, qword ptr [r12 + 0x10000000]",
2839		);
2840		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2841		.disp = @bitCast(u32, @as(i32, -4)),
2842		.base = .rbp,
2843		}), emit.code());
2844		try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]");
2845		try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2846		try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]");
2847		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2848		.disp = @bitCast(u32, @as(i32, -8)),
2849		.base = .rbp,
2850		.scale_index = .{
2851		.scale = 0,
2852		.index = .rcx,
2853		},
2854		}), emit.code());
2855		try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
2856		try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2857		.disp = @bitCast(u32, @as(i32, -4)),
2858		.base = .rbp,
2859		.scale_index = .{
2860		.scale = 2,
2861		.index = .rdx,
2862		},
2863		}), emit.code());
2864		try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
2865		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2866		.disp = @bitCast(u32, @as(i32, -8)),
2867		.base = .rbp,
2868		.scale_index = .{
2869		.scale = 3,
2870		.index = .rcx,
2871		},
2872		}), emit.code());
2873		try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
2874		try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
2875		.disp = @bitCast(u32, @as(i32, -24)),
2876		.base = .rsi,
2877		.scale_index = .{
2878		.scale = 0,
2879		.index = .rcx,
2880		},
2881		}), emit.code());
2882		try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
2883		try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
2884		.disp = 0,
2885		.base = .rbp,
2886		.scale_index = .{
2887		.scale = 0,
2888		.index = .rcx,
2889		},
2890		}), emit.code());
2891		try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
2892		}
2893
2894		test "lower MR encoding" {
2895		var emit = TestEmit.init();
2896		defer emit.deinit();
2897		try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code());
2898		try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx");
2899		try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2900		.disp = @bitCast(u32, @as(i32, -4)),
2901		.base = .rbp,
2902		}), .r11, emit.code());
2903		try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11");
2904		try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code());
2905		try expectEqualHexStrings(
2906		"\x44\x00\x24\x25\x00\x00\x00\x10",
2907		emit.lowered(),
2908		"add byte ptr [ds:0x10000000], r12b",
2909		);
2910		try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code());
2911		try expectEqualHexStrings(
2912		"\x44\x01\x24\x25\x00\x00\x00\x10",
2913		emit.lowered(),
2914		"add dword ptr [ds:0x10000000], r12d",
2915		);
2916		try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
2917		.disp = 0x10000000,
2918		.base = .r11,
2919		}), .r12, emit.code());
2920		try expectEqualHexStrings(
2921		"\x4D\x29\xA3\x00\x00\x00\x10",
2922		emit.lowered(),
2923		"sub qword ptr [r11 + 0x10000000], r12",
2924		);
2925		try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code());
2926		try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12");
2927		}
2928
2929		test "lower OI encoding" {
2930		var emit = TestEmit.init();
2931		defer emit.deinit();
2932		try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code());
2933		try expectEqualHexStrings(
2934		"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
2935		emit.lowered(),
2936		"movabs rax, 0x1000000000000000",
2937		);
2938		try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code());
2939		try expectEqualHexStrings(
2940		"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
2941		emit.lowered(),
2942		"movabs r11, 0x1000000000000000",
2943		);
2944		try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code());
2945		try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000");
2946		try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code());
2947		try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000");
2948		try lowerToOiEnc(.mov, .r11b, 0x10, emit.code());
2949		try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10");
2950		}
2951
2952		test "lower FD/TD encoding" {
2953		var emit = TestEmit.init();
2954		defer emit.deinit();
2955		try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code());
2956		try expectEqualHexStrings(
2957		"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2958		emit.lowered(),
2959		"mov rax, ds:0x1000000000000000",
2960		);
2961		try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code());
2962		try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000");
2963		try lowerToFdEnc(.mov, .ax, 0x1000, emit.code());
2964		try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000");
2965		try lowerToFdEnc(.mov, .al, 0x10, emit.code());
2966		try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10");
2967		}
2968
2969		test "lower M encoding" {
2970		var emit = TestEmit.init();
2971		defer emit.deinit();
2972		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), emit.code());
2973		try expectEqualHexStrings("\x41\xFF\xE4", emit.lowered(), "jmp r12");
2974		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), emit.code());
2975		try expectEqualHexStrings("\x66\x41\xFF\xE4", emit.lowered(), "jmp r12w");
2976		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2977		try expectEqualHexStrings("\x41\xFF\x24\x24", emit.lowered(), "jmp qword ptr [r12]");
2978		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2979		try expectEqualHexStrings("\x66\x41\xFF\x24\x24", emit.lowered(), "jmp word ptr [r12]");
2980		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), emit.code());
2981		try expectEqualHexStrings("\x41\xFF\x64\x24\x10", emit.lowered(), "jmp qword ptr [r12 + 0x10]");
2982		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2983		.disp = 0x1000,
2984		.base = .r12,
2985		}), emit.code());
2986		try expectEqualHexStrings(
2987		"\x41\xFF\xA4\x24\x00\x10\x00\x00",
2988		emit.lowered(),
2989		"jmp qword ptr [r12 + 0x1000]",
2990		);
2991		try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2992		try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [rip + 0x10]");
2993		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), emit.code());
2994		try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [ds:0x10]");
2995		try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), emit.code());
2996		try expectEqualHexStrings("\x41\x0F\x97\xC3", emit.lowered(), "seta r11b");
2997		try lowerToMEnc(.idiv, RegisterOrMemory.reg(.rax), emit.code());
2998		try expectEqualHexStrings("\x48\xF7\xF8", emit.lowered(), "idiv rax");
2999		try lowerToMEnc(.imul, RegisterOrMemory.reg(.al), emit.code());
3000		try expectEqualHexStrings("\xF6\xE8", emit.lowered(), "imul al");
3001		}
3002
3003		test "lower M1 and MC encodings" {
3004		var emit = TestEmit.init();
3005		defer emit.deinit();
3006		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12), emit.code());
3007		try expectEqualHexStrings("\x49\xD1\xE4", emit.lowered(), "sal r12, 1");
3008		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12d), emit.code());
3009		try expectEqualHexStrings("\x41\xD1\xE4", emit.lowered(), "sal r12d, 1");
3010		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12w), emit.code());
3011		try expectEqualHexStrings("\x66\x41\xD1\xE4", emit.lowered(), "sal r12w, 1");
3012		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12b), emit.code());
3013		try expectEqualHexStrings("\x41\xD0\xE4", emit.lowered(), "sal r12b, 1");
3014		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.rax), emit.code());
3015		try expectEqualHexStrings("\x48\xD1\xE0", emit.lowered(), "sal rax, 1");
3016		try lowerToM1Enc(.sal, RegisterOrMemory.reg(.eax), emit.code());
3017		try expectEqualHexStrings("\xD1\xE0", emit.lowered(), "sal eax, 1");
3018		try lowerToM1Enc(.sal, RegisterOrMemory.mem(.qword_ptr, .{
3019		.disp = @bitCast(u32, @as(i32, -0x10)),
3020		.base = .rbp,
3021		}), emit.code());
3022		try expectEqualHexStrings("\x48\xD1\x65\xF0", emit.lowered(), "sal qword ptr [rbp - 0x10], 1");
3023		try lowerToM1Enc(.sal, RegisterOrMemory.mem(.dword_ptr, .{
3024		.disp = @bitCast(u32, @as(i32, -0x10)),
3025		.base = .rbp,
3026		}), emit.code());
3027		try expectEqualHexStrings("\xD1\x65\xF0", emit.lowered(), "sal dword ptr [rbp - 0x10], 1");
3028
3029		try lowerToMcEnc(.shr, RegisterOrMemory.reg(.r12), emit.code());
3030		try expectEqualHexStrings("\x49\xD3\xEC", emit.lowered(), "shr r12, cl");
3031		try lowerToMcEnc(.shr, RegisterOrMemory.reg(.rax), emit.code());
3032		try expectEqualHexStrings("\x48\xD3\xE8", emit.lowered(), "shr rax, cl");
3033
3034		try lowerToMcEnc(.sar, RegisterOrMemory.reg(.rsi), emit.code());
3035		try expectEqualHexStrings("\x48\xD3\xFE", emit.lowered(), "sar rsi, cl");
3036		}
3037
3038		test "lower O encoding" {
3039		var emit = TestEmit.init();
3040		defer emit.deinit();
3041		try lowerToOEnc(.pop, .r12, emit.code());
3042		try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12");
3043		try lowerToOEnc(.push, .r12w, emit.code());
3044		try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w");
3045		}
3046
3047		test "lower RMI encoding" {
3048		var emit = TestEmit.init();
3049		defer emit.deinit();
3050		try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
3051		.disp = @bitCast(u32, @as(i32, -8)),
3052		.base = .rbp,
3053		}), 0x10, emit.code());
3054		try expectEqualHexStrings(
3055		"\x48\x69\x45\xF8\x10\x00\x00\x00",
3056		emit.lowered(),
3057		"imul rax, qword ptr [rbp - 8], 0x10",
3058		);
3059		try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
3060		.disp = @bitCast(u32, @as(i32, -4)),
3061		.base = .rbp,
3062		}), 0x10, emit.code());
3063		try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
3064		try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
3065		.disp = @bitCast(u32, @as(i32, -2)),
3066		.base = .rbp,
3067		}), 0x10, emit.code());
3068		try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
3069		try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code());
3070		try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10");
3071		try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
3072		try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
3073		}
3074
3075		test "lower MV encoding" {
3076		var emit = TestEmit.init();
3077		defer emit.deinit();
3078		try lowerToMvEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), .xmm1, emit.code());
3079		try expectEqualHexStrings(
3080		"\xC5\xFB\x11\x0D\x10\x00\x00\x00",
3081		emit.lowered(),
3082		"vmovsd qword ptr [rip + 0x10], xmm1",
3083		);
3084		}
3085
3086		test "lower VM encoding" {
3087		var emit = TestEmit.init();
3088		defer emit.deinit();
3089		try lowerToVmEnc(.vmovsd, .xmm1, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
3090		try expectEqualHexStrings(
3091		"\xC5\xFB\x10\x0D\x10\x00\x00\x00",
3092		emit.lowered(),
3093		"vmovsd xmm1, qword ptr [rip + 0x10]",
3094		);
3095		}
3096
3097		test "lower to RVM encoding" {
3098		var emit = TestEmit.init();
3099		defer emit.deinit();
3100		try lowerToRvmEnc(.vaddsd, .xmm0, .xmm1, RegisterOrMemory.reg(.xmm2), emit.code());
3101		try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2");
3102		try lowerToRvmEnc(.vaddsd, .xmm0, .xmm0, RegisterOrMemory.reg(.xmm1), emit.code());
3103		try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1");
3104		}

src/arch/x86_64/Encoding.zig created+587

...	...	@@ -0,0 +1,587 @@
	1	const Encoding = @This();
	2
	3	const std = @import("std");
	4	const assert = std.debug.assert;
	5	const math = std.math;
	6
	7	const bits = @import("bits.zig");
	8	const encoder = @import("encoder.zig");
	9	const Instruction = encoder.Instruction;
	10	const Register = bits.Register;
	11	const Rex = encoder.Rex;
	12	const LegacyPrefixes = encoder.LegacyPrefixes;
	13
	14	const table = @import("encodings.zig").table;
	15
	16	mnemonic: Mnemonic,
	17	op_en: OpEn,
	18	op1: Op,
	19	op2: Op,
	20	op3: Op,
	21	op4: Op,
	22	opc_len: u2,
	23	opc: [3]u8,
	24	modrm_ext: u3,
	25	mode: Mode,
	26
	27	pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
	28	op1: Instruction.Operand,
	29	op2: Instruction.Operand,
	30	op3: Instruction.Operand,
	31	op4: Instruction.Operand,
	32	}) !?Encoding {
	33	const input_op1 = Op.fromOperand(args.op1);
	34	const input_op2 = Op.fromOperand(args.op2);
	35	const input_op3 = Op.fromOperand(args.op3);
	36	const input_op4 = Op.fromOperand(args.op4);
	37
	38	const ops = &[_]Instruction.Operand{ args.op1, args.op2, args.op3, args.op4 };
	39	const rex_required = for (ops) \|op\| switch (op) {
	40	.reg => \|r\| switch (r) {
	41	.spl, .bpl, .sil, .dil => break true,
	42	else => {},
	43	},
	44	else => {},
	45	} else false;
	46	const rex_invalid = for (ops) \|op\| switch (op) {
	47	.reg => \|r\| switch (r) {
	48	.ah, .bh, .ch, .dh => break true,
	49	else => {},
	50	},
	51	else => {},
	52	} else false;
	53	const rex_extended = for (ops) \|op\| switch (op) {
	54	.reg => \|r\| if (r.isExtended()) break true,
	55	.mem => \|m\| {
	56	if (m.base()) \|base\| {
	57	if (base.isExtended()) break true;
	58	}
	59	if (m.scaleIndex()) \|si\| {
	60	if (si.index.isExtended()) break true;
	61	}
	62	},
	63	else => {},
	64	} else false;
	65
	66	if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;
	67
	68	// TODO work out what is the maximum number of variants we can actually find in one swoop.
	69	var candidates: [10]Encoding = undefined;
	70	var count: usize = 0;
	71	for (table) \|entry\| {
	72	const enc = Encoding{
	73	.mnemonic = entry[0],
	74	.op_en = entry[1],
	75	.op1 = entry[2],
	76	.op2 = entry[3],
	77	.op3 = entry[4],
	78	.op4 = entry[5],
	79	.opc_len = entry[6],
	80	.opc = .{ entry[7], entry[8], entry[9] },
	81	.modrm_ext = entry[10],
	82	.mode = entry[11],
	83	};
	84	if (enc.mnemonic == mnemonic and
	85	input_op1.isSubset(enc.op1, enc.mode) and
	86	input_op2.isSubset(enc.op2, enc.mode) and
	87	input_op3.isSubset(enc.op3, enc.mode) and
	88	input_op4.isSubset(enc.op4, enc.mode))
	89	{
	90	if (rex_required) {
	91	switch (enc.mode) {
	92	.rex, .long => {
	93	candidates[count] = enc;
	94	count += 1;
	95	},
	96	else => {},
	97	}
	98	} else {
	99	if (enc.mode != .rex) {
	100	candidates[count] = enc;
	101	count += 1;
	102	}
	103	}
	104	}
	105	}
	106
	107	if (count == 0) return null;
	108	if (count == 1) return candidates[0];
	109
	110	const EncodingLength = struct {
	111	fn estimate(encoding: Encoding, params: struct {
	112	op1: Instruction.Operand,
	113	op2: Instruction.Operand,
	114	op3: Instruction.Operand,
	115	op4: Instruction.Operand,
	116	}) usize {
	117	var inst = Instruction{
	118	.op1 = params.op1,
	119	.op2 = params.op2,
	120	.op3 = params.op3,
	121	.op4 = params.op4,
	122	.encoding = encoding,
	123	};
	124	var cwriter = std.io.countingWriter(std.io.null_writer);
	125	inst.encode(cwriter.writer()) catch unreachable; // Not allowed to fail here unless OOM.
	126	return @intCast(usize, cwriter.bytes_written);
	127	}
	128	};
	129
	130	var shortest_encoding: ?struct {
	131	index: usize,
	132	len: usize,
	133	} = null;
	134	var i: usize = 0;
	135	while (i < count) : (i += 1) {
	136	const candidate = candidates[i];
	137	switch (candidate.mode) {
	138	.long, .rex => if (rex_invalid) return error.CannotEncode,
	139	else => {},
	140	}
	141
	142	const len = EncodingLength.estimate(candidate, .{
	143	.op1 = args.op1,
	144	.op2 = args.op2,
	145	.op3 = args.op3,
	146	.op4 = args.op4,
	147	});
	148	const current = shortest_encoding orelse {
	149	shortest_encoding = .{ .index = i, .len = len };
	150	continue;
	151	};
	152	if (len < current.len) {
	153	shortest_encoding = .{ .index = i, .len = len };
	154	}
	155	}
	156
	157	return candidates[shortest_encoding.?.index];
	158	}
	159
	160	/// Returns first matching encoding by opcode.
	161	pub fn findByOpcode(opc: []const u8, prefixes: struct {
	162	legacy: LegacyPrefixes,
	163	rex: Rex,
	164	}, modrm_ext: ?u3) ?Encoding {
	165	for (table) \|entry\| {
	166	const enc = Encoding{
	167	.mnemonic = entry[0],
	168	.op_en = entry[1],
	169	.op1 = entry[2],
	170	.op2 = entry[3],
	171	.op3 = entry[4],
	172	.op4 = entry[5],
	173	.opc_len = entry[6],
	174	.opc = .{ entry[7], entry[8], entry[9] },
	175	.modrm_ext = entry[10],
	176	.mode = entry[11],
	177	};
	178	const match = match: {
	179	if (modrm_ext) \|ext\| {
	180	break :match ext == enc.modrm_ext and std.mem.eql(u8, enc.opcode(), opc);
	181	}
	182	break :match std.mem.eql(u8, enc.opcode(), opc);
	183	};
	184	if (match) {
	185	if (prefixes.rex.w) {
	186	switch (enc.mode) {
	187	.fpu, .sse, .sse2, .none => {},
	188	.long, .rex => return enc,
	189	}
	190	} else if (prefixes.rex.present and !prefixes.rex.isSet()) {
	191	if (enc.mode == .rex) return enc;
	192	} else if (prefixes.legacy.prefix_66) {
	193	switch (enc.operandBitSize()) {
	194	16 => return enc,
	195	else => {},
	196	}
	197	} else {
	198	if (enc.mode == .none) {
	199	switch (enc.operandBitSize()) {
	200	16 => {},
	201	else => return enc,
	202	}
	203	}
	204	}
	205	}
	206	}
	207	return null;
	208	}
	209
	210	pub fn opcode(encoding: *const Encoding) []const u8 {
	211	return encoding.opc[0..encoding.opc_len];
	212	}
	213
	214	pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 {
	215	const prefix = encoding.opc[0];
	216	return switch (prefix) {
	217	0x66, 0xf2, 0xf3 => prefix,
	218	else => null,
	219	};
	220	}
	221
	222	pub fn modRmExt(encoding: Encoding) u3 {
	223	return switch (encoding.op_en) {
	224	.m, .mi, .m1, .mc => encoding.modrm_ext,
	225	else => unreachable,
	226	};
	227	}
	228
	229	pub fn operandBitSize(encoding: Encoding) u64 {
	230	if (encoding.mode == .long) return 64;
	231	const bit_size: u64 = switch (encoding.op_en) {
	232	.np => switch (encoding.op1) {
	233	.o16 => 16,
	234	.o32 => 32,
	235	.o64 => 64,
	236	else => 32,
	237	},
	238	.td => encoding.op2.bitSize(),
	239	else => encoding.op1.bitSize(),
	240	};
	241	return bit_size;
	242	}
	243
	244	pub fn format(
	245	encoding: Encoding,
	246	comptime fmt: []const u8,
	247	options: std.fmt.FormatOptions,
	248	writer: anytype,
	249	) !void {
	250	_ = options;
	251	_ = fmt;
	252	switch (encoding.mode) {
	253	.long => try writer.writeAll("REX.W + "),
	254	else => {},
	255	}
	256
	257	for (encoding.opcode()) \|byte\| {
	258	try writer.print("{x:0>2} ", .{byte});
	259	}
	260
	261	switch (encoding.op_en) {
	262	.np, .fd, .td, .i, .zi, .d => {},
	263	.o, .oi => {
	264	const tag = switch (encoding.op1) {
	265	.r8 => "rb",
	266	.r16 => "rw",
	267	.r32 => "rd",
	268	.r64 => "rd",
	269	else => unreachable,
	270	};
	271	try writer.print("+{s} ", .{tag});
	272	},
	273	.m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
	274	.mr, .rm, .rmi => try writer.writeAll("/r "),
	275	}
	276
	277	switch (encoding.op_en) {
	278	.i, .d, .zi, .oi, .mi, .rmi => {
	279	const op = switch (encoding.op_en) {
	280	.i, .d => encoding.op1,
	281	.zi, .oi, .mi => encoding.op2,
	282	.rmi => encoding.op3,
	283	else => unreachable,
	284	};
	285	const tag = switch (op) {
	286	.imm8, .imm8s => "ib",
	287	.imm16, .imm16s => "iw",
	288	.imm32, .imm32s => "id",
	289	.imm64 => "io",
	290	.rel8 => "cb",
	291	.rel16 => "cw",
	292	.rel32 => "cd",
	293	else => unreachable,
	294	};
	295	try writer.print("{s} ", .{tag});
	296	},
	297	.np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm => {},
	298	}
	299
	300	try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
	301
	302	const ops = &[_]Op{ encoding.op1, encoding.op2, encoding.op3, encoding.op4 };
	303	for (ops) \|op\| switch (op) {
	304	.none, .o16, .o32, .o64 => break,
	305	else => try writer.print("{s} ", .{@tagName(op)}),
	306	};
	307
	308	const op_en = switch (encoding.op_en) {
	309	.zi => .i,
	310	else => \|op_en\| op_en,
	311	};
	312	try writer.print("{s}", .{@tagName(op_en)});
	313	}
	314
	315	pub const Mnemonic = enum {
	316	// zig fmt: off
	317	// General-purpose
	318	adc, add, @"and",
	319	call, cbw, cwde, cdqe, cwd, cdq, cqo, cmp,
	320	cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
	321	cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
	322	cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
	323	div,
	324	fisttp, fld,
	325	idiv, imul, int3,
	326	ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe,
	327	jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
	328	jmp,
	329	lea,
	330	mov, movsx, movsxd, movzx, mul,
	331	nop,
	332	@"or",
	333	pop, push,
	334	ret,
	335	sal, sar, sbb, shl, shr, sub, syscall,
	336	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
	337	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
	338	setnz, seto, setp, setpe, setpo, sets, setz,
	339	@"test",
	340	ud2,
	341	xor,
	342	// SSE
	343	addss,
	344	cmpss,
	345	movss,
	346	ucomiss,
	347	// SSE2
	348	addsd,
	349	cmpsd,
	350	movq, movsd,
	351	ucomisd,
	352	// zig fmt: on
	353	};
	354
	355	pub const OpEn = enum {
	356	// zig fmt: off
	357	np,
	358	o, oi,
	359	i, zi,
	360	d, m,
	361	fd, td,
	362	m1, mc, mi, mr, rm, rmi,
	363	// zig fmt: on
	364	};
	365
	366	pub const Op = enum {
	367	// zig fmt: off
	368	none,
	369	o16, o32, o64,
	370	unity,
	371	imm8, imm16, imm32, imm64,
	372	imm8s, imm16s, imm32s,
	373	al, ax, eax, rax,
	374	cl,
	375	r8, r16, r32, r64,
	376	rm8, rm16, rm32, rm64,
	377	m8, m16, m32, m64, m80,
	378	rel8, rel16, rel32,
	379	m,
	380	moffs,
	381	sreg,
	382	xmm, xmm_m32, xmm_m64,
	383	// zig fmt: on
	384
	385	pub fn fromOperand(operand: Instruction.Operand) Op {
	386	switch (operand) {
	387	.none => return .none,
	388
	389	.reg => \|reg\| {
	390	switch (reg.class()) {
	391	.segment => return .sreg,
	392	.floating_point => return switch (reg.bitSize()) {
	393	128 => .xmm,
	394	else => unreachable,
	395	},
	396	.general_purpose => {
	397	if (reg.to64() == .rax) return switch (reg) {
	398	.al => .al,
	399	.ax => .ax,
	400	.eax => .eax,
	401	.rax => .rax,
	402	else => unreachable,
	403	};
	404	if (reg == .cl) return .cl;
	405	return switch (reg.bitSize()) {
	406	8 => .r8,
	407	16 => .r16,
	408	32 => .r32,
	409	64 => .r64,
	410	else => unreachable,
	411	};
	412	},
	413	}
	414	},
	415
	416	.mem => \|mem\| switch (mem) {
	417	.moffs => return .moffs,
	418	.sib, .rip => {
	419	const bit_size = mem.bitSize();
	420	return switch (bit_size) {
	421	8 => .m8,
	422	16 => .m16,
	423	32 => .m32,
	424	64 => .m64,
	425	80 => .m80,
	426	else => unreachable,
	427	};
	428	},
	429	},
	430
	431	.imm => \|imm\| {
	432	switch (imm) {
	433	.signed => \|x\| {
	434	if (x == 1) return .unity;
	435	if (math.cast(i8, x)) \|_\| return .imm8s;
	436	if (math.cast(i16, x)) \|_\| return .imm16s;
	437	return .imm32s;
	438	},
	439	.unsigned => \|x\| {
	440	if (x == 1) return .unity;
	441	if (math.cast(i8, x)) \|_\| return .imm8s;
	442	if (math.cast(u8, x)) \|_\| return .imm8;
	443	if (math.cast(i16, x)) \|_\| return .imm16s;
	444	if (math.cast(u16, x)) \|_\| return .imm16;
	445	if (math.cast(i32, x)) \|_\| return .imm32s;
	446	if (math.cast(u32, x)) \|_\| return .imm32;
	447	return .imm64;
	448	},
	449	}
	450	},
	451	}
	452	}
	453
	454	pub fn bitSize(op: Op) u64 {
	455	return switch (op) {
	456	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
	457	.unity => 1,
	458	.imm8, .imm8s, .al, .cl, .r8, .m8, .rm8, .rel8 => 8,
	459	.imm16, .imm16s, .ax, .r16, .m16, .rm16, .rel16 => 16,
	460	.imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
	461	.imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
	462	.m80 => 80,
	463	.xmm => 128,
	464	};
	465	}
	466
	467	pub fn isSigned(op: Op) bool {
	468	return switch (op) {
	469	.unity, .imm8, .imm16, .imm32, .imm64 => false,
	470	.imm8s, .imm16s, .imm32s => true,
	471	else => unreachable,
	472	};
	473	}
	474
	475	pub fn isUnsigned(op: Op) bool {
	476	return !op.isSigned();
	477	}
	478
	479	pub fn isRegister(op: Op) bool {
	480	// zig fmt: off
	481	return switch (op) {
	482	.cl,
	483	.al, .ax, .eax, .rax,
	484	.r8, .r16, .r32, .r64,
	485	.rm8, .rm16, .rm32, .rm64,
	486	.xmm, .xmm_m32, .xmm_m64,
	487	=> true,
	488	else => false,
	489	};
	490	// zig fmt: on
	491	}
	492
	493	pub fn isImmediate(op: Op) bool {
	494	// zig fmt: off
	495	return switch (op) {
	496	.imm8, .imm16, .imm32, .imm64,
	497	.imm8s, .imm16s, .imm32s,
	498	.rel8, .rel16, .rel32,
	499	.unity,
	500	=> true,
	501	else => false,
	502	};
	503	// zig fmt: on
	504	}
	505
	506	pub fn isMemory(op: Op) bool {
	507	// zig fmt: off
	508	return switch (op) {
	509	.rm8, .rm16, .rm32, .rm64,
	510	.m8, .m16, .m32, .m64, .m80,
	511	.m,
	512	.xmm_m32, .xmm_m64,
	513	=> true,
	514	else => false,
	515	};
	516	// zig fmt: on
	517	}
	518
	519	pub fn isSegmentRegister(op: Op) bool {
	520	return switch (op) {
	521	.moffs, .sreg => true,
	522	else => false,
	523	};
	524	}
	525
	526	pub fn isFloatingPointRegister(op: Op) bool {
	527	return switch (op) {
	528	.xmm, .xmm_m32, .xmm_m64 => true,
	529	else => false,
	530	};
	531	}
	532
	533	/// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.
	534	pub fn isSubset(op: Op, target: Op, mode: Mode) bool {
	535	switch (op) {
	536	.m, .o16, .o32, .o64 => unreachable,
	537	.moffs, .sreg => return op == target,
	538	.none => switch (target) {
	539	.o16, .o32, .o64, .none => return true,
	540	else => return false,
	541	},
	542	else => {
	543	if (op.isRegister() and target.isRegister()) {
	544	switch (mode) {
	545	.sse, .sse2 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
	546	else => switch (target) {
	547	.cl, .al, .ax, .eax, .rax => return op == target,
	548	else => return op.bitSize() == target.bitSize(),
	549	},
	550	}
	551	}
	552	if (op.isMemory() and target.isMemory()) {
	553	switch (target) {
	554	.m => return true,
	555	else => return op.bitSize() == target.bitSize(),
	556	}
	557	}
	558	if (op.isImmediate() and target.isImmediate()) {
	559	switch (target) {
	560	.imm64 => if (op.bitSize() <= 64) return true,
	561	.imm32s, .rel32 => if (op.bitSize() < 32 or (op.bitSize() == 32 and op.isSigned()))
	562	return true,
	563	.imm32 => if (op.bitSize() <= 32) return true,
	564	.imm16s, .rel16 => if (op.bitSize() < 16 or (op.bitSize() == 16 and op.isSigned()))
	565	return true,
	566	.imm16 => if (op.bitSize() <= 16) return true,
	567	.imm8s, .rel8 => if (op.bitSize() < 8 or (op.bitSize() == 8 and op.isSigned()))
	568	return true,
	569	.imm8 => if (op.bitSize() <= 8) return true,
	570	else => {},
	571	}
	572	return op == target;
	573	}
	574	return false;
	575	},
	576	}
	577	}
	578	};
	579
	580	pub const Mode = enum {
	581	none,
	582	fpu,
	583	rex,
	584	long,
	585	sse,
	586	sse2,
	587	};

src/arch/x86_64/Mir.zig+358-454

...	...	@@ -12,9 +12,12 @@ const builtin = @import("builtin");
12	12	const assert = std.debug.assert;
13	13
14	14	const bits = @import("bits.zig");
	15	const encoder = @import("encoder.zig");
	16
15	17	const Air = @import("../../Air.zig");
16	18	const CodeGen = @import("CodeGen.zig");
17	19	const IntegerBitSet = std.bit_set.IntegerBitSet;
	20	const Memory = bits.Memory;
18	21	const Register = bits.Register;
19	22
20	23	instructions: std.MultiArrayList(Inst).Slice,
...	...	@@ -24,431 +27,300 @@ extra: []const u32,
24	27	pub const Inst = struct {
25	28	tag: Tag,
26	29	ops: Ops,
27		/// The meaning of this depends on `tag` and `ops`.
28	30	data: Data,
29	31
30		pub const Tag = enum(u16) {
31		/// ops flags: form:
32		/// 0b00 reg1, reg2
33		/// 0b00 reg1, imm32
34		/// 0b01 reg1, [reg2 + imm32]
35		/// 0b01 reg1, [ds:imm32]
36		/// 0b10 [reg1 + imm32], reg2
37		/// Notes:
38		/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
39		/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
40		adc,
41
42		/// ops flags: form:
43		/// 0b00 byte ptr [reg1 + imm32], imm8
44		/// 0b01 word ptr [reg1 + imm32], imm16
45		/// 0b10 dword ptr [reg1 + imm32], imm32
46		/// 0b11 qword ptr [reg1 + imm32], imm32 (sign-extended to imm64)
47		/// Notes:
48		/// * Uses `ImmPair` as payload
49		adc_mem_imm,
50
51		/// form: reg1, [reg2 + scale*index + imm32]
52		/// ops flags scale
53		/// 0b00 1
54		/// 0b01 2
55		/// 0b10 4
56		/// 0b11 8
57		/// Notes:
58		/// * Uses `IndexRegisterDisp` as payload
59		adc_scale_src,
60
61		/// form: [reg1 + scale*index + imm32], reg2
62		/// ops flags scale
63		/// 0b00 1
64		/// 0b01 2
65		/// 0b10 4
66		/// 0b11 8
67		/// Notes:
68		/// * Uses `IndexRegisterDisp` payload.
69		adc_scale_dst,
70
71		/// form: [reg1 + scale*rax + imm32], imm32
72		/// ops flags scale
73		/// 0b00 1
74		/// 0b01 2
75		/// 0b10 4
76		/// 0b11 8
77		/// Notes:
78		/// * Uses `IndexRegisterDispImm` payload.
79		adc_scale_imm,
80
81		/// ops flags: form:
82		/// 0b00 byte ptr [reg1 + index + imm32], imm8
83		/// 0b01 word ptr [reg1 + index + imm32], imm16
84		/// 0b10 dword ptr [reg1 + index + imm32], imm32
85		/// 0b11 qword ptr [reg1 + index + imm32], imm32 (sign-extended to imm64)
86		/// Notes:
87		/// * Uses `IndexRegisterDispImm` payload.
88		adc_mem_index_imm,
89
90		// The following instructions all have the same encoding as `adc`.
	32	pub const Index = u32;
91	33
	34	pub const Tag = enum(u8) {
	35	/// Add with carry
	36	adc,
	37	/// Add
92	38	add,
93		add_mem_imm,
94		add_scale_src,
95		add_scale_dst,
96		add_scale_imm,
97		add_mem_index_imm,
98		sub,
99		sub_mem_imm,
100		sub_scale_src,
101		sub_scale_dst,
102		sub_scale_imm,
103		sub_mem_index_imm,
104		xor,
105		xor_mem_imm,
106		xor_scale_src,
107		xor_scale_dst,
108		xor_scale_imm,
109		xor_mem_index_imm,
	39	/// Logical and
110	40	@"and",
111		and_mem_imm,
112		and_scale_src,
113		and_scale_dst,
114		and_scale_imm,
115		and_mem_index_imm,
116		@"or",
117		or_mem_imm,
118		or_scale_src,
119		or_scale_dst,
120		or_scale_imm,
121		or_mem_index_imm,
122		rol,
123		rol_mem_imm,
124		rol_scale_src,
125		rol_scale_dst,
126		rol_scale_imm,
127		rol_mem_index_imm,
128		ror,
129		ror_mem_imm,
130		ror_scale_src,
131		ror_scale_dst,
132		ror_scale_imm,
133		ror_mem_index_imm,
134		rcl,
135		rcl_mem_imm,
136		rcl_scale_src,
137		rcl_scale_dst,
138		rcl_scale_imm,
139		rcl_mem_index_imm,
140		rcr,
141		rcr_mem_imm,
142		rcr_scale_src,
143		rcr_scale_dst,
144		rcr_scale_imm,
145		rcr_mem_index_imm,
146		sbb,
147		sbb_mem_imm,
148		sbb_scale_src,
149		sbb_scale_dst,
150		sbb_scale_imm,
151		sbb_mem_index_imm,
	41	/// Call
	42	call,
	43	/// Convert byte to word
	44	cbw,
	45	/// Convert word to doubleword
	46	cwde,
	47	/// Convert doubleword to quadword
	48	cdqe,
	49	/// Convert word to doubleword
	50	cwd,
	51	/// Convert doubleword to quadword
	52	cdq,
	53	/// Convert doubleword to quadword
	54	cqo,
	55	/// Logical compare
152	56	cmp,
153		cmp_mem_imm,
154		cmp_scale_src,
155		cmp_scale_dst,
156		cmp_scale_imm,
157		cmp_mem_index_imm,
158		mov,
159		mov_mem_imm,
160		mov_scale_src,
161		mov_scale_dst,
162		mov_scale_imm,
163		mov_mem_index_imm,
164
165		/// ops flags: form:
166		/// 0b00 reg1, reg2,
167		/// 0b01 reg1, byte ptr [reg2 + imm32]
168		/// 0b10 reg1, word ptr [reg2 + imm32]
169		/// 0b11 reg1, dword ptr [reg2 + imm32]
170		mov_sign_extend,
171
172		/// ops flags: form:
173		/// 0b00 reg1, reg2
174		/// 0b01 reg1, byte ptr [reg2 + imm32]
175		/// 0b10 reg1, word ptr [reg2 + imm32]
176		mov_zero_extend,
177
178		/// ops flags: form:
179		/// 0b00 reg1, [reg2 + imm32]
180		/// 0b00 reg1, [ds:imm32]
181		/// 0b01 reg1, [rip + imm32]
182		/// 0b10 reg1, [reg2 + index + imm32]
183		/// Notes:
184		/// * 0b10 uses `IndexRegisterDisp` payload
185		lea,
186
187		/// ops flags: form:
188		/// 0b00 reg1, [rip + reloc] // via GOT PIC
189		/// 0b01 reg1, [rip + reloc] // direct load PIC
190		/// 0b10 reg1, [rip + reloc] // via imports table PIC
191		/// Notes:
192		/// * `Data` contains `relocation`
193		lea_pic,
194
195		/// ops flags: form:
196		/// 0b00 reg1, 1
197		/// 0b01 reg1, .cl
198		/// 0b10 reg1, imm8
199		/// Notes:
200		/// * If flags == 0b10, uses `imm`.
201		shl,
202		shl_mem_imm,
203		shl_scale_src,
204		shl_scale_dst,
205		shl_scale_imm,
206		shl_mem_index_imm,
207		sal,
208		sal_mem_imm,
209		sal_scale_src,
210		sal_scale_dst,
211		sal_scale_imm,
212		sal_mem_index_imm,
213		shr,
214		shr_mem_imm,
215		shr_scale_src,
216		shr_scale_dst,
217		shr_scale_imm,
218		shr_mem_index_imm,
219		sar,
220		sar_mem_imm,
221		sar_scale_src,
222		sar_scale_dst,
223		sar_scale_imm,
224		sar_mem_index_imm,
225
226		/// ops flags: form:
227		/// 0b00 reg1
228		/// 0b00 byte ptr [reg2 + imm32]
229		/// 0b01 word ptr [reg2 + imm32]
230		/// 0b10 dword ptr [reg2 + imm32]
231		/// 0b11 qword ptr [reg2 + imm32]
232		imul,
233		idiv,
234		mul,
	57	/// Unsigned division
235	58	div,
236
237		/// ops flags: form:
238		/// 0b00 AX <- AL
239		/// 0b01 DX:AX <- AX
240		/// 0b10 EDX:EAX <- EAX
241		/// 0b11 RDX:RAX <- RAX
242		cwd,
243
244		/// ops flags: form:
245		/// 0b00 reg1, reg2
246		/// 0b01 reg1, [reg2 + imm32]
247		/// 0b01 reg1, [imm32] if reg2 is none
248		/// 0b10 reg1, reg2, imm32
249		/// 0b11 reg1, [reg2 + imm32], imm32
250		imul_complex,
251
252		/// ops flags: form:
253		/// 0b00 reg1, imm64
254		/// 0b01 rax, moffs64
255		/// Notes:
256		/// * If reg1 is 64-bit, the immediate is 64-bit and stored
257		/// within extra data `Imm64`.
258		/// * For 0b01, reg1 (or reg2) need to be
259		/// a version of rax. If reg1 == .none, then reg2 == .rax,
260		/// or vice versa.
261		movabs,
262
263		/// ops flags: form:
264		/// 0b00 word ptr [reg1 + imm32]
265		/// 0b01 dword ptr [reg1 + imm32]
266		/// 0b10 qword ptr [reg1 + imm32]
267		/// Notes:
268		/// * source is always ST(0)
269		/// * only supports memory operands as destination
	59	/// Store integer with truncation
270	60	fisttp,
271
272		/// ops flags: form:
273		/// 0b01 dword ptr [reg1 + imm32]
274		/// 0b10 qword ptr [reg1 + imm32]
	61	/// Load floating-point value
275	62	fld,
276
277		/// ops flags: form:
278		/// 0b00 inst
279		/// 0b01 reg1
280		/// 0b01 [imm32] if reg1 is none
281		/// 0b10 [reg1 + imm32]
	63	/// Signed division
	64	idiv,
	65	/// Signed multiplication
	66	imul,
	67	///
	68	int3,
	69	/// Jump
282	70	jmp,
283		call,
284
285		/// ops flags:
286		/// unused
287		/// Notes:
288		/// * uses `inst_cc` in Data.
289		cond_jmp,
290
291		/// ops flags:
292		/// 0b00 reg1
293		/// Notes:
294		/// * uses condition code (CC) stored as part of data
295		cond_set_byte,
296
297		/// ops flags:
298		/// 0b00 reg1, reg2,
299		/// 0b01 reg1, word ptr [reg2 + imm]
300		/// 0b10 reg1, dword ptr [reg2 + imm]
301		/// 0b11 reg1, qword ptr [reg2 + imm]
302		/// Notes:
303		/// * uses condition code (CC) stored as part of data
304		cond_mov,
305
306		/// ops flags: form:
307		/// 0b00 reg1
308		/// 0b01 [reg1 + imm32]
309		/// 0b10 imm32
310		/// Notes:
311		/// * If 0b10 is specified and the tag is push, pushes immediate onto the stack
312		/// using the mnemonic PUSH imm32.
313		push,
	71	/// Load effective address
	72	lea,
	73	/// Move
	74	mov,
	75	/// Move with sign extension
	76	movsx,
	77	/// Move with zero extension
	78	movzx,
	79	/// Multiply
	80	mul,
	81	/// No-op
	82	nop,
	83	/// Logical or
	84	@"or",
	85	/// Pop
314	86	pop,
315
316		/// ops flags: form:
317		/// 0b00 retf imm16
318		/// 0b01 retf
319		/// 0b10 retn imm16
320		/// 0b11 retn
	87	/// Push
	88	push,
	89	/// Return
321	90	ret,
322
323		/// Fast system call
	91	/// Arithmetic shift left
	92	sal,
	93	/// Arithmetic shift right
	94	sar,
	95	/// Integer subtraction with borrow
	96	sbb,
	97	/// Logical shift left
	98	shl,
	99	/// Logical shift right
	100	shr,
	101	/// Subtract
	102	sub,
	103	/// Syscall
324	104	syscall,
325
326		/// ops flags: form:
327		/// 0b00 reg1, imm32 if reg2 == .none
328		/// 0b00 reg1, reg2
329		/// TODO handle more cases
	105	/// Test condition
330	106	@"test",
	107	/// Undefined instruction
	108	ud2,
	109	/// Logical exclusive-or
	110	xor,
331	111
332		/// Undefined Instruction
333		ud,
334
335		/// Breakpoint form:
336		/// 0b00 int3
337		interrupt,
338
339		/// Nop
340		nop,
341
342		/// SSE instructions
343		/// ops flags: form:
344		/// 0b00 reg1, qword ptr [reg2 + imm32]
345		/// 0b01 qword ptr [reg1 + imm32], reg2
346		/// 0b10 reg1, reg2
347		mov_f64_sse,
348		mov_f32_sse,
349
350		/// ops flags: form:
351		/// 0b00 reg1, reg2
352		add_f64_sse,
353		add_f32_sse,
354
355		/// ops flags: form:
356		/// 0b00 reg1, reg2
357		cmp_f64_sse,
358		cmp_f32_sse,
359
360		/// AVX instructions
361		/// ops flags: form:
362		/// 0b00 reg1, qword ptr [reg2 + imm32]
363		/// 0b01 qword ptr [reg1 + imm32], reg2
364		/// 0b10 reg1, reg1, reg2
365		mov_f64_avx,
366		mov_f32_avx,
367
368		/// ops flags: form:
369		/// 0b00 reg1, reg1, reg2
370		add_f64_avx,
371		add_f32_avx,
372
373		/// ops flags: form:
374		/// 0b00 reg1, reg1, reg2
375		cmp_f64_avx,
376		cmp_f32_avx,
377
378		/// Pseudo-instructions
379		/// call extern function
380		/// Notes:
381		/// * target of the call is stored as `relocation` in `Data` union.
	112	/// Add single precision floating point
	113	addss,
	114	/// Compare scalar single-precision floating-point values
	115	cmpss,
	116	/// Move scalar single-precision floating-point value
	117	movss,
	118	/// Unordered compare scalar single-precision floating-point values
	119	ucomiss,
	120	/// Add double precision floating point
	121	addsd,
	122	/// Compare scalar double-precision floating-point values
	123	cmpsd,
	124	/// Move scalar double-precision floating-point value
	125	movsd,
	126	/// Unordered compare scalar double-precision floating-point values
	127	ucomisd,
	128
	129	/// Conditional move
	130	cmovcc,
	131	/// Conditional jump
	132	jcc,
	133	/// Set byte on condition
	134	setcc,
	135
	136	/// Mov absolute to/from memory wrt segment register to/from rax
	137	mov_moffs,
	138
	139	/// Jump with relocation to another local MIR instruction
	140	/// Uses `inst` payload.
	141	jmp_reloc,
	142
	143	/// Call to an extern symbol via linker relocation.
	144	/// Uses `relocation` payload.
382	145	call_extern,
383	146
384		/// end of prologue
385		dbg_prologue_end,
	147	/// Load effective address of a symbol not yet allocated in VM.
	148	lea_linker,
386	149
387		/// start of epilogue
	150	/// End of prologue
	151	dbg_prologue_end,
	152	/// Start of epilogue
388	153	dbg_epilogue_begin,
389
390		/// update debug line
	154	/// Update debug line
	155	/// Uses `payload` payload with data of type `DbgLineColumn`.
391	156	dbg_line,
392
393		/// push registers
394		/// Uses `payload` field with `SaveRegisterList` as payload.
	157	/// Push registers
	158	/// Uses `payload` payload with data of type `SaveRegisterList`.
395	159	push_regs,
396
397		/// pop registers
398		/// Uses `payload` field with `SaveRegisterList` as payload.
	160	/// Pop registers
	161	/// Uses `payload` payload with data of type `SaveRegisterList`.
399	162	pop_regs,
400		};
401		/// The position of an MIR instruction within the `Mir` instructions array.
402		pub const Index = u32;
403	163
404		pub const Ops = packed struct {
405		reg1: u7,
406		reg2: u7,
407		flags: u2,
408
409		pub fn encode(vals: struct {
410		reg1: Register = .none,
411		reg2: Register = .none,
412		flags: u2 = 0b00,
413		}) Ops {
414		return .{
415		.reg1 = @enumToInt(vals.reg1),
416		.reg2 = @enumToInt(vals.reg2),
417		.flags = vals.flags,
418		};
419		}
	164	/// Tombstone
	165	/// Emitter should skip this instruction.
	166	dead,
	167	};
420	168
421		pub fn decode(ops: Ops) struct {
422		reg1: Register,
423		reg2: Register,
424		flags: u2,
425		} {
426		return .{
427		.reg1 = @intToEnum(Register, ops.reg1),
428		.reg2 = @intToEnum(Register, ops.reg2),
429		.flags = ops.flags,
430		};
431		}
	169	pub const Ops = enum(u8) {
	170	/// No data associated with this instruction (only mnemonic is used).
	171	none,
	172	/// Single register operand.
	173	/// Uses `r` payload.
	174	r,
	175	/// Register, register operands.
	176	/// Uses `rr` payload.
	177	rr,
	178	/// Register, register, register operands.
	179	/// Uses `rrr` payload.
	180	rrr,
	181	/// Register, register, immediate (sign-extended) operands.
	182	/// Uses `rri` payload.
	183	rri_s,
	184	/// Register, register, immediate (unsigned) operands.
	185	/// Uses `rri` payload.
	186	rri_u,
	187	/// Register with condition code (CC).
	188	/// Uses `r_c` payload.
	189	r_c,
	190	/// Register, register with condition code (CC).
	191	/// Uses `rr_c` payload.
	192	rr_c,
	193	/// Register, immediate (sign-extended) operands.
	194	/// Uses `ri` payload.
	195	ri_s,
	196	/// Register, immediate (unsigned) operands.
	197	/// Uses `ri` payload.
	198	ri_u,
	199	/// Register, 64-bit unsigned immediate operands.
	200	/// Uses `rx` payload with payload type `Imm64`.
	201	ri64,
	202	/// Immediate (sign-extended) operand.
	203	/// Uses `imm` payload.
	204	imm_s,
	205	/// Immediate (unsigned) operand.
	206	/// Uses `imm` payload.
	207	imm_u,
	208	/// Relative displacement operand.
	209	/// Uses `imm` payload.
	210	rel,
	211	/// Register, memory (SIB) operands.
	212	/// Uses `rx` payload.
	213	rm_sib,
	214	/// Register, memory (RIP) operands.
	215	/// Uses `rx` payload.
	216	rm_rip,
	217	/// Single memory (SIB) operand.
	218	/// Uses `payload` with extra data of type `MemorySib`.
	219	m_sib,
	220	/// Single memory (RIP) operand.
	221	/// Uses `payload` with extra data of type `MemoryRip`.
	222	m_rip,
	223	/// Memory (SIB), immediate (unsigned) operands.
	224	/// Uses `xi` payload with extra data of type `MemorySib`.
	225	mi_u_sib,
	226	/// Memory (RIP), immediate (unsigned) operands.
	227	/// Uses `xi` payload with extra data of type `MemoryRip`.
	228	mi_u_rip,
	229	/// Memory (SIB), immediate (sign-extend) operands.
	230	/// Uses `xi` payload with extra data of type `MemorySib`.
	231	mi_s_sib,
	232	/// Memory (RIP), immediate (sign-extend) operands.
	233	/// Uses `xi` payload with extra data of type `MemoryRip`.
	234	mi_s_rip,
	235	/// Memory (SIB), register operands.
	236	/// Uses `rx` payload with extra data of type `MemorySib`.
	237	mr_sib,
	238	/// Memory (RIP), register operands.
	239	/// Uses `rx` payload with extra data of type `MemoryRip`.
	240	mr_rip,
	241	/// Rax, Memory moffs.
	242	/// Uses `payload` with extra data of type `MemoryMoffs`.
	243	rax_moffs,
	244	/// Memory moffs, rax.
	245	/// Uses `payload` with extra data of type `MemoryMoffs`.
	246	moffs_rax,
	247	/// References another Mir instruction directly.
	248	/// Uses `inst` payload.
	249	inst,
	250	/// References another Mir instruction directly with condition code (CC).
	251	/// Uses `inst_cc` payload.
	252	inst_cc,
	253	/// Uses `payload` payload with data of type `MemoryConditionCode`.
	254	m_cc,
	255	/// Uses `rx` payload with extra data of type `MemoryConditionCode`.
	256	rm_cc,
	257	/// Uses `reloc` payload.
	258	reloc,
	259	/// Linker relocation - GOT indirection.
	260	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	261	got_reloc,
	262	/// Linker relocation - direct reference.
	263	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	264	direct_reloc,
	265	/// Linker relocation - imports table indirection (binding).
	266	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	267	import_reloc,
432	268	};
433	269
434		/// All instructions have a 4-byte payload, which is contained within
435		/// this union. `Tag` determines which union field is active, as well as
436		/// how to interpret the data within.
437	270	pub const Data = union {
438		/// Another instruction.
	271	/// References another Mir instruction.
439	272	inst: Index,
440		/// A 32-bit immediate value.
441		imm: u32,
442		/// A condition code for use with EFLAGS register.
443		cc: bits.Condition,
444		/// Another instruction with condition code.
445		/// Used by `cond_jmp`.
	273	/// Another instruction with condition code (CC).
	274	/// Used by `jcc`.
446	275	inst_cc: struct {
447	276	/// Another instruction.
448	277	inst: Index,
449	278	/// A condition code for use with EFLAGS register.
450	279	cc: bits.Condition,
451	280	},
	281	/// A 32-bit immediate value.
	282	imm: u32,
	283	r: Register,
	284	rr: struct {
	285	r1: Register,
	286	r2: Register,
	287	},
	288	rrr: struct {
	289	r1: Register,
	290	r2: Register,
	291	r3: Register,
	292	},
	293	rri: struct {
	294	r1: Register,
	295	r2: Register,
	296	imm: u32,
	297	},
	298	/// Register with condition code (CC).
	299	r_c: struct {
	300	r1: Register,
	301	cc: bits.Condition,
	302	},
	303	/// Register, register with condition code (CC).
	304	rr_c: struct {
	305	r1: Register,
	306	r2: Register,
	307	cc: bits.Condition,
	308	},
	309	/// Register, immediate.
	310	ri: struct {
	311	r1: Register,
	312	imm: u32,
	313	},
	314	/// Register, followed by custom payload found in extra.
	315	rx: struct {
	316	r1: Register,
	317	payload: u32,
	318	},
	319	/// Custom payload followed by an immediate.
	320	xi: struct {
	321	payload: u32,
	322	imm: u32,
	323	},
452	324	/// Relocation for the linker where:
453	325	/// * `atom_index` is the index of the source
454	326	/// * `sym_index` is the index of the target
...	...	@@ -471,62 +343,13 @@ pub const Inst = struct {
471	343	}
472	344	};
473	345
474		pub const IndexRegisterDisp = struct {
475		/// Index register to use with SIB-based encoding
476		index: u32,
477
478		/// Displacement value
479		disp: u32,
480
481		pub fn encode(index: Register, disp: u32) IndexRegisterDisp {
482		return .{
483		.index = @enumToInt(index),
484		.disp = disp,
485		};
486		}
487
488		pub fn decode(this: IndexRegisterDisp) struct {
489		index: Register,
490		disp: u32,
491		} {
492		return .{
493		.index = @intToEnum(Register, this.index),
494		.disp = this.disp,
495		};
496		}
497		};
498
499		/// TODO: would it be worth making `IndexRegisterDisp` and `IndexRegisterDispImm` a variable length list
500		/// instead of having two structs, one a superset of the other one?
501		pub const IndexRegisterDispImm = struct {
502		/// Index register to use with SIB-based encoding
503		index: u32,
504
505		/// Displacement value
506		disp: u32,
507
508		/// Immediate
509		imm: u32,
510
511		pub fn encode(index: Register, disp: u32, imm: u32) IndexRegisterDispImm {
512		return .{
513		.index = @enumToInt(index),
514		.disp = disp,
515		.imm = imm,
516		};
517		}
518
519		pub fn decode(this: IndexRegisterDispImm) struct {
520		index: Register,
521		disp: u32,
522		imm: u32,
523		} {
524		return .{
525		.index = @intToEnum(Register, this.index),
526		.disp = this.disp,
527		.imm = this.imm,
528		};
529		}
	346	pub const LeaRegisterReloc = struct {
	347	/// Destination register.
	348	reg: u32,
	349	/// Index of the containing atom.
	350	atom_index: u32,
	351	/// Index into the linker's symbol table.
	352	sym_index: u32,
530	353	};
531	354
532	355	/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers
...	...	@@ -570,16 +393,13 @@ pub const RegisterList = struct {
570	393	};
571	394
572	395	pub const SaveRegisterList = struct {
	396	/// Base register
	397	base_reg: u32,
573	398	/// Use `RegisterList` to populate.
574	399	register_list: u32,
575	400	stack_end: u32,
576	401	};
577	402
578		pub const ImmPair = struct {
579		dest_off: u32,
580		operand: u32,
581		};
582
583	403	pub const Imm64 = struct {
584	404	msb: u32,
585	405	lsb: u32,
...	...	@@ -599,6 +419,90 @@ pub const Imm64 = struct {
599	419	}
600	420	};
601	421
	422	// TODO this can be further compacted using packed struct
	423	pub const MemorySib = struct {
	424	/// Size of the pointer.
	425	ptr_size: u32,
	426	/// Base register. -1 means null, or no base register.
	427	base: i32,
	428	/// Scale for index register. -1 means null, or no scale.
	429	/// This has to be in sync with `index` field.
	430	scale: i32,
	431	/// Index register. -1 means null, or no index register.
	432	/// This has to be in sync with `scale` field.
	433	index: i32,
	434	/// Displacement value.
	435	disp: i32,
	436
	437	pub fn encode(mem: Memory) MemorySib {
	438	const sib = mem.sib;
	439	return .{
	440	.ptr_size = @enumToInt(sib.ptr_size),
	441	.base = if (sib.base) \|r\| @enumToInt(r) else -1,
	442	.scale = if (sib.scale_index) \|si\| si.scale else -1,
	443	.index = if (sib.scale_index) \|si\| @enumToInt(si.index) else -1,
	444	.disp = sib.disp,
	445	};
	446	}
	447
	448	pub fn decode(msib: MemorySib) Memory {
	449	const base: ?Register = if (msib.base == -1) null else @intToEnum(Register, msib.base);
	450	const scale_index: ?Memory.ScaleIndex = if (msib.index == -1) null else .{
	451	.scale = @intCast(u4, msib.scale),
	452	.index = @intToEnum(Register, msib.index),
	453	};
	454	const mem: Memory = .{ .sib = .{
	455	.ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),
	456	.base = base,
	457	.scale_index = scale_index,
	458	.disp = msib.disp,
	459	} };
	460	return mem;
	461	}
	462	};
	463
	464	pub const MemoryRip = struct {
	465	/// Size of the pointer.
	466	ptr_size: u32,
	467	/// Displacement value.
	468	disp: i32,
	469
	470	pub fn encode(mem: Memory) MemoryRip {
	471	return .{
	472	.ptr_size = @enumToInt(mem.rip.ptr_size),
	473	.disp = mem.rip.disp,
	474	};
	475	}
	476
	477	pub fn decode(mrip: MemoryRip) Memory {
	478	return .{ .rip = .{
	479	.ptr_size = @intToEnum(Memory.PtrSize, mrip.ptr_size),
	480	.disp = mrip.disp,
	481	} };
	482	}
	483	};
	484
	485	pub const MemoryMoffs = struct {
	486	/// Segment register.
	487	seg: u32,
	488	/// Absolute offset wrt to the segment register split between MSB and LSB parts much like
	489	/// `Imm64` payload.
	490	msb: u32,
	491	lsb: u32,
	492
	493	pub fn encodeOffset(moffs: *MemoryMoffs, v: u64) void {
	494	moffs.msb = @truncate(u32, v >> 32);
	495	moffs.lsb = @truncate(u32, v);
	496	}
	497
	498	pub fn decodeOffset(moffs: *const MemoryMoffs) u64 {
	499	var res: u64 = 0;
	500	res \|= (@intCast(u64, moffs.msb) << 32);
	501	res \|= @intCast(u64, moffs.lsb);
	502	return res;
	503	}
	504	};
	505
602	506	pub const DbgLineColumn = struct {
603	507	line: u32,
604	508	column: u32,
...	...	@@ -610,9 +514,9 @@ pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
610	514	mir.* = undefined;
611	515	}
612	516
613		pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
	517	pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end: u32 } {
614	518	const fields = std.meta.fields(T);
615		var i: usize = index;
	519	var i: u32 = index;
616	520	var result: T = undefined;
617	521	inline for (fields) \|field\| {
618	522	@field(result, field.name) = switch (field.type) {

src/arch/x86_64/bits.zig+301-845

...	...	@@ -1,9 +1,9 @@
1	1	const std = @import("std");
2		const testing = std.testing;
3		const mem = std.mem;
4	2	const assert = std.debug.assert;
5		const ArrayList = std.ArrayList;
	3	const expect = std.testing.expect;
	4
6	5	const Allocator = std.mem.Allocator;
	6	const ArrayList = std.ArrayList;
7	7	const DW = std.dwarf;
8	8
9	9	/// EFLAGS condition codes
...	...	@@ -135,131 +135,199 @@ pub const Condition = enum(u5) {
135	135	}
136	136	};
137	137
138		/// Definitions of all of the general purpose x64 registers. The order is semantically meaningful.
139		/// The registers are defined such that IDs go in descending order of 64-bit,
140		/// 32-bit, 16-bit, and then 8-bit, and each set contains exactly sixteen
141		/// registers. This results in some useful properties:
142		///
143		/// Any 64-bit register can be turned into its 32-bit form by adding 16, and
144		/// vice versa. This also works between 32-bit and 16-bit forms. With 8-bit, it
145		/// works for all except for sp, bp, si, and di, which do not have an 8-bit
146		/// form.
147		///
148		/// If (register & 8) is set, the register is extended.
149		///
150		/// The ID can be easily determined by figuring out what range the register is
151		/// in, and then subtracting the base.
152	138	pub const Register = enum(u7) {
153	139	// zig fmt: off
154		// 0 through 15, 64-bit registers. 8-15 are extended.
155		// id is just the int value.
156	140	rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi,
157	141	r8, r9, r10, r11, r12, r13, r14, r15,
158	142
159		// 16 through 31, 32-bit registers. 24-31 are extended.
160		// id is int value - 16.
161	143	eax, ecx, edx, ebx, esp, ebp, esi, edi,
162	144	r8d, r9d, r10d, r11d, r12d, r13d, r14d, r15d,
163	145
164		// 32-47, 16-bit registers. 40-47 are extended.
165		// id is int value - 32.
166	146	ax, cx, dx, bx, sp, bp, si, di,
167	147	r8w, r9w, r10w, r11w, r12w, r13w, r14w, r15w,
168	148
169		// 48-63, 8-bit registers. 56-63 are extended.
170		// id is int value - 48.
171		al, cl, dl, bl, ah, ch, dh, bh,
	149	al, cl, dl, bl, spl, bpl, sil, dil,
172	150	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,
173	151
174		// 64-79, 256-bit registers.
175		// id is int value - 64.
	152	ah, ch, dh, bh,
	153
176	154	ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,
177	155	ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
178	156
179		// 80-95, 128-bit registers.
180		// id is int value - 80.
181	157	xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
182	158	xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
183	159
184		// Pseudo-value for MIR instructions.
	160	es, cs, ss, ds, fs, gs,
	161
185	162	none,
186	163	// zig fmt: on
187	164
188		pub fn id(self: Register) u7 {
189		return switch (@enumToInt(self)) {
190		0...63 => @as(u7, @truncate(u4, @enumToInt(self))),
191		64...79 => @enumToInt(self),
	165	pub const Class = enum(u2) {
	166	general_purpose,
	167	floating_point,
	168	segment,
	169	};
	170
	171	pub fn class(reg: Register) Class {
	172	return switch (@enumToInt(reg)) {
	173	// zig fmt: off
	174	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => .general_purpose,
	175	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => .general_purpose,
	176	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => .general_purpose,
	177	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => .general_purpose,
	178	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => .general_purpose,
	179
	180	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => .floating_point,
	181	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => .floating_point,
	182
	183	@enumToInt(Register.es) ... @enumToInt(Register.gs) => .segment,
	184
192	185	else => unreachable,
	186	// zig fmt: on
193	187	};
194	188	}
195	189
196		/// Returns the bit-width of the register.
197		pub fn size(self: Register) u9 {
198		return switch (@enumToInt(self)) {
199		0...15 => 64,
200		16...31 => 32,
201		32...47 => 16,
202		48...63 => 8,
203		64...79 => 256,
204		80...95 => 128,
	190	pub fn id(reg: Register) u6 {
	191	const base = switch (@enumToInt(reg)) {
	192	// zig fmt: off
	193	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
	194	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
	195	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
	196	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
	197	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
	198
	199	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0) - 16,
	200	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0) - 16,
	201
	202	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es) - 32,
	203
205	204	else => unreachable,
	205	// zig fmt: on
206	206	};
	207	return @intCast(u6, @enumToInt(reg) - base);
207	208	}
208	209
209		/// Returns whether the register is extended. Extended registers are the
210		/// new registers added with amd64, r8 through r15. This also includes any
211		/// other variant of access to those registers, such as r8b, r15d, and so
212		/// on. This is needed because access to these registers requires special
213		/// handling via the REX prefix, via the B or R bits, depending on context.
214		pub fn isExtended(self: Register) bool {
215		return @enumToInt(self) & 0x08 != 0;
	210	pub fn bitSize(reg: Register) u64 {
	211	return switch (@enumToInt(reg)) {
	212	// zig fmt: off
	213	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => 64,
	214	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => 32,
	215	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => 16,
	216	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => 8,
	217	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => 8,
	218
	219	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => 256,
	220	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => 128,
	221
	222	@enumToInt(Register.es) ... @enumToInt(Register.gs) => 16,
	223
	224	else => unreachable,
	225	// zig fmt: on
	226	};
216	227	}
217	228
218		/// This returns the 4-bit register ID, which is used in practically every
219		/// opcode. Note that bit 3 (the highest bit) is never used directly in
220		/// an instruction (@see isExtended), and requires special handling. The
221		/// lower three bits are often embedded directly in instructions (such as
222		/// the B8 variant of moves), or used in R/M bytes.
223		pub fn enc(self: Register) u4 {
224		return @truncate(u4, @enumToInt(self));
	229	pub fn isExtended(reg: Register) bool {
	230	return switch (@enumToInt(reg)) {
	231	// zig fmt: off
	232	@enumToInt(Register.r8) ... @enumToInt(Register.r15) => true,
	233	@enumToInt(Register.r8d) ... @enumToInt(Register.r15d) => true,
	234	@enumToInt(Register.r8w) ... @enumToInt(Register.r15w) => true,
	235	@enumToInt(Register.r8b) ... @enumToInt(Register.r15b) => true,
	236
	237	@enumToInt(Register.ymm8) ... @enumToInt(Register.ymm15) => true,
	238	@enumToInt(Register.xmm8) ... @enumToInt(Register.xmm15) => true,
	239
	240	else => false,
	241	// zig fmt: on
	242	};
225	243	}
226	244
227		/// Like enc, but only returns the lower 3 bits.
228		pub fn lowEnc(self: Register) u3 {
229		return @truncate(u3, @enumToInt(self));
	245	pub fn enc(reg: Register) u4 {
	246	const base = switch (@enumToInt(reg)) {
	247	// zig fmt: off
	248	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
	249	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
	250	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
	251	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
	252	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
	253
	254	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
	255	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
	256
	257	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es),
	258
	259	else => unreachable,
	260	// zig fmt: on
	261	};
	262	return @truncate(u4, @enumToInt(reg) - base);
	263	}
	264
	265	pub fn lowEnc(reg: Register) u3 {
	266	return @truncate(u3, reg.enc());
	267	}
	268
	269	pub fn toBitSize(reg: Register, bit_size: u64) Register {
	270	return switch (bit_size) {
	271	8 => reg.to8(),
	272	16 => reg.to16(),
	273	32 => reg.to32(),
	274	64 => reg.to64(),
	275	128 => reg.to128(),
	276	256 => reg.to256(),
	277	else => unreachable,
	278	};
	279	}
	280
	281	fn gpBase(reg: Register) u7 {
	282	assert(reg.class() == .general_purpose);
	283	return switch (@enumToInt(reg)) {
	284	// zig fmt: off
	285	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
	286	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
	287	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
	288	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
	289	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
	290	else => unreachable,
	291	// zig fmt: on
	292	};
230	293	}
231	294
232		pub fn to256(self: Register) Register {
233		return @intToEnum(Register, @as(u8, self.enc()) + 64);
	295	pub fn to64(reg: Register) Register {
	296	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.rax));
234	297	}
235	298
236		pub fn to128(self: Register) Register {
237		return @intToEnum(Register, @as(u8, self.enc()) + 80);
	299	pub fn to32(reg: Register) Register {
	300	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.eax));
238	301	}
239	302
240		/// Convert from any register to its 64 bit alias.
241		pub fn to64(self: Register) Register {
242		return @intToEnum(Register, self.enc());
	303	pub fn to16(reg: Register) Register {
	304	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.ax));
243	305	}
244	306
245		/// Convert from any register to its 32 bit alias.
246		pub fn to32(self: Register) Register {
247		return @intToEnum(Register, @as(u8, self.enc()) + 16);
	307	pub fn to8(reg: Register) Register {
	308	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.al));
248	309	}
249	310
250		/// Convert from any register to its 16 bit alias.
251		pub fn to16(self: Register) Register {
252		return @intToEnum(Register, @as(u8, self.enc()) + 32);
	311	fn fpBase(reg: Register) u7 {
	312	assert(reg.class() == .floating_point);
	313	return switch (@enumToInt(reg)) {
	314	@enumToInt(Register.ymm0)...@enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
	315	@enumToInt(Register.xmm0)...@enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
	316	else => unreachable,
	317	};
253	318	}
254	319
255		/// Convert from any register to its 8 bit alias.
256		pub fn to8(self: Register) Register {
257		return @intToEnum(Register, @as(u8, self.enc()) + 48);
	320	pub fn to256(reg: Register) Register {
	321	return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.ymm0));
258	322	}
259	323
260		pub fn dwarfLocOp(self: Register) u8 {
261		switch (@enumToInt(self)) {
262		0...63 => return switch (self.to64()) {
	324	pub fn to128(reg: Register) Register {
	325	return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.xmm0));
	326	}
	327
	328	pub fn dwarfLocOp(reg: Register) u8 {
	329	return switch (reg.class()) {
	330	.general_purpose => switch (reg.to64()) {
263	331	.rax => DW.OP.reg0,
264	332	.rdx => DW.OP.reg1,
265	333	.rcx => DW.OP.reg2,
...	...	@@ -268,31 +336,19 @@ pub const Register = enum(u7) {
268	336	.rdi => DW.OP.reg5,
269	337	.rbp => DW.OP.reg6,
270	338	.rsp => DW.OP.reg7,
271
272		.r8 => DW.OP.reg8,
273		.r9 => DW.OP.reg9,
274		.r10 => DW.OP.reg10,
275		.r11 => DW.OP.reg11,
276		.r12 => DW.OP.reg12,
277		.r13 => DW.OP.reg13,
278		.r14 => DW.OP.reg14,
279		.r15 => DW.OP.reg15,
280
281		else => unreachable,
	339	else => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.reg0,
282	340	},
283
284		64...79 => return @as(u8, self.enc()) + DW.OP.reg17,
285
	341	.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.reg17,
286	342	else => unreachable,
287		}
	343	};
288	344	}
289	345
290	346	/// DWARF encodings that push a value onto the DWARF stack that is either
291	347	/// the contents of a register or the result of adding the contents a given
292	348	/// register to a given signed offset.
293		pub fn dwarfLocOpDeref(self: Register) u8 {
294		switch (@enumToInt(self)) {
295		0...63 => return switch (self.to64()) {
	349	pub fn dwarfLocOpDeref(reg: Register) u8 {
	350	return switch (reg.class()) {
	351	.general_purpose => switch (reg.to64()) {
296	352	.rax => DW.OP.breg0,
297	353	.rdx => DW.OP.breg1,
298	354	.rcx => DW.OP.breg2,
...	...	@@ -300,795 +356,195 @@ pub const Register = enum(u7) {
300	356	.rsi => DW.OP.breg4,
301	357	.rdi => DW.OP.breg5,
302	358	.rbp => DW.OP.breg6,
303		.rsp => DW.OP.fbreg,
304
305		.r8 => DW.OP.breg8,
306		.r9 => DW.OP.breg9,
307		.r10 => DW.OP.breg10,
308		.r11 => DW.OP.breg11,
309		.r12 => DW.OP.breg12,
310		.r13 => DW.OP.breg13,
311		.r14 => DW.OP.breg14,
312		.r15 => DW.OP.breg15,
313
314		else => unreachable,
	359	.rsp => DW.OP.breg7,
	360	else => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.breg0,
315	361	},
316
317		64...79 => return @as(u8, self.enc()) + DW.OP.breg17,
318
	362	.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.breg17,
319	363	else => unreachable,
320		}
	364	};
321	365	}
322	366	};
323	367
324		// zig fmt: on
325
326		/// Encoding helper functions for x86_64 instructions
327		///
328		/// Many of these helpers do very little, but they can help make things
329		/// slightly more readable with more descriptive field names / function names.
330		///
331		/// Some of them also have asserts to ensure that we aren't doing dumb things.
332		/// For example, trying to use register 4 (esp) in an indirect modr/m byte is illegal,
333		/// you need to encode it with an SIB byte.
334		///
335		/// Note that ALL of these helper functions will assume capacity,
336		/// so ensure that the `code` has sufficient capacity before using them.
337		/// The `init` method is the recommended way to ensure capacity.
338		pub const Encoder = struct {
339		/// Non-owning reference to the code array
340		code: *ArrayList(u8),
341
342		const Self = @This();
343
344		/// Wrap `code` in Encoder to make it easier to call these helper functions
345		///
346		/// maximum_inst_size should contain the maximum number of bytes
347		/// that the encoded instruction will take.
348		/// This is because the helper functions will assume capacity
349		/// in order to avoid bounds checking.
350		pub fn init(code: *ArrayList(u8), maximum_inst_size: u8) !Self {
351		try code.ensureUnusedCapacity(maximum_inst_size);
352		return Self{ .code = code };
353		}
354
355		/// Directly write a number to the code array with big endianness
356		pub fn writeIntBig(self: Self, comptime T: type, value: T) void {
357		mem.writeIntBig(
358		T,
359		self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
360		value,
361		);
362		}
363
364		/// Directly write a number to the code array with little endianness
365		pub fn writeIntLittle(self: Self, comptime T: type, value: T) void {
366		mem.writeIntLittle(
367		T,
368		self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
369		value,
370		);
371		}
	368	test "Register id - different classes" {
	369	try expect(Register.al.id() == Register.ax.id());
	370	try expect(Register.ah.id() == Register.spl.id());
	371	try expect(Register.ax.id() == Register.eax.id());
	372	try expect(Register.eax.id() == Register.rax.id());
372	373
373		// --------
374		// Prefixes
375		// --------
376
377		pub const LegacyPrefixes = packed struct {
378		/// LOCK
379		prefix_f0: bool = false,
380		/// REPNZ, REPNE, REP, Scalar Double-precision
381		prefix_f2: bool = false,
382		/// REPZ, REPE, REP, Scalar Single-precision
383		prefix_f3: bool = false,
384
385		/// CS segment override or Branch not taken
386		prefix_2e: bool = false,
387		/// DS segment override
388		prefix_36: bool = false,
389		/// ES segment override
390		prefix_26: bool = false,
391		/// FS segment override
392		prefix_64: bool = false,
393		/// GS segment override
394		prefix_65: bool = false,
395
396		/// Branch taken
397		prefix_3e: bool = false,
398
399		/// Operand size override (enables 16 bit operation)
400		prefix_66: bool = false,
401
402		/// Address size override (enables 16 bit address size)
403		prefix_67: bool = false,
404
405		padding: u5 = 0,
406		};
	374	try expect(Register.ymm0.id() == 0b10000);
	375	try expect(Register.ymm0.id() != Register.rax.id());
	376	try expect(Register.xmm0.id() == Register.ymm0.id());
407	377
408		/// Encodes legacy prefixes
409		pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) void {
410		if (@bitCast(u16, prefixes) != 0) {
411		// Hopefully this path isn't taken very often, so we'll do it the slow way for now
412
413		// LOCK
414		if (prefixes.prefix_f0) self.code.appendAssumeCapacity(0xf0);
415		// REPNZ, REPNE, REP, Scalar Double-precision
416		if (prefixes.prefix_f2) self.code.appendAssumeCapacity(0xf2);
417		// REPZ, REPE, REP, Scalar Single-precision
418		if (prefixes.prefix_f3) self.code.appendAssumeCapacity(0xf3);
419
420		// CS segment override or Branch not taken
421		if (prefixes.prefix_2e) self.code.appendAssumeCapacity(0x2e);
422		// DS segment override
423		if (prefixes.prefix_36) self.code.appendAssumeCapacity(0x36);
424		// ES segment override
425		if (prefixes.prefix_26) self.code.appendAssumeCapacity(0x26);
426		// FS segment override
427		if (prefixes.prefix_64) self.code.appendAssumeCapacity(0x64);
428		// GS segment override
429		if (prefixes.prefix_65) self.code.appendAssumeCapacity(0x65);
430
431		// Branch taken
432		if (prefixes.prefix_3e) self.code.appendAssumeCapacity(0x3e);
433
434		// Operand size override
435		if (prefixes.prefix_66) self.code.appendAssumeCapacity(0x66);
436
437		// Address size override
438		if (prefixes.prefix_67) self.code.appendAssumeCapacity(0x67);
439		}
440		}
441
442		/// Use 16 bit operand size
443		///
444		/// Note that this flag is overridden by REX.W, if both are present.
445		pub fn prefix16BitMode(self: Self) void {
446		self.code.appendAssumeCapacity(0x66);
447		}
448
449		pub const Vex = struct {
450		rex_prefix: Rex = .{},
451		lead_opc: u5 = 0b0_0001,
452		register: u4 = 0b1111,
453		length: u1 = 0b0,
454		simd_prefix: u2 = 0b00,
455		wig_desc: bool = false,
456		lig_desc: bool = false,
457		lz_desc: bool = false,
458
459		pub fn rex(self: *Vex, r: Rex) void {
460		self.rex_prefix = r;
461		}
462
463		pub fn lead_opc_0f(self: *Vex) void {
464		self.lead_opc = 0b0_0001;
465		}
466
467		pub fn lead_opc_0f_38(self: *Vex) void {
468		self.lead_opc = 0b0_0010;
469		}
470
471		pub fn lead_opc_0f_3a(self: *Vex) void {
472		self.lead_opc = 0b0_0011;
473		}
474
475		pub fn reg(self: *Vex, register: u4) void {
476		self.register = ~register;
477		}
478
479		pub fn len_128(self: *Vex) void {
480		self.length = 0;
481		}
482
483		pub fn len_256(self: *Vex) void {
484		assert(!self.lz_desc);
485		self.length = 1;
486		}
	378	try expect(Register.es.id() == 0b100000);
	379	}
487	380
488		pub fn simd_prefix_66(self: *Vex) void {
489		self.simd_prefix = 0b01;
490		}
	381	test "Register enc - different classes" {
	382	try expect(Register.al.enc() == Register.ax.enc());
	383	try expect(Register.ax.enc() == Register.eax.enc());
	384	try expect(Register.eax.enc() == Register.rax.enc());
	385	try expect(Register.ymm0.enc() == Register.rax.enc());
	386	try expect(Register.xmm0.enc() == Register.ymm0.enc());
	387	try expect(Register.es.enc() == Register.rax.enc());
	388	}
491	389
492		pub fn simd_prefix_f3(self: *Vex) void {
493		self.simd_prefix = 0b10;
494		}
	390	test "Register classes" {
	391	try expect(Register.r11.class() == .general_purpose);
	392	try expect(Register.ymm11.class() == .floating_point);
	393	try expect(Register.fs.class() == .segment);
	394	}
495	395
496		pub fn simd_prefix_f2(self: *Vex) void {
497		self.simd_prefix = 0b11;
498		}
	396	pub const Memory = union(enum) {
	397	sib: Sib,
	398	rip: Rip,
	399	moffs: Moffs,
499	400
500		pub fn wig(self: *Vex) void {
501		self.wig_desc = true;
502		}
	401	pub const ScaleIndex = packed struct {
	402	scale: u4,
	403	index: Register,
	404	};
503	405
504		pub fn lig(self: *Vex) void {
505		self.lig_desc = true;
	406	pub const PtrSize = enum {
	407	byte,
	408	word,
	409	dword,
	410	qword,
	411	tbyte,
	412
	413	pub fn fromSize(size: u32) PtrSize {
	414	return if (size <= 1)
	415	.byte
	416	else if (size <= 2)
	417	.word
	418	else if (size <= 4)
	419	.dword
	420	else if (size <= 8)
	421	.qword
	422	else if (size == 10)
	423	.tbyte
	424	else
	425	unreachable;
506	426	}
507	427
508		pub fn lz(self: *Vex) void {
509		self.lz_desc = true;
	428	pub fn fromBitSize(bit_size: u64) PtrSize {
	429	return switch (bit_size) {
	430	8 => .byte,
	431	16 => .word,
	432	32 => .dword,
	433	64 => .qword,
	434	80 => .tbyte,
	435	else => unreachable,
	436	};
510	437	}
511	438
512		pub fn write(self: Vex, writer: anytype) usize {
513		var buf: [3]u8 = .{0} ** 3;
514		const form_3byte: bool = blk: {
515		if (self.rex_prefix.w and !self.wig_desc) break :blk true;
516		if (self.rex_prefix.x or self.rex_prefix.b) break :blk true;
517		break :blk self.lead_opc != 0b0_0001;
	439	pub fn bitSize(s: PtrSize) u64 {
	440	return switch (s) {
	441	.byte => 8,
	442	.word => 16,
	443	.dword => 32,
	444	.qword => 64,
	445	.tbyte => 80,
518	446	};
519
520		if (self.lz_desc) {
521		assert(self.length == 0);
522		}
523
524		if (form_3byte) {
525		// First byte
526		buf[0] = 0xc4;
527		// Second byte
528		const rxb_mask: u3 = @intCast(u3, @boolToInt(!self.rex_prefix.r)) << 2 \|
529		@intCast(u2, @boolToInt(!self.rex_prefix.x)) << 1 \|
530		@boolToInt(!self.rex_prefix.b);
531		buf[1] \|= @intCast(u8, rxb_mask) << 5;
532		buf[1] \|= self.lead_opc;
533		// Third byte
534		buf[2] \|= @intCast(u8, @boolToInt(!self.rex_prefix.w)) << 7;
535		buf[2] \|= @intCast(u7, self.register) << 3;
536		buf[2] \|= @intCast(u3, self.length) << 2;
537		buf[2] \|= self.simd_prefix;
538		} else {
539		// First byte
540		buf[0] = 0xc5;
541		// Second byte
542		buf[1] \|= @intCast(u8, @boolToInt(!self.rex_prefix.r)) << 7;
543		buf[1] \|= @intCast(u7, self.register) << 3;
544		buf[1] \|= @intCast(u3, self.length) << 2;
545		buf[1] \|= self.simd_prefix;
546		}
547
548		const count: usize = if (form_3byte) 3 else 2;
549		_ = writer.writeAll(buf[0..count]) catch unreachable;
550		return count;
551	447	}
552	448	};
553	449
554		pub fn vex(self: Self, prefix: Vex) void {
555		_ = prefix.write(self.code.writer());
556		}
557
558		/// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB
559		pub const Rex = struct {
560		/// Wide, enables 64-bit operation
561		w: bool = false,
562		/// Extends the reg field in the ModR/M byte
563		r: bool = false,
564		/// Extends the index field in the SIB byte
565		x: bool = false,
566		/// Extends the r/m field in the ModR/M byte,
567		/// or the base field in the SIB byte,
568		/// or the reg field in the Opcode byte
569		b: bool = false,
	450	pub const Sib = struct {
	451	ptr_size: PtrSize,
	452	base: ?Register,
	453	scale_index: ?ScaleIndex,
	454	disp: i32,
570	455	};
571	456
572		/// Encodes a REX prefix byte given all the fields
573		///
574		/// Use this byte whenever you need 64 bit operation,
575		/// or one of reg, index, r/m, base, or opcode-reg might be extended.
576		///
577		/// See struct `Rex` for a description of each field.
578		///
579		/// Does not add a prefix byte if none of the fields are set!
580		pub fn rex(self: Self, byte: Rex) void {
581		var value: u8 = 0b0100_0000;
582
583		if (byte.w) value \|= 0b1000;
584		if (byte.r) value \|= 0b0100;
585		if (byte.x) value \|= 0b0010;
586		if (byte.b) value \|= 0b0001;
587
588		if (value != 0b0100_0000) {
589		self.code.appendAssumeCapacity(value);
590		}
591		}
592
593		// ------
594		// Opcode
595		// ------
596
597		/// Encodes a 1 byte opcode
598		pub fn opcode_1byte(self: Self, opcode: u8) void {
599		self.code.appendAssumeCapacity(opcode);
600		}
601
602		/// Encodes a 2 byte opcode
603		///
604		/// e.g. IMUL has the opcode 0x0f 0xaf, so you use
605		///
606		/// encoder.opcode_2byte(0x0f, 0xaf);
607		pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) void {
608		self.code.appendAssumeCapacity(prefix);
609		self.code.appendAssumeCapacity(opcode);
610		}
611
612		/// Encodes a 3 byte opcode
613		///
614		/// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
615		///
616		/// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
617		pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) void {
618		self.code.appendAssumeCapacity(prefix_1);
619		self.code.appendAssumeCapacity(prefix_2);
620		self.code.appendAssumeCapacity(opcode);
621		}
622
623		/// Encodes a 1 byte opcode with a reg field
624		///
625		/// Remember to add a REX prefix byte if reg is extended!
626		pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) void {
627		assert(opcode & 0b111 == 0);
628		self.code.appendAssumeCapacity(opcode \| reg);
629		}
630
631		// ------
632		// ModR/M
633		// ------
634
635		/// Construct a ModR/M byte given all the fields
636		///
637		/// Remember to add a REX prefix byte if reg or rm are extended!
638		pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) void {
639		self.code.appendAssumeCapacity(
640		@as(u8, mod) << 6 \| @as(u8, reg_or_opx) << 3 \| rm,
641		);
642		}
643
644		/// Construct a ModR/M byte using direct r/m addressing
645		/// r/m effective address: r/m
646		///
647		/// Note reg's effective address is always just reg for the ModR/M byte.
648		/// Remember to add a REX prefix byte if reg or rm are extended!
649		pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) void {
650		self.modRm(0b11, reg_or_opx, rm);
651		}
652
653		/// Construct a ModR/M byte using indirect r/m addressing
654		/// r/m effective address: [r/m]
655		///
656		/// Note reg's effective address is always just reg for the ModR/M byte.
657		/// Remember to add a REX prefix byte if reg or rm are extended!
658		pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) void {
659		assert(rm != 4 and rm != 5);
660		self.modRm(0b00, reg_or_opx, rm);
661		}
662
663		/// Construct a ModR/M byte using indirect SIB addressing
664		/// r/m effective address: [SIB]
665		///
666		/// Note reg's effective address is always just reg for the ModR/M byte.
667		/// Remember to add a REX prefix byte if reg or rm are extended!
668		pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) void {
669		self.modRm(0b00, reg_or_opx, 0b100);
670		}
671
672		/// Construct a ModR/M byte using RIP-relative addressing
673		/// r/m effective address: [RIP + disp32]
674		///
675		/// Note reg's effective address is always just reg for the ModR/M byte.
676		/// Remember to add a REX prefix byte if reg or rm are extended!
677		pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) void {
678		self.modRm(0b00, reg_or_opx, 0b101);
679		}
680
681		/// Construct a ModR/M byte using indirect r/m with a 8bit displacement
682		/// r/m effective address: [r/m + disp8]
683		///
684		/// Note reg's effective address is always just reg for the ModR/M byte.
685		/// Remember to add a REX prefix byte if reg or rm are extended!
686		pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) void {
687		assert(rm != 4);
688		self.modRm(0b01, reg_or_opx, rm);
689		}
690
691		/// Construct a ModR/M byte using indirect SIB with a 8bit displacement
692		/// r/m effective address: [SIB + disp8]
693		///
694		/// Note reg's effective address is always just reg for the ModR/M byte.
695		/// Remember to add a REX prefix byte if reg or rm are extended!
696		pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) void {
697		self.modRm(0b01, reg_or_opx, 0b100);
698		}
699
700		/// Construct a ModR/M byte using indirect r/m with a 32bit displacement
701		/// r/m effective address: [r/m + disp32]
702		///
703		/// Note reg's effective address is always just reg for the ModR/M byte.
704		/// Remember to add a REX prefix byte if reg or rm are extended!
705		pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) void {
706		assert(rm != 4);
707		self.modRm(0b10, reg_or_opx, rm);
708		}
709
710		/// Construct a ModR/M byte using indirect SIB with a 32bit displacement
711		/// r/m effective address: [SIB + disp32]
712		///
713		/// Note reg's effective address is always just reg for the ModR/M byte.
714		/// Remember to add a REX prefix byte if reg or rm are extended!
715		pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) void {
716		self.modRm(0b10, reg_or_opx, 0b100);
717		}
718
719		// ---
720		// SIB
721		// ---
722
723		/// Construct a SIB byte given all the fields
724		///
725		/// Remember to add a REX prefix byte if index or base are extended!
726		pub fn sib(self: Self, scale: u2, index: u3, base: u3) void {
727		self.code.appendAssumeCapacity(
728		@as(u8, scale) << 6 \| @as(u8, index) << 3 \| base,
729		);
730		}
731
732		/// Construct a SIB byte with scale * index + base, no frills.
733		/// r/m effective address: [base + scale * index]
734		///
735		/// Remember to add a REX prefix byte if index or base are extended!
736		pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) void {
737		assert(base != 5);
738
739		self.sib(scale, index, base);
740		}
741
742		/// Construct a SIB byte with scale * index + disp32
743		/// r/m effective address: [scale * index + disp32]
744		///
745		/// Remember to add a REX prefix byte if index or base are extended!
746		pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) void {
747		assert(index != 4);
748
749		// scale is actually ignored
750		// index = 4 means no index
751		// base = 5 means no base, if mod == 0.
752		self.sib(scale, index, 5);
753		}
754
755		/// Construct a SIB byte with just base
756		/// r/m effective address: [base]
757		///
758		/// Remember to add a REX prefix byte if index or base are extended!
759		pub fn sib_base(self: Self, base: u3) void {
760		assert(base != 5);
761
762		// scale is actually ignored
763		// index = 4 means no index
764		self.sib(0, 4, base);
765		}
766
767		/// Construct a SIB byte with just disp32
768		/// r/m effective address: [disp32]
769		///
770		/// Remember to add a REX prefix byte if index or base are extended!
771		pub fn sib_disp32(self: Self) void {
772		// scale is actually ignored
773		// index = 4 means no index
774		// base = 5 means no base, if mod == 0.
775		self.sib(0, 4, 5);
776		}
777
778		/// Construct a SIB byte with scale * index + base + disp8
779		/// r/m effective address: [base + scale * index + disp8]
780		///
781		/// Remember to add a REX prefix byte if index or base are extended!
782		pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) void {
783		self.sib(scale, index, base);
784		}
785
786		/// Construct a SIB byte with base + disp8, no index
787		/// r/m effective address: [base + disp8]
788		///
789		/// Remember to add a REX prefix byte if index or base are extended!
790		pub fn sib_baseDisp8(self: Self, base: u3) void {
791		// scale is ignored
792		// index = 4 means no index
793		self.sib(0, 4, base);
794		}
	457	pub const Rip = struct {
	458	ptr_size: PtrSize,
	459	disp: i32,
	460	};
795	461
796		/// Construct a SIB byte with scale * index + base + disp32
797		/// r/m effective address: [base + scale * index + disp32]
798		///
799		/// Remember to add a REX prefix byte if index or base are extended!
800		pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) void {
801		self.sib(scale, index, base);
802		}
	462	pub const Moffs = struct {
	463	seg: Register,
	464	offset: u64,
	465	};
803	466
804		/// Construct a SIB byte with base + disp32, no index
805		/// r/m effective address: [base + disp32]
806		///
807		/// Remember to add a REX prefix byte if index or base are extended!
808		pub fn sib_baseDisp32(self: Self, base: u3) void {
809		// scale is ignored
810		// index = 4 means no index
811		self.sib(0, 4, base);
	467	pub fn moffs(reg: Register, offset: u64) Memory {
	468	assert(reg.class() == .segment);
	469	return .{ .moffs = .{ .seg = reg, .offset = offset } };
812	470	}
813	471
814		// -------------------------
815		// Trivial (no bit fiddling)
816		// -------------------------
817
818		/// Encode an 8 bit immediate
819		///
820		/// It is sign-extended to 64 bits by the cpu.
821		pub fn imm8(self: Self, imm: i8) void {
822		self.code.appendAssumeCapacity(@bitCast(u8, imm));
	472	pub fn sib(ptr_size: PtrSize, args: struct {
	473	disp: i32,
	474	base: ?Register = null,
	475	scale_index: ?ScaleIndex = null,
	476	}) Memory {
	477	return .{ .sib = .{
	478	.base = args.base,
	479	.disp = args.disp,
	480	.ptr_size = ptr_size,
	481	.scale_index = args.scale_index,
	482	} };
823	483	}
824	484
825		/// Encode an 8 bit displacement
826		///
827		/// It is sign-extended to 64 bits by the cpu.
828		pub fn disp8(self: Self, disp: i8) void {
829		self.code.appendAssumeCapacity(@bitCast(u8, disp));
	485	pub fn rip(ptr_size: PtrSize, disp: i32) Memory {
	486	return .{ .rip = .{ .ptr_size = ptr_size, .disp = disp } };
830	487	}
831	488
832		/// Encode an 16 bit immediate
833		///
834		/// It is sign-extended to 64 bits by the cpu.
835		pub fn imm16(self: Self, imm: i16) void {
836		self.writeIntLittle(i16, imm);
	489	pub fn isSegmentRegister(mem: Memory) bool {
	490	return switch (mem) {
	491	.moffs => true,
	492	.rip => false,
	493	.sib => \|s\| if (s.base) \|r\| r.class() == .segment else false,
	494	};
837	495	}
838	496
839		/// Encode an 32 bit immediate
840		///
841		/// It is sign-extended to 64 bits by the cpu.
842		pub fn imm32(self: Self, imm: i32) void {
843		self.writeIntLittle(i32, imm);
	497	pub fn base(mem: Memory) ?Register {
	498	return switch (mem) {
	499	.moffs => \|m\| m.seg,
	500	.sib => \|s\| s.base,
	501	.rip => null,
	502	};
844	503	}
845	504
846		/// Encode an 32 bit displacement
847		///
848		/// It is sign-extended to 64 bits by the cpu.
849		pub fn disp32(self: Self, disp: i32) void {
850		self.writeIntLittle(i32, disp);
	505	pub fn scaleIndex(mem: Memory) ?ScaleIndex {
	506	return switch (mem) {
	507	.moffs, .rip => null,
	508	.sib => \|s\| s.scale_index,
	509	};
851	510	}
852	511
853		/// Encode an 64 bit immediate
854		///
855		/// It is sign-extended to 64 bits by the cpu.
856		pub fn imm64(self: Self, imm: u64) void {
857		self.writeIntLittle(u64, imm);
	512	pub fn bitSize(mem: Memory) u64 {
	513	return switch (mem) {
	514	.rip => \|r\| r.ptr_size.bitSize(),
	515	.sib => \|s\| s.ptr_size.bitSize(),
	516	.moffs => unreachable,
	517	};
858	518	}
859	519	};
860	520
861		test "Encoder helpers - general purpose registers" {
862		var code = ArrayList(u8).init(testing.allocator);
863		defer code.deinit();
864
865		// simple integer multiplication
866
867		// imul eax,edi
868		// 0faf c7
869		{
870		try code.resize(0);
871		const encoder = try Encoder.init(&code, 4);
872		encoder.rex(.{
873		.r = Register.eax.isExtended(),
874		.b = Register.edi.isExtended(),
875		});
876		encoder.opcode_2byte(0x0f, 0xaf);
877		encoder.modRm_direct(
878		Register.eax.lowEnc(),
879		Register.edi.lowEnc(),
880		);
881
882		try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);
883		}
884
885		// simple mov
886
887		// mov eax,edi
888		// 89 f8
889		{
890		try code.resize(0);
891		const encoder = try Encoder.init(&code, 3);
892		encoder.rex(.{
893		.r = Register.edi.isExtended(),
894		.b = Register.eax.isExtended(),
895		});
896		encoder.opcode_1byte(0x89);
897		encoder.modRm_direct(
898		Register.edi.lowEnc(),
899		Register.eax.lowEnc(),
900		);
901
902		try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);
903		}
904
905		// signed integer addition of 32-bit sign extended immediate to 64 bit register
906
907		// add rcx, 2147483647
908		//
909		// Using the following opcode: REX.W + 81 /0 id, we expect the following encoding
910		//
911		// 48 : REX.W set for 64 bit operand (rcx)
912		// 81 : opcode for "<arithmetic> with immediate"
913		// c1 : id = rcx,
914		// : c1 = 11 <-- mod = 11 indicates r/m is register (rcx)
915		// : 000 <-- opcode_extension = 0 because opcode extension is /0. /0 specifies ADD
916		// : 001 <-- 001 is rcx
917		// ffffff7f : 2147483647
918		{
919		try code.resize(0);
920		const encoder = try Encoder.init(&code, 7);
921		encoder.rex(.{ .w = true }); // use 64 bit operation
922		encoder.opcode_1byte(0x81);
923		encoder.modRm_direct(
924		0,
925		Register.rcx.lowEnc(),
926		);
927		encoder.imm32(2147483647);
928
929		try testing.expectEqualSlices(u8, &[_]u8{ 0x48, 0x81, 0xc1, 0xff, 0xff, 0xff, 0x7f }, code.items);
930		}
931		}
932
933		test "Encoder helpers - Vex prefix" {
934		var buf: [3]u8 = undefined;
935		var stream = std.io.fixedBufferStream(&buf);
936		const writer = stream.writer();
937
938		{
939		var vex_prefix = Encoder.Vex{};
940		vex_prefix.rex(.{
941		.r = true,
942		});
943		const nwritten = vex_prefix.write(writer);
944		try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, buf[0..nwritten]);
945		}
946
947		{
948		stream.reset();
949		var vex_prefix = Encoder.Vex{};
950		vex_prefix.reg(Register.xmm15.enc());
951		const nwritten = vex_prefix.write(writer);
952		try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]);
953		}
	521	pub const Immediate = union(enum) {
	522	signed: i32,
	523	unsigned: u64,
954	524
955		{
956		stream.reset();
957		var vex_prefix = Encoder.Vex{};
958		vex_prefix.rex(.{
959		.w = true,
960		.x = true,
961		});
962		const nwritten = vex_prefix.write(writer);
963		try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, buf[0..nwritten]);
	525	pub fn u(x: u64) Immediate {
	526	return .{ .unsigned = x };
964	527	}
965	528
966		{
967		stream.reset();
968		var vex_prefix = Encoder.Vex{};
969		vex_prefix.rex(.{
970		.w = true,
971		.r = true,
972		});
973		vex_prefix.len_256();
974		vex_prefix.lead_opc_0f();
975		vex_prefix.simd_prefix_66();
976		const nwritten = vex_prefix.write(writer);
977		try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, buf[0..nwritten]);
	529	pub fn s(x: i32) Immediate {
	530	return .{ .signed = x };
978	531	}
979	532
980		var code = ArrayList(u8).init(testing.allocator);
981		defer code.deinit();
982
983		{
984		// vmovapd xmm1, xmm2
985		const encoder = try Encoder.init(&code, 4);
986		var vex = Encoder.Vex{};
987		vex.simd_prefix_66();
988		encoder.vex(vex); // use 64 bit operation
989		encoder.opcode_1byte(0x28);
990		encoder.modRm_direct(0, Register.xmm1.lowEnc());
991		try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items);
992		}
993
994		{
995		try code.resize(0);
996
997		// vmovhpd xmm13, xmm1, qword ptr [rip]
998		const encoder = try Encoder.init(&code, 9);
999		var vex = Encoder.Vex{};
1000		vex.len_128();
1001		vex.simd_prefix_66();
1002		vex.lead_opc_0f();
1003		vex.rex(.{ .r = true });
1004		vex.reg(Register.xmm1.enc());
1005		encoder.vex(vex);
1006		encoder.opcode_1byte(0x16);
1007		encoder.modRm_RIPDisp32(Register.xmm13.lowEnc());
1008		encoder.disp32(0);
1009		try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items);
	533	pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 {
	534	return switch (imm) {
	535	.signed => \|x\| switch (bit_size) {
	536	1, 8 => @bitCast(u8, @intCast(i8, x)),
	537	16 => @bitCast(u16, @intCast(i16, x)),
	538	32, 64 => @bitCast(u32, x),
	539	else => unreachable,
	540	},
	541	.unsigned => \|x\| switch (bit_size) {
	542	1, 8 => @intCast(u8, x),
	543	16 => @intCast(u16, x),
	544	32 => @intCast(u32, x),
	545	64 => x,
	546	else => unreachable,
	547	},
	548	};
1010	549	}
1011		}
1012
1013		// TODO add these registers to the enum and populate dwarfLocOp
1014		// // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register.
1015		// RA = (16, "RA"),
1016		//
1017		// XMM0 = (17, "xmm0"),
1018		// XMM1 = (18, "xmm1"),
1019		// XMM2 = (19, "xmm2"),
1020		// XMM3 = (20, "xmm3"),
1021		// XMM4 = (21, "xmm4"),
1022		// XMM5 = (22, "xmm5"),
1023		// XMM6 = (23, "xmm6"),
1024		// XMM7 = (24, "xmm7"),
1025		//
1026		// XMM8 = (25, "xmm8"),
1027		// XMM9 = (26, "xmm9"),
1028		// XMM10 = (27, "xmm10"),
1029		// XMM11 = (28, "xmm11"),
1030		// XMM12 = (29, "xmm12"),
1031		// XMM13 = (30, "xmm13"),
1032		// XMM14 = (31, "xmm14"),
1033		// XMM15 = (32, "xmm15"),
1034		//
1035		// ST0 = (33, "st0"),
1036		// ST1 = (34, "st1"),
1037		// ST2 = (35, "st2"),
1038		// ST3 = (36, "st3"),
1039		// ST4 = (37, "st4"),
1040		// ST5 = (38, "st5"),
1041		// ST6 = (39, "st6"),
1042		// ST7 = (40, "st7"),
1043		//
1044		// MM0 = (41, "mm0"),
1045		// MM1 = (42, "mm1"),
1046		// MM2 = (43, "mm2"),
1047		// MM3 = (44, "mm3"),
1048		// MM4 = (45, "mm4"),
1049		// MM5 = (46, "mm5"),
1050		// MM6 = (47, "mm6"),
1051		// MM7 = (48, "mm7"),
1052		//
1053		// RFLAGS = (49, "rFLAGS"),
1054		// ES = (50, "es"),
1055		// CS = (51, "cs"),
1056		// SS = (52, "ss"),
1057		// DS = (53, "ds"),
1058		// FS = (54, "fs"),
1059		// GS = (55, "gs"),
1060		//
1061		// FS_BASE = (58, "fs.base"),
1062		// GS_BASE = (59, "gs.base"),
1063		//
1064		// TR = (62, "tr"),
1065		// LDTR = (63, "ldtr"),
1066		// MXCSR = (64, "mxcsr"),
1067		// FCW = (65, "fcw"),
1068		// FSW = (66, "fsw"),
1069		//
1070		// XMM16 = (67, "xmm16"),
1071		// XMM17 = (68, "xmm17"),
1072		// XMM18 = (69, "xmm18"),
1073		// XMM19 = (70, "xmm19"),
1074		// XMM20 = (71, "xmm20"),
1075		// XMM21 = (72, "xmm21"),
1076		// XMM22 = (73, "xmm22"),
1077		// XMM23 = (74, "xmm23"),
1078		// XMM24 = (75, "xmm24"),
1079		// XMM25 = (76, "xmm25"),
1080		// XMM26 = (77, "xmm26"),
1081		// XMM27 = (78, "xmm27"),
1082		// XMM28 = (79, "xmm28"),
1083		// XMM29 = (80, "xmm29"),
1084		// XMM30 = (81, "xmm30"),
1085		// XMM31 = (82, "xmm31"),
1086		//
1087		// K0 = (118, "k0"),
1088		// K1 = (119, "k1"),
1089		// K2 = (120, "k2"),
1090		// K3 = (121, "k3"),
1091		// K4 = (122, "k4"),
1092		// K5 = (123, "k5"),
1093		// K6 = (124, "k6"),
1094		// K7 = (125, "k7"),
	550	};

src/arch/x86_64/encoder.zig created+2275

...	...	@@ -0,0 +1,2275 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3	const log = std.log.scoped(.x86_64_encoder);
	4	const math = std.math;
	5	const testing = std.testing;
	6
	7	const bits = @import("bits.zig");
	8	const Encoding = @import("Encoding.zig");
	9	const Immediate = bits.Immediate;
	10	const Memory = bits.Memory;
	11	const Register = bits.Register;
	12
	13	pub const Instruction = struct {
	14	op1: Operand = .none,
	15	op2: Operand = .none,
	16	op3: Operand = .none,
	17	op4: Operand = .none,
	18	encoding: Encoding,
	19
	20	pub const Mnemonic = Encoding.Mnemonic;
	21
	22	pub const Operand = union(enum) {
	23	none,
	24	reg: Register,
	25	mem: Memory,
	26	imm: Immediate,
	27
	28	/// Returns the bitsize of the operand.
	29	pub fn bitSize(op: Operand) u64 {
	30	return switch (op) {
	31	.none => unreachable,
	32	.reg => \|reg\| reg.bitSize(),
	33	.mem => \|mem\| mem.bitSize(),
	34	.imm => unreachable,
	35	};
	36	}
	37
	38	/// Returns true if the operand is a segment register.
	39	/// Asserts the operand is either register or memory.
	40	pub fn isSegmentRegister(op: Operand) bool {
	41	return switch (op) {
	42	.none => unreachable,
	43	.reg => \|reg\| reg.class() == .segment,
	44	.mem => \|mem\| mem.isSegmentRegister(),
	45	.imm => unreachable,
	46	};
	47	}
	48
	49	pub fn fmtPrint(op: Operand, enc_op: Encoding.Op, writer: anytype) !void {
	50	switch (op) {
	51	.none => {},
	52	.reg => \|reg\| try writer.writeAll(@tagName(reg)),
	53	.mem => \|mem\| switch (mem) {
	54	.rip => \|rip\| {
	55	try writer.print("{s} ptr [rip", .{@tagName(rip.ptr_size)});
	56	if (rip.disp != 0) {
	57	const sign_bit = if (sign(rip.disp) < 0) "-" else "+";
	58	const disp_abs = try std.math.absInt(rip.disp);
	59	try writer.print(" {s} 0x{x}", .{ sign_bit, disp_abs });
	60	}
	61	try writer.writeByte(']');
	62	},
	63	.sib => \|sib\| {
	64	try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});
	65
	66	if (mem.isSegmentRegister()) {
	67	return writer.print("{s}:0x{x}", .{ @tagName(sib.base.?), sib.disp });
	68	}
	69
	70	try writer.writeByte('[');
	71
	72	if (sib.base) \|base\| {
	73	try writer.print("{s}", .{@tagName(base)});
	74	}
	75	if (sib.scale_index) \|si\| {
	76	if (sib.base != null) {
	77	try writer.writeAll(" + ");
	78	}
	79	try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });
	80	}
	81	if (sib.disp != 0) {
	82	if (sib.base != null or sib.scale_index != null) {
	83	try writer.writeByte(' ');
	84	}
	85	try writer.writeByte(if (sign(sib.disp) < 0) '-' else '+');
	86	const disp_abs = try std.math.absInt(sib.disp);
	87	try writer.print(" 0x{x}", .{disp_abs});
	88	}
	89
	90	try writer.writeByte(']');
	91	},
	92	.moffs => \|moffs\| try writer.print("{s}:0x{x}", .{ @tagName(moffs.seg), moffs.offset }),
	93	},
	94	.imm => \|imm\| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.bitSize())}),
	95	}
	96	}
	97	};
	98
	99	pub fn new(mnemonic: Mnemonic, args: struct {
	100	op1: Operand = .none,
	101	op2: Operand = .none,
	102	op3: Operand = .none,
	103	op4: Operand = .none,
	104	}) !Instruction {
	105	const encoding = (try Encoding.findByMnemonic(mnemonic, .{
	106	.op1 = args.op1,
	107	.op2 = args.op2,
	108	.op3 = args.op3,
	109	.op4 = args.op4,
	110	})) orelse {
	111	log.debug("no encoding found for: {s} {s} {s} {s} {s}", .{
	112	@tagName(mnemonic),
	113	@tagName(Encoding.Op.fromOperand(args.op1)),
	114	@tagName(Encoding.Op.fromOperand(args.op2)),
	115	@tagName(Encoding.Op.fromOperand(args.op3)),
	116	@tagName(Encoding.Op.fromOperand(args.op4)),
	117	});
	118	return error.InvalidInstruction;
	119	};
	120	log.debug("selected encoding: {}", .{encoding});
	121	return .{
	122	.op1 = args.op1,
	123	.op2 = args.op2,
	124	.op3 = args.op3,
	125	.op4 = args.op4,
	126	.encoding = encoding,
	127	};
	128	}
	129
	130	pub fn fmtPrint(inst: Instruction, writer: anytype) !void {
	131	try writer.print("{s}", .{@tagName(inst.encoding.mnemonic)});
	132	const ops = [_]struct { Operand, Encoding.Op }{
	133	.{ inst.op1, inst.encoding.op1 },
	134	.{ inst.op2, inst.encoding.op2 },
	135	.{ inst.op3, inst.encoding.op3 },
	136	.{ inst.op4, inst.encoding.op4 },
	137	};
	138	for (&ops, 0..) \|op, i\| {
	139	if (op[0] == .none) break;
	140	if (i > 0) {
	141	try writer.writeByte(',');
	142	}
	143	try writer.writeByte(' ');
	144	try op[0].fmtPrint(op[1], writer);
	145	}
	146	}
	147
	148	pub fn encode(inst: Instruction, writer: anytype) !void {
	149	const encoder = Encoder(@TypeOf(writer)){ .writer = writer };
	150	const encoding = inst.encoding;
	151
	152	try inst.encodeLegacyPrefixes(encoder);
	153	try inst.encodeMandatoryPrefix(encoder);
	154	try inst.encodeRexPrefix(encoder);
	155	try inst.encodeOpcode(encoder);
	156
	157	switch (encoding.op_en) {
	158	.np, .o => {},
	159	.i, .d => try encodeImm(inst.op1.imm, encoding.op1, encoder),
	160	.zi, .oi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
	161	.fd => try encoder.imm64(inst.op2.mem.moffs.offset),
	162	.td => try encoder.imm64(inst.op1.mem.moffs.offset),
	163	else => {
	164	const mem_op = switch (encoding.op_en) {
	165	.m, .mi, .m1, .mc, .mr => inst.op1,
	166	.rm, .rmi => inst.op2,
	167	else => unreachable,
	168	};
	169	switch (mem_op) {
	170	.reg => \|reg\| {
	171	const rm = switch (encoding.op_en) {
	172	.m, .mi, .m1, .mc => encoding.modRmExt(),
	173	.mr => inst.op2.reg.lowEnc(),
	174	.rm, .rmi => inst.op1.reg.lowEnc(),
	175	else => unreachable,
	176	};
	177	try encoder.modRm_direct(rm, reg.lowEnc());
	178	},
	179	.mem => \|mem\| {
	180	const op = switch (encoding.op_en) {
	181	.m, .mi, .m1, .mc => .none,
	182	.mr => inst.op2,
	183	.rm, .rmi => inst.op1,
	184	else => unreachable,
	185	};
	186	try encodeMemory(encoding, mem, op, encoder);
	187	},
	188	else => unreachable,
	189	}
	190
	191	switch (encoding.op_en) {
	192	.mi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
	193	.rmi => try encodeImm(inst.op3.imm, encoding.op3, encoder),
	194	else => {},
	195	}
	196	},
	197	}
	198	}
	199
	200	fn encodeOpcode(inst: Instruction, encoder: anytype) !void {
	201	const opcode = inst.encoding.opcode();
	202	switch (inst.encoding.op_en) {
	203	.o, .oi => try encoder.opcode_withReg(opcode[0], inst.op1.reg.lowEnc()),
	204	else => {
	205	const index: usize = if (inst.encoding.mandatoryPrefix()) \|_\| 1 else 0;
	206	for (opcode[index..]) \|byte\| {
	207	try encoder.opcode_1byte(byte);
	208	}
	209	},
	210	}
	211	}
	212
	213	fn encodeLegacyPrefixes(inst: Instruction, encoder: anytype) !void {
	214	const enc = inst.encoding;
	215	const op_en = enc.op_en;
	216
	217	var legacy = LegacyPrefixes{};
	218	if (enc.mode == .none) {
	219	const bit_size = enc.operandBitSize();
	220	if (bit_size == 16) {
	221	legacy.set16BitOverride();
	222	}
	223	}
	224
	225	const segment_override: ?Register = switch (op_en) {
	226	.i, .zi, .o, .oi, .d, .np => null,
	227	.fd => inst.op2.mem.base().?,
	228	.td => inst.op1.mem.base().?,
	229	.rm, .rmi => if (inst.op2.isSegmentRegister()) blk: {
	230	break :blk switch (inst.op2) {
	231	.reg => \|r\| r,
	232	.mem => \|m\| m.base().?,
	233	else => unreachable,
	234	};
	235	} else null,
	236	.m, .mi, .m1, .mc, .mr => if (inst.op1.isSegmentRegister()) blk: {
	237	break :blk switch (inst.op1) {
	238	.reg => \|r\| r,
	239	.mem => \|m\| m.base().?,
	240	else => unreachable,
	241	};
	242	} else null,
	243	};
	244	if (segment_override) \|seg\| {
	245	legacy.setSegmentOverride(seg);
	246	}
	247
	248	try encoder.legacyPrefixes(legacy);
	249	}
	250
	251	fn encodeRexPrefix(inst: Instruction, encoder: anytype) !void {
	252	const op_en = inst.encoding.op_en;
	253
	254	var rex = Rex{};
	255	rex.present = inst.encoding.mode == .rex;
	256	rex.w = inst.encoding.mode == .long;
	257
	258	switch (op_en) {
	259	.np, .i, .zi, .fd, .td, .d => {},
	260	.o, .oi => {
	261	rex.b = inst.op1.reg.isExtended();
	262	},
	263	.m, .mi, .m1, .mc, .mr, .rm, .rmi => {
	264	const r_op = switch (op_en) {
	265	.rm, .rmi => inst.op1,
	266	.mr => inst.op2,
	267	else => null,
	268	};
	269	if (r_op) \|op\| {
	270	rex.r = op.reg.isExtended();
	271	}
	272
	273	const b_x_op = switch (op_en) {
	274	.rm, .rmi => inst.op2,
	275	.m, .mi, .m1, .mc, .mr => inst.op1,
	276	else => unreachable,
	277	};
	278	switch (b_x_op) {
	279	.reg => \|r\| {
	280	rex.b = r.isExtended();
	281	},
	282	.mem => \|mem\| {
	283	rex.b = if (mem.base()) \|base\| base.isExtended() else false;
	284	rex.x = if (mem.scaleIndex()) \|si\| si.index.isExtended() else false;
	285	},
	286	else => unreachable,
	287	}
	288	},
	289	}
	290
	291	try encoder.rex(rex);
	292	}
	293
	294	fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void {
	295	const prefix = inst.encoding.mandatoryPrefix() orelse return;
	296	try encoder.opcode_1byte(prefix);
	297	}
	298
	299	fn encodeMemory(encoding: Encoding, mem: Memory, operand: Operand, encoder: anytype) !void {
	300	const operand_enc = switch (operand) {
	301	.reg => \|reg\| reg.lowEnc(),
	302	.none => encoding.modRmExt(),
	303	else => unreachable,
	304	};
	305
	306	switch (mem) {
	307	.moffs => unreachable,
	308	.sib => \|sib\| {
	309	if (sib.base) \|base\| {
	310	if (base.class() == .segment) {
	311	// TODO audit this wrt SIB
	312	try encoder.modRm_SIBDisp0(operand_enc);
	313	if (sib.scale_index) \|si\| {
	314	const scale = math.log2_int(u4, si.scale);
	315	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
	316	} else {
	317	try encoder.sib_disp32();
	318	}
	319	try encoder.disp32(sib.disp);
	320	} else {
	321	assert(base.class() == .general_purpose);
	322	const dst = base.lowEnc();
	323	const src = operand_enc;
	324	if (dst == 4 or sib.scale_index != null) {
	325	if (sib.disp == 0 and dst != 5) {
	326	try encoder.modRm_SIBDisp0(src);
	327	if (sib.scale_index) \|si\| {
	328	const scale = math.log2_int(u4, si.scale);
	329	try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
	330	} else {
	331	try encoder.sib_base(dst);
	332	}
	333	} else if (math.cast(i8, sib.disp)) \|_\| {
	334	try encoder.modRm_SIBDisp8(src);
	335	if (sib.scale_index) \|si\| {
	336	const scale = math.log2_int(u4, si.scale);
	337	try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
	338	} else {
	339	try encoder.sib_baseDisp8(dst);
	340	}
	341	try encoder.disp8(@truncate(i8, sib.disp));
	342	} else {
	343	try encoder.modRm_SIBDisp32(src);
	344	if (sib.scale_index) \|si\| {
	345	const scale = math.log2_int(u4, si.scale);
	346	try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
	347	} else {
	348	try encoder.sib_baseDisp32(dst);
	349	}
	350	try encoder.disp32(sib.disp);
	351	}
	352	} else {
	353	if (sib.disp == 0 and dst != 5) {
	354	try encoder.modRm_indirectDisp0(src, dst);
	355	} else if (math.cast(i8, sib.disp)) \|_\| {
	356	try encoder.modRm_indirectDisp8(src, dst);
	357	try encoder.disp8(@truncate(i8, sib.disp));
	358	} else {
	359	try encoder.modRm_indirectDisp32(src, dst);
	360	try encoder.disp32(sib.disp);
	361	}
	362	}
	363	}
	364	} else {
	365	try encoder.modRm_SIBDisp0(operand_enc);
	366	if (sib.scale_index) \|si\| {
	367	const scale = math.log2_int(u4, si.scale);
	368	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
	369	} else {
	370	try encoder.sib_disp32();
	371	}
	372	try encoder.disp32(sib.disp);
	373	}
	374	},
	375	.rip => \|rip\| {
	376	try encoder.modRm_RIPDisp32(operand_enc);
	377	try encoder.disp32(rip.disp);
	378	},
	379	}
	380	}
	381
	382	fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void {
	383	const raw = imm.asUnsigned(kind.bitSize());
	384	switch (kind.bitSize()) {
	385	8 => try encoder.imm8(@intCast(u8, raw)),
	386	16 => try encoder.imm16(@intCast(u16, raw)),
	387	32 => try encoder.imm32(@intCast(u32, raw)),
	388	64 => try encoder.imm64(raw),
	389	else => unreachable,
	390	}
	391	}
	392	};
	393
	394	inline fn sign(i: anytype) @TypeOf(i) {
	395	return @as(@TypeOf(i), @boolToInt(i > 0)) - @boolToInt(i < 0);
	396	}
	397
	398	pub const LegacyPrefixes = packed struct {
	399	/// LOCK
	400	prefix_f0: bool = false,
	401	/// REPNZ, REPNE, REP, Scalar Double-precision
	402	prefix_f2: bool = false,
	403	/// REPZ, REPE, REP, Scalar Single-precision
	404	prefix_f3: bool = false,
	405
	406	/// CS segment override or Branch not taken
	407	prefix_2e: bool = false,
	408	/// SS segment override
	409	prefix_36: bool = false,
	410	/// ES segment override
	411	prefix_26: bool = false,
	412	/// FS segment override
	413	prefix_64: bool = false,
	414	/// GS segment override
	415	prefix_65: bool = false,
	416
	417	/// Branch taken
	418	prefix_3e: bool = false,
	419
	420	/// Address size override (enables 16 bit address size)
	421	prefix_67: bool = false,
	422
	423	/// Operand size override (enables 16 bit operation)
	424	prefix_66: bool = false,
	425
	426	padding: u5 = 0,
	427
	428	pub fn setSegmentOverride(self: *LegacyPrefixes, reg: Register) void {
	429	assert(reg.class() == .segment);
	430	switch (reg) {
	431	.cs => self.prefix_2e = true,
	432	.ss => self.prefix_36 = true,
	433	.es => self.prefix_26 = true,
	434	.fs => self.prefix_64 = true,
	435	.gs => self.prefix_65 = true,
	436	.ds => {},
	437	else => unreachable,
	438	}
	439	}
	440
	441	pub fn set16BitOverride(self: *LegacyPrefixes) void {
	442	self.prefix_66 = true;
	443	}
	444	};
	445
	446	fn Encoder(comptime T: type) type {
	447	return struct {
	448	writer: T,
	449
	450	const Self = @This();
	451
	452	// --------
	453	// Prefixes
	454	// --------
	455
	456	/// Encodes legacy prefixes
	457	pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) !void {
	458	if (@bitCast(u16, prefixes) != 0) {
	459	// Hopefully this path isn't taken very often, so we'll do it the slow way for now
	460
	461	// LOCK
	462	if (prefixes.prefix_f0) try self.writer.writeByte(0xf0);
	463	// REPNZ, REPNE, REP, Scalar Double-precision
	464	if (prefixes.prefix_f2) try self.writer.writeByte(0xf2);
	465	// REPZ, REPE, REP, Scalar Single-precision
	466	if (prefixes.prefix_f3) try self.writer.writeByte(0xf3);
	467
	468	// CS segment override or Branch not taken
	469	if (prefixes.prefix_2e) try self.writer.writeByte(0x2e);
	470	// DS segment override
	471	if (prefixes.prefix_36) try self.writer.writeByte(0x36);
	472	// ES segment override
	473	if (prefixes.prefix_26) try self.writer.writeByte(0x26);
	474	// FS segment override
	475	if (prefixes.prefix_64) try self.writer.writeByte(0x64);
	476	// GS segment override
	477	if (prefixes.prefix_65) try self.writer.writeByte(0x65);
	478
	479	// Branch taken
	480	if (prefixes.prefix_3e) try self.writer.writeByte(0x3e);
	481
	482	// Operand size override
	483	if (prefixes.prefix_66) try self.writer.writeByte(0x66);
	484
	485	// Address size override
	486	if (prefixes.prefix_67) try self.writer.writeByte(0x67);
	487	}
	488	}
	489
	490	/// Use 16 bit operand size
	491	///
	492	/// Note that this flag is overridden by REX.W, if both are present.
	493	pub fn prefix16BitMode(self: Self) !void {
	494	try self.writer.writeByte(0x66);
	495	}
	496
	497	/// Encodes a REX prefix byte given all the fields
	498	///
	499	/// Use this byte whenever you need 64 bit operation,
	500	/// or one of reg, index, r/m, base, or opcode-reg might be extended.
	501	///
	502	/// See struct `Rex` for a description of each field.
	503	pub fn rex(self: Self, byte: Rex) !void {
	504	if (!byte.present and !byte.isSet()) return;
	505
	506	var value: u8 = 0b0100_0000;
	507
	508	if (byte.w) value \|= 0b1000;
	509	if (byte.r) value \|= 0b0100;
	510	if (byte.x) value \|= 0b0010;
	511	if (byte.b) value \|= 0b0001;
	512
	513	try self.writer.writeByte(value);
	514	}
	515
	516	// ------
	517	// Opcode
	518	// ------
	519
	520	/// Encodes a 1 byte opcode
	521	pub fn opcode_1byte(self: Self, opcode: u8) !void {
	522	try self.writer.writeByte(opcode);
	523	}
	524
	525	/// Encodes a 2 byte opcode
	526	///
	527	/// e.g. IMUL has the opcode 0x0f 0xaf, so you use
	528	///
	529	/// encoder.opcode_2byte(0x0f, 0xaf);
	530	pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) !void {
	531	try self.writer.writeAll(&.{ prefix, opcode });
	532	}
	533
	534	/// Encodes a 3 byte opcode
	535	///
	536	/// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
	537	///
	538	/// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
	539	pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) !void {
	540	try self.writer.writeAll(&.{ prefix_1, prefix_2, opcode });
	541	}
	542
	543	/// Encodes a 1 byte opcode with a reg field
	544	///
	545	/// Remember to add a REX prefix byte if reg is extended!
	546	pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) !void {
	547	assert(opcode & 0b111 == 0);
	548	try self.writer.writeByte(opcode \| reg);
	549	}
	550
	551	// ------
	552	// ModR/M
	553	// ------
	554
	555	/// Construct a ModR/M byte given all the fields
	556	///
	557	/// Remember to add a REX prefix byte if reg or rm are extended!
	558	pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) !void {
	559	try self.writer.writeByte(@as(u8, mod) << 6 \| @as(u8, reg_or_opx) << 3 \| rm);
	560	}
	561
	562	/// Construct a ModR/M byte using direct r/m addressing
	563	/// r/m effective address: r/m
	564	///
	565	/// Note reg's effective address is always just reg for the ModR/M byte.
	566	/// Remember to add a REX prefix byte if reg or rm are extended!
	567	pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) !void {
	568	try self.modRm(0b11, reg_or_opx, rm);
	569	}
	570
	571	/// Construct a ModR/M byte using indirect r/m addressing
	572	/// r/m effective address: [r/m]
	573	///
	574	/// Note reg's effective address is always just reg for the ModR/M byte.
	575	/// Remember to add a REX prefix byte if reg or rm are extended!
	576	pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) !void {
	577	assert(rm != 4 and rm != 5);
	578	try self.modRm(0b00, reg_or_opx, rm);
	579	}
	580
	581	/// Construct a ModR/M byte using indirect SIB addressing
	582	/// r/m effective address: [SIB]
	583	///
	584	/// Note reg's effective address is always just reg for the ModR/M byte.
	585	/// Remember to add a REX prefix byte if reg or rm are extended!
	586	pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) !void {
	587	try self.modRm(0b00, reg_or_opx, 0b100);
	588	}
	589
	590	/// Construct a ModR/M byte using RIP-relative addressing
	591	/// r/m effective address: [RIP + disp32]
	592	///
	593	/// Note reg's effective address is always just reg for the ModR/M byte.
	594	/// Remember to add a REX prefix byte if reg or rm are extended!
	595	pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) !void {
	596	try self.modRm(0b00, reg_or_opx, 0b101);
	597	}
	598
	599	/// Construct a ModR/M byte using indirect r/m with a 8bit displacement
	600	/// r/m effective address: [r/m + disp8]
	601	///
	602	/// Note reg's effective address is always just reg for the ModR/M byte.
	603	/// Remember to add a REX prefix byte if reg or rm are extended!
	604	pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) !void {
	605	assert(rm != 4);
	606	try self.modRm(0b01, reg_or_opx, rm);
	607	}
	608
	609	/// Construct a ModR/M byte using indirect SIB with a 8bit displacement
	610	/// r/m effective address: [SIB + disp8]
	611	///
	612	/// Note reg's effective address is always just reg for the ModR/M byte.
	613	/// Remember to add a REX prefix byte if reg or rm are extended!
	614	pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) !void {
	615	try self.modRm(0b01, reg_or_opx, 0b100);
	616	}
	617
	618	/// Construct a ModR/M byte using indirect r/m with a 32bit displacement
	619	/// r/m effective address: [r/m + disp32]
	620	///
	621	/// Note reg's effective address is always just reg for the ModR/M byte.
	622	/// Remember to add a REX prefix byte if reg or rm are extended!
	623	pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) !void {
	624	assert(rm != 4);
	625	try self.modRm(0b10, reg_or_opx, rm);
	626	}
	627
	628	/// Construct a ModR/M byte using indirect SIB with a 32bit displacement
	629	/// r/m effective address: [SIB + disp32]
	630	///
	631	/// Note reg's effective address is always just reg for the ModR/M byte.
	632	/// Remember to add a REX prefix byte if reg or rm are extended!
	633	pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) !void {
	634	try self.modRm(0b10, reg_or_opx, 0b100);
	635	}
	636
	637	// ---
	638	// SIB
	639	// ---
	640
	641	/// Construct a SIB byte given all the fields
	642	///
	643	/// Remember to add a REX prefix byte if index or base are extended!
	644	pub fn sib(self: Self, scale: u2, index: u3, base: u3) !void {
	645	try self.writer.writeByte(@as(u8, scale) << 6 \| @as(u8, index) << 3 \| base);
	646	}
	647
	648	/// Construct a SIB byte with scale * index + base, no frills.
	649	/// r/m effective address: [base + scale * index]
	650	///
	651	/// Remember to add a REX prefix byte if index or base are extended!
	652	pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) !void {
	653	assert(base != 5);
	654
	655	try self.sib(scale, index, base);
	656	}
	657
	658	/// Construct a SIB byte with scale * index + disp32
	659	/// r/m effective address: [scale * index + disp32]
	660	///
	661	/// Remember to add a REX prefix byte if index or base are extended!
	662	pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) !void {
	663	// scale is actually ignored
	664	// index = 4 means no index if and only if we haven't extended the register
	665	// TODO enforce this
	666	// base = 5 means no base, if mod == 0.
	667	try self.sib(scale, index, 5);
	668	}
	669
	670	/// Construct a SIB byte with just base
	671	/// r/m effective address: [base]
	672	///
	673	/// Remember to add a REX prefix byte if index or base are extended!
	674	pub fn sib_base(self: Self, base: u3) !void {
	675	assert(base != 5);
	676
	677	// scale is actually ignored
	678	// index = 4 means no index
	679	try self.sib(0, 4, base);
	680	}
	681
	682	/// Construct a SIB byte with just disp32
	683	/// r/m effective address: [disp32]
	684	///
	685	/// Remember to add a REX prefix byte if index or base are extended!
	686	pub fn sib_disp32(self: Self) !void {
	687	// scale is actually ignored
	688	// index = 4 means no index
	689	// base = 5 means no base, if mod == 0.
	690	try self.sib(0, 4, 5);
	691	}
	692
	693	/// Construct a SIB byte with scale * index + base + disp8
	694	/// r/m effective address: [base + scale * index + disp8]
	695	///
	696	/// Remember to add a REX prefix byte if index or base are extended!
	697	pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) !void {
	698	try self.sib(scale, index, base);
	699	}
	700
	701	/// Construct a SIB byte with base + disp8, no index
	702	/// r/m effective address: [base + disp8]
	703	///
	704	/// Remember to add a REX prefix byte if index or base are extended!
	705	pub fn sib_baseDisp8(self: Self, base: u3) !void {
	706	// scale is ignored
	707	// index = 4 means no index
	708	try self.sib(0, 4, base);
	709	}
	710
	711	/// Construct a SIB byte with scale * index + base + disp32
	712	/// r/m effective address: [base + scale * index + disp32]
	713	///
	714	/// Remember to add a REX prefix byte if index or base are extended!
	715	pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) !void {
	716	try self.sib(scale, index, base);
	717	}
	718
	719	/// Construct a SIB byte with base + disp32, no index
	720	/// r/m effective address: [base + disp32]
	721	///
	722	/// Remember to add a REX prefix byte if index or base are extended!
	723	pub fn sib_baseDisp32(self: Self, base: u3) !void {
	724	// scale is ignored
	725	// index = 4 means no index
	726	try self.sib(0, 4, base);
	727	}
	728
	729	// -------------------------
	730	// Trivial (no bit fiddling)
	731	// -------------------------
	732
	733	/// Encode an 8 bit displacement
	734	///
	735	/// It is sign-extended to 64 bits by the cpu.
	736	pub fn disp8(self: Self, disp: i8) !void {
	737	try self.writer.writeByte(@bitCast(u8, disp));
	738	}
	739
	740	/// Encode an 32 bit displacement
	741	///
	742	/// It is sign-extended to 64 bits by the cpu.
	743	pub fn disp32(self: Self, disp: i32) !void {
	744	try self.writer.writeIntLittle(i32, disp);
	745	}
	746
	747	/// Encode an 8 bit immediate
	748	///
	749	/// It is sign-extended to 64 bits by the cpu.
	750	pub fn imm8(self: Self, imm: u8) !void {
	751	try self.writer.writeByte(imm);
	752	}
	753
	754	/// Encode an 16 bit immediate
	755	///
	756	/// It is sign-extended to 64 bits by the cpu.
	757	pub fn imm16(self: Self, imm: u16) !void {
	758	try self.writer.writeIntLittle(u16, imm);
	759	}
	760
	761	/// Encode an 32 bit immediate
	762	///
	763	/// It is sign-extended to 64 bits by the cpu.
	764	pub fn imm32(self: Self, imm: u32) !void {
	765	try self.writer.writeIntLittle(u32, imm);
	766	}
	767
	768	/// Encode an 64 bit immediate
	769	///
	770	/// It is sign-extended to 64 bits by the cpu.
	771	pub fn imm64(self: Self, imm: u64) !void {
	772	try self.writer.writeIntLittle(u64, imm);
	773	}
	774	};
	775	}
	776
	777	pub const Rex = struct {
	778	w: bool = false,
	779	r: bool = false,
	780	x: bool = false,
	781	b: bool = false,
	782	present: bool = false,
	783
	784	pub fn isSet(rex: Rex) bool {
	785	return rex.w or rex.r or rex.x or rex.b;
	786	}
	787	};
	788
	789	// Tests
	790	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
	791	assert(expected.len > 0);
	792	if (std.mem.eql(u8, expected, given)) return;
	793	const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
	794	defer testing.allocator.free(expected_fmt);
	795	const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
	796	defer testing.allocator.free(given_fmt);
	797	const idx = std.mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
	798	var padding = try testing.allocator.alloc(u8, idx + 5);
	799	defer testing.allocator.free(padding);
	800	std.mem.set(u8, padding, ' ');
	801	std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
	802	assembly,
	803	expected_fmt,
	804	given_fmt,
	805	padding,
	806	});
	807	return error.TestFailed;
	808	}
	809
	810	const TestEncode = struct {
	811	buffer: [32]u8 = undefined,
	812	index: usize = 0,
	813
	814	fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: struct {
	815	op1: Instruction.Operand = .none,
	816	op2: Instruction.Operand = .none,
	817	op3: Instruction.Operand = .none,
	818	op4: Instruction.Operand = .none,
	819	}) !void {
	820	var stream = std.io.fixedBufferStream(&enc.buffer);
	821	var count_writer = std.io.countingWriter(stream.writer());
	822	const inst = try Instruction.new(mnemonic, .{
	823	.op1 = args.op1,
	824	.op2 = args.op2,
	825	.op3 = args.op3,
	826	.op4 = args.op4,
	827	});
	828	try inst.encode(count_writer.writer());
	829	enc.index = count_writer.bytes_written;
	830	}
	831
	832	fn code(enc: TestEncode) []const u8 {
	833	return enc.buffer[0..enc.index];
	834	}
	835	};
	836
	837	test "encode" {
	838	var buf = std.ArrayList(u8).init(testing.allocator);
	839	defer buf.deinit();
	840
	841	const inst = try Instruction.new(.mov, .{
	842	.op1 = .{ .reg = .rbx },
	843	.op2 = .{ .imm = Immediate.u(4) },
	844	});
	845	try inst.encode(buf.writer());
	846	try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items);
	847	}
	848
	849	test "lower I encoding" {
	850	var enc = TestEncode{};
	851
	852	try enc.encode(.push, .{ .op1 = .{ .imm = Immediate.u(0x10) } });
	853	try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10");
	854
	855	try enc.encode(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000) } });
	856	try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000");
	857
	858	try enc.encode(.push, .{ .op1 = .{ .imm = Immediate.u(0x10000000) } });
	859	try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000");
	860
	861	try enc.encode(.adc, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10000000) } });
	862	try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000");
	863
	864	try enc.encode(.add, .{ .op1 = .{ .reg = .al }, .op2 = .{ .imm = Immediate.u(0x10) } });
	865	try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10");
	866
	867	try enc.encode(.add, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
	868	try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
	869
	870	try enc.encode(.sbb, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .imm = Immediate.u(0x10) } });
	871	try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10");
	872
	873	try enc.encode(.xor, .{ .op1 = .{ .reg = .al }, .op2 = .{ .imm = Immediate.u(0x10) } });
	874	try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10");
	875	}
	876
	877	test "lower MI encoding" {
	878	var enc = TestEncode{};
	879
	880	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
	881	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
	882
	883	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
	884	.base = .r12,
	885	.disp = 0,
	886	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	887	try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
	888
	889	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
	890	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
	891
	892	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
	893	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
	894
	895	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
	896	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
	897
	898	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	899	.base = .r11,
	900	.disp = 0,
	901	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	902	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
	903
	904	try enc.encode(.mov, .{
	905	.op1 = .{ .mem = Memory.rip(.qword, 0x10) },
	906	.op2 = .{ .imm = Immediate.u(0x10) },
	907	});
	908	try expectEqualHexStrings(
	909	"\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
	910	enc.code(),
	911	"mov QWORD PTR [rip + 0x10], 0x10",
	912	);
	913
	914	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	915	.base = .rbp,
	916	.disp = -8,
	917	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	918	try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10");
	919
	920	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.word, .{
	921	.base = .rbp,
	922	.disp = -2,
	923	}) }, .op2 = .{ .imm = Immediate.s(-16) } });
	924	try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16");
	925
	926	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
	927	.base = .rbp,
	928	.disp = -1,
	929	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	930	try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10");
	931
	932	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	933	.base = .ds,
	934	.disp = 0x10000000,
	935	.scale_index = .{
	936	.scale = 2,
	937	.index = .rcx,
	938	},
	939	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	940	try expectEqualHexStrings(
	941	"\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
	942	enc.code(),
	943	"mov QWORD PTR [rcx*2 + 0x10000000], 0x10",
	944	);
	945
	946	try enc.encode(.adc, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
	947	.base = .rbp,
	948	.disp = -0x10,
	949	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	950	try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10");
	951
	952	try enc.encode(.adc, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	953	try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10");
	954
	955	try enc.encode(.adc, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
	956	try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10");
	957
	958	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	959	.base = .rdx,
	960	.disp = -8,
	961	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	962	try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10");
	963
	964	try enc.encode(.add, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
	965	try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
	966
	967	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	968	.base = .rbp,
	969	.disp = -0x10,
	970	}) }, .op2 = .{ .imm = Immediate.s(-0x10) } });
	971	try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10");
	972
	973	try enc.encode(.@"and", .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	974	.base = .ds,
	975	.disp = 0x10000000,
	976	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	977	try expectEqualHexStrings(
	978	"\x83\x24\x25\x00\x00\x00\x10\x10",
	979	enc.code(),
	980	"and DWORD PTR ds:0x10000000, 0x10",
	981	);
	982
	983	try enc.encode(.@"and", .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	984	.base = .es,
	985	.disp = 0x10000000,
	986	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	987	try expectEqualHexStrings(
	988	"\x26\x83\x24\x25\x00\x00\x00\x10\x10",
	989	enc.code(),
	990	"and DWORD PTR es:0x10000000, 0x10",
	991	);
	992
	993	try enc.encode(.@"and", .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	994	.base = .r12,
	995	.disp = 0x10000000,
	996	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	997	try expectEqualHexStrings(
	998	"\x41\x83\xA4\x24\x00\x00\x00\x10\x10",
	999	enc.code(),
	1000	"and DWORD PTR [r12 + 0x10000000], 0x10",
	1001	);
	1002
	1003	try enc.encode(.sub, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	1004	.base = .r11,
	1005	.disp = 0x10000000,
	1006	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	1007	try expectEqualHexStrings(
	1008	"\x41\x83\xAB\x00\x00\x00\x10\x10",
	1009	enc.code(),
	1010	"sub DWORD PTR [r11 + 0x10000000], 0x10",
	1011	);
	1012	}
	1013
	1014	test "lower RM encoding" {
	1015	var enc = TestEncode{};
	1016
	1017	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1018	.base = .r11,
	1019	.disp = 0,
	1020	}) } });
	1021	try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]");
	1022
	1023	try enc.encode(.mov, .{ .op1 = .{ .reg = .rbx }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1024	.base = .ds,
	1025	.disp = 0x10,
	1026	}) } });
	1027	try expectEqualHexStrings("\x48\x8B\x1C\x25\x10\x00\x00\x00", enc.code(), "mov rbx, QWORD PTR ds:0x10");
	1028
	1029	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1030	.base = .rbp,
	1031	.disp = -4,
	1032	}) } });
	1033	try expectEqualHexStrings("\x48\x8B\x45\xFC", enc.code(), "mov rax, QWORD PTR [rbp - 4]");
	1034
	1035	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1036	.base = .rbp,
	1037	.scale_index = .{
	1038	.scale = 1,
	1039	.index = .rcx,
	1040	},
	1041	.disp = -8,
	1042	}) } });
	1043	try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*1 - 8]");
	1044
	1045	try enc.encode(.mov, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.dword, .{
	1046	.base = .rbp,
	1047	.scale_index = .{
	1048	.scale = 4,
	1049	.index = .rdx,
	1050	},
	1051	.disp = -4,
	1052	}) } });
	1053	try expectEqualHexStrings("\x8B\x44\x95\xFC", enc.code(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
	1054
	1055	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1056	.base = .rbp,
	1057	.scale_index = .{
	1058	.scale = 8,
	1059	.index = .rcx,
	1060	},
	1061	.disp = -8,
	1062	}) } });
	1063	try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*8 - 8]");
	1064
	1065	try enc.encode(.mov, .{ .op1 = .{ .reg = .r8b }, .op2 = .{ .mem = Memory.sib(.byte, .{
	1066	.base = .rsi,
	1067	.scale_index = .{
	1068	.scale = 1,
	1069	.index = .rcx,
	1070	},
	1071	.disp = -24,
	1072	}) } });
	1073	try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", enc.code(), "mov r8b, BYTE PTR [rsi + rcx*1 - 24]");
	1074
	1075	// TODO this mnemonic needs cleanup as some prefixes are obsolete.
	1076	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .cs } });
	1077	try expectEqualHexStrings("\x48\x8C\xC8", enc.code(), "mov rax, cs");
	1078
	1079	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1080	.base = .rbp,
	1081	.disp = -16,
	1082	}) }, .op2 = .{ .reg = .fs } });
	1083	try expectEqualHexStrings("\x48\x8C\x65\xF0", enc.code(), "mov QWORD PTR [rbp - 16], fs");
	1084
	1085	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12w }, .op2 = .{ .reg = .cs } });
	1086	try expectEqualHexStrings("\x66\x41\x8C\xCC", enc.code(), "mov r12w, cs");
	1087
	1088	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.word, .{
	1089	.base = .rbp,
	1090	.disp = -16,
	1091	}) }, .op2 = .{ .reg = .fs } });
	1092	try expectEqualHexStrings("\x66\x8C\x65\xF0", enc.code(), "mov WORD PTR [rbp - 16], fs");
	1093
	1094	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .reg = .bx } });
	1095	try expectEqualHexStrings("\x0F\xBF\xC3", enc.code(), "movsx eax, bx");
	1096
	1097	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .reg = .bl } });
	1098	try expectEqualHexStrings("\x0F\xBE\xC3", enc.code(), "movsx eax, bl");
	1099
	1100	try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .reg = .bl } });
	1101	try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl");
	1102
	1103	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.word, .{
	1104	.base = .rbp,
	1105	.disp = 0,
	1106	}) } });
	1107	try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]");
	1108
	1109	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.byte, .{
	1110	.base = null,
	1111	.scale_index = .{
	1112	.index = .rax,
	1113	.scale = 2,
	1114	},
	1115	.disp = 0,
	1116	}) } });
	1117	try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]");
	1118
	1119	try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
	1120	try expectEqualHexStrings("\x66\x0F\xBE\x05\x10\x00\x00\x00", enc.code(), "movsx ax, BYTE PTR [rip + 0x10]");
	1121
	1122	try enc.encode(.movsx, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .bx } });
	1123	try expectEqualHexStrings("\x48\x0F\xBF\xC3", enc.code(), "movsx rax, bx");
	1124
	1125	try enc.encode(.movsxd, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .ebx } });
	1126	try expectEqualHexStrings("\x48\x63\xC3", enc.code(), "movsxd rax, ebx");
	1127
	1128	try enc.encode(.lea, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.rip(.qword, 0x10) } });
	1129	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, QWORD PTR [rip + 0x10]");
	1130
	1131	try enc.encode(.lea, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.rip(.dword, 0x10) } });
	1132	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, DWORD PTR [rip + 0x10]");
	1133
	1134	try enc.encode(.lea, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.rip(.dword, 0x10) } });
	1135	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, DWORD PTR [rip + 0x10]");
	1136
	1137	try enc.encode(.lea, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.rip(.word, 0x10) } });
	1138	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, WORD PTR [rip + 0x10]");
	1139
	1140	try enc.encode(.lea, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
	1141	try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]");
	1142
	1143	try enc.encode(.lea, .{ .op1 = .{ .reg = .rsi }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1144	.base = .rbp,
	1145	.scale_index = .{
	1146	.scale = 1,
	1147	.index = .rcx,
	1148	},
	1149	.disp = 0,
	1150	}) } });
	1151	try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]");
	1152
	1153	try enc.encode(.add, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1154	.base = .ds,
	1155	.disp = 0x10000000,
	1156	}) } });
	1157	try expectEqualHexStrings("\x4C\x03\x1C\x25\x00\x00\x00\x10", enc.code(), "add r11, QWORD PTR ds:0x10000000");
	1158
	1159	try enc.encode(.add, .{ .op1 = .{ .reg = .r12b }, .op2 = .{ .mem = Memory.sib(.byte, .{
	1160	.base = .ds,
	1161	.disp = 0x10000000,
	1162	}) } });
	1163	try expectEqualHexStrings("\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR ds:0x10000000");
	1164
	1165	try enc.encode(.add, .{ .op1 = .{ .reg = .r12b }, .op2 = .{ .mem = Memory.sib(.byte, .{
	1166	.base = .fs,
	1167	.disp = 0x10000000,
	1168	}) } });
	1169	try expectEqualHexStrings("\x64\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR fs:0x10000000");
	1170
	1171	try enc.encode(.sub, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1172	.base = .r13,
	1173	.disp = 0x10000000,
	1174	}) } });
	1175	try expectEqualHexStrings("\x4D\x2B\x9D\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r13 + 0x10000000]");
	1176
	1177	try enc.encode(.sub, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
	1178	.base = .r12,
	1179	.disp = 0x10000000,
	1180	}) } });
	1181	try expectEqualHexStrings("\x4D\x2B\x9C\x24\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r12 + 0x10000000]");
	1182
	1183	try enc.encode(.imul, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .reg = .r12 } });
	1184	try expectEqualHexStrings("\x4D\x0F\xAF\xDC", enc.code(), "mov r11, r12");
	1185	}
	1186
	1187	test "lower RMI encoding" {
	1188	var enc = TestEncode{};
	1189
	1190	try enc.encode(.imul, .{
	1191	.op1 = .{ .reg = .r11 },
	1192	.op2 = .{ .reg = .r12 },
	1193	.op3 = .{ .imm = Immediate.s(-2) },
	1194	});
	1195	try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2");
	1196
	1197	try enc.encode(.imul, .{
	1198	.op1 = .{ .reg = .r11 },
	1199	.op2 = .{ .mem = Memory.rip(.qword, -16) },
	1200	.op3 = .{ .imm = Immediate.s(-1024) },
	1201	});
	1202	try expectEqualHexStrings(
	1203	"\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF",
	1204	enc.code(),
	1205	"imul r11, QWORD PTR [rip - 16], -1024",
	1206	);
	1207
	1208	try enc.encode(.imul, .{
	1209	.op1 = .{ .reg = .bx },
	1210	.op2 = .{ .mem = Memory.sib(.word, .{
	1211	.base = .rbp,
	1212	.disp = -16,
	1213	}) },
	1214	.op3 = .{ .imm = Immediate.s(-1024) },
	1215	});
	1216	try expectEqualHexStrings(
	1217	"\x66\x69\x5D\xF0\x00\xFC",
	1218	enc.code(),
	1219	"imul bx, WORD PTR [rbp - 16], -1024",
	1220	);
	1221
	1222	try enc.encode(.imul, .{
	1223	.op1 = .{ .reg = .bx },
	1224	.op2 = .{ .mem = Memory.sib(.word, .{
	1225	.base = .rbp,
	1226	.disp = -16,
	1227	}) },
	1228	.op3 = .{ .imm = Immediate.u(1024) },
	1229	});
	1230	try expectEqualHexStrings(
	1231	"\x66\x69\x5D\xF0\x00\x04",
	1232	enc.code(),
	1233	"imul bx, WORD PTR [rbp - 16], 1024",
	1234	);
	1235	}
	1236
	1237	test "lower MR encoding" {
	1238	var enc = TestEncode{};
	1239
	1240	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .rbx } });
	1241	try expectEqualHexStrings("\x48\x89\xD8", enc.code(), "mov rax, rbx");
	1242
	1243	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1244	.base = .rbp,
	1245	.disp = -4,
	1246	}) }, .op2 = .{ .reg = .r11 } });
	1247	try expectEqualHexStrings("\x4c\x89\x5d\xfc", enc.code(), "mov QWORD PTR [rbp - 4], r11");
	1248
	1249	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.rip(.qword, 0x10) }, .op2 = .{ .reg = .r12 } });
	1250	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rip + 0x10], r12");
	1251
	1252	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1253	.base = .r11,
	1254	.scale_index = .{
	1255	.scale = 2,
	1256	.index = .r12,
	1257	},
	1258	.disp = 0x10,
	1259	}) }, .op2 = .{ .reg = .r13 } });
	1260	try expectEqualHexStrings("\x4F\x89\x6C\x63\x10", enc.code(), "mov QWORD PTR [r11 + 2 * r12 + 0x10], r13");
	1261
	1262	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.rip(.word, -0x10) }, .op2 = .{ .reg = .r12w } });
	1263	try expectEqualHexStrings("\x66\x44\x89\x25\xF0\xFF\xFF\xFF", enc.code(), "mov WORD PTR [rip - 0x10], r12w");
	1264
	1265	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
	1266	.base = .r11,
	1267	.scale_index = .{
	1268	.scale = 2,
	1269	.index = .r12,
	1270	},
	1271	.disp = 0x10,
	1272	}) }, .op2 = .{ .reg = .r13b } });
	1273	try expectEqualHexStrings("\x47\x88\x6C\x63\x10", enc.code(), "mov BYTE PTR [r11 + 2 * r12 + 0x10], r13b");
	1274
	1275	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
	1276	.base = .ds,
	1277	.disp = 0x10000000,
	1278	}) }, .op2 = .{ .reg = .r12b } });
	1279	try expectEqualHexStrings("\x44\x00\x24\x25\x00\x00\x00\x10", enc.code(), "add BYTE PTR ds:0x10000000, r12b");
	1280
	1281	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	1282	.base = .ds,
	1283	.disp = 0x10000000,
	1284	}) }, .op2 = .{ .reg = .r12d } });
	1285	try expectEqualHexStrings("\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [ds:0x10000000], r12d");
	1286
	1287	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
	1288	.base = .gs,
	1289	.disp = 0x10000000,
	1290	}) }, .op2 = .{ .reg = .r12d } });
	1291	try expectEqualHexStrings("\x65\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [gs:0x10000000], r12d");
	1292
	1293	try enc.encode(.sub, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1294	.base = .r11,
	1295	.disp = 0x10000000,
	1296	}) }, .op2 = .{ .reg = .r12 } });
	1297	try expectEqualHexStrings("\x4D\x29\xA3\x00\x00\x00\x10", enc.code(), "sub QWORD PTR [r11 + 0x10000000], r12");
	1298	}
	1299
	1300	test "lower M encoding" {
	1301	var enc = TestEncode{};
	1302
	1303	try enc.encode(.call, .{ .op1 = .{ .reg = .r12 } });
	1304	try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12");
	1305
	1306	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1307	.base = .r12,
	1308	.disp = 0,
	1309	}) } });
	1310	try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]");
	1311
	1312	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1313	.base = null,
	1314	.scale_index = .{
	1315	.index = .r11,
	1316	.scale = 2,
	1317	},
	1318	.disp = 0,
	1319	}) } });
	1320	try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]");
	1321
	1322	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1323	.base = null,
	1324	.scale_index = .{
	1325	.index = .r12,
	1326	.scale = 2,
	1327	},
	1328	.disp = 0,
	1329	}) } });
	1330	try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]");
	1331
	1332	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1333	.base = .gs,
	1334	.disp = 0,
	1335	}) } });
	1336	try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
	1337
	1338	try enc.encode(.call, .{ .op1 = .{ .imm = Immediate.s(0) } });
	1339	try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
	1340
	1341	try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
	1342	.base = .rbp,
	1343	.disp = 0,
	1344	}) } });
	1345	try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
	1346
	1347	try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{
	1348	.base = .rbp,
	1349	.disp = 0,
	1350	}) } });
	1351	try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
	1352
	1353	try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) } });
	1354	try expectEqualHexStrings("\x8F\x05\x00\x00\x00\x00", enc.code(), "pop QWORD PTR [rip]");
	1355
	1356	try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.word, 0) } });
	1357	try expectEqualHexStrings("\x66\x8F\x05\x00\x00\x00\x00", enc.code(), "pop WORD PTR [rbp]");
	1358
	1359	try enc.encode(.imul, .{ .op1 = .{ .reg = .rax } });
	1360	try expectEqualHexStrings("\x48\xF7\xE8", enc.code(), "imul rax");
	1361
	1362	try enc.encode(.imul, .{ .op1 = .{ .reg = .r12 } });
	1363	try expectEqualHexStrings("\x49\xF7\xEC", enc.code(), "imul r12");
	1364	}
	1365
	1366	test "lower O encoding" {
	1367	var enc = TestEncode{};
	1368
	1369	try enc.encode(.push, .{ .op1 = .{ .reg = .rax } });
	1370	try expectEqualHexStrings("\x50", enc.code(), "push rax");
	1371
	1372	try enc.encode(.push, .{ .op1 = .{ .reg = .r12w } });
	1373	try expectEqualHexStrings("\x66\x41\x54", enc.code(), "push r12w");
	1374
	1375	try enc.encode(.pop, .{ .op1 = .{ .reg = .r12 } });
	1376	try expectEqualHexStrings("\x41\x5c", enc.code(), "pop r12");
	1377	}
	1378
	1379	test "lower OI encoding" {
	1380	var enc = TestEncode{};
	1381
	1382	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x1000000000000000) } });
	1383	try expectEqualHexStrings(
	1384	"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
	1385	enc.code(),
	1386	"movabs rax, 0x1000000000000000",
	1387	);
	1388
	1389	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .imm = Immediate.u(0x1000000000000000) } });
	1390	try expectEqualHexStrings(
	1391	"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
	1392	enc.code(),
	1393	"movabs r11, 0x1000000000000000",
	1394	);
	1395
	1396	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11d }, .op2 = .{ .imm = Immediate.u(0x10000000) } });
	1397	try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000");
	1398
	1399	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11w }, .op2 = .{ .imm = Immediate.u(0x1000) } });
	1400	try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000");
	1401
	1402	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11b }, .op2 = .{ .imm = Immediate.u(0x10) } });
	1403	try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10");
	1404	}
	1405
	1406	test "lower FD/TD encoding" {
	1407	var enc = TestEncode{};
	1408
	1409	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.moffs(.cs, 0x10) } });
	1410	try expectEqualHexStrings("\x2E\x48\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs rax, cs:0x10");
	1411
	1412	try enc.encode(.mov, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.moffs(.fs, 0x10) } });
	1413	try expectEqualHexStrings("\x64\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs eax, fs:0x10");
	1414
	1415	try enc.encode(.mov, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.moffs(.gs, 0x10) } });
	1416	try expectEqualHexStrings("\x65\x66\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ax, gs:0x10");
	1417
	1418	try enc.encode(.mov, .{ .op1 = .{ .reg = .al }, .op2 = .{ .mem = Memory.moffs(.ds, 0x10) } });
	1419	try expectEqualHexStrings("\xA0\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs al, ds:0x10");
	1420
	1421	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.cs, 0x10) }, .op2 = .{ .reg = .rax } });
	1422	try expectEqualHexStrings("\x2E\x48\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs cs:0x10, rax");
	1423
	1424	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.fs, 0x10) }, .op2 = .{ .reg = .eax } });
	1425	try expectEqualHexStrings("\x64\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs fs:0x10, eax");
	1426
	1427	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.gs, 0x10) }, .op2 = .{ .reg = .ax } });
	1428	try expectEqualHexStrings("\x65\x66\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs gs:0x10, ax");
	1429
	1430	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.ds, 0x10) }, .op2 = .{ .reg = .al } });
	1431	try expectEqualHexStrings("\xA2\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ds:0x10, al");
	1432	}
	1433
	1434	test "lower NP encoding" {
	1435	var enc = TestEncode{};
	1436
	1437	try enc.encode(.int3, .{});
	1438	try expectEqualHexStrings("\xCC", enc.code(), "int3");
	1439
	1440	try enc.encode(.nop, .{});
	1441	try expectEqualHexStrings("\x90", enc.code(), "nop");
	1442
	1443	try enc.encode(.ret, .{});
	1444	try expectEqualHexStrings("\xC3", enc.code(), "ret");
	1445
	1446	try enc.encode(.syscall, .{});
	1447	try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall");
	1448	}
	1449
	1450	fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: struct {
	1451	op1: Instruction.Operand = .none,
	1452	op2: Instruction.Operand = .none,
	1453	op3: Instruction.Operand = .none,
	1454	op4: Instruction.Operand = .none,
	1455	}) !void {
	1456	const err = Instruction.new(mnemonic, .{
	1457	.op1 = args.op1,
	1458	.op2 = args.op2,
	1459	.op3 = args.op3,
	1460	.op4 = args.op4,
	1461	});
	1462	try testing.expectError(error.InvalidInstruction, err);
	1463	}
	1464
	1465	test "invalid instruction" {
	1466	try invalidInstruction(.call, .{ .op1 = .{ .reg = .eax } });
	1467	try invalidInstruction(.call, .{ .op1 = .{ .reg = .ax } });
	1468	try invalidInstruction(.call, .{ .op1 = .{ .reg = .al } });
	1469	try invalidInstruction(.call, .{ .op1 = .{ .mem = Memory.rip(.dword, 0) } });
	1470	try invalidInstruction(.call, .{ .op1 = .{ .mem = Memory.rip(.word, 0) } });
	1471	try invalidInstruction(.call, .{ .op1 = .{ .mem = Memory.rip(.byte, 0) } });
	1472	try invalidInstruction(.mov, .{ .op1 = .{ .mem = Memory.rip(.word, 0x10) }, .op2 = .{ .reg = .r12 } });
	1473	try invalidInstruction(.lea, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .rbx } });
	1474	try invalidInstruction(.lea, .{ .op1 = .{ .reg = .al }, .op2 = .{ .mem = Memory.rip(.byte, 0) } });
	1475	try invalidInstruction(.pop, .{ .op1 = .{ .reg = .r12b } });
	1476	try invalidInstruction(.pop, .{ .op1 = .{ .reg = .r12d } });
	1477	try invalidInstruction(.push, .{ .op1 = .{ .reg = .r12b } });
	1478	try invalidInstruction(.push, .{ .op1 = .{ .reg = .r12d } });
	1479	try invalidInstruction(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000000000000000) } });
	1480	}
	1481
	1482	fn cannotEncode(mnemonic: Instruction.Mnemonic, args: struct {
	1483	op1: Instruction.Operand = .none,
	1484	op2: Instruction.Operand = .none,
	1485	op3: Instruction.Operand = .none,
	1486	op4: Instruction.Operand = .none,
	1487	}) !void {
	1488	try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, .{
	1489	.op1 = args.op1,
	1490	.op2 = args.op2,
	1491	.op3 = args.op3,
	1492	.op4 = args.op4,
	1493	}));
	1494	}
	1495
	1496	test "cannot encode" {
	1497	try cannotEncode(.@"test", .{
	1498	.op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12, .disp = 0 }) },
	1499	.op2 = .{ .reg = .ah },
	1500	});
	1501	try cannotEncode(.@"test", .{
	1502	.op1 = .{ .reg = .r11b },
	1503	.op2 = .{ .reg = .bh },
	1504	});
	1505	try cannotEncode(.mov, .{
	1506	.op1 = .{ .reg = .sil },
	1507	.op2 = .{ .reg = .ah },
	1508	});
	1509	}
	1510
	1511	const Assembler = struct {
	1512	it: Tokenizer,
	1513
	1514	const Tokenizer = struct {
	1515	input: []const u8,
	1516	pos: usize = 0,
	1517
	1518	const Error = error{InvalidToken};
	1519
	1520	const Token = struct {
	1521	id: Id,
	1522	start: usize,
	1523	end: usize,
	1524
	1525	const Id = enum {
	1526	eof,
	1527
	1528	space,
	1529	new_line,
	1530
	1531	colon,
	1532	comma,
	1533	open_br,
	1534	close_br,
	1535	plus,
	1536	minus,
	1537	star,
	1538
	1539	string,
	1540	numeral,
	1541	};
	1542	};
	1543
	1544	const Iterator = struct {};
	1545
	1546	fn next(it: *Tokenizer) !Token {
	1547	var result = Token{
	1548	.id = .eof,
	1549	.start = it.pos,
	1550	.end = it.pos,
	1551	};
	1552
	1553	var state: enum {
	1554	start,
	1555	space,
	1556	new_line,
	1557	string,
	1558	numeral,
	1559	numeral_hex,
	1560	} = .start;
	1561
	1562	while (it.pos < it.input.len) : (it.pos += 1) {
	1563	const ch = it.input[it.pos];
	1564	switch (state) {
	1565	.start => switch (ch) {
	1566	',' => {
	1567	result.id = .comma;
	1568	it.pos += 1;
	1569	break;
	1570	},
	1571	':' => {
	1572	result.id = .colon;
	1573	it.pos += 1;
	1574	break;
	1575	},
	1576	'[' => {
	1577	result.id = .open_br;
	1578	it.pos += 1;
	1579	break;
	1580	},
	1581	']' => {
	1582	result.id = .close_br;
	1583	it.pos += 1;
	1584	break;
	1585	},
	1586	'+' => {
	1587	result.id = .plus;
	1588	it.pos += 1;
	1589	break;
	1590	},
	1591	'-' => {
	1592	result.id = .minus;
	1593	it.pos += 1;
	1594	break;
	1595	},
	1596	'*' => {
	1597	result.id = .star;
	1598	it.pos += 1;
	1599	break;
	1600	},
	1601	' ', '\t' => state = .space,
	1602	'\n', '\r' => state = .new_line,
	1603	'a'...'z', 'A'...'Z' => state = .string,
	1604	'0'...'9' => state = .numeral,
	1605	else => return error.InvalidToken,
	1606	},
	1607
	1608	.space => switch (ch) {
	1609	' ', '\t' => {},
	1610	else => {
	1611	result.id = .space;
	1612	break;
	1613	},
	1614	},
	1615
	1616	.new_line => switch (ch) {
	1617	'\n', '\r', ' ', '\t' => {},
	1618	else => {
	1619	result.id = .new_line;
	1620	break;
	1621	},
	1622	},
	1623
	1624	.string => switch (ch) {
	1625	'a'...'z', 'A'...'Z', '0'...'9' => {},
	1626	else => {
	1627	result.id = .string;
	1628	break;
	1629	},
	1630	},
	1631
	1632	.numeral => switch (ch) {
	1633	'x' => state = .numeral_hex,
	1634	'0'...'9' => {},
	1635	else => {
	1636	result.id = .numeral;
	1637	break;
	1638	},
	1639	},
	1640
	1641	.numeral_hex => switch (ch) {
	1642	'a'...'f' => {},
	1643	'0'...'9' => {},
	1644	else => {
	1645	result.id = .numeral;
	1646	break;
	1647	},
	1648	},
	1649	}
	1650	}
	1651
	1652	if (it.pos >= it.input.len) {
	1653	switch (state) {
	1654	.string => result.id = .string,
	1655	.numeral, .numeral_hex => result.id = .numeral,
	1656	else => {},
	1657	}
	1658	}
	1659
	1660	result.end = it.pos;
	1661	return result;
	1662	}
	1663
	1664	fn seekTo(it: *Tokenizer, pos: usize) void {
	1665	it.pos = pos;
	1666	}
	1667	};
	1668
	1669	pub fn init(input: []const u8) Assembler {
	1670	return .{
	1671	.it = Tokenizer{ .input = input },
	1672	};
	1673	}
	1674
	1675	pub fn assemble(as: *Assembler, writer: anytype) !void {
	1676	while (try as.next()) \|parsed_inst\| {
	1677	const inst = try Instruction.new(parsed_inst.mnemonic, .{
	1678	.op1 = parsed_inst.ops[0],
	1679	.op2 = parsed_inst.ops[1],
	1680	.op3 = parsed_inst.ops[2],
	1681	.op4 = parsed_inst.ops[3],
	1682	});
	1683	try inst.encode(writer);
	1684	}
	1685	}
	1686
	1687	const ParseResult = struct {
	1688	mnemonic: Instruction.Mnemonic,
	1689	ops: [4]Instruction.Operand,
	1690	};
	1691
	1692	const ParseError = error{
	1693	UnexpectedToken,
	1694	InvalidMnemonic,
	1695	InvalidOperand,
	1696	InvalidRegister,
	1697	InvalidPtrSize,
	1698	InvalidMemoryOperand,
	1699	InvalidScaleIndex,
	1700	} \|\| Tokenizer.Error \|\| std.fmt.ParseIntError;
	1701
	1702	fn next(as: *Assembler) ParseError!?ParseResult {
	1703	try as.skip(2, .{ .space, .new_line });
	1704	const mnemonic_tok = as.expect(.string) catch \|err\| switch (err) {
	1705	error.UnexpectedToken => return if (try as.peek() == .eof) null else err,
	1706	else => return err,
	1707	};
	1708	const mnemonic = mnemonicFromString(as.source(mnemonic_tok)) orelse
	1709	return error.InvalidMnemonic;
	1710	try as.skip(1, .{.space});
	1711
	1712	const rules = .{
	1713	.{},
	1714	.{.register},
	1715	.{.memory},
	1716	.{.immediate},
	1717	.{ .register, .register },
	1718	.{ .register, .memory },
	1719	.{ .memory, .register },
	1720	.{ .register, .immediate },
	1721	.{ .memory, .immediate },
	1722	.{ .register, .register, .immediate },
	1723	.{ .register, .memory, .immediate },
	1724	};
	1725
	1726	const pos = as.it.pos;
	1727	inline for (rules) \|rule\| {
	1728	var ops = [4]Instruction.Operand{ .none, .none, .none, .none };
	1729	if (as.parseOperandRule(rule, &ops)) {
	1730	return .{
	1731	.mnemonic = mnemonic,
	1732	.ops = ops,
	1733	};
	1734	} else \|_\| {
	1735	as.it.seekTo(pos);
	1736	}
	1737	}
	1738
	1739	return error.InvalidOperand;
	1740	}
	1741
	1742	fn source(as: *Assembler, token: Tokenizer.Token) []const u8 {
	1743	return as.it.input[token.start..token.end];
	1744	}
	1745
	1746	fn peek(as: *Assembler) Tokenizer.Error!Tokenizer.Token.Id {
	1747	const pos = as.it.pos;
	1748	const next_tok = try as.it.next();
	1749	const id = next_tok.id;
	1750	as.it.seekTo(pos);
	1751	return id;
	1752	}
	1753
	1754	fn expect(as: *Assembler, id: Tokenizer.Token.Id) ParseError!Tokenizer.Token {
	1755	const next_tok_id = try as.peek();
	1756	if (next_tok_id == id) return as.it.next();
	1757	return error.UnexpectedToken;
	1758	}
	1759
	1760	fn skip(as: *Assembler, comptime num: comptime_int, tok_ids: [num]Tokenizer.Token.Id) Tokenizer.Error!void {
	1761	outer: while (true) {
	1762	const pos = as.it.pos;
	1763	const next_tok = try as.it.next();
	1764	inline for (tok_ids) \|tok_id\| {
	1765	if (next_tok.id == tok_id) continue :outer;
	1766	}
	1767	as.it.seekTo(pos);
	1768	break;
	1769	}
	1770	}
	1771
	1772	fn mnemonicFromString(bytes: []const u8) ?Instruction.Mnemonic {
	1773	const ti = @typeInfo(Instruction.Mnemonic).Enum;
	1774	inline for (ti.fields) \|field\| {
	1775	if (std.mem.eql(u8, bytes, field.name)) {
	1776	return @field(Instruction.Mnemonic, field.name);
	1777	}
	1778	}
	1779	return null;
	1780	}
	1781
	1782	fn parseOperandRule(as: Assembler, rule: anytype, ops: [4]Instruction.Operand) ParseError!void {
	1783	inline for (rule, 0..) \|cond, i\| {
	1784	comptime assert(i < 4);
	1785	if (i > 0) {
	1786	_ = try as.expect(.comma);
	1787	try as.skip(1, .{.space});
	1788	}
	1789	if (@typeInfo(@TypeOf(cond)) != .EnumLiteral) {
	1790	@compileError("invalid condition in the rule: " ++ @typeName(@TypeOf(cond)));
	1791	}
	1792	switch (cond) {
	1793	.register => {
	1794	const reg_tok = try as.expect(.string);
	1795	const reg = registerFromString(as.source(reg_tok)) orelse
	1796	return error.InvalidOperand;
	1797	ops[i] = .{ .reg = reg };
	1798	},
	1799	.memory => {
	1800	const mem = try as.parseMemory();
	1801	ops[i] = .{ .mem = mem };
	1802	},
	1803	.immediate => {
	1804	const is_neg = if (as.expect(.minus)) \|_\| true else \|_\| false;
	1805	const imm_tok = try as.expect(.numeral);
	1806	const imm: Immediate = if (is_neg) blk: {
	1807	const imm = try std.fmt.parseInt(i32, as.source(imm_tok), 0);
	1808	break :blk .{ .signed = imm * -1 };
	1809	} else .{ .unsigned = try std.fmt.parseInt(u64, as.source(imm_tok), 0) };
	1810	ops[i] = .{ .imm = imm };
	1811	},
	1812	else => @compileError("unhandled enum literal " ++ @tagName(cond)),
	1813	}
	1814	try as.skip(1, .{.space});
	1815	}
	1816
	1817	try as.skip(1, .{.space});
	1818	const tok = try as.it.next();
	1819	switch (tok.id) {
	1820	.new_line, .eof => {},
	1821	else => return error.InvalidOperand,
	1822	}
	1823	}
	1824
	1825	fn registerFromString(bytes: []const u8) ?Register {
	1826	const ti = @typeInfo(Register).Enum;
	1827	inline for (ti.fields) \|field\| {
	1828	if (std.mem.eql(u8, bytes, field.name)) {
	1829	return @field(Register, field.name);
	1830	}
	1831	}
	1832	return null;
	1833	}
	1834
	1835	fn parseMemory(as: *Assembler) ParseError!Memory {
	1836	const ptr_size: ?Memory.PtrSize = blk: {
	1837	const pos = as.it.pos;
	1838	const ptr_size = as.parsePtrSize() catch \|err\| switch (err) {
	1839	error.UnexpectedToken => {
	1840	as.it.seekTo(pos);
	1841	break :blk null;
	1842	},
	1843	else => return err,
	1844	};
	1845	break :blk ptr_size;
	1846	};
	1847
	1848	try as.skip(1, .{.space});
	1849
	1850	// Supported rules and orderings.
	1851	const rules = .{
	1852	.{ .open_br, .base, .close_br }, // [ base ]
	1853	.{ .open_br, .base, .plus, .disp, .close_br }, // [ base + disp ]
	1854	.{ .open_br, .base, .minus, .disp, .close_br }, // [ base - disp ]
	1855	.{ .open_br, .disp, .plus, .base, .close_br }, // [ disp + base ]
	1856	.{ .open_br, .base, .plus, .index, .close_br }, // [ base + index ]
	1857	.{ .open_br, .base, .plus, .index, .star, .scale, .close_br }, // [ base + index * scale ]
	1858	.{ .open_br, .index, .star, .scale, .plus, .base, .close_br }, // [ index * scale + base ]
	1859	.{ .open_br, .base, .plus, .index, .star, .scale, .plus, .disp, .close_br }, // [ base + index * scale + disp ]
	1860	.{ .open_br, .base, .plus, .index, .star, .scale, .minus, .disp, .close_br }, // [ base + index * scale - disp ]
	1861	.{ .open_br, .index, .star, .scale, .plus, .base, .plus, .disp, .close_br }, // [ index * scale + base + disp ]
	1862	.{ .open_br, .index, .star, .scale, .plus, .base, .minus, .disp, .close_br }, // [ index * scale + base - disp ]
	1863	.{ .open_br, .disp, .plus, .index, .star, .scale, .plus, .base, .close_br }, // [ disp + index * scale + base ]
	1864	.{ .open_br, .disp, .plus, .base, .plus, .index, .star, .scale, .close_br }, // [ disp + base + index * scale ]
	1865	.{ .open_br, .base, .plus, .disp, .plus, .index, .star, .scale, .close_br }, // [ base + disp + index * scale ]
	1866	.{ .open_br, .base, .minus, .disp, .plus, .index, .star, .scale, .close_br }, // [ base - disp + index * scale ]
	1867	.{ .open_br, .base, .plus, .disp, .plus, .scale, .star, .index, .close_br }, // [ base + disp + scale * index ]
	1868	.{ .open_br, .base, .minus, .disp, .plus, .scale, .star, .index, .close_br }, // [ base - disp + scale * index ]
	1869	.{ .open_br, .rip, .plus, .disp, .close_br }, // [ rip + disp ]
	1870	.{ .open_br, .rip, .minus, .disp, .close_br }, // [ rig - disp ]
	1871	.{ .base, .colon, .disp }, // seg:disp
	1872	};
	1873
	1874	const pos = as.it.pos;
	1875	inline for (rules) \|rule\| {
	1876	if (as.parseMemoryRule(rule)) \|res\| {
	1877	if (res.rip) {
	1878	if (res.base != null or res.scale_index != null or res.offset != null)
	1879	return error.InvalidMemoryOperand;
	1880	return Memory.rip(ptr_size orelse .qword, res.disp orelse 0);
	1881	}
	1882	if (res.base) \|base\| {
	1883	if (res.rip)
	1884	return error.InvalidMemoryOperand;
	1885	if (res.offset) \|offset\| {
	1886	if (res.scale_index != null or res.disp != null)
	1887	return error.InvalidMemoryOperand;
	1888	return Memory.moffs(base, offset);
	1889	}
	1890	return Memory.sib(ptr_size orelse .qword, .{
	1891	.base = base,
	1892	.scale_index = res.scale_index,
	1893	.disp = res.disp orelse 0,
	1894	});
	1895	}
	1896	return error.InvalidMemoryOperand;
	1897	} else \|_\| {
	1898	as.it.seekTo(pos);
	1899	}
	1900	}
	1901
	1902	return error.InvalidOperand;
	1903	}
	1904
	1905	const MemoryParseResult = struct {
	1906	rip: bool = false,
	1907	base: ?Register = null,
	1908	scale_index: ?Memory.ScaleIndex = null,
	1909	disp: ?i32 = null,
	1910	offset: ?u64 = null,
	1911	};
	1912
	1913	fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!MemoryParseResult {
	1914	var res: MemoryParseResult = .{};
	1915	inline for (rule, 0..) \|cond, i\| {
	1916	if (@typeInfo(@TypeOf(cond)) != .EnumLiteral) {
	1917	@compileError("unsupported condition type in the rule: " ++ @typeName(@TypeOf(cond)));
	1918	}
	1919	switch (cond) {
	1920	.open_br, .close_br, .plus, .minus, .star, .colon => {
	1921	_ = try as.expect(cond);
	1922	},
	1923	.base => {
	1924	const tok = try as.expect(.string);
	1925	res.base = registerFromString(as.source(tok)) orelse return error.InvalidMemoryOperand;
	1926	},
	1927	.rip => {
	1928	const tok = try as.expect(.string);
	1929	if (!std.mem.eql(u8, as.source(tok), "rip")) return error.InvalidMemoryOperand;
	1930	res.rip = true;
	1931	},
	1932	.index => {
	1933	const tok = try as.expect(.string);
	1934	const index = registerFromString(as.source(tok)) orelse
	1935	return error.InvalidMemoryOperand;
	1936	if (res.scale_index) \|*si\| {
	1937	si.index = index;
	1938	} else {
	1939	res.scale_index = .{ .scale = 1, .index = index };
	1940	}
	1941	},
	1942	.scale => {
	1943	const tok = try as.expect(.numeral);
	1944	const scale = try std.fmt.parseInt(u2, as.source(tok), 0);
	1945	if (res.scale_index) \|*si\| {
	1946	si.scale = scale;
	1947	} else {
	1948	res.scale_index = .{ .scale = scale, .index = undefined };
	1949	}
	1950	},
	1951	.disp => {
	1952	const tok = try as.expect(.numeral);
	1953	const is_neg = blk: {
	1954	if (i > 0) {
	1955	if (rule[i - 1] == .minus) break :blk true;
	1956	}
	1957	break :blk false;
	1958	};
	1959	if (std.fmt.parseInt(i32, as.source(tok), 0)) \|disp\| {
	1960	res.disp = if (is_neg) -1 * disp else disp;
	1961	} else \|err\| switch (err) {
	1962	error.Overflow => {
	1963	if (is_neg) return err;
	1964	if (res.base) \|base\| {
	1965	if (base.class() != .segment) return err;
	1966	}
	1967	const offset = try std.fmt.parseInt(u64, as.source(tok), 0);
	1968	res.offset = offset;
	1969	},
	1970	else => return err,
	1971	}
	1972	},
	1973	else => @compileError("unhandled operand output type: " ++ @tagName(cond)),
	1974	}
	1975	try as.skip(1, .{.space});
	1976	}
	1977	return res;
	1978	}
	1979
	1980	fn parsePtrSize(as: *Assembler) ParseError!Memory.PtrSize {
	1981	const size = try as.expect(.string);
	1982	try as.skip(1, .{.space});
	1983	const ptr = try as.expect(.string);
	1984
	1985	const size_raw = as.source(size);
	1986	const ptr_raw = as.source(ptr);
	1987	const len = size_raw.len + ptr_raw.len + 1;
	1988	var buf: ["qword ptr".len]u8 = undefined;
	1989	if (len > buf.len) return error.InvalidPtrSize;
	1990
	1991	for (size_raw, 0..) \|c, i\| {
	1992	buf[i] = std.ascii.toLower(c);
	1993	}
	1994	buf[size_raw.len] = ' ';
	1995	for (ptr_raw, 0..) \|c, i\| {
	1996	buf[size_raw.len + i + 1] = std.ascii.toLower(c);
	1997	}
	1998
	1999	const slice = buf[0..len];
	2000	if (std.mem.eql(u8, slice, "qword ptr")) return .qword;
	2001	if (std.mem.eql(u8, slice, "dword ptr")) return .dword;
	2002	if (std.mem.eql(u8, slice, "word ptr")) return .word;
	2003	if (std.mem.eql(u8, slice, "byte ptr")) return .byte;
	2004	if (std.mem.eql(u8, slice, "tbyte ptr")) return .tbyte;
	2005	return error.InvalidPtrSize;
	2006	}
	2007	};
	2008
	2009	test "assemble" {
	2010	const input =
	2011	\\int3
	2012	\\mov rax, rbx
	2013	\\mov qword ptr [rbp], rax
	2014	\\mov qword ptr [rbp - 16], rax
	2015	\\mov qword ptr [16 + rbp], rax
	2016	\\mov rax, 0x10
	2017	\\mov byte ptr [rbp - 0x10], 0x10
	2018	\\mov word ptr [rbp + r12], r11w
	2019	\\mov word ptr [rbp + r12 * 2], r11w
	2020	\\mov word ptr [rbp + r12 * 2 - 16], r11w
	2021	\\mov dword ptr [rip - 16], r12d
	2022	\\mov rax, fs:0x0
	2023	\\mov rax, gs:0x1000000000000000
	2024	\\movzx r12, al
	2025	\\imul r12, qword ptr [rbp - 16], 6
	2026	\\jmp 0x0
	2027	\\jc 0x0
	2028	\\jb 0x0
	2029	\\sal rax, 1
	2030	\\sal rax, 63
	2031	\\shl rax, 63
	2032	\\sar rax, 63
	2033	\\shr rax, 63
	2034	\\test byte ptr [rbp - 16], r12b
	2035	\\sal r12, cl
	2036	\\mul qword ptr [rip - 16]
	2037	\\div r12
	2038	\\idiv byte ptr [rbp - 16]
	2039	\\cwde
	2040	\\cbw
	2041	\\cdqe
	2042	\\test byte ptr [rbp], ah
	2043	\\test byte ptr [r12], spl
	2044	\\cdq
	2045	\\cwd
	2046	\\cqo
	2047	\\test bl, 0x1
	2048	\\mov rbx,0x8000000000000000
	2049	\\movss xmm0, dword ptr [rbp]
	2050	\\movss xmm0, xmm1
	2051	\\movss dword ptr [rbp - 16 + rax * 2], xmm7
	2052	\\movss dword ptr [rbp - 16 + rax * 2], xmm8
	2053	\\movss xmm15, xmm9
	2054	\\movsd xmm8, qword ptr [rbp - 16]
	2055	\\movsd qword ptr [rbp - 8], xmm0
	2056	\\movq xmm8, qword ptr [rbp - 16]
	2057	\\movq qword ptr [rbp - 16], xmm8
	2058	\\ucomisd xmm0, qword ptr [rbp - 16]
	2059	\\fisttp qword ptr [rbp - 16]
	2060	\\fisttp word ptr [rip + 32]
	2061	\\fisttp dword ptr [rax]
	2062	\\fld tbyte ptr [rbp]
	2063	\\fld dword ptr [rbp]
	2064	\\xor bl, 0xff
	2065	\\ud2
	2066	\\add rsp, -1
	2067	\\add rsp, 0xff
	2068	\\mov sil, byte ptr [rax + rcx * 1]
	2069	\\
	2070	;
	2071
	2072	// zig fmt: off
	2073	const expected = &[_]u8{
	2074	0xCC,
	2075	0x48, 0x89, 0xD8,
	2076	0x48, 0x89, 0x45, 0x00,
	2077	0x48, 0x89, 0x45, 0xF0,
	2078	0x48, 0x89, 0x45, 0x10,
	2079	0x48, 0xC7, 0xC0, 0x10, 0x00, 0x00, 0x00,
	2080	0xC6, 0x45, 0xF0, 0x10,
	2081	0x66, 0x46, 0x89, 0x5C, 0x25, 0x00,
	2082	0x66, 0x46, 0x89, 0x5C, 0x65, 0x00,
	2083	0x66, 0x46, 0x89, 0x5C, 0x65, 0xF0,
	2084	0x44, 0x89, 0x25, 0xF0, 0xFF, 0xFF, 0xFF,
	2085	0x64, 0x48, 0x8B, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00,
	2086	0x65, 0x48, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,
	2087	0x4C, 0x0F, 0xB6, 0xE0,
	2088	0x4C, 0x6B, 0x65, 0xF0, 0x06,
	2089	0xE9, 0x00, 0x00, 0x00, 0x00,
	2090	0x0F, 0x82, 0x00, 0x00, 0x00, 0x00,
	2091	0x0F, 0x82, 0x00, 0x00, 0x00, 0x00,
	2092	0x48, 0xD1, 0xE0,
	2093	0x48, 0xC1, 0xE0, 0x3F,
	2094	0x48, 0xC1, 0xE0, 0x3F,
	2095	0x48, 0xC1, 0xF8, 0x3F,
	2096	0x48, 0xC1, 0xE8, 0x3F,
	2097	0x44, 0x84, 0x65, 0xF0,
	2098	0x49, 0xD3, 0xE4,
	2099	0x48, 0xF7, 0x25, 0xF0, 0xFF, 0xFF, 0xFF,
	2100	0x49, 0xF7, 0xF4,
	2101	0xF6, 0x7D, 0xF0,
	2102	0x98,
	2103	0x66, 0x98,
	2104	0x48, 0x98,
	2105	0x84, 0x65, 0x00,
	2106	0x41, 0x84, 0x24, 0x24,
	2107	0x99,
	2108	0x66, 0x99,
	2109	0x48, 0x99,
	2110	0xF6, 0xC3, 0x01,
	2111	0x48, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
	2112	0xF3, 0x0F, 0x10, 0x45, 0x00,
	2113	0xF3, 0x0F, 0x10, 0xC1,
	2114	0xF3, 0x0F, 0x11, 0x7C, 0x45, 0xF0,
	2115	0xF3, 0x44, 0x0F, 0x11, 0x44, 0x45, 0xF0,
	2116	0xF3, 0x45, 0x0F, 0x10, 0xF9,
	2117	0xF2, 0x44, 0x0F, 0x10, 0x45, 0xF0,
	2118	0xF2, 0x0F, 0x11, 0x45, 0xF8,
	2119	0xF3, 0x44, 0x0F, 0x7E, 0x45, 0xF0,
	2120	0x66, 0x44, 0x0F, 0xD6, 0x45, 0xF0,
	2121	0x66, 0x0F, 0x2E, 0x45, 0xF0,
	2122	0xDD, 0x4D, 0xF0,
	2123	0xDF, 0x0D, 0x20, 0x00, 0x00, 0x00,
	2124	0xDB, 0x08,
	2125	0xDB, 0x6D, 0x00,
	2126	0xD9, 0x45, 0x00,
	2127	0x80, 0xF3, 0xFF,
	2128	0x0F, 0x0B,
	2129	0x48, 0x83, 0xC4, 0xFF,
	2130	0x48, 0x81, 0xC4, 0xFF, 0x00, 0x00, 0x00,
	2131	0x40, 0x8A, 0x34, 0x08,
	2132	};
	2133	// zig fmt: on
	2134
	2135	var as = Assembler.init(input);
	2136	var output = std.ArrayList(u8).init(testing.allocator);
	2137	defer output.deinit();
	2138	try as.assemble(output.writer());
	2139	try expectEqualHexStrings(expected, output.items, input);
	2140	}
	2141
	2142	test "assemble - Jcc" {
	2143	const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{
	2144	.{ .ja, 0x87 },
	2145	.{ .jae, 0x83 },
	2146	.{ .jb, 0x82 },
	2147	.{ .jbe, 0x86 },
	2148	.{ .jc, 0x82 },
	2149	.{ .je, 0x84 },
	2150	.{ .jg, 0x8f },
	2151	.{ .jge, 0x8d },
	2152	.{ .jl, 0x8c },
	2153	.{ .jle, 0x8e },
	2154	.{ .jna, 0x86 },
	2155	.{ .jnae, 0x82 },
	2156	.{ .jnb, 0x83 },
	2157	.{ .jnbe, 0x87 },
	2158	.{ .jnc, 0x83 },
	2159	.{ .jne, 0x85 },
	2160	.{ .jng, 0x8e },
	2161	.{ .jnge, 0x8c },
	2162	.{ .jnl, 0x8d },
	2163	.{ .jnle, 0x8f },
	2164	.{ .jno, 0x81 },
	2165	.{ .jnp, 0x8b },
	2166	.{ .jns, 0x89 },
	2167	.{ .jnz, 0x85 },
	2168	.{ .jo, 0x80 },
	2169	.{ .jp, 0x8a },
	2170	.{ .jpe, 0x8a },
	2171	.{ .jpo, 0x8b },
	2172	.{ .js, 0x88 },
	2173	.{ .jz, 0x84 },
	2174	};
	2175
	2176	inline for (&mnemonics) \|mnemonic\| {
	2177	const input = @tagName(mnemonic[0]) ++ " 0x0";
	2178	const expected = [_]u8{ 0x0f, mnemonic[1], 0x0, 0x0, 0x0, 0x0 };
	2179	var as = Assembler.init(input);
	2180	var output = std.ArrayList(u8).init(testing.allocator);
	2181	defer output.deinit();
	2182	try as.assemble(output.writer());
	2183	try expectEqualHexStrings(&expected, output.items, input);
	2184	}
	2185	}
	2186
	2187	test "assemble - SETcc" {
	2188	const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{
	2189	.{ .seta, 0x97 },
	2190	.{ .setae, 0x93 },
	2191	.{ .setb, 0x92 },
	2192	.{ .setbe, 0x96 },
	2193	.{ .setc, 0x92 },
	2194	.{ .sete, 0x94 },
	2195	.{ .setg, 0x9f },
	2196	.{ .setge, 0x9d },
	2197	.{ .setl, 0x9c },
	2198	.{ .setle, 0x9e },
	2199	.{ .setna, 0x96 },
	2200	.{ .setnae, 0x92 },
	2201	.{ .setnb, 0x93 },
	2202	.{ .setnbe, 0x97 },
	2203	.{ .setnc, 0x93 },
	2204	.{ .setne, 0x95 },
	2205	.{ .setng, 0x9e },
	2206	.{ .setnge, 0x9c },
	2207	.{ .setnl, 0x9d },
	2208	.{ .setnle, 0x9f },
	2209	.{ .setno, 0x91 },
	2210	.{ .setnp, 0x9b },
	2211	.{ .setns, 0x99 },
	2212	.{ .setnz, 0x95 },
	2213	.{ .seto, 0x90 },
	2214	.{ .setp, 0x9a },
	2215	.{ .setpe, 0x9a },
	2216	.{ .setpo, 0x9b },
	2217	.{ .sets, 0x98 },
	2218	.{ .setz, 0x94 },
	2219	};
	2220
	2221	inline for (&mnemonics) \|mnemonic\| {
	2222	const input = @tagName(mnemonic[0]) ++ " al";
	2223	const expected = [_]u8{ 0x0f, mnemonic[1], 0xC0 };
	2224	var as = Assembler.init(input);
	2225	var output = std.ArrayList(u8).init(testing.allocator);
	2226	defer output.deinit();
	2227	try as.assemble(output.writer());
	2228	try expectEqualHexStrings(&expected, output.items, input);
	2229	}
	2230	}
	2231
	2232	test "assemble - CMOVcc" {
	2233	const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{
	2234	.{ .cmova, 0x47 },
	2235	.{ .cmovae, 0x43 },
	2236	.{ .cmovb, 0x42 },
	2237	.{ .cmovbe, 0x46 },
	2238	.{ .cmovc, 0x42 },
	2239	.{ .cmove, 0x44 },
	2240	.{ .cmovg, 0x4f },
	2241	.{ .cmovge, 0x4d },
	2242	.{ .cmovl, 0x4c },
	2243	.{ .cmovle, 0x4e },
	2244	.{ .cmovna, 0x46 },
	2245	.{ .cmovnae, 0x42 },
	2246	.{ .cmovnb, 0x43 },
	2247	.{ .cmovnbe, 0x47 },
	2248	.{ .cmovnc, 0x43 },
	2249	.{ .cmovne, 0x45 },
	2250	.{ .cmovng, 0x4e },
	2251	.{ .cmovnge, 0x4c },
	2252	.{ .cmovnl, 0x4d },
	2253	.{ .cmovnle, 0x4f },
	2254	.{ .cmovno, 0x41 },
	2255	.{ .cmovnp, 0x4b },
	2256	.{ .cmovns, 0x49 },
	2257	.{ .cmovnz, 0x45 },
	2258	.{ .cmovo, 0x40 },
	2259	.{ .cmovp, 0x4a },
	2260	.{ .cmovpe, 0x4a },
	2261	.{ .cmovpo, 0x4b },
	2262	.{ .cmovs, 0x48 },
	2263	.{ .cmovz, 0x44 },
	2264	};
	2265
	2266	inline for (&mnemonics) \|mnemonic\| {
	2267	const input = @tagName(mnemonic[0]) ++ " rax, rbx";
	2268	const expected = [_]u8{ 0x48, 0x0f, mnemonic[1], 0xC3 };
	2269	var as = Assembler.init(input);
	2270	var output = std.ArrayList(u8).init(testing.allocator);
	2271	defer output.deinit();
	2272	try as.assemble(output.writer());
	2273	try expectEqualHexStrings(&expected, output.items, input);
	2274	}
	2275	}

src/arch/x86_64/encodings.zig created+621

...	...	@@ -0,0 +1,621 @@
	1	const Encoding = @import("Encoding.zig");
	2	const Mnemonic = Encoding.Mnemonic;
	3	const OpEn = Encoding.OpEn;
	4	const Op = Encoding.Op;
	5	const Mode = Encoding.Mode;
	6
	7	const opcode_len = u2;
	8	const modrm_ext = u3;
	9
	10	const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, opcode_len, u8, u8, u8, modrm_ext, Mode };
	11
	12	// TODO move this into a .zon file when Zig is capable of importing .zon files
	13	// zig fmt: off
	14	pub const table = &[_]Entry{
	15	// General-purpose
	16	.{ .adc, .zi, .al, .imm8, .none, .none, 1, 0x14, 0x00, 0x00, 0, .none },
	17	.{ .adc, .zi, .ax, .imm16, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
	18	.{ .adc, .zi, .eax, .imm32, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
	19	.{ .adc, .zi, .rax, .imm32s, .none, .none, 1, 0x15, 0x00, 0x00, 0, .long },
	20	.{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .none },
	21	.{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .rex },
	22	.{ .adc, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
	23	.{ .adc, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
	24	.{ .adc, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 2, .long },
	25	.{ .adc, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
	26	.{ .adc, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
	27	.{ .adc, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .long },
	28	.{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .none },
	29	.{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .rex },
	30	.{ .adc, .mr, .rm16, .r16, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
	31	.{ .adc, .mr, .rm32, .r32, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
	32	.{ .adc, .mr, .rm64, .r64, .none, .none, 1, 0x11, 0x00, 0x00, 0, .long },
	33	.{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .none },
	34	.{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .rex },
	35	.{ .adc, .rm, .r16, .rm16, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
	36	.{ .adc, .rm, .r32, .rm32, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
	37	.{ .adc, .rm, .r64, .rm64, .none, .none, 1, 0x13, 0x00, 0x00, 0, .long },
	38
	39	.{ .add, .zi, .al, .imm8, .none, .none, 1, 0x04, 0x00, 0x00, 0, .none },
	40	.{ .add, .zi, .ax, .imm16, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
	41	.{ .add, .zi, .eax, .imm32, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
	42	.{ .add, .zi, .rax, .imm32s, .none, .none, 1, 0x05, 0x00, 0x00, 0, .long },
	43	.{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .none },
	44	.{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .rex },
	45	.{ .add, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
	46	.{ .add, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
	47	.{ .add, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 0, .long },
	48	.{ .add, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
	49	.{ .add, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
	50	.{ .add, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .long },
	51	.{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .none },
	52	.{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .rex },
	53	.{ .add, .mr, .rm16, .r16, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
	54	.{ .add, .mr, .rm32, .r32, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
	55	.{ .add, .mr, .rm64, .r64, .none, .none, 1, 0x01, 0x00, 0x00, 0, .long },
	56	.{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .none },
	57	.{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .rex },
	58	.{ .add, .rm, .r16, .rm16, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
	59	.{ .add, .rm, .r32, .rm32, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
	60	.{ .add, .rm, .r64, .rm64, .none, .none, 1, 0x03, 0x00, 0x00, 0, .long },
	61
	62	.{ .@"and", .zi, .al, .imm8, .none, .none, 1, 0x24, 0x00, 0x00, 0, .none },
	63	.{ .@"and", .zi, .ax, .imm16, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
	64	.{ .@"and", .zi, .eax, .imm32, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
	65	.{ .@"and", .zi, .rax, .imm32s, .none, .none, 1, 0x25, 0x00, 0x00, 0, .long },
	66	.{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .none },
	67	.{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .rex },
	68	.{ .@"and", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
	69	.{ .@"and", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
	70	.{ .@"and", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 4, .long },
	71	.{ .@"and", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
	72	.{ .@"and", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
	73	.{ .@"and", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .long },
	74	.{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .none },
	75	.{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .rex },
	76	.{ .@"and", .mr, .rm16, .r16, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
	77	.{ .@"and", .mr, .rm32, .r32, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
	78	.{ .@"and", .mr, .rm64, .r64, .none, .none, 1, 0x21, 0x00, 0x00, 0, .long },
	79	.{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .none },
	80	.{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .rex },
	81	.{ .@"and", .rm, .r16, .rm16, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
	82	.{ .@"and", .rm, .r32, .rm32, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
	83	.{ .@"and", .rm, .r64, .rm64, .none, .none, 1, 0x23, 0x00, 0x00, 0, .long },
	84
	85	// This is M encoding according to Intel, but D makes more sense here.
	86	.{ .call, .d, .rel32, .none, .none, .none, 1, 0xe8, 0x00, 0x00, 0, .none },
	87	.{ .call, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 2, .none },
	88
	89	.{ .cbw, .np, .o16, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
	90	.{ .cwde, .np, .o32, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
	91	.{ .cdqe, .np, .o64, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .long },
	92
	93	.{ .cwd, .np, .o16, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
	94	.{ .cdq, .np, .o32, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
	95	.{ .cqo, .np, .o64, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .long },
	96
	97	.{ .cmova, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	98	.{ .cmova, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	99	.{ .cmova, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
	100	.{ .cmovae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	101	.{ .cmovae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	102	.{ .cmovae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
	103	.{ .cmovb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	104	.{ .cmovb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	105	.{ .cmovb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
	106	.{ .cmovbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	107	.{ .cmovbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	108	.{ .cmovbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
	109	.{ .cmovc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	110	.{ .cmovc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	111	.{ .cmovc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
	112	.{ .cmove, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	113	.{ .cmove, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	114	.{ .cmove, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
	115	.{ .cmovg, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	116	.{ .cmovg, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	117	.{ .cmovg, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
	118	.{ .cmovge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	119	.{ .cmovge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	120	.{ .cmovge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
	121	.{ .cmovl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	122	.{ .cmovl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	123	.{ .cmovl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
	124	.{ .cmovle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	125	.{ .cmovle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	126	.{ .cmovle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
	127	.{ .cmovna, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	128	.{ .cmovna, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
	129	.{ .cmovna, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
	130	.{ .cmovnae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	131	.{ .cmovnae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
	132	.{ .cmovnae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
	133	.{ .cmovnb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	134	.{ .cmovnb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	135	.{ .cmovnb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
	136	.{ .cmovnbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	137	.{ .cmovnbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
	138	.{ .cmovnbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
	139	.{ .cmovnc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	140	.{ .cmovnc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
	141	.{ .cmovnc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
	142	.{ .cmovne, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	143	.{ .cmovne, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	144	.{ .cmovne, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
	145	.{ .cmovng, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	146	.{ .cmovng, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
	147	.{ .cmovng, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
	148	.{ .cmovnge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	149	.{ .cmovnge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
	150	.{ .cmovnge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
	151	.{ .cmovnl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	152	.{ .cmovnl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
	153	.{ .cmovnl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
	154	.{ .cmovnle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	155	.{ .cmovnle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
	156	.{ .cmovnle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
	157	.{ .cmovno, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
	158	.{ .cmovno, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
	159	.{ .cmovno, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .long },
	160	.{ .cmovnp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	161	.{ .cmovnp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	162	.{ .cmovnp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
	163	.{ .cmovns, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
	164	.{ .cmovns, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
	165	.{ .cmovns, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .long },
	166	.{ .cmovnz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	167	.{ .cmovnz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
	168	.{ .cmovnz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
	169	.{ .cmovo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
	170	.{ .cmovo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
	171	.{ .cmovo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .long },
	172	.{ .cmovp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	173	.{ .cmovp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	174	.{ .cmovp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
	175	.{ .cmovpe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	176	.{ .cmovpe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
	177	.{ .cmovpe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
	178	.{ .cmovpo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	179	.{ .cmovpo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
	180	.{ .cmovpo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
	181	.{ .cmovs, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
	182	.{ .cmovs, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
	183	.{ .cmovs, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .long },
	184	.{ .cmovz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	185	.{ .cmovz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
	186	.{ .cmovz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
	187
	188	.{ .cmp, .zi, .al, .imm8, .none, .none, 1, 0x3c, 0x00, 0x00, 0, .none },
	189	.{ .cmp, .zi, .ax, .imm16, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
	190	.{ .cmp, .zi, .eax, .imm32, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
	191	.{ .cmp, .zi, .rax, .imm32s, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .long },
	192	.{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .none },
	193	.{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .rex },
	194	.{ .cmp, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
	195	.{ .cmp, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
	196	.{ .cmp, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 7, .long },
	197	.{ .cmp, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
	198	.{ .cmp, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
	199	.{ .cmp, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .long },
	200	.{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .none },
	201	.{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .rex },
	202	.{ .cmp, .mr, .rm16, .r16, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
	203	.{ .cmp, .mr, .rm32, .r32, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
	204	.{ .cmp, .mr, .rm64, .r64, .none, .none, 1, 0x39, 0x00, 0x00, 0, .long },
	205	.{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .none },
	206	.{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .rex },
	207	.{ .cmp, .rm, .r16, .rm16, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
	208	.{ .cmp, .rm, .r32, .rm32, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
	209	.{ .cmp, .rm, .r64, .rm64, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .long },
	210
	211	.{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .none },
	212	.{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .rex },
	213	.{ .div, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
	214	.{ .div, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
	215	.{ .div, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .long },
	216
	217	.{ .fisttp, .m, .m16, .none, .none, .none, 1, 0xdf, 0x00, 0x00, 1, .fpu },
	218	.{ .fisttp, .m, .m32, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 1, .fpu },
	219	.{ .fisttp, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 1, .fpu },
	220
	221	.{ .fld, .m, .m32, .none, .none, .none, 1, 0xd9, 0x00, 0x00, 0, .fpu },
	222	.{ .fld, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 0, .fpu },
	223	.{ .fld, .m, .m80, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 5, .fpu },
	224
	225	.{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .none },
	226	.{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .rex },
	227	.{ .idiv, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
	228	.{ .idiv, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
	229	.{ .idiv, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .long },
	230
	231	.{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .none },
	232	.{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .rex },
	233	.{ .imul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
	234	.{ .imul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
	235	.{ .imul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .long },
	236	.{ .imul, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
	237	.{ .imul, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
	238	.{ .imul, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .long },
	239	.{ .imul, .rmi, .r16, .rm16, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
	240	.{ .imul, .rmi, .r32, .rm32, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
	241	.{ .imul, .rmi, .r64, .rm64, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .long },
	242	.{ .imul, .rmi, .r16, .rm16, .imm16, .none, 1, 0x69, 0x00, 0x00, 0, .none },
	243	.{ .imul, .rmi, .r32, .rm32, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .none },
	244	.{ .imul, .rmi, .r64, .rm64, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .long },
	245
	246	.{ .int3, .np, .none, .none, .none, .none, 1, 0xcc, 0x00, 0x00, 0, .none },
	247
	248	.{ .ja, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
	249	.{ .jae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
	250	.{ .jb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
	251	.{ .jbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
	252	.{ .jc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
	253	.{ .jrcxz, .d, .rel32, .none, .none, .none, 1, 0xe3, 0x00, 0x00, 0, .none },
	254	.{ .je, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
	255	.{ .jg, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
	256	.{ .jge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
	257	.{ .jl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
	258	.{ .jle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
	259	.{ .jna, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
	260	.{ .jnae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
	261	.{ .jnb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
	262	.{ .jnbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
	263	.{ .jnc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
	264	.{ .jne, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
	265	.{ .jng, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
	266	.{ .jnge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
	267	.{ .jnl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
	268	.{ .jnle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
	269	.{ .jno, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x81, 0x00, 0, .none },
	270	.{ .jnp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
	271	.{ .jns, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x89, 0x00, 0, .none },
	272	.{ .jnz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
	273	.{ .jo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x80, 0x00, 0, .none },
	274	.{ .jp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
	275	.{ .jpe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
	276	.{ .jpo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
	277	.{ .js, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x88, 0x00, 0, .none },
	278	.{ .jz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
	279
	280	.{ .jmp, .d, .rel32, .none, .none, .none, 1, 0xe9, 0x00, 0x00, 0, .none },
	281	.{ .jmp, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 4, .none },
	282
	283	.{ .lea, .rm, .r16, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
	284	.{ .lea, .rm, .r32, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
	285	.{ .lea, .rm, .r64, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .long },
	286
	287	.{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .none },
	288	.{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .rex },
	289	.{ .mov, .mr, .rm16, .r16, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
	290	.{ .mov, .mr, .rm32, .r32, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
	291	.{ .mov, .mr, .rm64, .r64, .none, .none, 1, 0x89, 0x00, 0x00, 0, .long },
	292	.{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .none },
	293	.{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .rex },
	294	.{ .mov, .rm, .r16, .rm16, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
	295	.{ .mov, .rm, .r32, .rm32, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
	296	.{ .mov, .rm, .r64, .rm64, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .long },
	297	.{ .mov, .mr, .rm16, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .none },
	298	.{ .mov, .mr, .rm64, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .long },
	299	.{ .mov, .rm, .sreg, .rm16, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .none },
	300	.{ .mov, .rm, .sreg, .rm64, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .long },
	301	.{ .mov, .fd, .al, .moffs, .none, .none, 1, 0xa0, 0x00, 0x00, 0, .none },
	302	.{ .mov, .fd, .ax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
	303	.{ .mov, .fd, .eax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
	304	.{ .mov, .fd, .rax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .long },
	305	.{ .mov, .td, .moffs, .al, .none, .none, 1, 0xa2, 0x00, 0x00, 0, .none },
	306	.{ .mov, .td, .moffs, .ax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
	307	.{ .mov, .td, .moffs, .eax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
	308	.{ .mov, .td, .moffs, .rax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .long },
	309	.{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .none },
	310	.{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .rex },
	311	.{ .mov, .oi, .r16, .imm16, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
	312	.{ .mov, .oi, .r32, .imm32, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
	313	.{ .mov, .oi, .r64, .imm64, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .long },
	314	.{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .none },
	315	.{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .rex },
	316	.{ .mov, .mi, .rm16, .imm16, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
	317	.{ .mov, .mi, .rm32, .imm32, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
	318	.{ .mov, .mi, .rm64, .imm32s, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .long },
	319
	320	.{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
	321	.{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex },
	322	.{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
	323	.{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex },
	324	.{ .movsx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .long },
	325	.{ .movsx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .none },
	326	.{ .movsx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .long },
	327
	328	// This instruction is discouraged.
	329	.{ .movsxd, .rm, .r32, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .none },
	330	.{ .movsxd, .rm, .r64, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .long },
	331
	332	.{ .movzx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
	333	.{ .movzx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
	334	.{ .movzx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .long },
	335	.{ .movzx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .none },
	336	.{ .movzx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .long },
	337
	338	.{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .none },
	339	.{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .rex },
	340	.{ .mul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
	341	.{ .mul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
	342	.{ .mul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .long },
	343
	344	.{ .nop, .np, .none, .none, .none, .none, 1, 0x90, 0x00, 0x00, 0, .none },
	345
	346	.{ .@"or", .zi, .al, .imm8, .none, .none, 1, 0x0c, 0x00, 0x00, 0, .none },
	347	.{ .@"or", .zi, .ax, .imm16, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
	348	.{ .@"or", .zi, .eax, .imm32, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
	349	.{ .@"or", .zi, .rax, .imm32s, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .long },
	350	.{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .none },
	351	.{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .rex },
	352	.{ .@"or", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
	353	.{ .@"or", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
	354	.{ .@"or", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 1, .long },
	355	.{ .@"or", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
	356	.{ .@"or", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
	357	.{ .@"or", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .long },
	358	.{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .none },
	359	.{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .rex },
	360	.{ .@"or", .mr, .rm16, .r16, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
	361	.{ .@"or", .mr, .rm32, .r32, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
	362	.{ .@"or", .mr, .rm64, .r64, .none, .none, 1, 0x09, 0x00, 0x00, 0, .long },
	363	.{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .none },
	364	.{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .rex },
	365	.{ .@"or", .rm, .r16, .rm16, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
	366	.{ .@"or", .rm, .r32, .rm32, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
	367	.{ .@"or", .rm, .r64, .rm64, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .long },
	368
	369	.{ .pop, .o, .r16, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
	370	.{ .pop, .o, .r64, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
	371	.{ .pop, .m, .rm16, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
	372	.{ .pop, .m, .rm64, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
	373
	374	.{ .push, .o, .r16, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
	375	.{ .push, .o, .r64, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
	376	.{ .push, .m, .rm16, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
	377	.{ .push, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
	378	.{ .push, .i, .imm8, .none, .none, .none, 1, 0x6a, 0x00, 0x00, 0, .none },
	379	.{ .push, .i, .imm16, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
	380	.{ .push, .i, .imm32, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
	381
	382	.{ .ret, .np, .none, .none, .none, .none, 1, 0xc3, 0x00, 0x00, 0, .none },
	383
	384	.{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
	385	.{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex },
	386	.{ .sal, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	387	.{ .sal, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	388	.{ .sal, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
	389	.{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
	390	.{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex },
	391	.{ .sal, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	392	.{ .sal, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	393	.{ .sal, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
	394	.{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
	395	.{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex },
	396	.{ .sal, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	397	.{ .sal, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	398	.{ .sal, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
	399
	400	.{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .none },
	401	.{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .rex },
	402	.{ .sar, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
	403	.{ .sar, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
	404	.{ .sar, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .long },
	405	.{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .none },
	406	.{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .rex },
	407	.{ .sar, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
	408	.{ .sar, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
	409	.{ .sar, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .long },
	410	.{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .none },
	411	.{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .rex },
	412	.{ .sar, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
	413	.{ .sar, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
	414	.{ .sar, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .long },
	415
	416	.{ .sbb, .zi, .al, .imm8, .none, .none, 1, 0x1c, 0x00, 0x00, 0, .none },
	417	.{ .sbb, .zi, .ax, .imm16, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
	418	.{ .sbb, .zi, .eax, .imm32, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
	419	.{ .sbb, .zi, .rax, .imm32s, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .long },
	420	.{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .none },
	421	.{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .rex },
	422	.{ .sbb, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
	423	.{ .sbb, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
	424	.{ .sbb, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 3, .long },
	425	.{ .sbb, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
	426	.{ .sbb, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
	427	.{ .sbb, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .long },
	428	.{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .none },
	429	.{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .rex },
	430	.{ .sbb, .mr, .rm16, .r16, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
	431	.{ .sbb, .mr, .rm32, .r32, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
	432	.{ .sbb, .mr, .rm64, .r64, .none, .none, 1, 0x19, 0x00, 0x00, 0, .long },
	433	.{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .none },
	434	.{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .rex },
	435	.{ .sbb, .rm, .r16, .rm16, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
	436	.{ .sbb, .rm, .r32, .rm32, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
	437	.{ .sbb, .rm, .r64, .rm64, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .long },
	438
	439	.{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
	440	.{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex },
	441	.{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
	442	.{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
	443	.{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
	444	.{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
	445	.{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
	446	.{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex },
	447	.{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
	448	.{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
	449	.{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
	450	.{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex },
	451	.{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
	452	.{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex },
	453	.{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
	454	.{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex },
	455	.{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
	456	.{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex },
	457	.{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
	458	.{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex },
	459	.{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
	460	.{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex },
	461	.{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
	462	.{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
	463	.{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
	464	.{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
	465	.{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
	466	.{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex },
	467	.{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
	468	.{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
	469	.{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
	470	.{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex },
	471	.{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
	472	.{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex },
	473	.{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
	474	.{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex },
	475	.{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
	476	.{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex },
	477	.{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
	478	.{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex },
	479	.{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .none },
	480	.{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .rex },
	481	.{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
	482	.{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex },
	483	.{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .none },
	484	.{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .rex },
	485	.{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
	486	.{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex },
	487	.{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .none },
	488	.{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .rex },
	489	.{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
	490	.{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex },
	491	.{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
	492	.{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex },
	493	.{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
	494	.{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex },
	495	.{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .none },
	496	.{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .rex },
	497	.{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
	498	.{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex },
	499
	500	.{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
	501	.{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex },
	502	.{ .shl, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	503	.{ .shl, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
	504	.{ .shl, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
	505	.{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
	506	.{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex },
	507	.{ .shl, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	508	.{ .shl, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
	509	.{ .shl, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
	510	.{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
	511	.{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex },
	512	.{ .shl, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	513	.{ .shl, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
	514	.{ .shl, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
	515
	516	.{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .none },
	517	.{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .rex },
	518	.{ .shr, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
	519	.{ .shr, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
	520	.{ .shr, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .long },
	521	.{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .none },
	522	.{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .rex },
	523	.{ .shr, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
	524	.{ .shr, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
	525	.{ .shr, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .long },
	526	.{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .none },
	527	.{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .rex },
	528	.{ .shr, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
	529	.{ .shr, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
	530	.{ .shr, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .long },
	531
	532	.{ .sub, .zi, .al, .imm8, .none, .none, 1, 0x2c, 0x00, 0x00, 0, .none },
	533	.{ .sub, .zi, .ax, .imm16, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
	534	.{ .sub, .zi, .eax, .imm32, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
	535	.{ .sub, .zi, .rax, .imm32s, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .long },
	536	.{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .none },
	537	.{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .rex },
	538	.{ .sub, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
	539	.{ .sub, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
	540	.{ .sub, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 5, .long },
	541	.{ .sub, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
	542	.{ .sub, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
	543	.{ .sub, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .long },
	544	.{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .none },
	545	.{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .rex },
	546	.{ .sub, .mr, .rm16, .r16, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
	547	.{ .sub, .mr, .rm32, .r32, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
	548	.{ .sub, .mr, .rm64, .r64, .none, .none, 1, 0x29, 0x00, 0x00, 0, .long },
	549	.{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .none },
	550	.{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .rex },
	551	.{ .sub, .rm, .r16, .rm16, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
	552	.{ .sub, .rm, .r32, .rm32, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
	553	.{ .sub, .rm, .r64, .rm64, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .long },
	554
	555	.{ .syscall, .np, .none, .none, .none, .none, 2, 0x0f, 0x05, 0x00, 0, .none },
	556
	557	.{ .@"test", .zi, .al, .imm8, .none, .none, 1, 0xa8, 0x00, 0x00, 0, .none },
	558	.{ .@"test", .zi, .ax, .imm16, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
	559	.{ .@"test", .zi, .eax, .imm32, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
	560	.{ .@"test", .zi, .rax, .imm32s, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .long },
	561	.{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .none },
	562	.{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .rex },
	563	.{ .@"test", .mi, .rm16, .imm16, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
	564	.{ .@"test", .mi, .rm32, .imm32, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
	565	.{ .@"test", .mi, .rm64, .imm32s, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .long },
	566	.{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .none },
	567	.{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .rex },
	568	.{ .@"test", .mr, .rm16, .r16, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
	569	.{ .@"test", .mr, .rm32, .r32, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
	570	.{ .@"test", .mr, .rm64, .r64, .none, .none, 1, 0x85, 0x00, 0x00, 0, .long },
	571
	572	.{ .ud2, .np, .none, .none, .none, .none, 2, 0x0f, 0x0b, 0x00, 0, .none },
	573
	574	.{ .xor, .zi, .al, .imm8, .none, .none, 1, 0x34, 0x00, 0x00, 0, .none },
	575	.{ .xor, .zi, .ax, .imm16, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
	576	.{ .xor, .zi, .eax, .imm32, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
	577	.{ .xor, .zi, .rax, .imm32s, .none, .none, 1, 0x35, 0x00, 0x00, 0, .long },
	578	.{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .none },
	579	.{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .rex },
	580	.{ .xor, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
	581	.{ .xor, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
	582	.{ .xor, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 6, .long },
	583	.{ .xor, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
	584	.{ .xor, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
	585	.{ .xor, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .long },
	586	.{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .none },
	587	.{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .rex },
	588	.{ .xor, .mr, .rm16, .r16, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
	589	.{ .xor, .mr, .rm32, .r32, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
	590	.{ .xor, .mr, .rm64, .r64, .none, .none, 1, 0x31, 0x00, 0x00, 0, .long },
	591	.{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .none },
	592	.{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .rex },
	593	.{ .xor, .rm, .r16, .rm16, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
	594	.{ .xor, .rm, .r32, .rm32, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
	595	.{ .xor, .rm, .r64, .rm64, .none, .none, 1, 0x33, 0x00, 0x00, 0, .long },
	596
	597	// SSE
	598	.{ .addss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x58, 0, .sse },
	599
	600	.{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, 3, 0xf3, 0x0f, 0xc2, 0, .sse },
	601
	602	.{ .movss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x10, 0, .sse },
	603	.{ .movss, .mr, .xmm_m32, .xmm, .none, .none, 3, 0xf3, 0x0f, 0x11, 0, .sse },
	604
	605	.{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, 2, 0x0f, 0x2e, 0x00, 0, .sse },
	606
	607	// SSE2
	608	.{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x58, 0, .sse2 },
	609
	610	.{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, 3, 0xf2, 0x0f, 0xc2, 0, .sse2 },
	611
	612	.{ .movq, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf3, 0x0f, 0x7e, 0, .sse2 },
	613	.{ .movq, .mr, .xmm_m64, .xmm, .none, .none, 3, 0x66, 0x0f, 0xd6, 0, .sse2 },
	614
	615	.{ .movsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x10, 0, .sse2 },
	616	.{ .movsd, .mr, .xmm_m64, .xmm, .none, .none, 3, 0xf2, 0x0f, 0x11, 0, .sse2 },
	617
	618	.{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0x66, 0x0f, 0x2e, 0, .sse2 },
	619	};
	620	// zig fmt: on
	621