Commit 704d38ba4981d46ac1d4687d8f6dcce72bf4067c | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2022-05-20 17:19:28+02:00
committer	GitHub <noreply@github.com> 2022-05-20 17:19:28+02:00
log	704d38ba4981d46ac1d4687d8f6dcce72bf4067c
tree	f80363ac870b07ab61ab41ff927c42785bfd20b9
parent	58943fc6276c49b138c98bda5abac8be2806f94f
parent	274654d73e36df29fdc017a009c8607fb46a15e4
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11681 from ziglang/x64-floats-tmp

x64: add prelim support for SSE/AVX-based floating-point ops

17 files changed, 2407 insertions(+), 848 deletions(-)

src
- arch
  - aarch64
    CodeGen.zig
    abi.zig
  - arm
    CodeGen.zig
    abi.zig
  - riscv64
    CodeGen.zig
    abi.zig
  - sparc64
    CodeGen.zig
    abi.zig
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig
    abi.zig
    bits.zig
- register_manager.zig
test
- behavior

src/arch/aarch64/CodeGen.zig+31-31

src/arch/aarch64/abi.zig+20

...	@@ -1,6 +1,8 @@	...	@@ -1,6 +1,8 @@
		1	const std = @import("std");
1	const builtin = @import("builtin");	2	const builtin = @import("builtin");
2	const bits = @import("bits.zig");	3	const bits = @import("bits.zig");
3	const Register = bits.Register;	4	const Register = bits.Register;
		5	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
4		6
5	const callee_preserved_regs_impl = if (builtin.os.tag.isDarwin()) struct {	7	const callee_preserved_regs_impl = if (builtin.os.tag.isDarwin()) struct {
6	pub const callee_preserved_regs = [_]Register{	8	pub const callee_preserved_regs = [_]Register{
...	@@ -18,3 +20,21 @@ pub const callee_preserved_regs = callee_preserved_regs_impl.callee_preserved_re	...	@@ -18,3 +20,21 @@ pub const callee_preserved_regs = callee_preserved_regs_impl.callee_preserved_re
18		20
19	pub const c_abi_int_param_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };	21	pub const c_abi_int_param_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
20	pub const c_abi_int_return_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };	22	pub const c_abi_int_return_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
		23
		24	const allocatable_registers = callee_preserved_regs;
		25	pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
		26
		27	// Register classes
		28	const RegisterBitSet = RegisterManager.RegisterBitSet;
		29	pub const RegisterClass = struct {
		30	pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
		31	// TODO uncomment once #11680 is fixed.
		32	// pub const gp: RegisterBitSet = blk: {
		33	// var set = RegisterBitSet.initEmpty();
		34	// set.setRangeValue(.{
		35	// .start = 0,
		36	// .end = callee_preserved_regs.len,
		37	// }, true);
		38	// break :blk set;
		39	// };
		40	};

src/arch/arm/CodeGen.zig+35-36

src/arch/arm/abi.zig+20-1

...	@@ -1,9 +1,28 @@	...	@@ -1,9 +1,28 @@
		1	const std = @import("std");
1	const bits = @import("bits.zig");	2	const bits = @import("bits.zig");
2	const Register = bits.Register;	3	const Register = bits.Register;
		4	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
3		5
4	pub const callee_preserved_regs = [_]Register{ .r4, .r5, .r6, .r7, .r8, .r10 };	6	pub const callee_preserved_regs = [_]Register{ .r4, .r5, .r6, .r7, .r8, .r10 };
5	pub const caller_preserved_regs = [_]Register{ .r0, .r1, .r2, .r3 };	7	pub const caller_preserved_regs = [_]Register{ .r0, .r1, .r2, .r3 };
6	pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
7		8
8	pub const c_abi_int_param_regs = [_]Register{ .r0, .r1, .r2, .r3 };	9	pub const c_abi_int_param_regs = [_]Register{ .r0, .r1, .r2, .r3 };
9	pub const c_abi_int_return_regs = [_]Register{ .r0, .r1 };	10	pub const c_abi_int_return_regs = [_]Register{ .r0, .r1 };
		11
		12	const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
		13	pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
		14
		15	// Register classes
		16	const RegisterBitSet = RegisterManager.RegisterBitSet;
		17	pub const RegisterClass = struct {
		18	pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
		19	// TODO uncomment once #11680 is fixed.
		20	// pub const gp: RegisterBitSet = blk: {
		21	// var set = RegisterBitSet.initEmpty();
		22	// set.setRangeValue(.{
		23	// .start = 0,
		24	// .end = caller_preserved_regs.len + callee_preserved_regs.len,
		25	// }, true);
		26	// break :blk set;
		27	// };
		28	};

src/arch/riscv64/CodeGen.zig+11-11

src/arch/riscv64/abi.zig+20

...	@@ -1,6 +1,26 @@	...	@@ -1,6 +1,26 @@
		1	const std = @import("std");
1	const bits = @import("bits.zig");	2	const bits = @import("bits.zig");
2	const Register = bits.Register;	3	const Register = bits.Register;
		4	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
3		5
4	pub const callee_preserved_regs = [_]Register{	6	pub const callee_preserved_regs = [_]Register{
5	.s0, .s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11,	7	.s0, .s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11,
6	};	8	};
		9
		10	const allocatable_registers = callee_preserved_regs;
		11	pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
		12
		13	// Register classes
		14	const RegisterBitSet = RegisterManager.RegisterBitSet;
		15	pub const RegisterClass = struct {
		16	pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
		17	// TODO uncomment once #11680 is fixed.
		18	// pub const gp: RegisterBitSet = blk: {
		19	// var set = RegisterBitSet.initEmpty();
		20	// set.setRangeValue(.{
		21	// .start = 0,
		22	// .end = callee_preserved_regs.len,
		23	// }, true);
		24	// break :blk set;
		25	// };
		26	};

src/arch/sparc64/CodeGen.zig+16-16

src/arch/sparc64/abi.zig+20-1

...	@@ -1,5 +1,7 @@	...	@@ -1,5 +1,7 @@
		1	const std = @import("std");
1	const bits = @import("bits.zig");	2	const bits = @import("bits.zig");
2	const Register = bits.Register;	3	const Register = bits.Register;
		4	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
3		5
4	// SPARCv9 stack constants.	6	// SPARCv9 stack constants.
5	// See: Registers and the Stack Frame, page 3P-8, SCD 2.4.1.	7	// See: Registers and the Stack Frame, page 3P-8, SCD 2.4.1.
...	@@ -21,7 +23,7 @@ pub const stack_save_area = 176;	...	@@ -21,7 +23,7 @@ pub const stack_save_area = 176;
21	pub const caller_preserved_regs = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5, .g1, .g4, .g5 };	23	pub const caller_preserved_regs = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5, .g1, .g4, .g5 };
22		24
23	// Try to allocate i, l, o, then g sets of registers, in order of priority.	25	// Try to allocate i, l, o, then g sets of registers, in order of priority.
24	pub const allocatable_regs = [_]Register{	26	const allocatable_regs = [_]Register{
25	// zig fmt: off	27	// zig fmt: off
26	.@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5",	28	.@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5",
27	.l0, .l1, .l2, .l3, .l4, .l5, .l6, .l7,	29	.l0, .l1, .l2, .l3, .l4, .l5, .l6, .l7,
...	@@ -35,3 +37,20 @@ pub const c_abi_int_param_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2"	...	@@ -35,3 +37,20 @@ pub const c_abi_int_param_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2"
35		37
36	pub const c_abi_int_return_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3 };	38	pub const c_abi_int_return_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3 };
37	pub const c_abi_int_return_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3" };	39	pub const c_abi_int_return_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3" };
		40
		41	pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_regs);
		42
		43	// Register classes
		44	const RegisterBitSet = RegisterManager.RegisterBitSet;
		45	pub const RegisterClass = struct {
		46	pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
		47	// TODO uncomment once #11680 is fixed.
		48	// pub const gp: RegisterBitSet = blk: {
		49	// var set = RegisterBitSet.initEmpty();
		50	// set.setRangeValue(.{
		51	// .start = 0,
		52	// .end = allocatable_regs.len,
		53	// }, true);
		54	// break :blk set;
		55	// };
		56	};

src/arch/x86_64/CodeGen.zig+673-434

...	@@ -21,9 +21,6 @@ const Emit = @import("Emit.zig");	...	@@ -21,9 +21,6 @@ const Emit = @import("Emit.zig");
21	const Liveness = @import("../../Liveness.zig");	21	const Liveness = @import("../../Liveness.zig");
22	const Mir = @import("Mir.zig");	22	const Mir = @import("Mir.zig");
23	const Module = @import("../../Module.zig");	23	const Module = @import("../../Module.zig");
24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25	const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
26	const RegisterLock = RegisterManager.RegisterLock;
27	const Target = std.Target;	24	const Target = std.Target;
28	const Type = @import("../../type.zig").Type;	25	const Type = @import("../../type.zig").Type;
29	const TypedValue = @import("../../TypedValue.zig");	26	const TypedValue = @import("../../TypedValue.zig");
...	@@ -31,13 +28,19 @@ const Value = @import("../../value.zig").Value;	...	@@ -31,13 +28,19 @@ const Value = @import("../../value.zig").Value;
31		28
32	const bits = @import("bits.zig");	29	const bits = @import("bits.zig");
33	const abi = @import("abi.zig");	30	const abi = @import("abi.zig");
34	const Register = bits.Register;	31
35	const callee_preserved_regs = abi.callee_preserved_regs;	32	const callee_preserved_regs = abi.callee_preserved_regs;
36	const caller_preserved_regs = abi.caller_preserved_regs;	33	const caller_preserved_regs = abi.caller_preserved_regs;
37	const allocatable_registers = abi.allocatable_registers;
38	const c_abi_int_param_regs = abi.c_abi_int_param_regs;	34	const c_abi_int_param_regs = abi.c_abi_int_param_regs;
39	const c_abi_int_return_regs = abi.c_abi_int_return_regs;	35	const c_abi_int_return_regs = abi.c_abi_int_return_regs;
40		36
		37	const RegisterManager = abi.RegisterManager;
		38	const RegisterLock = RegisterManager.RegisterLock;
		39	const Register = bits.Register;
		40
		41	const gp = abi.RegisterClass.gp;
		42	const sse = abi.RegisterClass.sse;
		43
41	const InnerError = error{	44	const InnerError = error{
42	OutOfMemory,	45	OutOfMemory,
43	CodegenFail,	46	CodegenFail,
...	@@ -119,12 +122,12 @@ pub const MCValue = union(enum) {	...	@@ -119,12 +122,12 @@ pub const MCValue = union(enum) {
119	/// A pointer-sized integer that fits in a register.	122	/// A pointer-sized integer that fits in a register.
120	/// If the type is a pointer, this is the pointer address in virtual address space.	123	/// If the type is a pointer, this is the pointer address in virtual address space.
121	immediate: u64,	124	immediate: u64,
122	/// The value is in a target-specific register.	125	/// The value is in a GP register.
123	register: Register,	126	register: Register,
124	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the register,	127	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register,
125	/// and the operation is an unsigned operation.	128	/// and the operation is an unsigned operation.
126	register_overflow_unsigned: Register,	129	register_overflow_unsigned: Register,
127	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the register,	130	/// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register,
128	/// and the operation is a signed operation.	131	/// and the operation is a signed operation.
129	register_overflow_signed: Register,	132	register_overflow_signed: Register,
130	/// The value is in memory at a hard-coded address.	133	/// The value is in memory at a hard-coded address.
...	@@ -387,17 +390,15 @@ fn gen(self: *Self) InnerError!void {	...	@@ -387,17 +390,15 @@ fn gen(self: *Self) InnerError!void {
387	if (cc != .Naked) {	390	if (cc != .Naked) {
388	_ = try self.addInst(.{	391	_ = try self.addInst(.{
389	.tag = .push,	392	.tag = .push,
390	.ops = (Mir.Ops{	393	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
391	.reg1 = .rbp,
392	}).encode(),
393	.data = undefined, // unused for push reg,	394	.data = undefined, // unused for push reg,
394	});	395	});
395	_ = try self.addInst(.{	396	_ = try self.addInst(.{
396	.tag = .mov,	397	.tag = .mov,
397	.ops = (Mir.Ops{	398	.ops = Mir.Inst.Ops.encode(.{
398	.reg1 = .rbp,	399	.reg1 = .rbp,
399	.reg2 = .rsp,	400	.reg2 = .rsp,
400	}).encode(),	401	}),
401	.data = undefined,	402	.data = undefined,
402	});	403	});
403	// We want to subtract the aligned stack frame size from rsp here, but we don't	404	// We want to subtract the aligned stack frame size from rsp here, but we don't
...	@@ -434,9 +435,7 @@ fn gen(self: *Self) InnerError!void {	...	@@ -434,9 +435,7 @@ fn gen(self: *Self) InnerError!void {
434	// push the callee_preserved_regs that were used	435	// push the callee_preserved_regs that were used
435	const backpatch_push_callee_preserved_regs_i = try self.addInst(.{	436	const backpatch_push_callee_preserved_regs_i = try self.addInst(.{
436	.tag = .push_regs_from_callee_preserved_regs,	437	.tag = .push_regs_from_callee_preserved_regs,
437	.ops = (Mir.Ops{	438	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
438	.reg1 = .rbp,
439	}).encode(),
440	.data = .{ .payload = undefined }, // to be backpatched	439	.data = .{ .payload = undefined }, // to be backpatched
441	});	440	});
442		441
...	@@ -476,9 +475,7 @@ fn gen(self: *Self) InnerError!void {	...	@@ -476,9 +475,7 @@ fn gen(self: *Self) InnerError!void {
476	// pop the callee_preserved_regs	475	// pop the callee_preserved_regs
477	_ = try self.addInst(.{	476	_ = try self.addInst(.{
478	.tag = .pop_regs_from_callee_preserved_regs,	477	.tag = .pop_regs_from_callee_preserved_regs,
479	.ops = (Mir.Ops{	478	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
480	.reg1 = .rbp,
481	}).encode(),
482	.data = .{ .payload = callee_preserved_regs_payload },	479	.data = .{ .payload = callee_preserved_regs_payload },
483	});	480	});
484		481
...	@@ -497,17 +494,13 @@ fn gen(self: *Self) InnerError!void {	...	@@ -497,17 +494,13 @@ fn gen(self: *Self) InnerError!void {
497		494
498	_ = try self.addInst(.{	495	_ = try self.addInst(.{
499	.tag = .pop,	496	.tag = .pop,
500	.ops = (Mir.Ops{	497	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
501	.reg1 = .rbp,
502	}).encode(),
503	.data = undefined,	498	.data = undefined,
504	});	499	});
505		500
506	_ = try self.addInst(.{	501	_ = try self.addInst(.{
507	.tag = .ret,	502	.tag = .ret,
508	.ops = (Mir.Ops{	503	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
509	.flags = 0b11,
510	}).encode(),
511	.data = undefined,	504	.data = undefined,
512	});	505	});
513		506
...	@@ -521,16 +514,12 @@ fn gen(self: *Self) InnerError!void {	...	@@ -521,16 +514,12 @@ fn gen(self: *Self) InnerError!void {
521	if (aligned_stack_end > 0) {	514	if (aligned_stack_end > 0) {
522	self.mir_instructions.set(backpatch_stack_sub, .{	515	self.mir_instructions.set(backpatch_stack_sub, .{
523	.tag = .sub,	516	.tag = .sub,
524	.ops = (Mir.Ops{	517	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
525	.reg1 = .rsp,
526	}).encode(),
527	.data = .{ .imm = aligned_stack_end },	518	.data = .{ .imm = aligned_stack_end },
528	});	519	});
529	self.mir_instructions.set(backpatch_stack_add, .{	520	self.mir_instructions.set(backpatch_stack_add, .{
530	.tag = .add,	521	.tag = .add,
531	.ops = (Mir.Ops{	522	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
532	.reg1 = .rsp,
533	}).encode(),
534	.data = .{ .imm = aligned_stack_end },	523	.data = .{ .imm = aligned_stack_end },
535	});	524	});
536	}	525	}
...	@@ -889,13 +878,30 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {	...	@@ -889,13 +878,30 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
889	self.stack_align = abi_align;	878	self.stack_align = abi_align;
890		879
891	if (reg_ok) {	880	if (reg_ok) {
892	// Make sure the type can fit in a register before we try to allocate one.	881	switch (elem_ty.zigTypeTag()) {
893	const ptr_bits = self.target.cpu.arch.ptrBitWidth();	882	.Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}),
894	const ptr_bytes: u64 = @divExact(ptr_bits, 8);	883	.Float => {
895	if (abi_size <= ptr_bytes) {	884	if (intrinsicsAllowed(self.target.*, elem_ty)) {
896	if (self.register_manager.tryAllocReg(inst)) \|reg\| {	885	const ptr_bytes: u64 = 32;
897	return MCValue{ .register = registerAlias(reg, abi_size) };	886	if (abi_size <= ptr_bytes) {
898	}	887	if (self.register_manager.tryAllocReg(inst, sse)) \|reg\| {
		888	return MCValue{ .register = registerAlias(reg, abi_size) };
		889	}
		890	}
		891	}
		892
		893	return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{});
		894	},
		895	else => {
		896	// Make sure the type can fit in a register before we try to allocate one.
		897	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
		898	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
		899	if (abi_size <= ptr_bytes) {
		900	if (self.register_manager.tryAllocReg(inst, gp)) \|reg\| {
		901	return MCValue{ .register = registerAlias(reg, abi_size) };
		902	}
		903	}
		904	},
899	}	905	}
900	}	906	}
901	const stack_offset = try self.allocMem(inst, abi_size, abi_align);	907	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
...	@@ -962,7 +968,14 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou	...	@@ -962,7 +968,14 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou
962	/// allocated. A second call to `copyToTmpRegister` may return the same register.	968	/// allocated. A second call to `copyToTmpRegister` may return the same register.
963	/// This can have a side effect of spilling instructions to the stack to free up a register.	969	/// This can have a side effect of spilling instructions to the stack to free up a register.
964	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {	970	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
965	const reg = try self.register_manager.allocReg(null);	971	const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
		972	.Float => blk: {
		973	if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
		974	return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
		975	},
		976	else => gp,
		977	};
		978	const reg: Register = try self.register_manager.allocReg(null, reg_class);
966	try self.genSetReg(ty, reg, mcv);	979	try self.genSetReg(ty, reg, mcv);
967	return reg;	980	return reg;
968	}	981	}
...	@@ -972,7 +985,14 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {	...	@@ -972,7 +985,14 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
972	/// This can have a side effect of spilling instructions to the stack to free up a register.	985	/// This can have a side effect of spilling instructions to the stack to free up a register.
973	/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!	986	/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
974	fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {	987	fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {
975	const reg = try self.register_manager.allocReg(reg_owner);	988	const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
		989	.Float => blk: {
		990	if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
		991	return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
		992	},
		993	else => gp,
		994	};
		995	const reg: Register = try self.register_manager.allocReg(reg_owner, reg_class);
976	try self.genSetReg(ty, reg, mcv);	996	try self.genSetReg(ty, reg, mcv);
977	return MCValue{ .register = reg };	997	return MCValue{ .register = reg };
978	}	998	}
...	@@ -1028,7 +1048,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1028,7 +1048,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1028	};	1048	};
1029	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);	1049	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
1030		1050
1031	const reg = try self.register_manager.allocReg(inst);	1051	const reg = try self.register_manager.allocReg(inst, gp);
1032	try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });	1052	try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });
1033	try self.genSetReg(operand_ty, reg, operand);	1053	try self.genSetReg(operand_ty, reg, operand);
1034	break :blk MCValue{ .register = reg };	1054	break :blk MCValue{ .register = reg };
...	@@ -1192,10 +1212,10 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1192,10 +1212,10 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1192	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv);	1212	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv);
1193	_ = try self.addInst(.{	1213	_ = try self.addInst(.{
1194	.tag = if (signedness == .signed) .cond_mov_lt else .cond_mov_below,	1214	.tag = if (signedness == .signed) .cond_mov_lt else .cond_mov_below,
1195	.ops = (Mir.Ops{	1215	.ops = Mir.Inst.Ops.encode(.{
1196	.reg1 = dst_mcv.register,	1216	.reg1 = dst_mcv.register,
1197	.reg2 = lhs_reg,	1217	.reg2 = lhs_reg,
1198	}).encode(),	1218	}),
1199	.data = undefined,	1219	.data = undefined,
1200	});	1220	});
1201		1221
...	@@ -1383,7 +1403,7 @@ fn genSetStackTruncatedOverflowCompare(	...	@@ -1383,7 +1403,7 @@ fn genSetStackTruncatedOverflowCompare(
1383	.unsigned => ty,	1403	.unsigned => ty,
1384	};	1404	};
1385		1405
1386	const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null });	1406	const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
1387	const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);	1407	const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
1388	defer for (temp_regs_locks) \|rreg\| {	1408	defer for (temp_regs_locks) \|rreg\| {
1389	self.register_manager.unlockReg(rreg);	1409	self.register_manager.unlockReg(rreg);
...	@@ -1396,10 +1416,10 @@ fn genSetStackTruncatedOverflowCompare(	...	@@ -1396,10 +1416,10 @@ fn genSetStackTruncatedOverflowCompare(
1396	};	1416	};
1397	_ = try self.addInst(.{	1417	_ = try self.addInst(.{
1398	.tag = .cond_set_byte_overflow,	1418	.tag = .cond_set_byte_overflow,
1399	.ops = (Mir.Ops{	1419	.ops = Mir.Inst.Ops.encode(.{
1400	.reg1 = overflow_reg.to8(),	1420	.reg1 = overflow_reg.to8(),
1401	.flags = flags,	1421	.flags = flags,
1402	}).encode(),	1422	}),
1403	.data = undefined,	1423	.data = undefined,
1404	});	1424	});
1405		1425
...	@@ -1416,10 +1436,7 @@ fn genSetStackTruncatedOverflowCompare(	...	@@ -1416,10 +1436,7 @@ fn genSetStackTruncatedOverflowCompare(
1416	const eq_reg = temp_regs[2];	1436	const eq_reg = temp_regs[2];
1417	_ = try self.addInst(.{	1437	_ = try self.addInst(.{
1418	.tag = .cond_set_byte_eq_ne,	1438	.tag = .cond_set_byte_eq_ne,
1419	.ops = (Mir.Ops{	1439	.ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }),
1420	.reg1 = eq_reg.to8(),
1421	.flags = 0b00,
1422	}).encode(),
1423	.data = undefined,	1440	.data = undefined,
1424	});	1441	});
1425		1442
...	@@ -1565,19 +1582,17 @@ fn genIntMulDivOpMir(	...	@@ -1565,19 +1582,17 @@ fn genIntMulDivOpMir(
1565	.signed => {	1582	.signed => {
1566	_ = try self.addInst(.{	1583	_ = try self.addInst(.{
1567	.tag = .cwd,	1584	.tag = .cwd,
1568	.ops = (Mir.Ops{	1585	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
1569	.flags = 0b11,
1570	}).encode(),
1571	.data = undefined,	1586	.data = undefined,
1572	});	1587	});
1573	},	1588	},
1574	.unsigned => {	1589	.unsigned => {
1575	_ = try self.addInst(.{	1590	_ = try self.addInst(.{
1576	.tag = .xor,	1591	.tag = .xor,
1577	.ops = (Mir.Ops{	1592	.ops = Mir.Inst.Ops.encode(.{
1578	.reg1 = .rdx,	1593	.reg1 = .rdx,
1579	.reg2 = .rdx,	1594	.reg2 = .rdx,
1580	}).encode(),	1595	}),
1581	.data = undefined,	1596	.data = undefined,
1582	});	1597	});
1583	},	1598	},
...	@@ -1596,16 +1611,14 @@ fn genIntMulDivOpMir(	...	@@ -1596,16 +1611,14 @@ fn genIntMulDivOpMir(
1596	.register => \|reg\| {	1611	.register => \|reg\| {
1597	_ = try self.addInst(.{	1612	_ = try self.addInst(.{
1598	.tag = tag,	1613	.tag = tag,
1599	.ops = (Mir.Ops{	1614	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
1600	.reg1 = reg,
1601	}).encode(),
1602	.data = undefined,	1615	.data = undefined,
1603	});	1616	});
1604	},	1617	},
1605	.stack_offset => \|off\| {	1618	.stack_offset => \|off\| {
1606	_ = try self.addInst(.{	1619	_ = try self.addInst(.{
1607	.tag = tag,	1620	.tag = tag,
1608	.ops = (Mir.Ops{	1621	.ops = Mir.Inst.Ops.encode(.{
1609	.reg2 = .rbp,	1622	.reg2 = .rbp,
1610	.flags = switch (abi_size) {	1623	.flags = switch (abi_size) {
1611	1 => 0b00,	1624	1 => 0b00,
...	@@ -1614,7 +1627,7 @@ fn genIntMulDivOpMir(	...	@@ -1614,7 +1627,7 @@ fn genIntMulDivOpMir(
1614	8 => 0b11,	1627	8 => 0b11,
1615	else => unreachable,	1628	else => unreachable,
1616	},	1629	},
1617	}).encode(),	1630	}),
1618	.data = .{ .imm = @bitCast(u32, -off) },	1631	.data = .{ .imm = @bitCast(u32, -off) },
1619	});	1632	});
1620	},	1633	},
...	@@ -1647,34 +1660,34 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -1647,34 +1660,34 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
1647		1660
1648	_ = try self.addInst(.{	1661	_ = try self.addInst(.{
1649	.tag = .xor,	1662	.tag = .xor,
1650	.ops = (Mir.Ops{	1663	.ops = Mir.Inst.Ops.encode(.{
1651	.reg1 = divisor.to64(),	1664	.reg1 = divisor.to64(),
1652	.reg2 = dividend.to64(),	1665	.reg2 = dividend.to64(),
1653	}).encode(),	1666	}),
1654	.data = undefined,	1667	.data = undefined,
1655	});	1668	});
1656	_ = try self.addInst(.{	1669	_ = try self.addInst(.{
1657	.tag = .sar,	1670	.tag = .sar,
1658	.ops = (Mir.Ops{	1671	.ops = Mir.Inst.Ops.encode(.{
1659	.reg1 = divisor.to64(),	1672	.reg1 = divisor.to64(),
1660	.flags = 0b10,	1673	.flags = 0b10,
1661	}).encode(),	1674	}),
1662	.data = .{ .imm = 63 },	1675	.data = .{ .imm = 63 },
1663	});	1676	});
1664	_ = try self.addInst(.{	1677	_ = try self.addInst(.{
1665	.tag = .@"test",	1678	.tag = .@"test",
1666	.ops = (Mir.Ops{	1679	.ops = Mir.Inst.Ops.encode(.{
1667	.reg1 = .rdx,	1680	.reg1 = .rdx,
1668	.reg2 = .rdx,	1681	.reg2 = .rdx,
1669	}).encode(),	1682	}),
1670	.data = undefined,	1683	.data = undefined,
1671	});	1684	});
1672	_ = try self.addInst(.{	1685	_ = try self.addInst(.{
1673	.tag = .cond_mov_eq,	1686	.tag = .cond_mov_eq,
1674	.ops = (Mir.Ops{	1687	.ops = Mir.Inst.Ops.encode(.{
1675	.reg1 = divisor.to64(),	1688	.reg1 = divisor.to64(),
1676	.reg2 = .rdx,	1689	.reg2 = .rdx,
1677	}).encode(),	1690	}),
1678	.data = undefined,	1691	.data = undefined,
1679	});	1692	});
1680	try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });	1693	try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
...	@@ -2052,17 +2065,17 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	...	@@ -2052,17 +2065,17 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2052	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);	2065	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2053	defer self.register_manager.unlockReg(offset_reg_lock);	2066	defer self.register_manager.unlockReg(offset_reg_lock);
2054		2067
2055	const addr_reg = try self.register_manager.allocReg(null);	2068	const addr_reg = try self.register_manager.allocReg(null, gp);
2056	switch (slice_mcv) {	2069	switch (slice_mcv) {
2057	.stack_offset => \|off\| {	2070	.stack_offset => \|off\| {
2058	// mov reg, [rbp - 8]	2071	// mov reg, [rbp - 8]
2059	_ = try self.addInst(.{	2072	_ = try self.addInst(.{
2060	.tag = .mov,	2073	.tag = .mov,
2061	.ops = (Mir.Ops{	2074	.ops = Mir.Inst.Ops.encode(.{
2062	.reg1 = addr_reg.to64(),	2075	.reg1 = addr_reg.to64(),
2063	.reg2 = .rbp,	2076	.reg2 = .rbp,
2064	.flags = 0b01,	2077	.flags = 0b01,
2065	}).encode(),	2078	}),
2066	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },	2079	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
2067	});	2080	});
2068	},	2081	},
...	@@ -2131,7 +2144,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2131,7 +2144,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2131	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);	2144	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2132	defer self.register_manager.unlockReg(offset_reg_lock);	2145	defer self.register_manager.unlockReg(offset_reg_lock);
2133		2146
2134	const addr_reg = try self.register_manager.allocReg(null);	2147	const addr_reg = try self.register_manager.allocReg(null, gp);
2135	switch (array) {	2148	switch (array) {
2136	.register => {	2149	.register => {
2137	const off = @intCast(i32, try self.allocMem(	2150	const off = @intCast(i32, try self.allocMem(
...	@@ -2143,10 +2156,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2143,10 +2156,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2143	// lea reg, [rbp]	2156	// lea reg, [rbp]
2144	_ = try self.addInst(.{	2157	_ = try self.addInst(.{
2145	.tag = .lea,	2158	.tag = .lea,
2146	.ops = (Mir.Ops{	2159	.ops = Mir.Inst.Ops.encode(.{
2147	.reg1 = addr_reg.to64(),	2160	.reg1 = addr_reg.to64(),
2148	.reg2 = .rbp,	2161	.reg2 = .rbp,
2149	}).encode(),	2162	}),
2150	.data = .{ .imm = @bitCast(u32, -off) },	2163	.data = .{ .imm = @bitCast(u32, -off) },
2151	});	2164	});
2152	},	2165	},
...	@@ -2154,10 +2167,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2154,10 +2167,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2154	// lea reg, [rbp]	2167	// lea reg, [rbp]
2155	_ = try self.addInst(.{	2168	_ = try self.addInst(.{
2156	.tag = .lea,	2169	.tag = .lea,
2157	.ops = (Mir.Ops{	2170	.ops = Mir.Inst.Ops.encode(.{
2158	.reg1 = addr_reg.to64(),	2171	.reg1 = addr_reg.to64(),
2159	.reg2 = .rbp,	2172	.reg2 = .rbp,
2160	}).encode(),	2173	}),
2161	.data = .{ .imm = @bitCast(u32, -off) },	2174	.data = .{ .imm = @bitCast(u32, -off) },
2162	});	2175	});
2163	},	2176	},
...	@@ -2222,11 +2235,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2222,11 +2235,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2222	// mov dst_mcv, [dst_mcv]	2235	// mov dst_mcv, [dst_mcv]
2223	_ = try self.addInst(.{	2236	_ = try self.addInst(.{
2224	.tag = .mov,	2237	.tag = .mov,
2225	.ops = (Mir.Ops{	2238	.ops = Mir.Inst.Ops.encode(.{
2226	.flags = 0b01,
2227	.reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),	2239	.reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),
2228	.reg2 = dst_mcv.register,	2240	.reg2 = dst_mcv.register,
2229	}).encode(),	2241	.flags = 0b01,
		2242	}),
2230	.data = .{ .imm = 0 },	2243	.data = .{ .imm = 0 },
2231	});	2244	});
2232	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };	2245	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
...	@@ -2488,17 +2501,17 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -2488,17 +2501,17 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2488	// mov dst_reg, [reg]	2501	// mov dst_reg, [reg]
2489	_ = try self.addInst(.{	2502	_ = try self.addInst(.{
2490	.tag = .mov,	2503	.tag = .mov,
2491	.ops = (Mir.Ops{	2504	.ops = Mir.Inst.Ops.encode(.{
2492	.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),	2505	.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
2493	.reg2 = reg,	2506	.reg2 = reg,
2494	.flags = 0b01,	2507	.flags = 0b01,
2495	}).encode(),	2508	}),
2496	.data = .{ .imm = 0 },	2509	.data = .{ .imm = 0 },
2497	});	2510	});
2498	},	2511	},
2499	.stack_offset => \|off\| {	2512	.stack_offset => \|off\| {
2500	if (abi_size <= 8) {	2513	if (abi_size <= 8) {
2501	const tmp_reg = try self.register_manager.allocReg(null);	2514	const tmp_reg = try self.register_manager.allocReg(null, gp);
2502	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);	2515	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
2503	return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});	2516	return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
2504	}	2517	}
...	@@ -2559,10 +2572,10 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue	...	@@ -2559,10 +2572,10 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
2559	const fn_owner_decl = mod.declPtr(self.mod_fn.owner_decl);	2572	const fn_owner_decl = mod.declPtr(self.mod_fn.owner_decl);
2560	_ = try self.addInst(.{	2573	_ = try self.addInst(.{
2561	.tag = .lea_pie,	2574	.tag = .lea_pie,
2562	.ops = (Mir.Ops{	2575	.ops = Mir.Inst.Ops.encode(.{
2563	.reg1 = registerAlias(reg, abi_size),	2576	.reg1 = registerAlias(reg, abi_size),
2564	.flags = flags,	2577	.flags = flags,
2565	}).encode(),	2578	}),
2566	.data = .{	2579	.data = .{
2567	.load_reloc = .{	2580	.load_reloc = .{
2568	.atom_index = fn_owner_decl.link.macho.local_sym_index,	2581	.atom_index = fn_owner_decl.link.macho.local_sym_index,
...	@@ -2623,7 +2636,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2623,7 +2636,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2623	});	2636	});
2624	_ = try self.addInst(.{	2637	_ = try self.addInst(.{
2625	.tag = .mov_mem_imm,	2638	.tag = .mov_mem_imm,
2626	.ops = (Mir.Ops{	2639	.ops = Mir.Inst.Ops.encode(.{
2627	.reg1 = reg.to64(),	2640	.reg1 = reg.to64(),
2628	.flags = switch (abi_size) {	2641	.flags = switch (abi_size) {
2629	1 => 0b00,	2642	1 => 0b00,
...	@@ -2631,7 +2644,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2631,7 +2644,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2631	4 => 0b10,	2644	4 => 0b10,
2632	else => unreachable,	2645	else => unreachable,
2633	},	2646	},
2634	}).encode(),	2647	}),
2635	.data = .{ .payload = payload },	2648	.data = .{ .payload = payload },
2636	});	2649	});
2637	},	2650	},
...	@@ -2645,11 +2658,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2645,11 +2658,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2645	const tmp_reg = try self.copyToTmpRegister(value_ty, value);	2658	const tmp_reg = try self.copyToTmpRegister(value_ty, value);
2646	_ = try self.addInst(.{	2659	_ = try self.addInst(.{
2647	.tag = .mov,	2660	.tag = .mov,
2648	.ops = (Mir.Ops{	2661	.ops = Mir.Inst.Ops.encode(.{
2649	.reg1 = reg.to64(),	2662	.reg1 = reg.to64(),
2650	.reg2 = tmp_reg.to64(),	2663	.reg2 = tmp_reg.to64(),
2651	.flags = 0b10,	2664	.flags = 0b10,
2652	}).encode(),	2665	}),
2653	.data = .{ .imm = 0 },	2666	.data = .{ .imm = 0 },
2654	});	2667	});
2655	},	2668	},
...	@@ -2661,11 +2674,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2661,11 +2674,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2661	.register => \|src_reg\| {	2674	.register => \|src_reg\| {
2662	_ = try self.addInst(.{	2675	_ = try self.addInst(.{
2663	.tag = .mov,	2676	.tag = .mov,
2664	.ops = (Mir.Ops{	2677	.ops = Mir.Inst.Ops.encode(.{
2665	.reg1 = reg.to64(),	2678	.reg1 = reg.to64(),
2666	.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),	2679	.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
2667	.flags = 0b10,	2680	.flags = 0b10,
2668	}).encode(),	2681	}),
2669	.data = .{ .imm = 0 },	2682	.data = .{ .imm = 0 },
2670	});	2683	});
2671	},	2684	},
...	@@ -2699,7 +2712,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2699,7 +2712,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2699	};	2712	};
2700	defer if (value_lock) \|lock\| self.register_manager.unlockReg(lock);	2713	defer if (value_lock) \|lock\| self.register_manager.unlockReg(lock);
2701		2714
2702	const addr_reg = try self.register_manager.allocReg(null);	2715	const addr_reg = try self.register_manager.allocReg(null, gp);
2703	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	2716	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
2704	defer self.register_manager.unlockReg(addr_reg_lock);	2717	defer self.register_manager.unlockReg(addr_reg_lock);
2705		2718
...	@@ -2709,11 +2722,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2709,11 +2722,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2709	// mov reg, [reg]	2722	// mov reg, [reg]
2710	_ = try self.addInst(.{	2723	_ = try self.addInst(.{
2711	.tag = .mov,	2724	.tag = .mov,
2712	.ops = (Mir.Ops{	2725	.ops = Mir.Inst.Ops.encode(.{
2713	.reg1 = addr_reg.to64(),	2726	.reg1 = addr_reg.to64(),
2714	.reg2 = addr_reg.to64(),	2727	.reg2 = addr_reg.to64(),
2715	.flags = 0b01,	2728	.flags = 0b01,
2716	}).encode(),	2729	}),
2717	.data = .{ .imm = 0 },	2730	.data = .{ .imm = 0 },
2718	});	2731	});
2719		2732
...	@@ -2748,21 +2761,21 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2748,21 +2761,21 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2748	}	2761	}
2749	_ = try self.addInst(.{	2762	_ = try self.addInst(.{
2750	.tag = .mov_mem_imm,	2763	.tag = .mov_mem_imm,
2751	.ops = (Mir.Ops{	2764	.ops = Mir.Inst.Ops.encode(.{
2752	.reg1 = addr_reg.to64(),	2765	.reg1 = addr_reg.to64(),
2753	.flags = flags,	2766	.flags = flags,
2754	}).encode(),	2767	}),
2755	.data = .{ .payload = payload },	2768	.data = .{ .payload = payload },
2756	});	2769	});
2757	},	2770	},
2758	.register => \|reg\| {	2771	.register => \|reg\| {
2759	_ = try self.addInst(.{	2772	_ = try self.addInst(.{
2760	.tag = .mov,	2773	.tag = .mov,
2761	.ops = (Mir.Ops{	2774	.ops = Mir.Inst.Ops.encode(.{
2762	.reg1 = addr_reg.to64(),	2775	.reg1 = addr_reg.to64(),
2763	.reg2 = reg,	2776	.reg2 = reg,
2764	.flags = 0b10,	2777	.flags = 0b10,
2765	}).encode(),	2778	}),
2766	.data = .{ .imm = 0 },	2779	.data = .{ .imm = 0 },
2767	});	2780	});
2768	},	2781	},
...	@@ -2771,7 +2784,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2771,7 +2784,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2771	.memory,	2784	.memory,
2772	=> {	2785	=> {
2773	if (abi_size <= 8) {	2786	if (abi_size <= 8) {
2774	const tmp_reg = try self.register_manager.allocReg(null);	2787	const tmp_reg = try self.register_manager.allocReg(null, gp);
2775	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	2788	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2776	defer self.register_manager.unlockReg(tmp_reg_lock);	2789	defer self.register_manager.unlockReg(tmp_reg_lock);
2777		2790
...	@@ -2779,20 +2792,20 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2779,20 +2792,20 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2779		2792
2780	_ = try self.addInst(.{	2793	_ = try self.addInst(.{
2781	.tag = .mov,	2794	.tag = .mov,
2782	.ops = (Mir.Ops{	2795	.ops = Mir.Inst.Ops.encode(.{
2783	.reg1 = tmp_reg,	2796	.reg1 = tmp_reg,
2784	.reg2 = tmp_reg,	2797	.reg2 = tmp_reg,
2785	.flags = 0b01,	2798	.flags = 0b01,
2786	}).encode(),	2799	}),
2787	.data = .{ .imm = 0 },	2800	.data = .{ .imm = 0 },
2788	});	2801	});
2789	_ = try self.addInst(.{	2802	_ = try self.addInst(.{
2790	.tag = .mov,	2803	.tag = .mov,
2791	.ops = (Mir.Ops{	2804	.ops = Mir.Inst.Ops.encode(.{
2792	.reg1 = addr_reg.to64(),	2805	.reg1 = addr_reg.to64(),
2793	.reg2 = tmp_reg,	2806	.reg2 = tmp_reg,
2794	.flags = 0b10,	2807	.flags = 0b10,
2795	}).encode(),	2808	}),
2796	.data = .{ .imm = 0 },	2809	.data = .{ .imm = 0 },
2797	});	2810	});
2798	return;	2811	return;
...	@@ -2806,11 +2819,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2806,11 +2819,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2806	const tmp_reg = try self.copyToTmpRegister(value_ty, value);	2819	const tmp_reg = try self.copyToTmpRegister(value_ty, value);
2807	_ = try self.addInst(.{	2820	_ = try self.addInst(.{
2808	.tag = .mov,	2821	.tag = .mov,
2809	.ops = (Mir.Ops{	2822	.ops = Mir.Inst.Ops.encode(.{
2810	.reg1 = addr_reg.to64(),	2823	.reg1 = addr_reg.to64(),
2811	.reg2 = tmp_reg,	2824	.reg2 = tmp_reg,
2812	.flags = 0b10,	2825	.flags = 0b10,
2813	}).encode(),	2826	}),
2814	.data = .{ .imm = 0 },	2827	.data = .{ .imm = 0 },
2815	});	2828	});
2816	return;	2829	return;
...	@@ -2889,7 +2902,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde	...	@@ -2889,7 +2902,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
2889	if (can_reuse_operand) {	2902	if (can_reuse_operand) {
2890	break :blk reg;	2903	break :blk reg;
2891	} else {	2904	} else {
2892	const result_reg = try self.register_manager.allocReg(inst);	2905	const result_reg = try self.register_manager.allocReg(inst, gp);
2893	try self.genSetReg(ptr_ty, result_reg, mcv);	2906	try self.genSetReg(ptr_ty, result_reg, mcv);
2894	break :blk result_reg;	2907	break :blk result_reg;
2895	}	2908	}
...	@@ -2967,10 +2980,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2967,10 +2980,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
2967	if (signedness == .signed and field_size < 8) {	2980	if (signedness == .signed and field_size < 8) {
2968	_ = try self.addInst(.{	2981	_ = try self.addInst(.{
2969	.tag = .mov_sign_extend,	2982	.tag = .mov_sign_extend,
2970	.ops = (Mir.Ops{	2983	.ops = Mir.Inst.Ops.encode(.{
2971	.reg1 = dst_mcv.register,	2984	.reg1 = dst_mcv.register,
2972	.reg2 = registerAlias(dst_mcv.register, field_size),	2985	.reg2 = registerAlias(dst_mcv.register, field_size),
2973	}).encode(),	2986	}),
2974	.data = undefined,	2987	.data = undefined,
2975	});	2988	});
2976	}	2989	}
...	@@ -2990,7 +3003,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2990,7 +3003,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
2990	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);	3003	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
2991	defer self.register_manager.unlockReg(reg_lock);	3004	defer self.register_manager.unlockReg(reg_lock);
2992		3005
2993	const dst_reg = try self.register_manager.allocReg(inst);	3006	const dst_reg = try self.register_manager.allocReg(inst, gp);
2994	const flags: u2 = switch (mcv) {	3007	const flags: u2 = switch (mcv) {
2995	.register_overflow_unsigned => 0b10,	3008	.register_overflow_unsigned => 0b10,
2996	.register_overflow_signed => 0b00,	3009	.register_overflow_signed => 0b00,
...	@@ -2998,10 +3011,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2998,10 +3011,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
2998	};	3011	};
2999	_ = try self.addInst(.{	3012	_ = try self.addInst(.{
3000	.tag = .cond_set_byte_overflow,	3013	.tag = .cond_set_byte_overflow,
3001	.ops = (Mir.Ops{	3014	.ops = Mir.Inst.Ops.encode(.{
3002	.reg1 = dst_reg.to8(),	3015	.reg1 = dst_reg.to8(),
3003	.flags = flags,	3016	.flags = flags,
3004	}).encode(),	3017	}),
3005	.data = undefined,	3018	.data = undefined,
3006	});	3019	});
3007	break :result MCValue{ .register = dst_reg.to8() };	3020	break :result MCValue{ .register = dst_reg.to8() };
...	@@ -3042,10 +3055,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi	...	@@ -3042,10 +3055,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3042	1 => {	3055	1 => {
3043	_ = try self.addInst(.{	3056	_ = try self.addInst(.{
3044	.tag = tag,	3057	.tag = tag,
3045	.ops = (Mir.Ops{	3058	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
3046	.reg1 = registerAlias(reg, abi_size),
3047	.flags = 0b00,
3048	}).encode(),
3049	.data = undefined,	3059	.data = undefined,
3050	});	3060	});
3051	return;	3061	return;
...	@@ -3053,10 +3063,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi	...	@@ -3053,10 +3063,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3053	else => {	3063	else => {
3054	_ = try self.addInst(.{	3064	_ = try self.addInst(.{
3055	.tag = tag,	3065	.tag = tag,
3056	.ops = (Mir.Ops{	3066	.ops = Mir.Inst.Ops.encode(.{
3057	.reg1 = registerAlias(reg, abi_size),	3067	.reg1 = registerAlias(reg, abi_size),
3058	.flags = 0b10,	3068	.flags = 0b10,
3059	}).encode(),	3069	}),
3060	.data = .{ .imm = @intCast(u8, imm) },	3070	.data = .{ .imm = @intCast(u8, imm) },
3061	});	3071	});
3062	return;	3072	return;
...	@@ -3074,10 +3084,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi	...	@@ -3074,10 +3084,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3074		3084
3075	_ = try self.addInst(.{	3085	_ = try self.addInst(.{
3076	.tag = tag,	3086	.tag = tag,
3077	.ops = (Mir.Ops{	3087	.ops = Mir.Inst.Ops.encode(.{
3078	.reg1 = registerAlias(reg, abi_size),	3088	.reg1 = registerAlias(reg, abi_size),
3079	.flags = 0b01,	3089	.flags = 0b01,
3080	}).encode(),	3090	}),
3081	.data = undefined,	3091	.data = undefined,
3082	});	3092	});
3083	}	3093	}
...	@@ -3326,7 +3336,7 @@ fn genBinOp(	...	@@ -3326,7 +3336,7 @@ fn genBinOp(
3326	const rhs = try self.resolveInst(rhs_air);	3336	const rhs = try self.resolveInst(rhs_air);
3327	const lhs_ty = self.air.typeOf(lhs_air);	3337	const lhs_ty = self.air.typeOf(lhs_air);
3328	const rhs_ty = self.air.typeOf(rhs_air);	3338	const rhs_ty = self.air.typeOf(rhs_air);
3329	if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) {	3339	if (lhs_ty.zigTypeTag() == .Vector) {
3330	return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});	3340	return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
3331	}	3341	}
3332	if (lhs_ty.abiSize(self.target.*) > 8) {	3342	if (lhs_ty.abiSize(self.target.*) > 8) {
...	@@ -3450,22 +3460,62 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu	...	@@ -3450,22 +3460,62 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3450	const reg = try self.copyToTmpRegister(dst_ty, src_mcv);	3460	const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
3451	return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });	3461	return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
3452	},	3462	},
3453	.register => \|src_reg\| {	3463	.register => \|src_reg\| switch (dst_ty.zigTypeTag()) {
3454	_ = try self.addInst(.{	3464	.Float => {
3455	.tag = mir_tag,	3465	if (intrinsicsAllowed(self.target.*, dst_ty)) {
3456	.ops = (Mir.Ops{	3466	const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3457	.reg1 = registerAlias(dst_reg, abi_size),	3467	.f32 => switch (mir_tag) {
3458	.reg2 = registerAlias(src_reg, abi_size),	3468	.add => if (hasAvxSupport(self.target.*))
3459	}).encode(),	3469	Mir.Inst.Tag.add_f32_avx
3460	.data = undefined,	3470	else
3461	});	3471	Mir.Inst.Tag.add_f32_sse,
		3472	.cmp => if (hasAvxSupport(self.target.*))
		3473	Mir.Inst.Tag.cmp_f32_avx
		3474	else
		3475	Mir.Inst.Tag.cmp_f32_sse,
		3476	else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
		3477	},
		3478	.f64 => switch (mir_tag) {
		3479	.add => if (hasAvxSupport(self.target.*))
		3480	Mir.Inst.Tag.add_f64_avx
		3481	else
		3482	Mir.Inst.Tag.add_f64_sse,
		3483	.cmp => if (hasAvxSupport(self.target.*))
		3484	Mir.Inst.Tag.cmp_f64_avx
		3485	else
		3486	Mir.Inst.Tag.cmp_f64_sse,
		3487	else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
		3488	},
		3489	else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
		3490	};
		3491	_ = try self.addInst(.{
		3492	.tag = actual_tag,
		3493	.ops = Mir.Inst.Ops.encode(.{
		3494	.reg1 = dst_reg.to128(),
		3495	.reg2 = src_reg.to128(),
		3496	}),
		3497	.data = undefined,
		3498	});
		3499	return;
		3500	}
		3501
		3502	return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
		3503	},
		3504	else => {
		3505	_ = try self.addInst(.{
		3506	.tag = mir_tag,
		3507	.ops = Mir.Inst.Ops.encode(.{
		3508	.reg1 = registerAlias(dst_reg, abi_size),
		3509	.reg2 = registerAlias(src_reg, abi_size),
		3510	}),
		3511	.data = undefined,
		3512	});
		3513	},
3462	},	3514	},
3463	.immediate => \|imm\| {	3515	.immediate => \|imm\| {
3464	_ = try self.addInst(.{	3516	_ = try self.addInst(.{
3465	.tag = mir_tag,	3517	.tag = mir_tag,
3466	.ops = (Mir.Ops{	3518	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }),
3467	.reg1 = registerAlias(dst_reg, abi_size),
3468	}).encode(),
3469	.data = .{ .imm = @truncate(u32, imm) },	3519	.data = .{ .imm = @truncate(u32, imm) },
3470	});	3520	});
3471	},	3521	},
...	@@ -3488,11 +3538,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu	...	@@ -3488,11 +3538,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3488	}	3538	}
3489	_ = try self.addInst(.{	3539	_ = try self.addInst(.{
3490	.tag = mir_tag,	3540	.tag = mir_tag,
3491	.ops = (Mir.Ops{	3541	.ops = Mir.Inst.Ops.encode(.{
3492	.reg1 = registerAlias(dst_reg, abi_size),	3542	.reg1 = registerAlias(dst_reg, abi_size),
3493	.reg2 = .rbp,	3543	.reg2 = .rbp,
3494	.flags = 0b01,	3544	.flags = 0b01,
3495	}).encode(),	3545	}),
3496	.data = .{ .imm = @bitCast(u32, -off) },	3546	.data = .{ .imm = @bitCast(u32, -off) },
3497	});	3547	});
3498	},	3548	},
...	@@ -3515,11 +3565,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu	...	@@ -3515,11 +3565,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3515	.register => \|src_reg\| {	3565	.register => \|src_reg\| {
3516	_ = try self.addInst(.{	3566	_ = try self.addInst(.{
3517	.tag = mir_tag,	3567	.tag = mir_tag,
3518	.ops = (Mir.Ops{	3568	.ops = Mir.Inst.Ops.encode(.{
3519	.reg1 = .rbp,	3569	.reg1 = .rbp,
3520	.reg2 = registerAlias(src_reg, abi_size),	3570	.reg2 = registerAlias(src_reg, abi_size),
3521	.flags = 0b10,	3571	.flags = 0b10,
3522	}).encode(),	3572	}),
3523	.data = .{ .imm = @bitCast(u32, -off) },	3573	.data = .{ .imm = @bitCast(u32, -off) },
3524	});	3574	});
3525	},	3575	},
...	@@ -3546,10 +3596,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu	...	@@ -3546,10 +3596,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3546	});	3596	});
3547	_ = try self.addInst(.{	3597	_ = try self.addInst(.{
3548	.tag = tag,	3598	.tag = tag,
3549	.ops = (Mir.Ops{	3599	.ops = Mir.Inst.Ops.encode(.{
3550	.reg1 = .rbp,	3600	.reg1 = .rbp,
3551	.flags = flags,	3601	.flags = flags,
3552	}).encode(),	3602	}),
3553	.data = .{ .payload = payload },	3603	.data = .{ .payload = payload },
3554	});	3604	});
3555	},	3605	},
...	@@ -3604,10 +3654,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -3604,10 +3654,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3604	// register, register	3654	// register, register
3605	_ = try self.addInst(.{	3655	_ = try self.addInst(.{
3606	.tag = .imul_complex,	3656	.tag = .imul_complex,
3607	.ops = (Mir.Ops{	3657	.ops = Mir.Inst.Ops.encode(.{
3608	.reg1 = registerAlias(dst_reg, abi_size),	3658	.reg1 = registerAlias(dst_reg, abi_size),
3609	.reg2 = registerAlias(src_reg, abi_size),	3659	.reg2 = registerAlias(src_reg, abi_size),
3610	}).encode(),	3660	}),
3611	.data = undefined,	3661	.data = undefined,
3612	});	3662	});
3613	},	3663	},
...	@@ -3617,11 +3667,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -3617,11 +3667,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3617	if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {	3667	if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
3618	_ = try self.addInst(.{	3668	_ = try self.addInst(.{
3619	.tag = .imul_complex,	3669	.tag = .imul_complex,
3620	.ops = (Mir.Ops{	3670	.ops = Mir.Inst.Ops.encode(.{
3621	.reg1 = dst_reg.to32(),	3671	.reg1 = dst_reg.to32(),
3622	.reg2 = dst_reg.to32(),	3672	.reg2 = dst_reg.to32(),
3623	.flags = 0b10,	3673	.flags = 0b10,
3624	}).encode(),	3674	}),
3625	.data = .{ .imm = @truncate(u32, imm) },	3675	.data = .{ .imm = @truncate(u32, imm) },
3626	});	3676	});
3627	} else {	3677	} else {
...	@@ -3633,11 +3683,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -3633,11 +3683,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3633	.stack_offset => \|off\| {	3683	.stack_offset => \|off\| {
3634	_ = try self.addInst(.{	3684	_ = try self.addInst(.{
3635	.tag = .imul_complex,	3685	.tag = .imul_complex,
3636	.ops = (Mir.Ops{	3686	.ops = Mir.Inst.Ops.encode(.{
3637	.reg1 = registerAlias(dst_reg, abi_size),	3687	.reg1 = registerAlias(dst_reg, abi_size),
3638	.reg2 = .rbp,	3688	.reg2 = .rbp,
3639	.flags = 0b01,	3689	.flags = 0b01,
3640	}).encode(),	3690	}),
3641	.data = .{ .imm = @bitCast(u32, -off) },	3691	.data = .{ .imm = @bitCast(u32, -off) },
3642	});	3692	});
3643	},	3693	},
...	@@ -3670,10 +3720,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -3670,10 +3720,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3670	// register, register	3720	// register, register
3671	_ = try self.addInst(.{	3721	_ = try self.addInst(.{
3672	.tag = .imul_complex,	3722	.tag = .imul_complex,
3673	.ops = (Mir.Ops{	3723	.ops = Mir.Inst.Ops.encode(.{
3674	.reg1 = registerAlias(dst_reg, abi_size),	3724	.reg1 = registerAlias(dst_reg, abi_size),
3675	.reg2 = registerAlias(src_reg, abi_size),	3725	.reg2 = registerAlias(src_reg, abi_size),
3676	}).encode(),	3726	}),
3677	.data = undefined,	3727	.data = undefined,
3678	});	3728	});
3679	// copy dst_reg back out	3729	// copy dst_reg back out
...	@@ -3780,9 +3830,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3780,9 +3830,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
3780	fn airBreakpoint(self: *Self) !void {	3830	fn airBreakpoint(self: *Self) !void {
3781	_ = try self.addInst(.{	3831	_ = try self.addInst(.{
3782	.tag = .interrupt,	3832	.tag = .interrupt,
3783	.ops = (Mir.Ops{	3833	.ops = Mir.Inst.Ops.encode(.{}),
3784	.flags = 0b00,
3785	}).encode(),
3786	.data = undefined,	3834	.data = undefined,
3787	});	3835	});
3788	return self.finishAirBookkeeping();	3836	return self.finishAirBookkeeping();
...	@@ -3883,9 +3931,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -3883,9 +3931,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3883	// Adjust the stack	3931	// Adjust the stack
3884	_ = try self.addInst(.{	3932	_ = try self.addInst(.{
3885	.tag = .sub,	3933	.tag = .sub,
3886	.ops = (Mir.Ops{	3934	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
3887	.reg1 = .rsp,
3888	}).encode(),
3889	.data = .{ .imm = info.stack_byte_count },	3935	.data = .{ .imm = info.stack_byte_count },
3890	});	3936	});
3891	}	3937	}
...	@@ -3909,9 +3955,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -3909,9 +3955,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3909	unreachable;	3955	unreachable;
3910	_ = try self.addInst(.{	3956	_ = try self.addInst(.{
3911	.tag = .call,	3957	.tag = .call,
3912	.ops = (Mir.Ops{	3958	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
3913	.flags = 0b01,
3914	}).encode(),
3915	.data = .{ .imm = @truncate(u32, got_addr) },	3959	.data = .{ .imm = @truncate(u32, got_addr) },
3916	});	3960	});
3917	} else if (func_value.castTag(.extern_fn)) \|_\| {	3961	} else if (func_value.castTag(.extern_fn)) \|_\| {
...	@@ -3925,10 +3969,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -3925,10 +3969,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3925	try self.genSetReg(Type.initTag(.usize), .rax, mcv);	3969	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
3926	_ = try self.addInst(.{	3970	_ = try self.addInst(.{
3927	.tag = .call,	3971	.tag = .call,
3928	.ops = (Mir.Ops{	3972	.ops = Mir.Inst.Ops.encode(.{
3929	.reg1 = .rax,	3973	.reg1 = .rax,
3930	.flags = 0b01,	3974	.flags = 0b01,
3931	}).encode(),	3975	}),
3932	.data = undefined,	3976	.data = undefined,
3933	});	3977	});
3934	}	3978	}
...	@@ -3943,10 +3987,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -3943,10 +3987,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3943	// callq *%rax	3987	// callq *%rax
3944	_ = try self.addInst(.{	3988	_ = try self.addInst(.{
3945	.tag = .call,	3989	.tag = .call,
3946	.ops = (Mir.Ops{	3990	.ops = Mir.Inst.Ops.encode(.{
3947	.reg1 = .rax,	3991	.reg1 = .rax,
3948	.flags = 0b01,	3992	.flags = 0b01,
3949	}).encode(),	3993	}),
3950	.data = undefined,	3994	.data = undefined,
3951	});	3995	});
3952	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {	3996	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
...	@@ -3978,10 +4022,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -3978,10 +4022,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3978	try self.genSetReg(Type.initTag(.usize), .rax, mcv);	4022	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
3979	_ = try self.addInst(.{	4023	_ = try self.addInst(.{
3980	.tag = .call,	4024	.tag = .call,
3981	.ops = (Mir.Ops{	4025	.ops = Mir.Inst.Ops.encode(.{
3982	.reg1 = .rax,	4026	.reg1 = .rax,
3983	.flags = 0b01,	4027	.flags = 0b01,
3984	}).encode(),	4028	}),
3985	.data = undefined,	4029	.data = undefined,
3986	});	4030	});
3987	}	4031	}
...	@@ -3996,9 +4040,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -3996,9 +4040,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3996	const fn_got_addr = got_addr + got_index * ptr_bytes;	4040	const fn_got_addr = got_addr + got_index * ptr_bytes;
3997	_ = try self.addInst(.{	4041	_ = try self.addInst(.{
3998	.tag = .call,	4042	.tag = .call,
3999	.ops = (Mir.Ops{	4043	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4000	.flags = 0b01,
4001	}).encode(),
4002	.data = .{ .imm = @intCast(u32, fn_got_addr) },	4044	.data = .{ .imm = @intCast(u32, fn_got_addr) },
4003	});	4045	});
4004	} else return self.fail("TODO implement calling extern fn on plan9", .{});	4046	} else return self.fail("TODO implement calling extern fn on plan9", .{});
...	@@ -4008,10 +4050,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -4008,10 +4050,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4008	try self.genSetReg(Type.initTag(.usize), .rax, mcv);	4050	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
4009	_ = try self.addInst(.{	4051	_ = try self.addInst(.{
4010	.tag = .call,	4052	.tag = .call,
4011	.ops = (Mir.Ops{	4053	.ops = Mir.Inst.Ops.encode(.{
4012	.reg1 = .rax,	4054	.reg1 = .rax,
4013	.flags = 0b01,	4055	.flags = 0b01,
4014	}).encode(),	4056	}),
4015	.data = undefined,	4057	.data = undefined,
4016	});	4058	});
4017	}	4059	}
...	@@ -4021,9 +4063,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -4021,9 +4063,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
4021	// Readjust the stack	4063	// Readjust the stack
4022	_ = try self.addInst(.{	4064	_ = try self.addInst(.{
4023	.tag = .add,	4065	.tag = .add,
4024	.ops = (Mir.Ops{	4066	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4025	.reg1 = .rsp,
4026	}).encode(),
4027	.data = .{ .imm = info.stack_byte_count },	4067	.data = .{ .imm = info.stack_byte_count },
4028	});	4068	});
4029	}	4069	}
...	@@ -4081,9 +4121,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4081,9 +4121,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
4081	// which is available if the jump is 127 bytes or less forward.	4121	// which is available if the jump is 127 bytes or less forward.
4082	const jmp_reloc = try self.addInst(.{	4122	const jmp_reloc = try self.addInst(.{
4083	.tag = .jmp,	4123	.tag = .jmp,
4084	.ops = (Mir.Ops{	4124	.ops = Mir.Inst.Ops.encode(.{}),
4085	.flags = 0b00,
4086	}).encode(),
4087	.data = .{ .inst = undefined },	4125	.data = .{ .inst = undefined },
4088	});	4126	});
4089	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);	4127	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
...	@@ -4116,9 +4154,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4116,9 +4154,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
4116	// which is available if the jump is 127 bytes or less forward.	4154	// which is available if the jump is 127 bytes or less forward.
4117	const jmp_reloc = try self.addInst(.{	4155	const jmp_reloc = try self.addInst(.{
4118	.tag = .jmp,	4156	.tag = .jmp,
4119	.ops = (Mir.Ops{	4157	.ops = Mir.Inst.Ops.encode(.{}),
4120	.flags = 0b00,
4121	}).encode(),
4122	.data = .{ .inst = undefined },	4158	.data = .{ .inst = undefined },
4123	});	4159	});
4124	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);	4160	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
...	@@ -4162,8 +4198,28 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -4162,8 +4198,28 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
4162		4198
4163	const dst_mcv = MCValue{ .register = dst_reg };	4199	const dst_mcv = MCValue{ .register = dst_reg };
4164		4200
		4201	const rhs_ty = self.air.typeOf(bin_op.rhs);
4165	// This instruction supports only signed 32-bit immediates at most.	4202	// This instruction supports only signed 32-bit immediates at most.
4166	const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);	4203	const src_mcv: MCValue = blk: {
		4204	switch (rhs_ty.zigTypeTag()) {
		4205	.Float => {
		4206	const rhs = try self.resolveInst(bin_op.rhs);
		4207	const rhs_lock: ?RegisterLock = switch (rhs) {
		4208	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		4209	else => null,
		4210	};
		4211	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
		4212	const src_reg = try self.copyToTmpRegister(rhs_ty, rhs);
		4213	break :blk MCValue{ .register = src_reg };
		4214	},
		4215	else => break :blk try self.limitImmediateType(bin_op.rhs, i32),
		4216	}
		4217	};
		4218	const src_lock: ?RegisterLock = switch (src_mcv) {
		4219	.register => \|reg\| self.register_manager.lockReg(reg),
		4220	else => null,
		4221	};
		4222	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
4167		4223
4168	try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);	4224	try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
4169	break :result switch (signedness) {	4225	break :result switch (signedness) {
...	@@ -4362,9 +4418,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {	...	@@ -4362,9 +4418,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4362	Mir.Inst.Tag.cond_jmp_greater_less;	4418	Mir.Inst.Tag.cond_jmp_greater_less;
4363	return self.addInst(.{	4419	return self.addInst(.{
4364	.tag = tag,	4420	.tag = tag,
4365	.ops = (Mir.Ops{	4421	.ops = Mir.Inst.Ops.encode(.{ .flags = flags }),
4366	.flags = flags,
4367	}).encode(),
4368	.data = .{ .inst = undefined },	4422	.data = .{ .inst = undefined },
4369	});	4423	});
4370	},	4424	},
...	@@ -4372,17 +4426,12 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {	...	@@ -4372,17 +4426,12 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4372	try self.spillCompareFlagsIfOccupied();	4426	try self.spillCompareFlagsIfOccupied();
4373	_ = try self.addInst(.{	4427	_ = try self.addInst(.{
4374	.tag = .@"test",	4428	.tag = .@"test",
4375	.ops = (Mir.Ops{	4429	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
4376	.reg1 = reg,
4377	.flags = 0b00,
4378	}).encode(),
4379	.data = .{ .imm = 1 },	4430	.data = .{ .imm = 1 },
4380	});	4431	});
4381	return self.addInst(.{	4432	return self.addInst(.{
4382	.tag = .cond_jmp_eq_ne,	4433	.tag = .cond_jmp_eq_ne,
4383	.ops = (Mir.Ops{	4434	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4384	.flags = 0b01,
4385	}).encode(),
4386	.data = .{ .inst = undefined },	4435	.data = .{ .inst = undefined },
4387	});	4436	});
4388	},	4437	},
...	@@ -4776,9 +4825,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4776,9 +4825,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
4776	try self.genBody(body);	4825	try self.genBody(body);
4777	_ = try self.addInst(.{	4826	_ = try self.addInst(.{
4778	.tag = .jmp,	4827	.tag = .jmp,
4779	.ops = (Mir.Ops{	4828	.ops = Mir.Inst.Ops.encode(.{}),
4780	.flags = 0b00,
4781	}).encode(),
4782	.data = .{ .inst = jmp_target },	4829	.data = .{ .inst = jmp_target },
4783	});	4830	});
4784	return self.finishAirBookkeeping();	4831	return self.finishAirBookkeeping();
...	@@ -4829,19 +4876,17 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u	...	@@ -4829,19 +4876,17 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4829	.immediate => \|imm\| {	4876	.immediate => \|imm\| {
4830	_ = try self.addInst(.{	4877	_ = try self.addInst(.{
4831	.tag = .xor,	4878	.tag = .xor,
4832	.ops = (Mir.Ops{	4879	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }),
4833	.reg1 = registerAlias(cond_reg, abi_size),
4834	}).encode(),
4835	.data = .{ .imm = @intCast(u32, imm) },	4880	.data = .{ .imm = @intCast(u32, imm) },
4836	});	4881	});
4837	},	4882	},
4838	.register => \|reg\| {	4883	.register => \|reg\| {
4839	_ = try self.addInst(.{	4884	_ = try self.addInst(.{
4840	.tag = .xor,	4885	.tag = .xor,
4841	.ops = (Mir.Ops{	4886	.ops = Mir.Inst.Ops.encode(.{
4842	.reg1 = registerAlias(cond_reg, abi_size),	4887	.reg1 = registerAlias(cond_reg, abi_size),
4843	.reg2 = registerAlias(reg, abi_size),	4888	.reg2 = registerAlias(reg, abi_size),
4844	}).encode(),	4889	}),
4845	.data = undefined,	4890	.data = undefined,
4846	});	4891	});
4847	},	4892	},
...	@@ -4860,17 +4905,15 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u	...	@@ -4860,17 +4905,15 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4860		4905
4861	_ = try self.addInst(.{	4906	_ = try self.addInst(.{
4862	.tag = .@"test",	4907	.tag = .@"test",
4863	.ops = (Mir.Ops{	4908	.ops = Mir.Inst.Ops.encode(.{
4864	.reg1 = registerAlias(cond_reg, abi_size),	4909	.reg1 = registerAlias(cond_reg, abi_size),
4865	.reg2 = registerAlias(cond_reg, abi_size),	4910	.reg2 = registerAlias(cond_reg, abi_size),
4866	}).encode(),	4911	}),
4867	.data = undefined,	4912	.data = undefined,
4868	});	4913	});
4869	return self.addInst(.{	4914	return self.addInst(.{
4870	.tag = .cond_jmp_eq_ne,	4915	.tag = .cond_jmp_eq_ne,
4871	.ops = (Mir.Ops{	4916	.ops = Mir.Inst.Ops.encode(.{}),
4872	.flags = 0b00,
4873	}).encode(),
4874	.data = .{ .inst = undefined },	4917	.data = .{ .inst = undefined },
4875	});	4918	});
4876	},	4919	},
...	@@ -5036,9 +5079,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {	...	@@ -5036,9 +5079,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
5036	// Leave the jump offset undefined	5079	// Leave the jump offset undefined
5037	const jmp_reloc = try self.addInst(.{	5080	const jmp_reloc = try self.addInst(.{
5038	.tag = .jmp,	5081	.tag = .jmp,
5039	.ops = (Mir.Ops{	5082	.ops = Mir.Inst.Ops.encode(.{}),
5040	.flags = 0b00,
5041	}).encode(),
5042	.data = .{ .inst = undefined },	5083	.data = .{ .inst = undefined },
5043	});	5084	});
5044	block_data.relocs.appendAssumeCapacity(jmp_reloc);	5085	block_data.relocs.appendAssumeCapacity(jmp_reloc);
...	@@ -5126,9 +5167,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5126,9 +5167,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5126	};	5167	};
5127	_ = try self.addInst(.{	5168	_ = try self.addInst(.{
5128	.tag = .push,	5169	.tag = .push,
5129	.ops = (Mir.Ops{	5170	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b10 }),
5130	.flags = 0b10,
5131	}).encode(),
5132	.data = .{ .imm = n },	5171	.data = .{ .imm = n },
5133	});	5172	});
5134	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {	5173	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
...	@@ -5137,9 +5176,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5137,9 +5176,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5137	return self.fail("unrecognized register: '{s}'", .{reg_name});	5176	return self.fail("unrecognized register: '{s}'", .{reg_name});
5138	_ = try self.addInst(.{	5177	_ = try self.addInst(.{
5139	.tag = .push,	5178	.tag = .push,
5140	.ops = (Mir.Ops{	5179	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5141	.reg1 = reg,
5142	}).encode(),
5143	.data = undefined,	5180	.data = undefined,
5144	});	5181	});
5145	} else return self.fail("TODO more push operands", .{});	5182	} else return self.fail("TODO more push operands", .{});
...	@@ -5151,9 +5188,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5151,9 +5188,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5151	return self.fail("unrecognized register: '{s}'", .{reg_name});	5188	return self.fail("unrecognized register: '{s}'", .{reg_name});
5152	_ = try self.addInst(.{	5189	_ = try self.addInst(.{
5153	.tag = .pop,	5190	.tag = .pop,
5154	.ops = (Mir.Ops{	5191	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5155	.reg1 = reg,
5156	}).encode(),
5157	.data = undefined,	5192	.data = undefined,
5158	});	5193	});
5159	} else return self.fail("TODO more pop operands", .{});	5194	} else return self.fail("TODO more pop operands", .{});
...	@@ -5265,7 +5300,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE	...	@@ -5265,7 +5300,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5265	});	5300	});
5266	_ = try self.addInst(.{	5301	_ = try self.addInst(.{
5267	.tag = .mov_mem_imm,	5302	.tag = .mov_mem_imm,
5268	.ops = (Mir.Ops{	5303	.ops = Mir.Inst.Ops.encode(.{
5269	.reg1 = .rsp,	5304	.reg1 = .rsp,
5270	.flags = switch (abi_size) {	5305	.flags = switch (abi_size) {
5271	1 => 0b00,	5306	1 => 0b00,
...	@@ -5273,7 +5308,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE	...	@@ -5273,7 +5308,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5273	4 => 0b10,	5308	4 => 0b10,
5274	else => unreachable,	5309	else => unreachable,
5275	},	5310	},
5276	}).encode(),	5311	}),
5277	.data = .{ .payload = payload },	5312	.data = .{ .payload = payload },
5278	});	5313	});
5279	},	5314	},
...	@@ -5299,15 +5334,50 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE	...	@@ -5299,15 +5334,50 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5299	});	5334	});
5300	},	5335	},
5301	.register => \|reg\| {	5336	.register => \|reg\| {
5302	_ = try self.addInst(.{	5337	switch (ty.zigTypeTag()) {
5303	.tag = .mov,	5338	.Float => {
5304	.ops = (Mir.Ops{	5339	if (intrinsicsAllowed(self.target.*, ty)) {
5305	.reg1 = .rsp,	5340	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5306	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),	5341	.f32 => if (hasAvxSupport(self.target.*))
5307	.flags = 0b10,	5342	Mir.Inst.Tag.mov_f32_avx
5308	}).encode(),	5343	else
5309	.data = .{ .imm = @bitCast(u32, -stack_offset) },	5344	Mir.Inst.Tag.mov_f32_sse,
5310	});	5345	.f64 => if (hasAvxSupport(self.target.*))
		5346	Mir.Inst.Tag.mov_f64_avx
		5347	else
		5348	Mir.Inst.Tag.mov_f64_sse,
		5349	else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
		5350	};
		5351	_ = try self.addInst(.{
		5352	.tag = tag,
		5353	.ops = Mir.Inst.Ops.encode(.{
		5354	.reg1 = switch (ty.tag()) {
		5355	.f32 => .esp,
		5356	.f64 => .rsp,
		5357	else => unreachable,
		5358	},
		5359	.reg2 = reg.to128(),
		5360	.flags = 0b01,
		5361	}),
		5362	.data = .{ .imm = @bitCast(u32, -stack_offset) },
		5363	});
		5364	return;
		5365	}
		5366
		5367	return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
		5368	},
		5369	else => {
		5370	_ = try self.addInst(.{
		5371	.tag = .mov,
		5372	.ops = Mir.Inst.Ops.encode(.{
		5373	.reg1 = .rsp,
		5374	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
		5375	.flags = 0b10,
		5376	}),
		5377	.data = .{ .imm = @bitCast(u32, -stack_offset) },
		5378	});
		5379	},
		5380	}
5311	},	5381	},
5312	.ptr_stack_offset => {	5382	.ptr_stack_offset => {
5313	const reg = try self.copyToTmpRegister(ty, mcv);	5383	const reg = try self.copyToTmpRegister(ty, mcv);
...	@@ -5360,7 +5430,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5360,7 +5430,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5360		5430
5361	const overflow_bit_ty = ty.structFieldType(1);	5431	const overflow_bit_ty = ty.structFieldType(1);
5362	const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);	5432	const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
5363	const tmp_reg = try self.register_manager.allocReg(null);	5433	const tmp_reg = try self.register_manager.allocReg(null, gp);
5364	const flags: u2 = switch (mcv) {	5434	const flags: u2 = switch (mcv) {
5365	.register_overflow_unsigned => 0b10,	5435	.register_overflow_unsigned => 0b10,
5366	.register_overflow_signed => 0b00,	5436	.register_overflow_signed => 0b00,
...	@@ -5368,10 +5438,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5368,10 +5438,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5368	};	5438	};
5369	_ = try self.addInst(.{	5439	_ = try self.addInst(.{
5370	.tag = .cond_set_byte_overflow,	5440	.tag = .cond_set_byte_overflow,
5371	.ops = (Mir.Ops{	5441	.ops = Mir.Inst.Ops.encode(.{
5372	.reg1 = tmp_reg.to8(),	5442	.reg1 = tmp_reg.to8(),
5373	.flags = flags,	5443	.flags = flags,
5374	}).encode(),	5444	}),
5375	.data = undefined,	5445	.data = undefined,
5376	});	5446	});
5377		5447
...	@@ -5398,7 +5468,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5398,7 +5468,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5398	});	5468	});
5399	_ = try self.addInst(.{	5469	_ = try self.addInst(.{
5400	.tag = .mov_mem_imm,	5470	.tag = .mov_mem_imm,
5401	.ops = (Mir.Ops{	5471	.ops = Mir.Inst.Ops.encode(.{
5402	.reg1 = base_reg,	5472	.reg1 = base_reg,
5403	.flags = switch (abi_size) {	5473	.flags = switch (abi_size) {
5404	1 => 0b00,	5474	1 => 0b00,
...	@@ -5406,7 +5476,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5406,7 +5476,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5406	4 => 0b10,	5476	4 => 0b10,
5407	else => unreachable,	5477	else => unreachable,
5408	},	5478	},
5409	}).encode(),	5479	}),
5410	.data = .{ .payload = payload },	5480	.data = .{ .payload = payload },
5411	});	5481	});
5412	},	5482	},
...	@@ -5420,10 +5490,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5420,10 +5490,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5420	});	5490	});
5421	_ = try self.addInst(.{	5491	_ = try self.addInst(.{
5422	.tag = .mov_mem_imm,	5492	.tag = .mov_mem_imm,
5423	.ops = (Mir.Ops{	5493	.ops = Mir.Inst.Ops.encode(.{
5424	.reg1 = base_reg,	5494	.reg1 = base_reg,
5425	.flags = 0b10,	5495	.flags = 0b10,
5426	}).encode(),	5496	}),
5427	.data = .{ .payload = payload },	5497	.data = .{ .payload = payload },
5428	});	5498	});
5429	}	5499	}
...	@@ -5434,10 +5504,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5434,10 +5504,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5434	});	5504	});
5435	_ = try self.addInst(.{	5505	_ = try self.addInst(.{
5436	.tag = .mov_mem_imm,	5506	.tag = .mov_mem_imm,
5437	.ops = (Mir.Ops{	5507	.ops = Mir.Inst.Ops.encode(.{
5438	.reg1 = base_reg,	5508	.reg1 = base_reg,
5439	.flags = 0b10,	5509	.flags = 0b10,
5440	}).encode(),	5510	}),
5441	.data = .{ .payload = payload },	5511	.data = .{ .payload = payload },
5442	});	5512	});
5443	}	5513	}
...	@@ -5453,44 +5523,80 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5453,44 +5523,80 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5453	}	5523	}
5454		5524
5455	const base_reg = opts.dest_stack_base orelse .rbp;	5525	const base_reg = opts.dest_stack_base orelse .rbp;
5456	if (!math.isPowerOfTwo(abi_size)) {
5457	const reg_lock = self.register_manager.lockReg(reg);
5458	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
5459		5526
5460	const tmp_reg = try self.copyToTmpRegister(ty, mcv);	5527	switch (ty.zigTypeTag()) {
		5528	.Float => {
		5529	if (intrinsicsAllowed(self.target.*, ty)) {
		5530	const tag: Mir.Inst.Tag = switch (ty.tag()) {
		5531	.f32 => if (hasAvxSupport(self.target.*))
		5532	Mir.Inst.Tag.mov_f32_avx
		5533	else
		5534	Mir.Inst.Tag.mov_f32_sse,
		5535	.f64 => if (hasAvxSupport(self.target.*))
		5536	Mir.Inst.Tag.mov_f64_avx
		5537	else
		5538	Mir.Inst.Tag.mov_f64_sse,
		5539	else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
		5540	};
		5541	_ = try self.addInst(.{
		5542	.tag = tag,
		5543	.ops = Mir.Inst.Ops.encode(.{
		5544	.reg1 = switch (ty.tag()) {
		5545	.f32 => base_reg.to32(),
		5546	.f64 => base_reg.to64(),
		5547	else => unreachable,
		5548	},
		5549	.reg2 = reg.to128(),
		5550	.flags = 0b01,
		5551	}),
		5552	.data = .{ .imm = @bitCast(u32, -stack_offset) },
		5553	});
		5554	return;
		5555	}
		5556
		5557	return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
		5558	},
		5559	else => {
		5560	if (!math.isPowerOfTwo(abi_size)) {
		5561	const reg_lock = self.register_manager.lockReg(reg);
		5562	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
		5563
		5564	const tmp_reg = try self.copyToTmpRegister(ty, mcv);
5461		5565
5462	var next_offset = stack_offset;	5566	var next_offset = stack_offset;
5463	var remainder = abi_size;	5567	var remainder = abi_size;
5464	while (remainder > 0) {	5568	while (remainder > 0) {
5465	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));	5569	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5466		5570
5467	_ = try self.addInst(.{	5571	_ = try self.addInst(.{
5468	.tag = .mov,	5572	.tag = .mov,
5469	.ops = (Mir.Ops{	5573	.ops = Mir.Inst.Ops.encode(.{
5470	.reg1 = base_reg,	5574	.reg1 = base_reg,
5471	.reg2 = registerAlias(tmp_reg, nearest_power_of_two),	5575	.reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5472	.flags = 0b10,	5576	.flags = 0b10,
5473	}).encode(),	5577	}),
5474	.data = .{ .imm = @bitCast(u32, -next_offset) },	5578	.data = .{ .imm = @bitCast(u32, -next_offset) },
5475	});	5579	});
5476		5580
5477	if (nearest_power_of_two > 1) {	5581	if (nearest_power_of_two > 1) {
5478	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 });	5582	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 });
5479	}	5583	}
5480		5584
5481	remainder -= nearest_power_of_two;	5585	remainder -= nearest_power_of_two;
5482	next_offset -= nearest_power_of_two;	5586	next_offset -= nearest_power_of_two;
5483	}	5587	}
5484	} else {	5588	} else {
5485	_ = try self.addInst(.{	5589	_ = try self.addInst(.{
5486	.tag = .mov,	5590	.tag = .mov,
5487	.ops = (Mir.Ops{	5591	.ops = Mir.Inst.Ops.encode(.{
5488	.reg1 = base_reg,	5592	.reg1 = base_reg,
5489	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),	5593	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5490	.flags = 0b10,	5594	.flags = 0b10,
5491	}).encode(),	5595	}),
5492	.data = .{ .imm = @bitCast(u32, -stack_offset) },	5596	.data = .{ .imm = @bitCast(u32, -stack_offset) },
5493	});	5597	});
		5598	}
		5599	},
5494	}	5600	}
5495	},	5601	},
5496	.memory,	5602	.memory,
...	@@ -5558,7 +5664,7 @@ fn genInlineMemcpy(	...	@@ -5558,7 +5664,7 @@ fn genInlineMemcpy(
5558	null;	5664	null;
5559	defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);	5665	defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
5560		5666
5561	const dst_addr_reg = try self.register_manager.allocReg(null);	5667	const dst_addr_reg = try self.register_manager.allocReg(null, gp);
5562	switch (dst_ptr) {	5668	switch (dst_ptr) {
5563	.memory,	5669	.memory,
5564	.got_load,	5670	.got_load,
...	@@ -5569,20 +5675,20 @@ fn genInlineMemcpy(	...	@@ -5569,20 +5675,20 @@ fn genInlineMemcpy(
5569	.ptr_stack_offset, .stack_offset => \|off\| {	5675	.ptr_stack_offset, .stack_offset => \|off\| {
5570	_ = try self.addInst(.{	5676	_ = try self.addInst(.{
5571	.tag = .lea,	5677	.tag = .lea,
5572	.ops = (Mir.Ops{	5678	.ops = Mir.Inst.Ops.encode(.{
5573	.reg1 = dst_addr_reg.to64(),	5679	.reg1 = dst_addr_reg.to64(),
5574	.reg2 = opts.dest_stack_base orelse .rbp,	5680	.reg2 = opts.dest_stack_base orelse .rbp,
5575	}).encode(),	5681	}),
5576	.data = .{ .imm = @bitCast(u32, -off) },	5682	.data = .{ .imm = @bitCast(u32, -off) },
5577	});	5683	});
5578	},	5684	},
5579	.register => \|reg\| {	5685	.register => \|reg\| {
5580	_ = try self.addInst(.{	5686	_ = try self.addInst(.{
5581	.tag = .mov,	5687	.tag = .mov,
5582	.ops = (Mir.Ops{	5688	.ops = Mir.Inst.Ops.encode(.{
5583	.reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),	5689	.reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),
5584	.reg2 = reg,	5690	.reg2 = reg,
5585	}).encode(),	5691	}),
5586	.data = undefined,	5692	.data = undefined,
5587	});	5693	});
5588	},	5694	},
...	@@ -5593,7 +5699,7 @@ fn genInlineMemcpy(	...	@@ -5593,7 +5699,7 @@ fn genInlineMemcpy(
5593	const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);	5699	const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
5594	defer self.register_manager.unlockReg(dst_addr_reg_lock);	5700	defer self.register_manager.unlockReg(dst_addr_reg_lock);
5595		5701
5596	const src_addr_reg = try self.register_manager.allocReg(null);	5702	const src_addr_reg = try self.register_manager.allocReg(null, gp);
5597	switch (src_ptr) {	5703	switch (src_ptr) {
5598	.memory,	5704	.memory,
5599	.got_load,	5705	.got_load,
...	@@ -5604,20 +5710,20 @@ fn genInlineMemcpy(	...	@@ -5604,20 +5710,20 @@ fn genInlineMemcpy(
5604	.ptr_stack_offset, .stack_offset => \|off\| {	5710	.ptr_stack_offset, .stack_offset => \|off\| {
5605	_ = try self.addInst(.{	5711	_ = try self.addInst(.{
5606	.tag = .lea,	5712	.tag = .lea,
5607	.ops = (Mir.Ops{	5713	.ops = Mir.Inst.Ops.encode(.{
5608	.reg1 = src_addr_reg.to64(),	5714	.reg1 = src_addr_reg.to64(),
5609	.reg2 = opts.source_stack_base orelse .rbp,	5715	.reg2 = opts.source_stack_base orelse .rbp,
5610	}).encode(),	5716	}),
5611	.data = .{ .imm = @bitCast(u32, -off) },	5717	.data = .{ .imm = @bitCast(u32, -off) },
5612	});	5718	});
5613	},	5719	},
5614	.register => \|reg\| {	5720	.register => \|reg\| {
5615	_ = try self.addInst(.{	5721	_ = try self.addInst(.{
5616	.tag = .mov,	5722	.tag = .mov,
5617	.ops = (Mir.Ops{	5723	.ops = Mir.Inst.Ops.encode(.{
5618	.reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),	5724	.reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),
5619	.reg2 = reg,	5725	.reg2 = reg,
5620	}).encode(),	5726	}),
5621	.data = undefined,	5727	.data = undefined,
5622	});	5728	});
5623	},	5729	},
...	@@ -5628,7 +5734,7 @@ fn genInlineMemcpy(	...	@@ -5628,7 +5734,7 @@ fn genInlineMemcpy(
5628	const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);	5734	const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
5629	defer self.register_manager.unlockReg(src_addr_reg_lock);	5735	defer self.register_manager.unlockReg(src_addr_reg_lock);
5630		5736
5631	const regs = try self.register_manager.allocRegs(2, .{ null, null });	5737	const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
5632	const count_reg = regs[0].to64();	5738	const count_reg = regs[0].to64();
5633	const tmp_reg = regs[1].to8();	5739	const tmp_reg = regs[1].to8();
5634		5740
...	@@ -5637,18 +5743,14 @@ fn genInlineMemcpy(	...	@@ -5637,18 +5743,14 @@ fn genInlineMemcpy(
5637	// mov rcx, 0	5743	// mov rcx, 0
5638	_ = try self.addInst(.{	5744	_ = try self.addInst(.{
5639	.tag = .mov,	5745	.tag = .mov,
5640	.ops = (Mir.Ops{	5746	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }),
5641	.reg1 = .rcx,
5642	}).encode(),
5643	.data = .{ .imm = 0 },	5747	.data = .{ .imm = 0 },
5644	});	5748	});
5645		5749
5646	// mov rax, 0	5750	// mov rax, 0
5647	_ = try self.addInst(.{	5751	_ = try self.addInst(.{
5648	.tag = .mov,	5752	.tag = .mov,
5649	.ops = (Mir.Ops{	5753	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5650	.reg1 = .rax,
5651	}).encode(),
5652	.data = .{ .imm = 0 },	5754	.data = .{ .imm = 0 },
5653	});	5755	});
5654		5756
...	@@ -5656,70 +5758,62 @@ fn genInlineMemcpy(	...	@@ -5656,70 +5758,62 @@ fn genInlineMemcpy(
5656	// cmp count, 0	5758	// cmp count, 0
5657	const loop_start = try self.addInst(.{	5759	const loop_start = try self.addInst(.{
5658	.tag = .cmp,	5760	.tag = .cmp,
5659	.ops = (Mir.Ops{	5761	.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5660	.reg1 = count_reg,
5661	}).encode(),
5662	.data = .{ .imm = 0 },	5762	.data = .{ .imm = 0 },
5663	});	5763	});
5664		5764
5665	// je end	5765	// je end
5666	const loop_reloc = try self.addInst(.{	5766	const loop_reloc = try self.addInst(.{
5667	.tag = .cond_jmp_eq_ne,	5767	.tag = .cond_jmp_eq_ne,
5668	.ops = (Mir.Ops{ .flags = 0b01 }).encode(),	5768	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
5669	.data = .{ .inst = undefined },	5769	.data = .{ .inst = undefined },
5670	});	5770	});
5671		5771
5672	// mov tmp, [addr + rcx]	5772	// mov tmp, [addr + rcx]
5673	_ = try self.addInst(.{	5773	_ = try self.addInst(.{
5674	.tag = .mov_scale_src,	5774	.tag = .mov_scale_src,
5675	.ops = (Mir.Ops{	5775	.ops = Mir.Inst.Ops.encode(.{
5676	.reg1 = tmp_reg.to8(),	5776	.reg1 = tmp_reg.to8(),
5677	.reg2 = src_addr_reg,	5777	.reg2 = src_addr_reg,
5678	}).encode(),	5778	}),
5679	.data = .{ .imm = 0 },	5779	.data = .{ .imm = 0 },
5680	});	5780	});
5681		5781
5682	// mov [stack_offset + rax], tmp	5782	// mov [stack_offset + rax], tmp
5683	_ = try self.addInst(.{	5783	_ = try self.addInst(.{
5684	.tag = .mov_scale_dst,	5784	.tag = .mov_scale_dst,
5685	.ops = (Mir.Ops{	5785	.ops = Mir.Inst.Ops.encode(.{
5686	.reg1 = dst_addr_reg,	5786	.reg1 = dst_addr_reg,
5687	.reg2 = tmp_reg.to8(),	5787	.reg2 = tmp_reg.to8(),
5688	}).encode(),	5788	}),
5689	.data = .{ .imm = 0 },	5789	.data = .{ .imm = 0 },
5690	});	5790	});
5691		5791
5692	// add rcx, 1	5792	// add rcx, 1
5693	_ = try self.addInst(.{	5793	_ = try self.addInst(.{
5694	.tag = .add,	5794	.tag = .add,
5695	.ops = (Mir.Ops{	5795	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }),
5696	.reg1 = .rcx,
5697	}).encode(),
5698	.data = .{ .imm = 1 },	5796	.data = .{ .imm = 1 },
5699	});	5797	});
5700		5798
5701	// add rax, 1	5799	// add rax, 1
5702	_ = try self.addInst(.{	5800	_ = try self.addInst(.{
5703	.tag = .add,	5801	.tag = .add,
5704	.ops = (Mir.Ops{	5802	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5705	.reg1 = .rax,
5706	}).encode(),
5707	.data = .{ .imm = 1 },	5803	.data = .{ .imm = 1 },
5708	});	5804	});
5709		5805
5710	// sub count, 1	5806	// sub count, 1
5711	_ = try self.addInst(.{	5807	_ = try self.addInst(.{
5712	.tag = .sub,	5808	.tag = .sub,
5713	.ops = (Mir.Ops{	5809	.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5714	.reg1 = count_reg,
5715	}).encode(),
5716	.data = .{ .imm = 1 },	5810	.data = .{ .imm = 1 },
5717	});	5811	});
5718		5812
5719	// jmp loop	5813	// jmp loop
5720	_ = try self.addInst(.{	5814	_ = try self.addInst(.{
5721	.tag = .jmp,	5815	.tag = .jmp,
5722	.ops = (Mir.Ops{ .flags = 0b00 }).encode(),	5816	.ops = Mir.Inst.Ops.encode(.{}),
5723	.data = .{ .inst = loop_start },	5817	.data = .{ .inst = loop_start },
5724	});	5818	});
5725		5819
...	@@ -5740,7 +5834,7 @@ fn genInlineMemset(	...	@@ -5740,7 +5834,7 @@ fn genInlineMemset(
5740	const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);	5834	const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
5741	defer self.register_manager.unlockReg(rax_lock);	5835	defer self.register_manager.unlockReg(rax_lock);
5742		5836
5743	const addr_reg = try self.register_manager.allocReg(null);	5837	const addr_reg = try self.register_manager.allocReg(null, gp);
5744	switch (dst_ptr) {	5838	switch (dst_ptr) {
5745	.memory,	5839	.memory,
5746	.got_load,	5840	.got_load,
...	@@ -5751,20 +5845,20 @@ fn genInlineMemset(	...	@@ -5751,20 +5845,20 @@ fn genInlineMemset(
5751	.ptr_stack_offset, .stack_offset => \|off\| {	5845	.ptr_stack_offset, .stack_offset => \|off\| {
5752	_ = try self.addInst(.{	5846	_ = try self.addInst(.{
5753	.tag = .lea,	5847	.tag = .lea,
5754	.ops = (Mir.Ops{	5848	.ops = Mir.Inst.Ops.encode(.{
5755	.reg1 = addr_reg.to64(),	5849	.reg1 = addr_reg.to64(),
5756	.reg2 = opts.dest_stack_base orelse .rbp,	5850	.reg2 = opts.dest_stack_base orelse .rbp,
5757	}).encode(),	5851	}),
5758	.data = .{ .imm = @bitCast(u32, -off) },	5852	.data = .{ .imm = @bitCast(u32, -off) },
5759	});	5853	});
5760	},	5854	},
5761	.register => \|reg\| {	5855	.register => \|reg\| {
5762	_ = try self.addInst(.{	5856	_ = try self.addInst(.{
5763	.tag = .mov,	5857	.tag = .mov,
5764	.ops = (Mir.Ops{	5858	.ops = Mir.Inst.Ops.encode(.{
5765	.reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),	5859	.reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
5766	.reg2 = reg,	5860	.reg2 = reg,
5767	}).encode(),	5861	}),
5768	.data = undefined,	5862	.data = undefined,
5769	});	5863	});
5770	},	5864	},
...	@@ -5782,16 +5876,14 @@ fn genInlineMemset(	...	@@ -5782,16 +5876,14 @@ fn genInlineMemset(
5782	// cmp rax, -1	5876	// cmp rax, -1
5783	const loop_start = try self.addInst(.{	5877	const loop_start = try self.addInst(.{
5784	.tag = .cmp,	5878	.tag = .cmp,
5785	.ops = (Mir.Ops{	5879	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5786	.reg1 = .rax,
5787	}).encode(),
5788	.data = .{ .imm = @bitCast(u32, @as(i32, -1)) },	5880	.data = .{ .imm = @bitCast(u32, @as(i32, -1)) },
5789	});	5881	});
5790		5882
5791	// je end	5883	// je end
5792	const loop_reloc = try self.addInst(.{	5884	const loop_reloc = try self.addInst(.{
5793	.tag = .cond_jmp_eq_ne,	5885	.tag = .cond_jmp_eq_ne,
5794	.ops = (Mir.Ops{ .flags = 0b01 }).encode(),	5886	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
5795	.data = .{ .inst = undefined },	5887	.data = .{ .inst = undefined },
5796	});	5888	});
5797		5889
...	@@ -5807,9 +5899,7 @@ fn genInlineMemset(	...	@@ -5807,9 +5899,7 @@ fn genInlineMemset(
5807	});	5899	});
5808	_ = try self.addInst(.{	5900	_ = try self.addInst(.{
5809	.tag = .mov_mem_index_imm,	5901	.tag = .mov_mem_index_imm,
5810	.ops = (Mir.Ops{	5902	.ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg }),
5811	.reg1 = addr_reg,
5812	}).encode(),
5813	.data = .{ .payload = payload },	5903	.data = .{ .payload = payload },
5814	});	5904	});
5815	},	5905	},
...	@@ -5819,16 +5909,14 @@ fn genInlineMemset(	...	@@ -5819,16 +5909,14 @@ fn genInlineMemset(
5819	// sub rax, 1	5909	// sub rax, 1
5820	_ = try self.addInst(.{	5910	_ = try self.addInst(.{
5821	.tag = .sub,	5911	.tag = .sub,
5822	.ops = (Mir.Ops{	5912	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
5823	.reg1 = .rax,
5824	}).encode(),
5825	.data = .{ .imm = 1 },	5913	.data = .{ .imm = 1 },
5826	});	5914	});
5827		5915
5828	// jmp loop	5916	// jmp loop
5829	_ = try self.addInst(.{	5917	_ = try self.addInst(.{
5830	.tag = .jmp,	5918	.tag = .jmp,
5831	.ops = (Mir.Ops{ .flags = 0b00 }).encode(),	5919	.ops = Mir.Inst.Ops.encode(.{}),
5832	.data = .{ .inst = loop_start },	5920	.data = .{ .inst = loop_start },
5833	});	5921	});
5834		5922
...	@@ -5849,10 +5937,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -5849,10 +5937,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5849	}	5937	}
5850	_ = try self.addInst(.{	5938	_ = try self.addInst(.{
5851	.tag = .lea,	5939	.tag = .lea,
5852	.ops = (Mir.Ops{	5940	.ops = Mir.Inst.Ops.encode(.{
5853	.reg1 = registerAlias(reg, abi_size),	5941	.reg1 = registerAlias(reg, abi_size),
5854	.reg2 = .rbp,	5942	.reg2 = .rbp,
5855	}).encode(),	5943	}),
5856	.data = .{ .imm = @bitCast(u32, -off) },	5944	.data = .{ .imm = @bitCast(u32, -off) },
5857	});	5945	});
5858	},	5946	},
...	@@ -5889,10 +5977,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -5889,10 +5977,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5889	};	5977	};
5890	_ = try self.addInst(.{	5978	_ = try self.addInst(.{
5891	.tag = tag,	5979	.tag = tag,
5892	.ops = (Mir.Ops{	5980	.ops = Mir.Inst.Ops.encode(.{
5893	.reg1 = reg.to8(),	5981	.reg1 = reg.to8(),
5894	.flags = flags,	5982	.flags = flags,
5895	}).encode(),	5983	}),
5896	.data = undefined,	5984	.data = undefined,
5897	});	5985	});
5898	},	5986	},
...	@@ -5902,10 +5990,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -5902,10 +5990,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5902	if (x == 0) {	5990	if (x == 0) {
5903	_ = try self.addInst(.{	5991	_ = try self.addInst(.{
5904	.tag = .xor,	5992	.tag = .xor,
5905	.ops = (Mir.Ops{	5993	.ops = Mir.Inst.Ops.encode(.{
5906	.reg1 = reg.to32(),	5994	.reg1 = reg.to32(),
5907	.reg2 = reg.to32(),	5995	.reg2 = reg.to32(),
5908	}).encode(),	5996	}),
5909	.data = undefined,	5997	.data = undefined,
5910	});	5998	});
5911	return;	5999	return;
...	@@ -5914,9 +6002,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -5914,9 +6002,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5914	// Next best case: if we set the lower four bytes, the upper four will be zeroed.	6002	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
5915	_ = try self.addInst(.{	6003	_ = try self.addInst(.{
5916	.tag = .mov,	6004	.tag = .mov,
5917	.ops = (Mir.Ops{	6005	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
5918	.reg1 = registerAlias(reg, abi_size),
5919	}).encode(),
5920	.data = .{ .imm = @truncate(u32, x) },	6006	.data = .{ .imm = @truncate(u32, x) },
5921	});	6007	});
5922	return;	6008	return;
...	@@ -5931,9 +6017,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -5931,9 +6017,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5931	const payload = try self.addExtra(Mir.Imm64.encode(x));	6017	const payload = try self.addExtra(Mir.Imm64.encode(x));
5932	_ = try self.addInst(.{	6018	_ = try self.addInst(.{
5933	.tag = .movabs,	6019	.tag = .movabs,
5934	.ops = (Mir.Ops{	6020	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }),
5935	.reg1 = reg.to64(),
5936	}).encode(),
5937	.data = .{ .payload = payload },	6021	.data = .{ .payload = payload },
5938	});	6022	});
5939	},	6023	},
...	@@ -5942,150 +6026,290 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -5942,150 +6026,290 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5942	if (src_reg.id() == reg.id())	6026	if (src_reg.id() == reg.id())
5943	return;	6027	return;
5944		6028
5945	if (ty.zigTypeTag() == .Int) blk: {	6029	switch (ty.zigTypeTag()) {
5946	switch (ty.intInfo(self.target.*).signedness) {	6030	.Int => switch (ty.intInfo(self.target.*).signedness) {
5947	.signed => {	6031	.signed => {
5948	if (abi_size > 4) break :blk;	6032	if (abi_size <= 4) {
5949	_ = try self.addInst(.{	6033	_ = try self.addInst(.{
5950	.tag = .mov_sign_extend,	6034	.tag = .mov_sign_extend,
5951	.ops = (Mir.Ops{	6035	.ops = Mir.Inst.Ops.encode(.{
5952	.reg1 = reg.to64(),	6036	.reg1 = reg.to64(),
5953	.reg2 = registerAlias(src_reg, abi_size),	6037	.reg2 = registerAlias(src_reg, abi_size),
5954	}).encode(),	6038	}),
5955	.data = undefined,	6039	.data = undefined,
5956	});	6040	});
		6041	return;
		6042	}
5957	},	6043	},
5958	.unsigned => {	6044	.unsigned => {
5959	if (abi_size > 2) break :blk;	6045	if (abi_size <= 2) {
		6046	_ = try self.addInst(.{
		6047	.tag = .mov_zero_extend,
		6048	.ops = Mir.Inst.Ops.encode(.{
		6049	.reg1 = reg.to64(),
		6050	.reg2 = registerAlias(src_reg, abi_size),
		6051	}),
		6052	.data = undefined,
		6053	});
		6054	return;
		6055	}
		6056	},
		6057	},
		6058	.Float => {
		6059	if (intrinsicsAllowed(self.target.*, ty)) {
		6060	const tag: Mir.Inst.Tag = switch (ty.tag()) {
		6061	.f32 => if (hasAvxSupport(self.target.*))
		6062	Mir.Inst.Tag.mov_f32_avx
		6063	else
		6064	Mir.Inst.Tag.mov_f32_sse,
		6065	.f64 => if (hasAvxSupport(self.target.*))
		6066	Mir.Inst.Tag.mov_f64_avx
		6067	else
		6068	Mir.Inst.Tag.mov_f64_sse,
		6069	else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
		6070	};
5960	_ = try self.addInst(.{	6071	_ = try self.addInst(.{
5961	.tag = .mov_zero_extend,	6072	.tag = tag,
5962	.ops = (Mir.Ops{	6073	.ops = Mir.Inst.Ops.encode(.{
5963	.reg1 = reg.to64(),	6074	.reg1 = reg.to128(),
5964	.reg2 = registerAlias(src_reg, abi_size),	6075	.reg2 = src_reg.to128(),
5965	}).encode(),	6076	.flags = 0b10,
		6077	}),
5966	.data = undefined,	6078	.data = undefined,
5967	});	6079	});
5968	},	6080	return;
5969	}	6081	}
5970	return;	6082
		6083	return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
		6084	},
		6085	else => {},
5971	}	6086	}
5972		6087
5973	_ = try self.addInst(.{	6088	_ = try self.addInst(.{
5974	.tag = .mov,	6089	.tag = .mov,
5975	.ops = (Mir.Ops{	6090	.ops = Mir.Inst.Ops.encode(.{
5976	.reg1 = registerAlias(reg, abi_size),	6091	.reg1 = registerAlias(reg, abi_size),
5977	.reg2 = registerAlias(src_reg, abi_size),	6092	.reg2 = registerAlias(src_reg, abi_size),
5978	}).encode(),	6093	}),
5979	.data = undefined,	6094	.data = undefined,
5980	});	6095	});
5981	},	6096	},
5982	.direct_load,	6097	.direct_load,
5983	.got_load,	6098	.got_load,
5984	=> {	6099	=> {
5985	try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);	6100	switch (ty.zigTypeTag()) {
5986	_ = try self.addInst(.{	6101	.Float => {
5987	.tag = .mov,	6102	const base_reg = try self.register_manager.allocReg(null, gp);
5988	.ops = (Mir.Ops{	6103	try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
5989	.reg1 = registerAlias(reg, abi_size),	6104
5990	.reg2 = reg.to64(),	6105	if (intrinsicsAllowed(self.target.*, ty)) {
5991	.flags = 0b01,	6106	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5992	}).encode(),	6107	.f32 => if (hasAvxSupport(self.target.*))
5993	.data = .{ .imm = 0 },	6108	Mir.Inst.Tag.mov_f32_avx
5994	});	6109	else
		6110	Mir.Inst.Tag.mov_f32_sse,
		6111	.f64 => if (hasAvxSupport(self.target.*))
		6112	Mir.Inst.Tag.mov_f64_avx
		6113	else
		6114	Mir.Inst.Tag.mov_f64_sse,
		6115	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
		6116	};
		6117
		6118	_ = try self.addInst(.{
		6119	.tag = tag,
		6120	.ops = Mir.Inst.Ops.encode(.{
		6121	.reg1 = reg.to128(),
		6122	.reg2 = switch (ty.tag()) {
		6123	.f32 => base_reg.to32(),
		6124	.f64 => base_reg.to64(),
		6125	else => unreachable,
		6126	},
		6127	}),
		6128	.data = .{ .imm = 0 },
		6129	});
		6130	return;
		6131	}
		6132
		6133	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
		6134	},
		6135	else => {
		6136	try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
		6137	_ = try self.addInst(.{
		6138	.tag = .mov,
		6139	.ops = Mir.Inst.Ops.encode(.{
		6140	.reg1 = registerAlias(reg, abi_size),
		6141	.reg2 = reg.to64(),
		6142	.flags = 0b01,
		6143	}),
		6144	.data = .{ .imm = 0 },
		6145	});
		6146	},
		6147	}
5995	},	6148	},
5996	.memory => \|x\| {	6149	.memory => \|x\| switch (ty.zigTypeTag()) {
5997	if (x <= math.maxInt(i32)) {	6150	.Float => {
5998	// mov reg, [ds:imm32]	6151	const base_reg = try self.register_manager.allocReg(null, gp);
5999	_ = try self.addInst(.{	6152	try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
6000	.tag = .mov,	6153
6001	.ops = (Mir.Ops{	6154	if (intrinsicsAllowed(self.target.*, ty)) {
6002	.reg1 = registerAlias(reg, abi_size),	6155	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6003	.flags = 0b01,	6156	.f32 => if (hasAvxSupport(self.target.*))
6004	}).encode(),	6157	Mir.Inst.Tag.mov_f32_avx
6005	.data = .{ .imm = @truncate(u32, x) },	6158	else
6006	});	6159	Mir.Inst.Tag.mov_f32_sse,
6007	} else {	6160	.f64 => if (hasAvxSupport(self.target.*))
6008	// If this is RAX, we can use a direct load.	6161	Mir.Inst.Tag.mov_f64_avx
6009	// Otherwise, we need to load the address, then indirectly load the value.	6162	else
6010	if (reg.id() == 0) {	6163	Mir.Inst.Tag.mov_f64_sse,
6011	// movabs rax, ds:moffs64	6164	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6012	const payload = try self.addExtra(Mir.Imm64.encode(x));	6165	};
		6166
6013	_ = try self.addInst(.{	6167	_ = try self.addInst(.{
6014	.tag = .movabs,	6168	.tag = tag,
6015	.ops = (Mir.Ops{	6169	.ops = Mir.Inst.Ops.encode(.{
6016	.reg1 = .rax,	6170	.reg1 = reg.to128(),
6017	.flags = 0b01, // imm64 will become moffs64	6171	.reg2 = switch (ty.tag()) {
6018	}).encode(),	6172	.f32 => base_reg.to32(),
6019	.data = .{ .payload = payload },	6173	.f64 => base_reg.to64(),
		6174	else => unreachable,
		6175	},
		6176	}),
		6177	.data = .{ .imm = 0 },
6020	});	6178	});
6021	} else {	6179	return;
6022	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.	6180	}
6023	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
6024		6181
6025	// mov reg, [reg + 0x0]	6182	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
		6183	},
		6184	else => {
		6185	if (x <= math.maxInt(i32)) {
		6186	// mov reg, [ds:imm32]
6026	_ = try self.addInst(.{	6187	_ = try self.addInst(.{
6027	.tag = .mov,	6188	.tag = .mov,
6028	.ops = (Mir.Ops{	6189	.ops = Mir.Inst.Ops.encode(.{
6029	.reg1 = registerAlias(reg, abi_size),	6190	.reg1 = registerAlias(reg, abi_size),
6030	.reg2 = reg.to64(),
6031	.flags = 0b01,	6191	.flags = 0b01,
6032	}).encode(),	6192	}),
6033	.data = .{ .imm = 0 },	6193	.data = .{ .imm = @truncate(u32, x) },
6034	});	6194	});
		6195	} else {
		6196	// If this is RAX, we can use a direct load.
		6197	// Otherwise, we need to load the address, then indirectly load the value.
		6198	if (reg.id() == 0) {
		6199	// movabs rax, ds:moffs64
		6200	const payload = try self.addExtra(Mir.Imm64.encode(x));
		6201	_ = try self.addInst(.{
		6202	.tag = .movabs,
		6203	.ops = Mir.Inst.Ops.encode(.{
		6204	.reg1 = .rax,
		6205	.flags = 0b01, // imm64 will become moffs64
		6206	}),
		6207	.data = .{ .payload = payload },
		6208	});
		6209	} else {
		6210	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
		6211	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
		6212
		6213	// mov reg, [reg + 0x0]
		6214	_ = try self.addInst(.{
		6215	.tag = .mov,
		6216	.ops = Mir.Inst.Ops.encode(.{
		6217	.reg1 = registerAlias(reg, abi_size),
		6218	.reg2 = reg.to64(),
		6219	.flags = 0b01,
		6220	}),
		6221	.data = .{ .imm = 0 },
		6222	});
		6223	}
6035	}	6224	}
6036	}	6225	},
6037	},	6226	},
6038	.stack_offset => \|off\| {	6227	.stack_offset => \|off\| {
6039	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {	6228	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
6040	return self.fail("stack offset too large", .{});	6229	return self.fail("stack offset too large", .{});
6041	}	6230	}
6042		6231
6043	if (ty.zigTypeTag() == .Int) blk: {	6232	switch (ty.zigTypeTag()) {
6044	switch (ty.intInfo(self.target.*).signedness) {	6233	.Int => switch (ty.intInfo(self.target.*).signedness) {
6045	.signed => {	6234	.signed => {
6046	const flags: u2 = switch (abi_size) {	6235	if (abi_size <= 4) {
6047	1 => 0b01,	6236	const flags: u2 = switch (abi_size) {
6048	2 => 0b10,	6237	1 => 0b01,
6049	4 => 0b11,	6238	2 => 0b10,
6050	else => break :blk,	6239	4 => 0b11,
6051	};	6240	else => unreachable,
6052	_ = try self.addInst(.{	6241	};
6053	.tag = .mov_sign_extend,	6242	_ = try self.addInst(.{
6054	.ops = (Mir.Ops{	6243	.tag = .mov_sign_extend,
6055	.reg1 = reg.to64(),	6244	.ops = Mir.Inst.Ops.encode(.{
6056	.reg2 = .rbp,	6245	.reg1 = reg.to64(),
6057	.flags = flags,	6246	.reg2 = .rbp,
6058	}).encode(),	6247	.flags = flags,
6059	.data = .{ .imm = @bitCast(u32, -off) },	6248	}),
6060	});	6249	.data = .{ .imm = @bitCast(u32, -off) },
		6250	});
		6251	return;
		6252	}
6061	},	6253	},
6062	.unsigned => {	6254	.unsigned => {
6063	const flags: u2 = switch (abi_size) {	6255	if (abi_size <= 2) {
6064	1 => 0b01,	6256	const flags: u2 = switch (abi_size) {
6065	2 => 0b10,	6257	1 => 0b01,
6066	else => break :blk,	6258	2 => 0b10,
		6259	else => unreachable,
		6260	};
		6261	_ = try self.addInst(.{
		6262	.tag = .mov_zero_extend,
		6263	.ops = Mir.Inst.Ops.encode(.{
		6264	.reg1 = reg.to64(),
		6265	.reg2 = .rbp,
		6266	.flags = flags,
		6267	}),
		6268	.data = .{ .imm = @bitCast(u32, -off) },
		6269	});
		6270	return;
		6271	}
		6272	},
		6273	},
		6274	.Float => {
		6275	if (intrinsicsAllowed(self.target.*, ty)) {
		6276	const tag: Mir.Inst.Tag = switch (ty.tag()) {
		6277	.f32 => if (hasAvxSupport(self.target.*))
		6278	Mir.Inst.Tag.mov_f32_avx
		6279	else
		6280	Mir.Inst.Tag.mov_f32_sse,
		6281	.f64 => if (hasAvxSupport(self.target.*))
		6282	Mir.Inst.Tag.mov_f64_avx
		6283	else
		6284	Mir.Inst.Tag.mov_f64_sse,
		6285	else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
6067	};	6286	};
6068	_ = try self.addInst(.{	6287	_ = try self.addInst(.{
6069	.tag = .mov_zero_extend,	6288	.tag = tag,
6070	.ops = (Mir.Ops{	6289	.ops = Mir.Inst.Ops.encode(.{
6071	.reg1 = reg.to64(),	6290	.reg1 = reg.to128(),
6072	.reg2 = .rbp,	6291	.reg2 = switch (ty.tag()) {
6073	.flags = flags,	6292	.f32 => .ebp,
6074	}).encode(),	6293	.f64 => .rbp,
		6294	else => unreachable,
		6295	},
		6296	}),
6075	.data = .{ .imm = @bitCast(u32, -off) },	6297	.data = .{ .imm = @bitCast(u32, -off) },
6076	});	6298	});
6077	},	6299	return;
6078	}	6300	}
6079	return;	6301	return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
		6302	},
		6303	else => {},
6080	}	6304	}
6081		6305
6082	_ = try self.addInst(.{	6306	_ = try self.addInst(.{
6083	.tag = .mov,	6307	.tag = .mov,
6084	.ops = (Mir.Ops{	6308	.ops = Mir.Inst.Ops.encode(.{
6085	.reg1 = registerAlias(reg, abi_size),	6309	.reg1 = registerAlias(reg, abi_size),
6086	.reg2 = .rbp,	6310	.reg2 = .rbp,
6087	.flags = 0b01,	6311	.flags = 0b01,
6088	}).encode(),	6312	}),
6089	.data = .{ .imm = @bitCast(u32, -off) },	6313	.data = .{ .imm = @bitCast(u32, -off) },
6090	});	6314	});
6091	},	6315	},
...	@@ -6152,14 +6376,14 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6152,14 +6376,14 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6152	};	6376	};
6153	_ = try self.addInst(.{	6377	_ = try self.addInst(.{
6154	.tag = .fld,	6378	.tag = .fld,
6155	.ops = (Mir.Ops{	6379	.ops = Mir.Inst.Ops.encode(.{
		6380	.reg1 = .rbp,
6156	.flags = switch (src_ty.abiSize(self.target.*)) {	6381	.flags = switch (src_ty.abiSize(self.target.*)) {
6157	4 => 0b01,	6382	4 => 0b01,
6158	8 => 0b10,	6383	8 => 0b10,
6159	else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),	6384	else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
6160	},	6385	},
6161	.reg1 = .rbp,	6386	}),
6162	}).encode(),
6163	.data = .{ .imm = @bitCast(u32, -stack_offset) },	6387	.data = .{ .imm = @bitCast(u32, -stack_offset) },
6164	});	6388	});
6165		6389
...	@@ -6167,15 +6391,15 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6167,15 +6391,15 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6167	const stack_dst = try self.allocRegOrMem(inst, false);	6391	const stack_dst = try self.allocRegOrMem(inst, false);
6168	_ = try self.addInst(.{	6392	_ = try self.addInst(.{
6169	.tag = .fisttp,	6393	.tag = .fisttp,
6170	.ops = (Mir.Ops{	6394	.ops = Mir.Inst.Ops.encode(.{
		6395	.reg1 = .rbp,
6171	.flags = switch (dst_ty.abiSize(self.target.*)) {	6396	.flags = switch (dst_ty.abiSize(self.target.*)) {
6172	1...2 => 0b00,	6397	1...2 => 0b00,
6173	3...4 => 0b01,	6398	3...4 => 0b01,
6174	5...8 => 0b10,	6399	5...8 => 0b10,
6175	else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),	6400	else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),
6176	},	6401	},
6177	.reg1 = .rbp,	6402	}),
6178	}).encode(),
6179	.data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) },	6403	.data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) },
6180	});	6404	});
6181		6405
...	@@ -6267,15 +6491,15 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6267,15 +6491,15 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6267	const src: MCValue = blk: {	6491	const src: MCValue = blk: {
6268	switch (src_ptr) {	6492	switch (src_ptr) {
6269	.got_load, .direct_load, .memory => {	6493	.got_load, .direct_load, .memory => {
6270	const reg = try self.register_manager.allocReg(null);	6494	const reg = try self.register_manager.allocReg(null, gp);
6271	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);	6495	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
6272	_ = try self.addInst(.{	6496	_ = try self.addInst(.{
6273	.tag = .mov,	6497	.tag = .mov,
6274	.ops = (Mir.Ops{	6498	.ops = Mir.Inst.Ops.encode(.{
6275	.reg1 = reg,	6499	.reg1 = reg,
6276	.reg2 = reg,	6500	.reg2 = reg,
6277	.flags = 0b01,	6501	.flags = 0b01,
6278	}).encode(),	6502	}),
6279	.data = .{ .imm = 0 },	6503	.data = .{ .imm = 0 },
6280	});	6504	});
6281	break :blk MCValue{ .register = reg };	6505	break :blk MCValue{ .register = reg };
...	@@ -6839,9 +7063,11 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {	...	@@ -6839,9 +7063,11 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
6839	return reg.to32();	7063	return reg.to32();
6840	} else if (size_bytes <= 8) {	7064	} else if (size_bytes <= 8) {
6841	return reg.to64();	7065	return reg.to64();
6842	} else {	7066	} else if (size_bytes <= 16) {
6843	unreachable; // TODO handle floating-point registers	7067	return reg.to128();
6844	}	7068	} else if (size_bytes <= 32) {
		7069	return reg.to256();
		7070	} else unreachable;
6845	}	7071	}
6846		7072
6847	/// Truncates the value in the register in place.	7073	/// Truncates the value in the register in place.
...	@@ -6867,3 +7093,16 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	...	@@ -6867,3 +7093,16 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
6867	},	7093	},
6868	}	7094	}
6869	}	7095	}
		7096
		7097	fn intrinsicsAllowed(target: Target, ty: Type) bool {
		7098	return switch (ty.tag()) {
		7099	.f32,
		7100	.f64,
		7101	=> Target.x86.featureSetHasAny(target.cpu.features, .{ .sse2, .avx, .avx2 }),
		7102	else => unreachable, // TODO finish this off
		7103	};
		7104	}
		7105
		7106	fn hasAvxSupport(target: Target) bool {
		7107	return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
		7108	}

src/arch/x86_64/Emit.zig+785-236

 @@ -25,8 +25,8 @@ const MCValue = @import("CodeGen.zig").MCValue;
 const Mir = @import("Mir.zig");
 const Module = @import("../../Module.zig");
 const Instruction = bits.Instruction;
-const Register = bits.Register;
 const Type = @import("../../type.zig").Type;
+const Register = bits.Register;
 mir: Mir,
 bin_file: *link.File,
 @@ -67,6 +67,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
         const inst = @intCast(u32, index);
         try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
         switch (tag) {
+            // GPR instructions
             .adc => try emit.mirArith(.adc, inst),
             .add => try emit.mirArith(.add, inst),
             .sub => try emit.mirArith(.sub, inst),
 @@ -181,6 +182,27 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
             .interrupt => try emit.mirInterrupt(inst),
             .nop => try emit.mirNop(),
+            // SSE instructions
+            .mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),
+            .mov_f32_sse => try emit.mirMovFloatSse(.movss, inst),
+            .add_f64_sse => try emit.mirAddFloatSse(.addsd, inst),
+            .add_f32_sse => try emit.mirAddFloatSse(.addss, inst),
+            .cmp_f64_sse => try emit.mirCmpFloatSse(.ucomisd, inst),
+            .cmp_f32_sse => try emit.mirCmpFloatSse(.ucomiss, inst),
+            // AVX instructions
+            .mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst),
+            .mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst),
+            .add_f64_avx => try emit.mirAddFloatAvx(.vaddsd, inst),
+            .add_f32_avx => try emit.mirAddFloatAvx(.vaddss, inst),
+            .cmp_f64_avx => try emit.mirCmpFloatAvx(.vucomisd, inst),
+            .cmp_f32_avx => try emit.mirCmpFloatAvx(.vucomiss, inst),
+            // Pseudo-instructions
             .call_extern => try emit.mirCallExtern(inst),
             .dbg_line => try emit.mirDbgLine(inst),
 @@ -228,7 +250,7 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
 fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .interrupt);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => return lowerToZoEnc(.int3, emit.code),
         else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}),
 @@ -244,7 +266,7 @@ fn mirSyscall(emit: *Emit) InnerError!void {
 }
 fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             // PUSH/POP reg
 @@ -271,8 +293,9 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
         0b11 => unreachable,
     }
 }
 fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const payload = emit.mir.instructions.items(.data)[inst].payload;
     const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data;
     const regs = data.regs;
 @@ -295,7 +318,7 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
 }
 fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             const target = emit.mir.instructions.items(.data)[inst].inst;
 @@ -324,7 +347,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
         0b10 => {
             // JMP/CALL r/m64
             const imm = emit.mir.instructions.items(.data)[inst].imm;
-            return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
+            return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
                 .disp = imm,
                 .base = ops.reg1,
             }), emit.code);
 @@ -334,7 +357,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
 }
 fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const target = emit.mir.instructions.items(.data)[inst].inst;
     const tag = switch (mir_tag) {
         .cond_jmp_greater_less => switch (ops.flags) {
 @@ -366,7 +389,7 @@ fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerErr
 }
 fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const tag = switch (mir_tag) {
         .cond_set_byte_greater_less => switch (ops.flags) {
             0b00 => Tag.setge,
 @@ -396,7 +419,7 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne
 }
 fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     if (ops.flags == 0b00) {
         return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
     }
 @@ -416,7 +439,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
 fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .@"test");
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             if (ops.reg2 == .none) {
 @@ -440,7 +463,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .ret);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             // RETF imm16
 @@ -464,7 +487,7 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 }
 fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             if (ops.reg2 == .none) {
 @@ -481,8 +504,8 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
             // mov reg1, [reg2 + imm32]
             // RM
             const imm = emit.mir.instructions.items(.data)[inst].imm;
-            const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
+            const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
-            return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
+            return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
                 .disp = imm,
                 .base = src_reg,
             }), emit.code);
 @@ -494,7 +517,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
             // mov [reg1 + imm32], reg2
             // MR
             const imm = emit.mir.instructions.items(.data)[inst].imm;
-            return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
+            return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
                 .disp = imm,
                 .base = ops.reg1,
             }), ops.reg2, emit.code);
 @@ -506,7 +529,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
 }
 fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     assert(ops.reg2 == .none);
     const payload = emit.mir.instructions.items(.data)[inst].payload;
     const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
 @@ -523,14 +546,15 @@ fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
 }
 inline fn setRexWRegister(reg: Register) bool {
+    if (reg.size() > 64) return false;
     if (reg.size() == 64) return true;
     return switch (reg) {
-        .ah, .bh, .ch, .dh => true,
+        .ah, .ch, .dh, .bh => true,
         else => false,
     };
 }
-inline fn immOpSize(u_imm: u32) u8 {
+inline fn immOpSize(u_imm: u32) u6 {
     const imm = @bitCast(i32, u_imm);
     if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
         return 8;
 @@ -542,7 +566,7 @@ inline fn immOpSize(u_imm: u32) u8 {
 }
 fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const scale = ops.flags;
     const imm = emit.mir.instructions.items(.data)[inst].imm;
     // OP reg1, [reg2 + scale*rcx + imm32]
 @@ -550,7 +574,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
         .scale = scale,
         .index = .rcx,
     };
-    return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
+    return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
         .disp = imm,
         .base = ops.reg2,
         .scale_index = scale_index,
 @@ -558,7 +582,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
 }
 fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const scale = ops.flags;
     const imm = emit.mir.instructions.items(.data)[inst].imm;
     const scale_index = ScaleIndex{
 @@ -574,7 +598,7 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
         }), imm, emit.code);
     }
     // OP [reg1 + scale*rax + imm32], reg2
-    return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
+    return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
         .disp = imm,
         .base = ops.reg1,
         .scale_index = scale_index,
 @@ -582,7 +606,7 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
 }
 fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const scale = ops.flags;
     const payload = emit.mir.instructions.items(.data)[inst].payload;
     const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
 @@ -599,7 +623,7 @@ fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
 }
 fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     assert(ops.reg2 == .none);
     const payload = emit.mir.instructions.items(.data)[inst].payload;
     const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
 @@ -624,7 +648,7 @@ fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!v
 fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const mir_tag = emit.mir.instructions.items(.tag)[inst];
     assert(mir_tag == .mov_sign_extend);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
     switch (ops.flags) {
         0b00 => {
 @@ -655,7 +679,7 @@ fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const mir_tag = emit.mir.instructions.items(.tag)[inst];
     assert(mir_tag == .mov_zero_extend);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
     switch (ops.flags) {
         0b00 => {
 @@ -682,31 +706,46 @@ fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .movabs);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
-    const imm: u64 = if (ops.reg1.size() == 64) blk: {
+    switch (ops.flags) {
-        const payload = emit.mir.instructions.items(.data)[inst].payload;
+        0b00 => {
-        const imm = emit.mir.extraData(Mir.Imm64, payload).data;
+            const imm: u64 = if (ops.reg1.size() == 64) blk: {
-        break :blk imm.decode();
+                const payload = emit.mir.instructions.items(.data)[inst].payload;
-    } else emit.mir.instructions.items(.data)[inst].imm;
+                const imm = emit.mir.extraData(Mir.Imm64, payload).data;
-    if (ops.flags == 0b00) {
+                break :blk imm.decode();
-        // movabs reg, imm64
+            } else emit.mir.instructions.items(.data)[inst].imm;
-        // OI
+            // movabs reg, imm64
-        return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
+            // OI
-    }
+            return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
-    if (ops.reg1 == .none) {
+        },
-        // movabs moffs64, rax
+        0b01 => {
-        // TD
+            if (ops.reg1 == .none) {
-        return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
+                const imm: u64 = if (ops.reg2.size() == 64) blk: {
+                    const payload = emit.mir.instructions.items(.data)[inst].payload;
+                    const imm = emit.mir.extraData(Mir.Imm64, payload).data;
+                    break :blk imm.decode();
+                } else emit.mir.instructions.items(.data)[inst].imm;
+                // movabs moffs64, rax
+                // TD
+                return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
+            }
+            const imm: u64 = if (ops.reg1.size() == 64) blk: {
+                const payload = emit.mir.instructions.items(.data)[inst].payload;
+                const imm = emit.mir.extraData(Mir.Imm64, payload).data;
+                break :blk imm.decode();
+            } else emit.mir.instructions.items(.data)[inst].imm;
+            // movabs rax, moffs64
+            // FD
+            return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
+        },
+        else => return emit.fail("TODO unused variant: movabs 0b{b}", .{ops.flags}),
     }
-    // movabs rax, moffs64
-    // FD
-    return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
 }
 fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .fisttp);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     // the selecting between operand sizes for this particular `fisttp` instruction
     // is done via opcode instead of the usual prefixes.
 @@ -728,7 +767,7 @@ fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .fld);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     // the selecting between operand sizes for this particular `fisttp` instruction
     // is done via opcode instead of the usual prefixes.
 @@ -745,8 +784,9 @@ fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     };
     return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
 }
 fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             // sal reg1, 1
 @@ -771,12 +811,11 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
 }
 fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     if (ops.reg1 != .none) {
         assert(ops.reg2 == .none);
         return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
     }
-    assert(ops.reg1 == .none);
     assert(ops.reg2 != .none);
     const imm = emit.mir.instructions.items(.data)[inst].imm;
     const ptr_size: Memory.PtrSize = switch (ops.flags) {
 @@ -794,14 +833,14 @@ fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
 fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .imul_complex);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
         },
         0b01 => {
             const imm = emit.mir.instructions.items(.data)[inst].imm;
-            const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
+            const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
             return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
                 .disp = imm,
                 .base = src_reg,
 @@ -823,7 +862,7 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 }
 fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const tag: Tag = switch (ops.flags) {
         0b00 => .cbw,
         0b01 => .cwd,
 @@ -836,17 +875,17 @@ fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .lea);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     switch (ops.flags) {
         0b00 => {
             // lea reg1, [reg2 + imm32]
             // RM
             const imm = emit.mir.instructions.items(.data)[inst].imm;
-            const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
+            const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
             return lowerToRmEnc(
                 .lea,
                 ops.reg1,
-                RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
+                RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
                     .disp = imm,
                     .base = src_reg,
                 }),
 @@ -860,7 +899,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
             try lowerToRmEnc(
                 .lea,
                 ops.reg1,
-                RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
+                RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
                 emit.code,
             );
             const end_offset = emit.code.items.len;
 @@ -873,7 +912,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
         0b10 => {
             // lea reg, [rbp + rcx + imm32]
             const imm = emit.mir.instructions.items(.data)[inst].imm;
-            const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
+            const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
             const scale_index = ScaleIndex{
                 .scale = 0,
                 .index = .rcx,
 @@ -881,7 +920,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
             return lowerToRmEnc(
                 .lea,
                 ops.reg1,
-                RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
+                RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
                     .disp = imm,
                     .base = src_reg,
                     .scale_index = scale_index,
 @@ -896,7 +935,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .lea_pie);
-    const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
     const load_reloc = emit.mir.instructions.items(.data)[inst].load_reloc;
     // lea reg1, [rip + reloc]
 @@ -904,7 +943,7 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     try lowerToRmEnc(
         .lea,
         ops.reg1,
-        RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
+        RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
         emit.code,
     );
 @@ -935,6 +974,99 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     }
 }
+// SSE instructions
+fn mirMovFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
+    switch (ops.flags) {
+        0b00 => {
+            const imm = emit.mir.instructions.items(.data)[inst].imm;
+            return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
+                .disp = imm,
+                .base = ops.reg2,
+            }), emit.code);
+        },
+        0b01 => {
+            const imm = emit.mir.instructions.items(.data)[inst].imm;
+            return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
+                .disp = imm,
+                .base = ops.reg1,
+            }), ops.reg2, emit.code);
+        },
+        0b10 => {
+            return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
+        },
+        else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
+    }
+}
+fn mirAddFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
+    switch (ops.flags) {
+        0b00 => {
+            return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
+        },
+        else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
+    }
+}
+fn mirCmpFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
+    switch (ops.flags) {
+        0b00 => {
+            return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
+        },
+        else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
+    }
+}
+// AVX instructions
+fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
+    switch (ops.flags) {
+        0b00 => {
+            const imm = emit.mir.instructions.items(.data)[inst].imm;
+            return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
+                .disp = imm,
+                .base = ops.reg2,
+            }), emit.code);
+        },
+        0b01 => {
+            const imm = emit.mir.instructions.items(.data)[inst].imm;
+            return lowerToMvEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
+                .disp = imm,
+                .base = ops.reg1,
+            }), ops.reg2, emit.code);
+        },
+        0b10 => {
+            return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
+        },
+        else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
+    }
+}
+fn mirAddFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
+    switch (ops.flags) {
+        0b00 => {
+            return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
+        },
+        else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
+    }
+}
+fn mirCmpFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst].decode();
+    switch (ops.flags) {
+        0b00 => {
+            return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
+        },
+        else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
+    }
+}
+// Pseudo-instructions
 fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     assert(tag == .call_extern);
 @@ -1168,6 +1300,54 @@ const Tag = enum {
     cmovng,
     cmovb,
     cmovnae,
+    movsd,
+    movss,
+    addsd,
+    addss,
+    cmpsd,
+    cmpss,
+    ucomisd,
+    ucomiss,
+    vmovsd,
+    vmovss,
+    vaddsd,
+    vaddss,
+    vcmpsd,
+    vcmpss,
+    vucomisd,
+    vucomiss,
+    fn isSse(tag: Tag) bool {
+        return switch (tag) {
+            .movsd,
+            .movss,
+            .addsd,
+            .addss,
+            .cmpsd,
+            .cmpss,
+            .ucomisd,
+            .ucomiss,
+            => true,
+            else => false,
+        };
+    }
+    fn isAvx(tag: Tag) bool {
+        return switch (tag) {
+            .vmovsd,
+            .vmovss,
+            .vaddsd,
+            .vaddss,
+            .vcmpsd,
+            .vcmpss,
+            .vucomisd,
+            .vucomiss,
+            => true,
+            else => false,
+        };
+    }
     fn isSetCC(tag: Tag) bool {
         return switch (tag) {
 @@ -1252,177 +1432,273 @@ const Encoding = enum {
     /// OP r64, r/m64, imm32
     rmi,
-};
-const OpCode = union(enum) {
+    /// OP xmm1, xmm2/m64
-    one_byte: u8,
+    vm,
-    two_byte: struct { _1: u8, _2: u8 },
-    fn oneByte(opc: u8) OpCode {
+    /// OP m64, xmm1
-        return .{ .one_byte = opc };
+    mv,
-    }
-    fn twoByte(opc1: u8, opc2: u8) OpCode {
+    /// OP xmm1, xmm2, xmm3/m64
-        return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
+    rvm,
+    /// OP xmm1, xmm2, xmm3/m64, imm8
+    rvmi,
+};
+const OpCode = struct {
+    bytes: [3]u8,
+    count: usize,
+    fn init(comptime in_bytes: []const u8) OpCode {
+        comptime assert(in_bytes.len <= 3);
+        comptime var bytes: [3]u8 = undefined;
+        inline for (in_bytes) |x, i| {
+            bytes[i] = x;
+        }
+        return .{ .bytes = bytes, .count = in_bytes.len };
     }
     fn encode(opc: OpCode, encoder: Encoder) void {
-        switch (opc) {
+        switch (opc.count) {
-            .one_byte => |v| encoder.opcode_1byte(v),
+            1 => encoder.opcode_1byte(opc.bytes[0]),
-            .two_byte => |v| encoder.opcode_2byte(v._1, v._2),
+            2 => encoder.opcode_2byte(opc.bytes[0], opc.bytes[1]),
+            3 => encoder.opcode_3byte(opc.bytes[0], opc.bytes[1], opc.bytes[2]),
+            else => unreachable,
         }
     }
     fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
-        assert(opc == .one_byte);
+        assert(opc.count == 1);
-        encoder.opcode_withReg(opc.one_byte, reg.lowId());
+        encoder.opcode_withReg(opc.bytes[0], reg.lowEnc());
     }
 };
-inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
+inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) OpCode {
+    // zig fmt: off
     switch (enc) {
         .zo => return switch (tag) {
-            .ret_near => OpCode.oneByte(0xc3),
+            .ret_near => OpCode.init(&.{0xc3}),
-            .ret_far => OpCode.oneByte(0xcb),
+            .ret_far  => OpCode.init(&.{0xcb}),
-            .int3 => OpCode.oneByte(0xcc),
+            .int3     => OpCode.init(&.{0xcc}),
-            .nop => OpCode.oneByte(0x90),
+            .nop      => OpCode.init(&.{0x90}),
-            .syscall => OpCode.twoByte(0x0f, 0x05),
+            .syscall  => OpCode.init(&.{ 0x0f, 0x05 }),
-            .cbw => OpCode.oneByte(0x98),
+            .cbw      => OpCode.init(&.{0x98}),
-            .cwd, .cdq, .cqo => OpCode.oneByte(0x99),
+            .cwd,
-            else => null,
+            .cdq,
+            .cqo      => OpCode.init(&.{0x99}),
+            else      => unreachable,
         },
         .d => return switch (tag) {
-            .jmp_near => OpCode.oneByte(0xe9),
+            .jmp_near  =>                  OpCode.init(&.{0xe9}),
-            .call_near => OpCode.oneByte(0xe8),
+            .call_near =>                  OpCode.init(&.{0xe8}),
-            .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
+            .jo        => if (is_one_byte) OpCode.init(&.{0x70}) else OpCode.init(&.{0x0f,0x80}),
-            .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
+            .jno       => if (is_one_byte) OpCode.init(&.{0x71}) else OpCode.init(&.{0x0f,0x81}),
-            .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
+            .jb,
-            .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
+            .jc,
-            .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
+            .jnae      => if (is_one_byte) OpCode.init(&.{0x72}) else OpCode.init(&.{0x0f,0x82}),
-            .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
+            .jnb,
-            .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
+            .jnc,
-            .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
+            .jae       => if (is_one_byte) OpCode.init(&.{0x73}) else OpCode.init(&.{0x0f,0x83}),
-            .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
+            .je,
-            .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
+            .jz        => if (is_one_byte) OpCode.init(&.{0x74}) else OpCode.init(&.{0x0f,0x84}),
-            .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
+            .jne,
-            .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
+            .jnz       => if (is_one_byte) OpCode.init(&.{0x75}) else OpCode.init(&.{0x0f,0x85}),
-            .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
+            .jna,
-            .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
+            .jbe       => if (is_one_byte) OpCode.init(&.{0x76}) else OpCode.init(&.{0x0f,0x86}),
-            .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
+            .jnbe,
-            .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
+            .ja        => if (is_one_byte) OpCode.init(&.{0x77}) else OpCode.init(&.{0x0f,0x87}),
-            else => null,
+            .js        => if (is_one_byte) OpCode.init(&.{0x78}) else OpCode.init(&.{0x0f,0x88}),
+            .jns       => if (is_one_byte) OpCode.init(&.{0x79}) else OpCode.init(&.{0x0f,0x89}),
+            .jpe,
+            .jp        => if (is_one_byte) OpCode.init(&.{0x7a}) else OpCode.init(&.{0x0f,0x8a}),
+            .jpo,
+            .jnp       => if (is_one_byte) OpCode.init(&.{0x7b}) else OpCode.init(&.{0x0f,0x8b}),
+            .jnge,
+            .jl        => if (is_one_byte) OpCode.init(&.{0x7c}) else OpCode.init(&.{0x0f,0x8c}),
+            .jge,
+            .jnl       => if (is_one_byte) OpCode.init(&.{0x7d}) else OpCode.init(&.{0x0f,0x8d}),
+            .jle,
+            .jng       => if (is_one_byte) OpCode.init(&.{0x7e}) else OpCode.init(&.{0x0f,0x8e}),
+            .jg,
+            .jnle      => if (is_one_byte) OpCode.init(&.{0x7f}) else OpCode.init(&.{0x0f,0x8f}),
+            else       => unreachable,
         },
         .m => return switch (tag) {
-            .jmp_near, .call_near, .push => OpCode.oneByte(0xff),
+            .jmp_near,
-            .pop => OpCode.oneByte(0x8f),
+            .call_near,
-            .seto => OpCode.twoByte(0x0f, 0x90),
+            .push       =>                  OpCode.init(&.{0xff}),
-            .setno => OpCode.twoByte(0x0f, 0x91),
+            .pop        =>                  OpCode.init(&.{0x8f}),
-            .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
+            .seto       =>                  OpCode.init(&.{0x0f,0x90}),
-            .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
+            .setno      =>                  OpCode.init(&.{0x0f,0x91}),
-            .sete, .setz => OpCode.twoByte(0x0f, 0x94),
+            .setb,
-            .setne, .setnz => OpCode.twoByte(0x0f, 0x95),
+            .setc,
-            .setbe, .setna => OpCode.twoByte(0x0f, 0x96),
+            .setnae     =>                  OpCode.init(&.{0x0f,0x92}),
-            .seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
+            .setnb,
-            .sets => OpCode.twoByte(0x0f, 0x98),
+            .setnc,
-            .setns => OpCode.twoByte(0x0f, 0x99),
+            .setae      =>                  OpCode.init(&.{0x0f,0x93}),
-            .setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
+            .sete,
-            .setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
+            .setz       =>                  OpCode.init(&.{0x0f,0x94}),
-            .setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
+            .setne,
-            .setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
+            .setnz      =>                  OpCode.init(&.{0x0f,0x95}),
-            .setle, .setng => OpCode.twoByte(0x0f, 0x9e),
+            .setbe,
-            .setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
+            .setna      =>                  OpCode.init(&.{0x0f,0x96}),
-            .idiv, .div, .imul, .mul => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
+            .seta,
-            .fisttp16 => OpCode.oneByte(0xdf),
+            .setnbe     =>                  OpCode.init(&.{0x0f,0x97}),
-            .fisttp32 => OpCode.oneByte(0xdb),
+            .sets       =>                  OpCode.init(&.{0x0f,0x98}),
-            .fisttp64 => OpCode.oneByte(0xdd),
+            .setns      =>                  OpCode.init(&.{0x0f,0x99}),
-            .fld32 => OpCode.oneByte(0xd9),
+            .setp,
-            .fld64 => OpCode.oneByte(0xdd),
+            .setpe      =>                  OpCode.init(&.{0x0f,0x9a}),
-            else => null,
+            .setnp,
+            .setop      =>                  OpCode.init(&.{0x0f,0x9b}),
+            .setl,
+            .setnge     =>                  OpCode.init(&.{0x0f,0x9c}),
+            .setnl,
+            .setge      =>                  OpCode.init(&.{0x0f,0x9d}),
+            .setle,
+            .setng      =>                  OpCode.init(&.{0x0f,0x9e}),
+            .setnle,
+            .setg       =>                  OpCode.init(&.{0x0f,0x9f}),
+            .idiv,
+            .div,
+            .imul,
+            .mul        => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
+            .fisttp16   =>                  OpCode.init(&.{0xdf}),
+            .fisttp32   =>                  OpCode.init(&.{0xdb}),
+            .fisttp64   =>                  OpCode.init(&.{0xdd}),
+            .fld32      =>                  OpCode.init(&.{0xd9}),
+            .fld64      =>                  OpCode.init(&.{0xdd}),
+            else        => unreachable,
         },
         .o => return switch (tag) {
-            .push => OpCode.oneByte(0x50),
+            .push => OpCode.init(&.{0x50}),
-            .pop => OpCode.oneByte(0x58),
+            .pop  => OpCode.init(&.{0x58}),
-            else => null,
+            else  => unreachable,
         },
         .i => return switch (tag) {
-            .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
+            .push     => if (is_one_byte) OpCode.init(&.{0x6a}) else OpCode.init(&.{0x68}),
-            .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
+            .@"test"  => if (is_one_byte) OpCode.init(&.{0xa8}) else OpCode.init(&.{0xa9}),
-            .ret_near => OpCode.oneByte(0xc2),
+            .ret_near => OpCode.init(&.{0xc2}),
-            .ret_far => OpCode.oneByte(0xca),
+            .ret_far  => OpCode.init(&.{0xca}),
-            else => null,
+            else      => unreachable,
         },
         .m1 => return switch (tag) {
-            .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd0 else 0xd1),
+            .shl, .sal,
-            else => null,
+            .shr, .sar  => if (is_one_byte) OpCode.init(&.{0xd0}) else OpCode.init(&.{0xd1}),
+            else        => unreachable,
         },
         .mc => return switch (tag) {
-            .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd2 else 0xd3),
+            .shl, .sal,
-            else => null,
+            .shr, .sar  => if (is_one_byte) OpCode.init(&.{0xd2}) else OpCode.init(&.{0xd3}),
+            else        => unreachable,
         },
         .mi => return switch (tag) {
-            .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
+            .adc, .add,
-            .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
+            .sub, .xor,
-            .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
+            .@"and", .@"or",
-            else => null,
+            .sbb, .cmp       => if (is_one_byte) OpCode.init(&.{0x80}) else OpCode.init(&.{0x81}),
+            .mov             => if (is_one_byte) OpCode.init(&.{0xc6}) else OpCode.init(&.{0xc7}),
+            .@"test"         => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
+            else             => unreachable,
         },
         .mi8 => return switch (tag) {
-            .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(0x83),
+            .adc, .add,
-            .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xc0 else 0xc1),
+            .sub, .xor,
-            else => null,
+            .@"and", .@"or",
+            .sbb, .cmp        =>                  OpCode.init(&.{0x83}),
+            .shl, .sal,
+            .shr, .sar        => if (is_one_byte) OpCode.init(&.{0xc0}) else OpCode.init(&.{0xc1}),
+            else              => unreachable,
         },
         .mr => return switch (tag) {
-            .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
+            .adc     => if (is_one_byte) OpCode.init(&.{0x10}) else OpCode.init(&.{0x11}),
-            .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
+            .add     => if (is_one_byte) OpCode.init(&.{0x00}) else OpCode.init(&.{0x01}),
-            .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
+            .sub     => if (is_one_byte) OpCode.init(&.{0x28}) else OpCode.init(&.{0x29}),
-            .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
+            .xor     => if (is_one_byte) OpCode.init(&.{0x30}) else OpCode.init(&.{0x31}),
-            .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
+            .@"and"  => if (is_one_byte) OpCode.init(&.{0x20}) else OpCode.init(&.{0x21}),
-            .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
+            .@"or"   => if (is_one_byte) OpCode.init(&.{0x08}) else OpCode.init(&.{0x09}),
-            .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
+            .sbb     => if (is_one_byte) OpCode.init(&.{0x18}) else OpCode.init(&.{0x19}),
-            .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
+            .cmp     => if (is_one_byte) OpCode.init(&.{0x38}) else OpCode.init(&.{0x39}),
-            .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
+            .mov     => if (is_one_byte) OpCode.init(&.{0x88}) else OpCode.init(&.{0x89}),
-            .@"test" => OpCode.oneByte(if (is_one_byte) 0x84 else 0x85),
+            .@"test" => if (is_one_byte) OpCode.init(&.{0x84}) else OpCode.init(&.{0x85}),
-            else => null,
+            .movsd   =>                  OpCode.init(&.{0xf2,0x0f,0x11}),
+            .movss   =>                  OpCode.init(&.{0xf3,0x0f,0x11}),
+            else     => unreachable,
         },
         .rm => return switch (tag) {
-            .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
+            .adc      => if (is_one_byte) OpCode.init(&.{0x12})      else OpCode.init(&.{0x13}),
-            .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
+            .add      => if (is_one_byte) OpCode.init(&.{0x02})      else OpCode.init(&.{0x03}),
-            .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
+            .sub      => if (is_one_byte) OpCode.init(&.{0x2a})      else OpCode.init(&.{0x2b}),
-            .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
+            .xor      => if (is_one_byte) OpCode.init(&.{0x32})      else OpCode.init(&.{0x33}),
-            .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
+            .@"and"   => if (is_one_byte) OpCode.init(&.{0x22})      else OpCode.init(&.{0x23}),
-            .@"or" => OpCode.oneByte(if (is_one_byte) 0x0a else 0x0b),
+            .@"or"    => if (is_one_byte) OpCode.init(&.{0x0a})      else OpCode.init(&.{0x0b}),
-            .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
+            .sbb      => if (is_one_byte) OpCode.init(&.{0x1a})      else OpCode.init(&.{0x1b}),
-            .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
+            .cmp      => if (is_one_byte) OpCode.init(&.{0x3a})      else OpCode.init(&.{0x3b}),
-            .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
+            .mov      => if (is_one_byte) OpCode.init(&.{0x8a})      else OpCode.init(&.{0x8b}),
-            .movsx => OpCode.twoByte(0x0f, if (is_one_byte) 0xbe else 0xbf),
+            .movsx    => if (is_one_byte) OpCode.init(&.{0x0f,0xbe}) else OpCode.init(&.{0x0f,0xbf}),
-            .movsxd => OpCode.oneByte(0x63),
+            .movsxd   =>                  OpCode.init(&.{0x63}),
-            .movzx => OpCode.twoByte(0x0f, if (is_one_byte) 0xb6 else 0xb7),
+            .movzx    => if (is_one_byte) OpCode.init(&.{0x0f,0xb6}) else OpCode.init(&.{0x0f,0xb7}),
-            .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
+            .lea      => if (is_one_byte) OpCode.init(&.{0x8c})      else OpCode.init(&.{0x8d}),
-            .imul => OpCode.twoByte(0x0f, 0xaf),
+            .imul     =>                  OpCode.init(&.{0x0f,0xaf}),
-            .cmove, .cmovz => OpCode.twoByte(0x0f, 0x44),
+            .cmove,
-            .cmovb, .cmovnae => OpCode.twoByte(0x0f, 0x42),
+            .cmovz    =>                  OpCode.init(&.{0x0f,0x44}),
-            .cmovl, .cmovng => OpCode.twoByte(0x0f, 0x4c),
+            .cmovb,
-            else => null,
+            .cmovnae  =>                  OpCode.init(&.{0x0f,0x42}),
+            .cmovl,
+            .cmovng   =>                  OpCode.init(&.{0x0f,0x4c}),
+            .movsd    =>                  OpCode.init(&.{0xf2,0x0f,0x10}),
+            .movss    =>                  OpCode.init(&.{0xf3,0x0f,0x10}),
+            .addsd    =>                  OpCode.init(&.{0xf2,0x0f,0x58}),
+            .addss    =>                  OpCode.init(&.{0xf3,0x0f,0x58}),
+            .ucomisd  =>                  OpCode.init(&.{0x66,0x0f,0x2e}),
+            .ucomiss  =>                  OpCode.init(&.{0x0f,0x2e}),
+            else => unreachable,
         },
         .oi => return switch (tag) {
-            .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
+            .mov => if (is_one_byte) OpCode.init(&.{0xb0}) else OpCode.init(&.{0xb8}),
-            else => null,
+            else => unreachable,
         },
         .fd => return switch (tag) {
-            .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
+            .mov => if (is_one_byte) OpCode.init(&.{0xa0}) else OpCode.init(&.{0xa1}),
-            else => null,
+            else => unreachable,
         },
         .td => return switch (tag) {
-            .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
+            .mov => if (is_one_byte) OpCode.init(&.{0xa2}) else OpCode.init(&.{0xa3}),
-            else => null,
+            else => unreachable,
         },
         .rmi => return switch (tag) {
-            .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
+            .imul => if (is_one_byte) OpCode.init(&.{0x6b}) else OpCode.init(&.{0x69}),
-            else => null,
+            else  => unreachable,
+        },
+        .mv => return switch (tag) {
+            .vmovsd,
+            .vmovss => OpCode.init(&.{0x11}),
+            else => unreachable,
+        },
+        .vm => return switch (tag) {
+            .vmovsd,
+            .vmovss   => OpCode.init(&.{0x10}),
+            .vucomisd,
+            .vucomiss => OpCode.init(&.{0x2e}),
+            else => unreachable,
+        },
+        .rvm => return switch (tag) {
+            .vaddsd,
+            .vaddss  => OpCode.init(&.{0x58}),
+            .vmovsd,
+            .vmovss  => OpCode.init(&.{0x10}),
+            else => unreachable,
+        },
+        .rvmi => return switch (tag) {
+            .vcmpsd,
+            .vcmpss  => OpCode.init(&.{0xc2}),
+            else     => unreachable,
         },
     }
+    // zig fmt: on
 }
-inline fn getModRmExt(tag: Tag) ?u3 {
+inline fn getModRmExt(tag: Tag) u3 {
     return switch (tag) {
         .adc => 0x2,
         .add => 0x0,
 @@ -1483,11 +1759,101 @@ inline fn getModRmExt(tag: Tag) ?u3 {
         .fisttp64 => 0x1,
         .fld32 => 0x0,
         .fld64 => 0x0,
-        else => null,
+        else => unreachable,
     };
 }
-const ScaleIndex = struct {
+const VexEncoding = struct {
+    prefix: Encoder.Vex,
+    reg: ?enum {
+        ndd,
+        nds,
+        dds,
+    },
+};
+inline fn getVexEncoding(tag: Tag, enc: Encoding) VexEncoding {
+    const desc: struct {
+        reg: enum {
+            none,
+            ndd,
+            nds,
+            dds,
+        } = .none,
+        len_256: bool = false,
+        wig: bool = false,
+        lig: bool = false,
+        lz: bool = false,
+        lead_opc: enum {
+            l_0f,
+            l_0f_3a,
+            l_0f_38,
+        } = .l_0f,
+        simd_prefix: enum {
+            none,
+            p_66,
+            p_f2,
+            p_f3,
+        } = .none,
+    } = blk: {
+        switch (enc) {
+            .mv => switch (tag) {
+                .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
+                .vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
+                else => unreachable,
+            },
+            .vm => switch (tag) {
+                .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
+                .vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
+                .vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true },
+                .vucomiss => break :blk .{ .lig = true, .wig = true },
+                else => unreachable,
+            },
+            .rvm => switch (tag) {
+                .vaddsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
+                .vaddss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
+                .vmovsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
+                .vmovss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
+                else => unreachable,
+            },
+            .rvmi => switch (tag) {
+                .vcmpsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
+                .vcmpss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
+                else => unreachable,
+            },
+            else => unreachable,
+        }
+    };
+    var vex: Encoder.Vex = .{};
+    if (desc.len_256) vex.len_256();
+    if (desc.wig) vex.wig();
+    if (desc.lig) vex.lig();
+    if (desc.lz) vex.lz();
+    switch (desc.lead_opc) {
+        .l_0f => {},
+        .l_0f_3a => vex.lead_opc_0f_3a(),
+        .l_0f_38 => vex.lead_opc_0f_38(),
+    }
+    switch (desc.simd_prefix) {
+        .none => {},
+        .p_66 => vex.simd_prefix_66(),
+        .p_f2 => vex.simd_prefix_f2(),
+        .p_f3 => vex.simd_prefix_f3(),
+    }
+    return VexEncoding{ .prefix = vex, .reg = switch (desc.reg) {
+        .none => null,
+        .nds => .nds,
+        .dds => .dds,
+        .ndd => .ndd,
+    } };
+}
+const ScaleIndex = packed struct {
     scale: u2,
     index: Register,
 };
 @@ -1499,49 +1865,38 @@ const Memory = struct {
     ptr_size: PtrSize,
     scale_index: ?ScaleIndex = null,
-    const PtrSize = enum {
+    const PtrSize = enum(u2) {
-        byte_ptr,
+        byte_ptr = 0b00,
-        word_ptr,
+        word_ptr = 0b01,
-        dword_ptr,
+        dword_ptr = 0b10,
-        qword_ptr,
+        qword_ptr = 0b11,
-        fn fromBits(in_bits: u64) PtrSize {
+        fn new(bit_size: u64) PtrSize {
-            return switch (in_bits) {
+            return @intToEnum(PtrSize, math.log2_int(u4, @intCast(u4, @divExact(bit_size, 8))));
-                8 => .byte_ptr,
-                16 => .word_ptr,
-                32 => .dword_ptr,
-                64 => .qword_ptr,
-                else => unreachable,
-            };
         }
         /// Returns size in bits.
         fn size(ptr_size: PtrSize) u64 {
-            return switch (ptr_size) {
+            return 8 * (math.powi(u8, 2, @enumToInt(ptr_size)) catch unreachable);
-                .byte_ptr => 8,
-                .word_ptr => 16,
-                .dword_ptr => 32,
-                .qword_ptr => 64,
-            };
         }
     };
     fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
         if (mem_op.base) |base| {
-            const dst = base.lowId();
+            const dst = base.lowEnc();
             const src = operand;
             if (dst == 4 or mem_op.scale_index != null) {
                 if (mem_op.disp == 0 and dst != 5) {
                     encoder.modRm_SIBDisp0(src);
                     if (mem_op.scale_index) |si| {
-                        encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
+                        encoder.sib_scaleIndexBase(si.scale, si.index.lowEnc(), dst);
                     } else {
                         encoder.sib_base(dst);
                     }
                 } else if (immOpSize(mem_op.disp) == 8) {
                     encoder.modRm_SIBDisp8(src);
                     if (mem_op.scale_index) |si| {
-                        encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
+                        encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowEnc(), dst);
                     } else {
                         encoder.sib_baseDisp8(dst);
                     }
 @@ -1549,7 +1904,7 @@ const Memory = struct {
                 } else {
                     encoder.modRm_SIBDisp32(src);
                     if (mem_op.scale_index) |si| {
-                        encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
+                        encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowEnc(), dst);
                     } else {
                         encoder.sib_baseDisp32(dst);
                     }
 @@ -1572,7 +1927,7 @@ const Memory = struct {
             } else {
                 encoder.modRm_SIBDisp0(operand);
                 if (mem_op.scale_index) |si| {
-                    encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
+                    encoder.sib_scaleIndexDisp32(si.scale, si.index.lowEnc());
                 } else {
                     encoder.sib_disp32();
                 }
 @@ -1581,6 +1936,7 @@ const Memory = struct {
         }
     }
+    /// Returns size in bits.
     fn size(memory: Memory) u64 {
         return memory.ptr_size.size();
     }
 @@ -1629,6 +1985,7 @@ const RegisterOrMemory = union(enum) {
         };
     }
+    /// Returns size in bits.
     fn size(reg_or_mem: RegisterOrMemory) u64 {
         return switch (reg_or_mem) {
             .register => |reg| reg.size(),
 @@ -1638,7 +1995,8 @@ const RegisterOrMemory = union(enum) {
 };
 fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
-    const opc = getOpCode(tag, .zo, false).?;
+    assert(!tag.isAvx());
+    const opc = getOpCode(tag, .zo, false);
     const encoder = try Encoder.init(code, 2);
     switch (tag) {
         .cqo => {
 @@ -1652,14 +2010,15 @@ fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
 }
 fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
+    assert(!tag.isAvx());
     if (tag == .ret_far or tag == .ret_near) {
         const encoder = try Encoder.init(code, 3);
-        const opc = getOpCode(tag, .i, false).?;
+        const opc = getOpCode(tag, .i, false);
         opc.encode(encoder);
         encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
         return;
     }
-    const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
+    const opc = getOpCode(tag, .i, immOpSize(imm) == 8);
     const encoder = try Encoder.init(code, 5);
     if (immOpSize(imm) == 16) {
         encoder.prefix16BitMode();
 @@ -1669,7 +2028,8 @@ fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
 }
 fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
-    const opc = getOpCode(tag, .o, false).?;
+    assert(!tag.isAvx());
+    const opc = getOpCode(tag, .o, false);
     const encoder = try Encoder.init(code, 3);
     if (reg.size() == 16) {
         encoder.prefix16BitMode();
 @@ -1682,15 +2042,17 @@ fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!voi
 }
 fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
-    const opc = getOpCode(tag, .d, false).?;
+    assert(!tag.isAvx());
+    const opc = getOpCode(tag, .d, false);
     const encoder = try Encoder.init(code, 6);
     opc.encode(encoder);
     encoder.imm32(@bitCast(i32, imm));
 }
 fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void {
-    const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?;
+    assert(!tag.isAvx());
-    const modrm_ext = getModRmExt(tag).?;
+    const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
+    const modrm_ext = getModRmExt(tag);
     switch (reg_or_mem) {
         .register => |reg| {
             const encoder = try Encoder.init(code, 4);
 @@ -1703,7 +2065,7 @@ fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *st
                 .b = reg.isExtended(),
             });
             opc.encode(encoder);
-            encoder.modRm_direct(modrm_ext, reg.lowId());
+            encoder.modRm_direct(modrm_ext, reg.lowEnc());
         },
         .memory => |mem_op| {
             const encoder = try Encoder.init(code, 8);
 @@ -1744,10 +2106,8 @@ fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) I
 }
 fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void {
-    const opc = if (td)
+    assert(!tag.isAvx());
-        getOpCode(tag, .td, reg.size() == 8).?
+    const opc = if (td) getOpCode(tag, .td, reg.size() == 8) else getOpCode(tag, .fd, reg.size() == 8);
-    else
-        getOpCode(tag, .fd, reg.size() == 8).?;
     const encoder = try Encoder.init(code, 10);
     if (reg.size() == 16) {
         encoder.prefix16BitMode();
 @@ -1766,7 +2126,8 @@ fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8),
 }
 fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void {
-    const opc = getOpCode(tag, .oi, reg.size() == 8).?;
+    assert(!tag.isAvx());
+    const opc = getOpCode(tag, .oi, reg.size() == 8);
     const encoder = try Encoder.init(code, 10);
     if (reg.size() == 16) {
         encoder.prefix16BitMode();
 @@ -1792,8 +2153,9 @@ fn lowerToMiXEnc(
     enc: Encoding,
     code: *std.ArrayList(u8),
 ) InnerError!void {
-    const modrm_ext = getModRmExt(tag).?;
+    assert(!tag.isAvx());
-    const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?;
+    const modrm_ext = getModRmExt(tag);
+    const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
     switch (reg_or_mem) {
         .register => |dst_reg| {
             const encoder = try Encoder.init(code, 7);
 @@ -1808,7 +2170,7 @@ fn lowerToMiXEnc(
                 .b = dst_reg.isExtended(),
             });
             opc.encode(encoder);
-            encoder.modRm_direct(modrm_ext, dst_reg.lowId());
+            encoder.modRm_direct(modrm_ext, dst_reg.lowEnc());
             encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_reg.size());
         },
         .memory => |dst_mem| {
 @@ -1847,10 +2209,11 @@ fn lowerToRmEnc(
     reg_or_mem: RegisterOrMemory,
     code: *std.ArrayList(u8),
 ) InnerError!void {
-    const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8).?;
+    assert(!tag.isAvx());
+    const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8);
     switch (reg_or_mem) {
         .register => |src_reg| {
-            const encoder = try Encoder.init(code, 4);
+            const encoder = try Encoder.init(code, 5);
             if (reg.size() == 16) {
                 encoder.prefix16BitMode();
             }
 @@ -1860,7 +2223,7 @@ fn lowerToRmEnc(
                 .b = src_reg.isExtended(),
             });
             opc.encode(encoder);
-            encoder.modRm_direct(reg.lowId(), src_reg.lowId());
+            encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
         },
         .memory => |src_mem| {
             const encoder = try Encoder.init(code, 9);
 @@ -1882,7 +2245,7 @@ fn lowerToRmEnc(
                 });
             }
             opc.encode(encoder);
-            src_mem.encode(encoder, reg.lowId());
+            src_mem.encode(encoder, reg.lowEnc());
         },
     }
 }
 @@ -1893,7 +2256,8 @@ fn lowerToMrEnc(
     reg: Register,
     code: *std.ArrayList(u8),
 ) InnerError!void {
-    const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8).?;
+    assert(!tag.isAvx());
+    const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8);
     switch (reg_or_mem) {
         .register => |dst_reg| {
             const encoder = try Encoder.init(code, 4);
 @@ -1906,7 +2270,7 @@ fn lowerToMrEnc(
                 .b = dst_reg.isExtended(),
             });
             opc.encode(encoder);
-            encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
+            encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
         },
         .memory => |dst_mem| {
             const encoder = try Encoder.init(code, 9);
 @@ -1926,7 +2290,7 @@ fn lowerToMrEnc(
                 });
             }
             opc.encode(encoder);
-            dst_mem.encode(encoder, reg.lowId());
+            dst_mem.encode(encoder, reg.lowEnc());
         },
     }
 }
 @@ -1938,7 +2302,8 @@ fn lowerToRmiEnc(
     imm: u32,
     code: *std.ArrayList(u8),
 ) InnerError!void {
-    const opc = getOpCode(tag, .rmi, false).?;
+    assert(!tag.isAvx());
+    const opc = getOpCode(tag, .rmi, false);
     const encoder = try Encoder.init(code, 13);
     if (reg.size() == 16) {
         encoder.prefix16BitMode();
 @@ -1951,7 +2316,7 @@ fn lowerToRmiEnc(
                 .b = src_reg.isExtended(),
             });
             opc.encode(encoder);
-            encoder.modRm_direct(reg.lowId(), src_reg.lowId());
+            encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
         },
         .memory => |src_mem| {
             if (src_mem.base) |base| {
 @@ -1969,12 +2334,165 @@ fn lowerToRmiEnc(
                 });
             }
             opc.encode(encoder);
-            src_mem.encode(encoder, reg.lowId());
+            src_mem.encode(encoder, reg.lowEnc());
         },
     }
     encodeImm(encoder, imm, reg.size());
 }
+/// Also referred to as XM encoding in Intel manual.
+fn lowerToVmEnc(
+    tag: Tag,
+    reg: Register,
+    reg_or_mem: RegisterOrMemory,
+    code: *std.ArrayList(u8),
+) InnerError!void {
+    const opc = getOpCode(tag, .vm, false);
+    var enc = getVexEncoding(tag, .vm);
+    const vex = &enc.prefix;
+    switch (reg_or_mem) {
+        .register => |src_reg| {
+            const encoder = try Encoder.init(code, 5);
+            vex.rex(.{
+                .r = reg.isExtended(),
+                .b = src_reg.isExtended(),
+            });
+            encoder.vex(enc.prefix);
+            opc.encode(encoder);
+            encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
+        },
+        .memory => |src_mem| {
+            const encoder = try Encoder.init(code, 10);
+            if (src_mem.base) |base| {
+                vex.rex(.{
+                    .r = reg.isExtended(),
+                    .b = base.isExtended(),
+                });
+            } else {
+                vex.rex(.{
+                    .r = reg.isExtended(),
+                });
+            }
+            encoder.vex(enc.prefix);
+            opc.encode(encoder);
+            src_mem.encode(encoder, reg.lowEnc());
+        },
+    }
+}
+/// Usually referred to as MR encoding with V/V in Intel manual.
+fn lowerToMvEnc(
+    tag: Tag,
+    reg_or_mem: RegisterOrMemory,
+    reg: Register,
+    code: *std.ArrayList(u8),
+) InnerError!void {
+    const opc = getOpCode(tag, .mv, false);
+    var enc = getVexEncoding(tag, .mv);
+    const vex = &enc.prefix;
+    switch (reg_or_mem) {
+        .register => |dst_reg| {
+            const encoder = try Encoder.init(code, 4);
+            vex.rex(.{
+                .r = reg.isExtended(),
+                .b = dst_reg.isExtended(),
+            });
+            encoder.vex(enc.prefix);
+            opc.encode(encoder);
+            encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
+        },
+        .memory => |dst_mem| {
+            const encoder = try Encoder.init(code, 10);
+            if (dst_mem.base) |base| {
+                vex.rex(.{
+                    .r = reg.isExtended(),
+                    .b = base.isExtended(),
+                });
+            } else {
+                vex.rex(.{
+                    .r = reg.isExtended(),
+                });
+            }
+            encoder.vex(enc.prefix);
+            opc.encode(encoder);
+            dst_mem.encode(encoder, reg.lowEnc());
+        },
+    }
+}
+fn lowerToRvmEnc(
+    tag: Tag,
+    reg1: Register,
+    reg2: Register,
+    reg_or_mem: RegisterOrMemory,
+    code: *std.ArrayList(u8),
+) InnerError!void {
+    const opc = getOpCode(tag, .rvm, false);
+    var enc = getVexEncoding(tag, .rvm);
+    const vex = &enc.prefix;
+    switch (reg_or_mem) {
+        .register => |reg3| {
+            if (enc.reg) |vvvv| {
+                switch (vvvv) {
+                    .nds => vex.reg(reg2.enc()),
+                    else => unreachable, // TODO
+                }
+            }
+            const encoder = try Encoder.init(code, 5);
+            vex.rex(.{
+                .r = reg1.isExtended(),
+                .b = reg3.isExtended(),
+            });
+            encoder.vex(enc.prefix);
+            opc.encode(encoder);
+            encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
+        },
+        .memory => |dst_mem| {
+            _ = dst_mem;
+            unreachable; // TODO
+        },
+    }
+}
+fn lowerToRvmiEnc(
+    tag: Tag,
+    reg1: Register,
+    reg2: Register,
+    reg_or_mem: RegisterOrMemory,
+    imm: u32,
+    code: *std.ArrayList(u8),
+) InnerError!void {
+    const opc = getOpCode(tag, .rvmi, false);
+    var enc = getVexEncoding(tag, .rvmi);
+    const vex = &enc.prefix;
+    const encoder: Encoder = blk: {
+        switch (reg_or_mem) {
+            .register => |reg3| {
+                if (enc.reg) |vvvv| {
+                    switch (vvvv) {
+                        .nds => vex.reg(reg2.enc()),
+                        else => unreachable, // TODO
+                    }
+                }
+                const encoder = try Encoder.init(code, 5);
+                vex.rex(.{
+                    .r = reg1.isExtended(),
+                    .b = reg3.isExtended(),
+                });
+                encoder.vex(enc.prefix);
+                opc.encode(encoder);
+                encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
+                break :blk encoder;
+            },
+            .memory => |dst_mem| {
+                _ = dst_mem;
+                unreachable; // TODO
+            },
+        }
+    };
+    encodeImm(encoder, imm, 8); // TODO
+}
 fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
     assert(expected.len > 0);
     if (mem.eql(u8, expected, given)) return;
 @@ -2369,3 +2887,34 @@ test "lower RMI encoding" {
     try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
     try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
 }
+test "lower MV encoding" {
+    var emit = TestEmit.init();
+    defer emit.deinit();
+    try lowerToMvEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), .xmm1, emit.code());
+    try expectEqualHexStrings(
+        "\xC5\xFB\x11\x0D\x10\x00\x00\x00",
+        emit.lowered(),
+        "vmovsd qword ptr [rip + 0x10], xmm1",
+    );
+}
+test "lower VM encoding" {
+    var emit = TestEmit.init();
+    defer emit.deinit();
+    try lowerToVmEnc(.vmovsd, .xmm1, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
+    try expectEqualHexStrings(
+        "\xC5\xFB\x10\x0D\x10\x00\x00\x00",
+        emit.lowered(),
+        "vmovsd xmm1, qword ptr [rip + 0x10]",
+    );
+}
+test "lower to RVM encoding" {
+    var emit = TestEmit.init();
+    defer emit.deinit();
+    try lowerToRvmEnc(.vaddsd, .xmm0, .xmm1, RegisterOrMemory.reg(.xmm2), emit.code());
+    try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2");
+    try lowerToRvmEnc(.vaddsd, .xmm0, .xmm0, RegisterOrMemory.reg(.xmm1), emit.code());
+    try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1");
+}

src/arch/x86_64/Mir.zig+67-30

...	@@ -22,11 +22,7 @@ extra: []const u32,	...	@@ -22,11 +22,7 @@ extra: []const u32,
22		22
23	pub const Inst = struct {	23	pub const Inst = struct {
24	tag: Tag,	24	tag: Tag,
25	/// This is 3 fields, and the meaning of each depends on `tag`.	25	ops: Ops,
26	/// reg1: Register
27	/// reg2: Register
28	/// flags: u2
29	ops: u16,
30	/// The meaning of this depends on `tag` and `ops`.	26	/// The meaning of this depends on `tag` and `ops`.
31	data: Data,	27	data: Data,
32		28
...	@@ -349,6 +345,42 @@ pub const Inst = struct {	...	@@ -349,6 +345,42 @@ pub const Inst = struct {
349	/// Nop	345	/// Nop
350	nop,	346	nop,
351		347
		348	/// SSE instructions
		349	/// ops flags: form:
		350	/// 0b00 reg1, qword ptr [reg2 + imm32]
		351	/// 0b01 qword ptr [reg1 + imm32], reg2
		352	/// 0b10 reg1, reg2
		353	mov_f64_sse,
		354	mov_f32_sse,
		355
		356	/// ops flags: form:
		357	/// 0b00 reg1, reg2
		358	add_f64_sse,
		359	add_f32_sse,
		360
		361	/// ops flags: form:
		362	/// 0b00 reg1, reg2
		363	cmp_f64_sse,
		364	cmp_f32_sse,
		365
		366	/// AVX instructions
		367	/// ops flags: form:
		368	/// 0b00 reg1, qword ptr [reg2 + imm32]
		369	/// 0b01 qword ptr [reg1 + imm32], reg2
		370	/// 0b10 reg1, reg1, reg2
		371	mov_f64_avx,
		372	mov_f32_avx,
		373
		374	/// ops flags: form:
		375	/// 0b00 reg1, reg1, reg2
		376	add_f64_avx,
		377	add_f32_avx,
		378
		379	/// ops flags: form:
		380	/// 0b00 reg1, reg1, reg2
		381	cmp_f64_avx,
		382	cmp_f32_avx,
		383
352	/// Pseudo-instructions	384	/// Pseudo-instructions
353	/// call extern function	385	/// call extern function
354	/// Notes:	386	/// Notes:
...	@@ -381,6 +413,36 @@ pub const Inst = struct {	...	@@ -381,6 +413,36 @@ pub const Inst = struct {
381	/// The position of an MIR instruction within the `Mir` instructions array.	413	/// The position of an MIR instruction within the `Mir` instructions array.
382	pub const Index = u32;	414	pub const Index = u32;
383		415
		416	pub const Ops = packed struct {
		417	reg1: u7,
		418	reg2: u7,
		419	flags: u2,
		420
		421	pub fn encode(vals: struct {
		422	reg1: Register = .none,
		423	reg2: Register = .none,
		424	flags: u2 = 0b00,
		425	}) Ops {
		426	return .{
		427	.reg1 = @enumToInt(vals.reg1),
		428	.reg2 = @enumToInt(vals.reg2),
		429	.flags = vals.flags,
		430	};
		431	}
		432
		433	pub fn decode(ops: Ops) struct {
		434	reg1: Register,
		435	reg2: Register,
		436	flags: u2,
		437	} {
		438	return .{
		439	.reg1 = @intToEnum(Register, ops.reg1),
		440	.reg2 = @intToEnum(Register, ops.reg2),
		441	.flags = ops.flags,
		442	};
		443	}
		444	};
		445
384	/// All instructions have a 4-byte payload, which is contained within	446	/// All instructions have a 4-byte payload, which is contained within
385	/// this union. `Tag` determines which union field is active, as well as	447	/// this union. `Tag` determines which union field is active, as well as
386	/// how to interpret the data within.	448	/// how to interpret the data within.
...	@@ -450,31 +512,6 @@ pub const DbgLineColumn = struct {	...	@@ -450,31 +512,6 @@ pub const DbgLineColumn = struct {
450	column: u32,	512	column: u32,
451	};	513	};
452		514
453	pub const Ops = struct {
454	reg1: Register = .none,
455	reg2: Register = .none,
456	flags: u2 = 0b00,
457
458	pub fn encode(self: Ops) u16 {
459	var ops: u16 = 0;
460	ops \|= @intCast(u16, @enumToInt(self.reg1)) << 9;
461	ops \|= @intCast(u16, @enumToInt(self.reg2)) << 2;
462	ops \|= self.flags;
463	return ops;
464	}
465
466	pub fn decode(ops: u16) Ops {
467	const reg1 = @intToEnum(Register, @truncate(u7, ops >> 9));
468	const reg2 = @intToEnum(Register, @truncate(u7, ops >> 2));
469	const flags = @truncate(u2, ops);
470	return .{
471	.reg1 = reg1,
472	.reg2 = reg2,
473	.flags = flags,
474	};
475	}
476	};
477
478	pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {	515	pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
479	mir.instructions.deinit(gpa);	516	mir.instructions.deinit(gpa);
480	gpa.free(mir.extra);	517	gpa.free(mir.extra);

src/arch/x86_64/abi.zig+36-1

...	@@ -3,6 +3,7 @@ const Type = @import("../../type.zig").Type;	...	@@ -3,6 +3,7 @@ const Type = @import("../../type.zig").Type;
3	const Target = std.Target;	3	const Target = std.Target;
4	const assert = std.debug.assert;	4	const assert = std.debug.assert;
5	const Register = @import("bits.zig").Register;	5	const Register = @import("bits.zig").Register;
		6	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
6		7
7	pub const Class = enum { integer, sse, sseup, x87, x87up, complex_x87, memory, none };	8	pub const Class = enum { integer, sse, sseup, x87, x87up, complex_x87, memory, none };
8		9
...	@@ -378,6 +379,40 @@ pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 };	...	@@ -378,6 +379,40 @@ pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 };
378	/// the caller relinquishes control to a subroutine via call instruction (or similar).	379	/// the caller relinquishes control to a subroutine via call instruction (or similar).
379	/// In other words, these registers are free to use by the callee.	380	/// In other words, these registers are free to use by the callee.
380	pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 };	381	pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 };
381	pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;	382
382	pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };	383	pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
383	pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };	384	pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
		385
		386	const sse_avx_regs = [_]Register{
		387	.ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7,
		388	.ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15,
		389	};
		390	const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ sse_avx_regs;
		391	pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
		392
		393	// Register classes
		394	const RegisterBitSet = RegisterManager.RegisterBitSet;
		395	pub const RegisterClass = struct {
		396	pub const gp: RegisterBitSet = @as(RegisterBitSet, std.math.maxInt(std.meta.Int(
		397	.unsigned,
		398	caller_preserved_regs.len + callee_preserved_regs.len,
		399	)));
		400	pub const sse: RegisterBitSet = std.math.maxInt(RegisterBitSet) - gp;
		401	// TODO uncomment once #11680 is fixed.
		402	// pub const gp: RegisterBitSet = blk: {
		403	// var set = RegisterBitSet.initEmpty();
		404	// set.setRangeValue(.{
		405	// .start = 0,
		406	// .end = caller_preserved_regs.len + callee_preserved_regs.len,
		407	// }, true);
		408	// break :blk set;
		409	// };
		410	// pub const sse: RegisterBitSet = blk: {
		411	// var set = RegisterBitSet.initEmpty();
		412	// set.setRangeValue(.{
		413	// .start = caller_preserved_regs.len + callee_preserved_regs.len,
		414	// .end = allocatable_registers.len,
		415	// }, true);
		416	// break :blk set;
		417	// };
		418	};

src/arch/x86_64/bits.zig+245-18

...	@@ -8,7 +8,7 @@ const DW = std.dwarf;	...	@@ -8,7 +8,7 @@ const DW = std.dwarf;
8		8
9	// zig fmt: off	9	// zig fmt: off
10		10
11	/// Definitions of all of the x64 registers. The order is semantically meaningful.	11	/// Definitions of all of the general purpose x64 registers. The order is semantically meaningful.
12	/// The registers are defined such that IDs go in descending order of 64-bit,	12	/// The registers are defined such that IDs go in descending order of 64-bit,
13	/// 32-bit, 16-bit, and then 8-bit, and each set contains exactly sixteen	13	/// 32-bit, 16-bit, and then 8-bit, and each set contains exactly sixteen
14	/// registers. This results in some useful properties:	14	/// registers. This results in some useful properties:
...	@@ -43,17 +43,36 @@ pub const Register = enum(u7) {	...	@@ -43,17 +43,36 @@ pub const Register = enum(u7) {
43	al, cl, dl, bl, ah, ch, dh, bh,	43	al, cl, dl, bl, ah, ch, dh, bh,
44	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,	44	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,
45		45
46	// Pseudo, used only for MIR to signify that the	46	// 64-79, 256-bit registers.
47	// operand is not a register but an immediate, etc.	47	// id is int value - 64.
		48	ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,
		49	ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
		50
		51	// 80-95, 128-bit registers.
		52	// id is int value - 80.
		53	xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
		54	xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
		55
		56	// Pseudo-value for MIR instructions.
48	none,	57	none,
49		58
		59	pub fn id(self: Register) u7 {
		60	return switch (@enumToInt(self)) {
		61	0...63 => @as(u7, @truncate(u4, @enumToInt(self))),
		62	64...79 => @enumToInt(self),
		63	else => unreachable,
		64	};
		65	}
		66
50	/// Returns the bit-width of the register.	67	/// Returns the bit-width of the register.
51	pub fn size(self: Register) u7 {	68	pub fn size(self: Register) u9 {
52	return switch (@enumToInt(self)) {	69	return switch (@enumToInt(self)) {
53	0...15 => 64,	70	0...15 => 64,
54	16...31 => 32,	71	16...31 => 32,
55	32...47 => 16,	72	32...47 => 16,
56	48...64 => 8,	73	48...63 => 8,
		74	64...79 => 256,
		75	80...95 => 128,
57	else => unreachable,	76	else => unreachable,
58	};	77	};
59	}	78	}
...	@@ -72,33 +91,41 @@ pub const Register = enum(u7) {	...	@@ -72,33 +91,41 @@ pub const Register = enum(u7) {
72	/// an instruction (@see isExtended), and requires special handling. The	91	/// an instruction (@see isExtended), and requires special handling. The
73	/// lower three bits are often embedded directly in instructions (such as	92	/// lower three bits are often embedded directly in instructions (such as
74	/// the B8 variant of moves), or used in R/M bytes.	93	/// the B8 variant of moves), or used in R/M bytes.
75	pub fn id(self: Register) u4 {	94	pub fn enc(self: Register) u4 {
76	return @truncate(u4, @enumToInt(self));	95	return @truncate(u4, @enumToInt(self));
77	}	96	}
78		97
79	/// Like id, but only returns the lower 3 bits.	98	/// Like enc, but only returns the lower 3 bits.
80	pub fn lowId(self: Register) u3 {	99	pub fn lowEnc(self: Register) u3 {
81	return @truncate(u3, @enumToInt(self));	100	return @truncate(u3, @enumToInt(self));
82	}	101	}
83		102
		103	pub fn to256(self: Register) Register {
		104	return @intToEnum(Register, @as(u8, self.enc()) + 64);
		105	}
		106
		107	pub fn to128(self: Register) Register {
		108	return @intToEnum(Register, @as(u8, self.enc()) + 80);
		109	}
		110
84	/// Convert from any register to its 64 bit alias.	111	/// Convert from any register to its 64 bit alias.
85	pub fn to64(self: Register) Register {	112	pub fn to64(self: Register) Register {
86	return @intToEnum(Register, self.id());	113	return @intToEnum(Register, self.enc());
87	}	114	}
88		115
89	/// Convert from any register to its 32 bit alias.	116	/// Convert from any register to its 32 bit alias.
90	pub fn to32(self: Register) Register {	117	pub fn to32(self: Register) Register {
91	return @intToEnum(Register, @as(u8, self.id()) + 16);	118	return @intToEnum(Register, @as(u8, self.enc()) + 16);
92	}	119	}
93		120
94	/// Convert from any register to its 16 bit alias.	121	/// Convert from any register to its 16 bit alias.
95	pub fn to16(self: Register) Register {	122	pub fn to16(self: Register) Register {
96	return @intToEnum(Register, @as(u8, self.id()) + 32);	123	return @intToEnum(Register, @as(u8, self.enc()) + 32);
97	}	124	}
98		125
99	/// Convert from any register to its 8 bit alias.	126	/// Convert from any register to its 8 bit alias.
100	pub fn to8(self: Register) Register {	127	pub fn to8(self: Register) Register {
101	return @intToEnum(Register, @as(u8, self.id()) + 48);	128	return @intToEnum(Register, @as(u8, self.enc()) + 48);
102	}	129	}
103		130
104	pub fn dwarfLocOp(self: Register) u8 {	131	pub fn dwarfLocOp(self: Register) u8 {
...	@@ -251,6 +278,115 @@ pub const Encoder = struct {	...	@@ -251,6 +278,115 @@ pub const Encoder = struct {
251	self.code.appendAssumeCapacity(0x66);	278	self.code.appendAssumeCapacity(0x66);
252	}	279	}
253		280
		281	pub const Vex = struct {
		282	rex_prefix: Rex = .{},
		283	lead_opc: u5 = 0b0_0001,
		284	register: u4 = 0b1111,
		285	length: u1 = 0b0,
		286	simd_prefix: u2 = 0b00,
		287	wig_desc: bool = false,
		288	lig_desc: bool = false,
		289	lz_desc: bool = false,
		290
		291	pub fn rex(self: *Vex, r: Rex) void {
		292	self.rex_prefix = r;
		293	}
		294
		295	pub fn lead_opc_0f(self: *Vex) void {
		296	self.lead_opc = 0b0_0001;
		297	}
		298
		299	pub fn lead_opc_0f_38(self: *Vex) void {
		300	self.lead_opc = 0b0_0010;
		301	}
		302
		303	pub fn lead_opc_0f_3a(self: *Vex) void {
		304	self.lead_opc = 0b0_0011;
		305	}
		306
		307	pub fn reg(self: *Vex, register: u4) void {
		308	self.register = ~register;
		309	}
		310
		311	pub fn len_128(self: *Vex) void {
		312	self.length = 0;
		313	}
		314
		315	pub fn len_256(self: *Vex) void {
		316	assert(!self.lz_desc);
		317	self.length = 1;
		318	}
		319
		320	pub fn simd_prefix_66(self: *Vex) void {
		321	self.simd_prefix = 0b01;
		322	}
		323
		324	pub fn simd_prefix_f3(self: *Vex) void {
		325	self.simd_prefix = 0b10;
		326	}
		327
		328	pub fn simd_prefix_f2(self: *Vex) void {
		329	self.simd_prefix = 0b11;
		330	}
		331
		332	pub fn wig(self: *Vex) void {
		333	self.wig_desc = true;
		334	}
		335
		336	pub fn lig(self: *Vex) void {
		337	self.lig_desc = true;
		338	}
		339
		340	pub fn lz(self: *Vex) void {
		341	self.lz_desc = true;
		342	}
		343
		344	pub fn write(self: Vex, writer: anytype) usize {
		345	var buf: [3]u8 = .{0} ** 3;
		346	const form_3byte: bool = blk: {
		347	if (self.rex_prefix.w and !self.wig_desc) break :blk true;
		348	if (self.rex_prefix.x or self.rex_prefix.b) break :blk true;
		349	break :blk self.lead_opc != 0b0_0001;
		350	};
		351
		352	if (self.lz_desc) {
		353	assert(self.length == 0);
		354	}
		355
		356	if (form_3byte) {
		357	// First byte
		358	buf[0] = 0xc4;
		359	// Second byte
		360	const rxb_mask: u3 = @intCast(u3, @boolToInt(!self.rex_prefix.r)) << 2 \|
		361	@intCast(u2, @boolToInt(!self.rex_prefix.x)) << 1 \|
		362	@boolToInt(!self.rex_prefix.b);
		363	buf[1] \|= @intCast(u8, rxb_mask) << 5;
		364	buf[1] \|= self.lead_opc;
		365	// Third byte
		366	buf[2] \|= @intCast(u8, @boolToInt(!self.rex_prefix.w)) << 7;
		367	buf[2] \|= @intCast(u7, self.register) << 3;
		368	buf[2] \|= @intCast(u3, self.length) << 2;
		369	buf[2] \|= self.simd_prefix;
		370	} else {
		371	// First byte
		372	buf[0] = 0xc5;
		373	// Second byte
		374	buf[1] \|= @intCast(u8, @boolToInt(!self.rex_prefix.r)) << 7;
		375	buf[1] \|= @intCast(u7, self.register) << 3;
		376	buf[1] \|= @intCast(u3, self.length) << 2;
		377	buf[1] \|= self.simd_prefix;
		378	}
		379
		380	const count: usize = if (form_3byte) 3 else 2;
		381	_ = writer.writeAll(buf[0..count]) catch unreachable;
		382	return count;
		383	}
		384	};
		385
		386	pub fn vex(self: Self, prefix: Vex) void {
		387	_ = prefix.write(self.code.writer());
		388	}
		389
254	/// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB	390	/// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB
255	pub const Rex = struct {	391	pub const Rex = struct {
256	/// Wide, enables 64-bit operation	392	/// Wide, enables 64-bit operation
...	@@ -305,6 +441,17 @@ pub const Encoder = struct {	...	@@ -305,6 +441,17 @@ pub const Encoder = struct {
305	self.code.appendAssumeCapacity(opcode);	441	self.code.appendAssumeCapacity(opcode);
306	}	442	}
307		443
		444	/// Encodes a 3 byte opcode
		445	///
		446	/// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
		447	///
		448	/// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
		449	pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) void {
		450	self.code.appendAssumeCapacity(prefix_1);
		451	self.code.appendAssumeCapacity(prefix_2);
		452	self.code.appendAssumeCapacity(opcode);
		453	}
		454
308	/// Encodes a 1 byte opcode with a reg field	455	/// Encodes a 1 byte opcode with a reg field
309	///	456	///
310	/// Remember to add a REX prefix byte if reg is extended!	457	/// Remember to add a REX prefix byte if reg is extended!
...	@@ -543,7 +690,7 @@ pub const Encoder = struct {	...	@@ -543,7 +690,7 @@ pub const Encoder = struct {
543	}	690	}
544	};	691	};
545		692
546	test "x86_64 Encoder helpers" {	693	test "Encoder helpers - general purpose registers" {
547	var code = ArrayList(u8).init(testing.allocator);	694	var code = ArrayList(u8).init(testing.allocator);
548	defer code.deinit();	695	defer code.deinit();
549		696
...	@@ -560,8 +707,8 @@ test "x86_64 Encoder helpers" {	...	@@ -560,8 +707,8 @@ test "x86_64 Encoder helpers" {
560	});	707	});
561	encoder.opcode_2byte(0x0f, 0xaf);	708	encoder.opcode_2byte(0x0f, 0xaf);
562	encoder.modRm_direct(	709	encoder.modRm_direct(
563	Register.eax.lowId(),	710	Register.eax.lowEnc(),
564	Register.edi.lowId(),	711	Register.edi.lowEnc(),
565	);	712	);
566		713
567	try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);	714	try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);
...	@@ -580,8 +727,8 @@ test "x86_64 Encoder helpers" {	...	@@ -580,8 +727,8 @@ test "x86_64 Encoder helpers" {
580	});	727	});
581	encoder.opcode_1byte(0x89);	728	encoder.opcode_1byte(0x89);
582	encoder.modRm_direct(	729	encoder.modRm_direct(
583	Register.edi.lowId(),	730	Register.edi.lowEnc(),
584	Register.eax.lowId(),	731	Register.eax.lowEnc(),
585	);	732	);
586		733
587	try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);	734	try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);
...	@@ -607,7 +754,7 @@ test "x86_64 Encoder helpers" {	...	@@ -607,7 +754,7 @@ test "x86_64 Encoder helpers" {
607	encoder.opcode_1byte(0x81);	754	encoder.opcode_1byte(0x81);
608	encoder.modRm_direct(	755	encoder.modRm_direct(
609	0,	756	0,
610	Register.rcx.lowId(),	757	Register.rcx.lowEnc(),
611	);	758	);
612	encoder.imm32(2147483647);	759	encoder.imm32(2147483647);
613		760
...	@@ -615,6 +762,86 @@ test "x86_64 Encoder helpers" {	...	@@ -615,6 +762,86 @@ test "x86_64 Encoder helpers" {
615	}	762	}
616	}	763	}
617		764
		765	test "Encoder helpers - Vex prefix" {
		766	var buf: [3]u8 = undefined;
		767	var stream = std.io.fixedBufferStream(&buf);
		768	const writer = stream.writer();
		769
		770	{
		771	var vex_prefix = Encoder.Vex{};
		772	vex_prefix.rex(.{
		773	.r = true,
		774	});
		775	const nwritten = vex_prefix.write(writer);
		776	try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, buf[0..nwritten]);
		777	}
		778
		779	{
		780	stream.reset();
		781	var vex_prefix = Encoder.Vex{};
		782	vex_prefix.reg(Register.xmm15.enc());
		783	const nwritten = vex_prefix.write(writer);
		784	try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]);
		785	}
		786
		787	{
		788	stream.reset();
		789	var vex_prefix = Encoder.Vex{};
		790	vex_prefix.rex(.{
		791	.w = true,
		792	.x = true,
		793	});
		794	const nwritten = vex_prefix.write(writer);
		795	try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, buf[0..nwritten]);
		796	}
		797
		798	{
		799	stream.reset();
		800	var vex_prefix = Encoder.Vex{};
		801	vex_prefix.rex(.{
		802	.w = true,
		803	.r = true,
		804	});
		805	vex_prefix.len_256();
		806	vex_prefix.lead_opc_0f();
		807	vex_prefix.simd_prefix_66();
		808	const nwritten = vex_prefix.write(writer);
		809	try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, buf[0..nwritten]);
		810	}
		811
		812	var code = ArrayList(u8).init(testing.allocator);
		813	defer code.deinit();
		814
		815	{
		816	// vmovapd xmm1, xmm2
		817	const encoder = try Encoder.init(&code, 4);
		818	var vex = Encoder.Vex{};
		819	vex.simd_prefix_66();
		820	encoder.vex(vex); // use 64 bit operation
		821	encoder.opcode_1byte(0x28);
		822	encoder.modRm_direct(0, Register.xmm1.lowEnc());
		823	try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items);
		824	}
		825
		826	{
		827	try code.resize(0);
		828
		829	// vmovhpd xmm13, xmm1, qword ptr [rip]
		830	const encoder = try Encoder.init(&code, 9);
		831	var vex = Encoder.Vex{};
		832	vex.len_128();
		833	vex.simd_prefix_66();
		834	vex.lead_opc_0f();
		835	vex.rex(.{ .r = true });
		836	vex.reg(Register.xmm1.enc());
		837	encoder.vex(vex);
		838	encoder.opcode_1byte(0x16);
		839	encoder.modRm_RIPDisp32(Register.xmm13.lowEnc());
		840	encoder.disp32(0);
		841	try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items);
		842	}
		843	}
		844
618	// TODO add these registers to the enum and populate dwarfLocOp	845	// TODO add these registers to the enum and populate dwarfLocOp
619	// // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register.	846	// // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register.
620	// RA = (16, "RA"),	847	// RA = (16, "RA"),

src/register_manager.zig+424-32

...	@@ -41,28 +41,33 @@ pub fn RegisterManager(	...	@@ -41,28 +41,33 @@ pub fn RegisterManager(
41	registers: [tracked_registers.len]Air.Inst.Index = undefined,	41	registers: [tracked_registers.len]Air.Inst.Index = undefined,
42	/// Tracks which registers are free (in which case the	42	/// Tracks which registers are free (in which case the
43	/// corresponding bit is set to 1)	43	/// corresponding bit is set to 1)
44	free_registers: FreeRegInt = math.maxInt(FreeRegInt),	44	free_registers: RegisterBitSet = math.maxInt(RegisterBitSet),
45	/// Tracks all registers allocated in the course of this	45	/// Tracks all registers allocated in the course of this
46	/// function	46	/// function
47	allocated_registers: FreeRegInt = 0,	47	allocated_registers: RegisterBitSet = 0,
48	/// Tracks registers which are locked from being allocated	48	/// Tracks registers which are locked from being allocated
49	locked_registers: FreeRegInt = 0,	49	locked_registers: RegisterBitSet = 0,
50		50
51	const Self = @This();	51	const Self = @This();
52		52
53	/// An integer whose bits represent all the registers and	53	/// An integer whose bits represent all the registers and
54	/// whether they are free.	54	/// whether they are free.
55	const FreeRegInt = std.meta.Int(.unsigned, tracked_registers.len);	55	pub const RegisterBitSet = std.meta.Int(.unsigned, tracked_registers.len);
56	const ShiftInt = math.Log2Int(FreeRegInt);	56	const ShiftInt = math.Log2Int(RegisterBitSet);
57		57
58	fn getFunction(self: Self) Function {	58	fn getFunction(self: Self) Function {
59	return @fieldParentPtr(Function, "register_manager", self);	59	return @fieldParentPtr(Function, "register_manager", self);
60	}	60	}
61		61
62	fn getRegisterMask(reg: Register) ?FreeRegInt {	62	fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool {
		63	const mask = getRegisterMask(reg) orelse return true;
		64	return mask & register_class == 0;
		65	}
		66
		67	fn getRegisterMask(reg: Register) ?RegisterBitSet {
63	const index = indexOfRegIntoTracked(reg) orelse return null;	68	const index = indexOfRegIntoTracked(reg) orelse return null;
64	const shift = @intCast(ShiftInt, index);	69	const shift = @intCast(ShiftInt, index);
65	const mask = @as(FreeRegInt, 1) << shift;	70	const mask = @as(RegisterBitSet, 1) << shift;
66	return mask;	71	return mask;
67	}	72	}
68		73
...	@@ -81,7 +86,10 @@ pub fn RegisterManager(	...	@@ -81,7 +86,10 @@ pub fn RegisterManager(
81	self.free_registers \|= mask;	86	self.free_registers \|= mask;
82	}	87	}
83		88
84	pub fn indexOfReg(comptime registers: []const Register, reg: Register) ?std.math.IntFittingRange(0, registers.len - 1) {	89	pub fn indexOfReg(
		90	comptime registers: []const Register,
		91	reg: Register,
		92	) ?std.math.IntFittingRange(0, registers.len - 1) {
85	inline for (tracked_registers) \|cpreg, i\| {	93	inline for (tracked_registers) \|cpreg, i\| {
86	if (reg.id() == cpreg.id()) return i;	94	if (reg.id() == cpreg.id()) return i;
87	}	95	}
...	@@ -180,17 +188,20 @@ pub fn RegisterManager(	...	@@ -180,17 +188,20 @@ pub fn RegisterManager(
180	self: *Self,	188	self: *Self,
181	comptime count: comptime_int,	189	comptime count: comptime_int,
182	insts: [count]?Air.Inst.Index,	190	insts: [count]?Air.Inst.Index,
		191	register_class: RegisterBitSet,
183	) ?[count]Register {	192	) ?[count]Register {
184	comptime assert(count > 0 and count <= tracked_registers.len);	193	comptime assert(count > 0 and count <= tracked_registers.len);
185		194
186	const free_and_not_locked_registers = self.free_registers & ~self.locked_registers;	195	const free_registers = self.free_registers & register_class;
187	const free_and_not_locked_registers_count = @popCount(FreeRegInt, free_and_not_locked_registers);	196	const free_and_not_locked_registers = free_registers & ~self.locked_registers;
		197	const free_and_not_locked_registers_count = @popCount(RegisterBitSet, free_and_not_locked_registers);
188	if (free_and_not_locked_registers_count < count) return null;	198	if (free_and_not_locked_registers_count < count) return null;
189		199
190	var regs: [count]Register = undefined;	200	var regs: [count]Register = undefined;
191	var i: usize = 0;	201	var i: usize = 0;
192	for (tracked_registers) \|reg\| {	202	for (tracked_registers) \|reg\| {
193	if (i >= count) break;	203	if (i >= count) break;
		204	if (excludeRegister(reg, register_class)) continue;
194	if (self.isRegLocked(reg)) continue;	205	if (self.isRegLocked(reg)) continue;
195	if (!self.isRegFree(reg)) continue;	206	if (!self.isRegFree(reg)) continue;
196		207
...	@@ -216,8 +227,8 @@ pub fn RegisterManager(	...	@@ -216,8 +227,8 @@ pub fn RegisterManager(
216	/// Allocates a register and optionally tracks it with a	227	/// Allocates a register and optionally tracks it with a
217	/// corresponding instruction. Returns `null` if all registers	228	/// corresponding instruction. Returns `null` if all registers
218	/// are allocated.	229	/// are allocated.
219	pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index) ?Register {	230	pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, register_class: RegisterBitSet) ?Register {
220	return if (tryAllocRegs(self, 1, .{inst})) \|regs\| regs[0] else null;	231	return if (tryAllocRegs(self, 1, .{inst}, register_class)) \|regs\| regs[0] else null;
221	}	232	}
222		233
223	/// Allocates a specified number of registers, optionally	234	/// Allocates a specified number of registers, optionally
...	@@ -227,12 +238,16 @@ pub fn RegisterManager(	...	@@ -227,12 +238,16 @@ pub fn RegisterManager(
227	self: *Self,	238	self: *Self,
228	comptime count: comptime_int,	239	comptime count: comptime_int,
229	insts: [count]?Air.Inst.Index,	240	insts: [count]?Air.Inst.Index,
		241	register_class: RegisterBitSet,
230	) AllocateRegistersError![count]Register {	242	) AllocateRegistersError![count]Register {
231	comptime assert(count > 0 and count <= tracked_registers.len);	243	comptime assert(count > 0 and count <= tracked_registers.len);
232	const locked_registers_count = @popCount(FreeRegInt, self.locked_registers);
233	if (count > tracked_registers.len - locked_registers_count) return error.OutOfRegisters;
234		244
235	const result = self.tryAllocRegs(count, insts) orelse blk: {	245	const available_registers_count = @popCount(RegisterBitSet, register_class);
		246	const locked_registers = self.locked_registers & register_class;
		247	const locked_registers_count = @popCount(RegisterBitSet, locked_registers);
		248	if (count > available_registers_count - locked_registers_count) return error.OutOfRegisters;
		249
		250	const result = self.tryAllocRegs(count, insts, register_class) orelse blk: {
236	// We'll take over the first count registers. Spill	251	// We'll take over the first count registers. Spill
237	// the instructions that were previously there to a	252	// the instructions that were previously there to a
238	// stack allocations.	253	// stack allocations.
...	@@ -240,6 +255,7 @@ pub fn RegisterManager(	...	@@ -240,6 +255,7 @@ pub fn RegisterManager(
240	var i: usize = 0;	255	var i: usize = 0;
241	for (tracked_registers) \|reg\| {	256	for (tracked_registers) \|reg\| {
242	if (i >= count) break;	257	if (i >= count) break;
		258	if (excludeRegister(reg, register_class)) continue;
243	if (self.isRegLocked(reg)) continue;	259	if (self.isRegLocked(reg)) continue;
244		260
245	regs[i] = reg;	261	regs[i] = reg;
...	@@ -275,8 +291,12 @@ pub fn RegisterManager(	...	@@ -275,8 +291,12 @@ pub fn RegisterManager(
275		291
276	/// Allocates a register and optionally tracks it with a	292	/// Allocates a register and optionally tracks it with a
277	/// corresponding instruction.	293	/// corresponding instruction.
278	pub fn allocReg(self: *Self, inst: ?Air.Inst.Index) AllocateRegistersError!Register {	294	pub fn allocReg(
279	return (try self.allocRegs(1, .{inst}))[0];	295	self: *Self,
		296	inst: ?Air.Inst.Index,
		297	register_class: RegisterBitSet,
		298	) AllocateRegistersError!Register {
		299	return (try self.allocRegs(1, .{inst}, register_class))[0];
280	}	300	}
281		301
282	/// Spills the register if it is currently allocated. If a	302	/// Spills the register if it is currently allocated. If a
...	@@ -332,6 +352,334 @@ pub fn RegisterManager(	...	@@ -332,6 +352,334 @@ pub fn RegisterManager(
332	};	352	};
333	}	353	}
334		354
		355	// TODO delete current implementation of RegisterManager above, and uncomment the one
		356	// below once #11680 is fixed:
		357	// https://github.com/ziglang/zig/issues/11680
		358
		359	//pub fn RegisterManager(
		360	// comptime Function: type,
		361	// comptime Register: type,
		362	// comptime tracked_registers: []const Register,
		363	//) type {
		364	// // architectures which do not have a concept of registers should
		365	// // refrain from using RegisterManager
		366	// assert(tracked_registers.len > 0); // see note above
		367
		368	// return struct {
		369	// /// Tracks the AIR instruction allocated to every register. If
		370	// /// no instruction is allocated to a register (i.e. the
		371	// /// register is free), the value in that slot is undefined.
		372	// ///
		373	// /// The key must be canonical register.
		374	// registers: [tracked_registers.len]Air.Inst.Index = undefined,
		375	// /// Tracks which registers are free (in which case the
		376	// /// corresponding bit is set to 1)
		377	// free_registers: RegisterBitSet = RegisterBitSet.initFull(),
		378	// /// Tracks all registers allocated in the course of this
		379	// /// function
		380	// allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
		381	// /// Tracks registers which are locked from being allocated
		382	// locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
		383
		384	// const Self = @This();
		385
		386	// pub const RegisterBitSet = StaticBitSet(tracked_registers.len);
		387
		388	// fn getFunction(self: Self) Function {
		389	// return @fieldParentPtr(Function, "register_manager", self);
		390	// }
		391
		392	// fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool {
		393	// const index = indexOfRegIntoTracked(reg) orelse return true;
		394	// return !register_class.isSet(index);
		395	// }
		396
		397	// fn markRegAllocated(self: *Self, reg: Register) void {
		398	// const index = indexOfRegIntoTracked(reg) orelse return;
		399	// self.allocated_registers.set(index);
		400	// }
		401
		402	// fn markRegUsed(self: *Self, reg: Register) void {
		403	// const index = indexOfRegIntoTracked(reg) orelse return;
		404	// self.free_registers.unset(index);
		405	// }
		406
		407	// fn markRegFree(self: *Self, reg: Register) void {
		408	// const index = indexOfRegIntoTracked(reg) orelse return;
		409	// self.free_registers.set(index);
		410	// }
		411
		412	// pub fn indexOfReg(
		413	// comptime registers: []const Register,
		414	// reg: Register,
		415	// ) ?std.math.IntFittingRange(0, registers.len - 1) {
		416	// inline for (tracked_registers) \|cpreg, i\| {
		417	// if (reg.id() == cpreg.id()) return i;
		418	// }
		419	// return null;
		420	// }
		421
		422	// pub fn indexOfRegIntoTracked(reg: Register) ?RegisterBitSet.ShiftInt {
		423	// return indexOfReg(tracked_registers, reg);
		424	// }
		425
		426	// /// Returns true when this register is not tracked
		427	// pub fn isRegFree(self: Self, reg: Register) bool {
		428	// const index = indexOfRegIntoTracked(reg) orelse return true;
		429	// return self.free_registers.isSet(index);
		430	// }
		431
		432	// /// Returns whether this register was allocated in the course
		433	// /// of this function.
		434	// ///
		435	// /// Returns false when this register is not tracked
		436	// pub fn isRegAllocated(self: Self, reg: Register) bool {
		437	// const index = indexOfRegIntoTracked(reg) orelse return false;
		438	// return self.allocated_registers.isSet(index);
		439	// }
		440
		441	// /// Returns whether this register is locked
		442	// ///
		443	// /// Returns false when this register is not tracked
		444	// pub fn isRegLocked(self: Self, reg: Register) bool {
		445	// const index = indexOfRegIntoTracked(reg) orelse return false;
		446	// return self.locked_registers.isSet(index);
		447	// }
		448
		449	// pub const RegisterLock = struct {
		450	// register: Register,
		451	// };
		452
		453	// /// Prevents the register from being allocated until they are
		454	// /// unlocked again.
		455	// /// Returns `RegisterLock` if the register was not already
		456	// /// locked, or `null` otherwise.
		457	// /// Only the owner of the `RegisterLock` can unlock the
		458	// /// register later.
		459	// pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
		460	// log.debug("locking {}", .{reg});
		461	// if (self.isRegLocked(reg)) {
		462	// log.debug(" register already locked", .{});
		463	// return null;
		464	// }
		465	// const index = indexOfRegIntoTracked(reg) orelse return null;
		466	// self.locked_registers.set(index);
		467	// return RegisterLock{ .register = reg };
		468	// }
		469
		470	// /// Like `lockReg` but asserts the register was unused always
		471	// /// returning a valid lock.
		472	// pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
		473	// log.debug("locking asserting free {}", .{reg});
		474	// assert(!self.isRegLocked(reg));
		475	// const index = indexOfRegIntoTracked(reg) orelse unreachable;
		476	// self.locked_registers.set(index);
		477	// return RegisterLock{ .register = reg };
		478	// }
		479
		480	// /// Like `lockRegAssumeUnused` but locks multiple registers.
		481	// pub fn lockRegsAssumeUnused(
		482	// self: *Self,
		483	// comptime count: comptime_int,
		484	// regs: [count]Register,
		485	// ) [count]RegisterLock {
		486	// var buf: [count]RegisterLock = undefined;
		487	// for (regs) \|reg, i\| {
		488	// buf[i] = self.lockRegAssumeUnused(reg);
		489	// }
		490	// return buf;
		491	// }
		492
		493	// /// Unlocks the register allowing its re-allocation and re-use.
		494	// /// Requires `RegisterLock` to unlock a register.
		495	// /// Call `lockReg` to obtain the lock first.
		496	// pub fn unlockReg(self: *Self, lock: RegisterLock) void {
		497	// log.debug("unlocking {}", .{lock.register});
		498	// const index = indexOfRegIntoTracked(lock.register) orelse return;
		499	// self.locked_registers.unset(index);
		500	// }
		501
		502	// /// Returns true when at least one register is locked
		503	// pub fn lockedRegsExist(self: Self) bool {
		504	// return self.locked_registers.count() > 0;
		505	// }
		506
		507	// /// Allocates a specified number of registers, optionally
		508	// /// tracking them. Returns `null` if not enough registers are
		509	// /// free.
		510	// pub fn tryAllocRegs(
		511	// self: *Self,
		512	// comptime count: comptime_int,
		513	// insts: [count]?Air.Inst.Index,
		514	// register_class: RegisterBitSet,
		515	// ) ?[count]Register {
		516	// comptime assert(count > 0 and count <= tracked_registers.len);
		517
		518	// var free_and_not_locked_registers = self.free_registers;
		519	// free_and_not_locked_registers.setIntersection(register_class);
		520
		521	// var unlocked_registers = self.locked_registers;
		522	// unlocked_registers.toggleAll();
		523
		524	// free_and_not_locked_registers.setIntersection(unlocked_registers);
		525
		526	// if (free_and_not_locked_registers.count() < count) return null;
		527
		528	// var regs: [count]Register = undefined;
		529	// var i: usize = 0;
		530	// for (tracked_registers) \|reg\| {
		531	// if (i >= count) break;
		532	// if (excludeRegister(reg, register_class)) continue;
		533	// if (self.isRegLocked(reg)) continue;
		534	// if (!self.isRegFree(reg)) continue;
		535
		536	// regs[i] = reg;
		537	// i += 1;
		538	// }
		539	// assert(i == count);
		540
		541	// for (regs) \|reg, j\| {
		542	// self.markRegAllocated(reg);
		543
		544	// if (insts[j]) \|inst\| {
		545	// // Track the register
		546	// const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null
		547	// self.registers[index] = inst;
		548	// self.markRegUsed(reg);
		549	// }
		550	// }
		551
		552	// return regs;
		553	// }
		554
		555	// /// Allocates a register and optionally tracks it with a
		556	// /// corresponding instruction. Returns `null` if all registers
		557	// /// are allocated.
		558	// pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, register_class: RegisterBitSet) ?Register {
		559	// return if (tryAllocRegs(self, 1, .{inst}, register_class)) \|regs\| regs[0] else null;
		560	// }
		561
		562	// /// Allocates a specified number of registers, optionally
		563	// /// tracking them. Asserts that count is not
		564	// /// larger than the total number of registers available.
		565	// pub fn allocRegs(
		566	// self: *Self,
		567	// comptime count: comptime_int,
		568	// insts: [count]?Air.Inst.Index,
		569	// register_class: RegisterBitSet,
		570	// ) AllocateRegistersError![count]Register {
		571	// comptime assert(count > 0 and count <= tracked_registers.len);
		572
		573	// var locked_registers = self.locked_registers;
		574	// locked_registers.setIntersection(register_class);
		575
		576	// if (count > register_class.count() - locked_registers.count()) return error.OutOfRegisters;
		577
		578	// const result = self.tryAllocRegs(count, insts, register_class) orelse blk: {
		579	// // We'll take over the first count registers. Spill
		580	// // the instructions that were previously there to a
		581	// // stack allocations.
		582	// var regs: [count]Register = undefined;
		583	// var i: usize = 0;
		584	// for (tracked_registers) \|reg\| {
		585	// if (i >= count) break;
		586	// if (excludeRegister(reg, register_class)) break;
		587	// if (self.isRegLocked(reg)) continue;
		588
		589	// regs[i] = reg;
		590	// self.markRegAllocated(reg);
		591	// const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null
		592	// if (insts[i]) \|inst\| {
		593	// // Track the register
		594	// if (self.isRegFree(reg)) {
		595	// self.markRegUsed(reg);
		596	// } else {
		597	// const spilled_inst = self.registers[index];
		598	// try self.getFunction().spillInstruction(reg, spilled_inst);
		599	// }
		600	// self.registers[index] = inst;
		601	// } else {
		602	// // Don't track the register
		603	// if (!self.isRegFree(reg)) {
		604	// const spilled_inst = self.registers[index];
		605	// try self.getFunction().spillInstruction(reg, spilled_inst);
		606	// self.freeReg(reg);
		607	// }
		608	// }
		609
		610	// i += 1;
		611	// }
		612
		613	// break :blk regs;
		614	// };
		615
		616	// log.debug("allocated registers {any} for insts {any}", .{ result, insts });
		617	// return result;
		618	// }
		619
		620	// /// Allocates a register and optionally tracks it with a
		621	// /// corresponding instruction.
		622	// pub fn allocReg(
		623	// self: *Self,
		624	// inst: ?Air.Inst.Index,
		625	// register_class: RegisterBitSet,
		626	// ) AllocateRegistersError!Register {
		627	// return (try self.allocRegs(1, .{inst}, register_class))[0];
		628	// }
		629
		630	// /// Spills the register if it is currently allocated. If a
		631	// /// corresponding instruction is passed, will also track this
		632	// /// register.
		633	// pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
		634	// const index = indexOfRegIntoTracked(reg) orelse return;
		635	// log.debug("getReg {} for inst {}", .{ reg, inst });
		636	// self.markRegAllocated(reg);
		637
		638	// if (inst) \|tracked_inst\|
		639	// if (!self.isRegFree(reg)) {
		640	// // Move the instruction that was previously there to a
		641	// // stack allocation.
		642	// const spilled_inst = self.registers[index];
		643	// self.registers[index] = tracked_inst;
		644	// try self.getFunction().spillInstruction(reg, spilled_inst);
		645	// } else {
		646	// self.getRegAssumeFree(reg, tracked_inst);
		647	// }
		648	// else {
		649	// if (!self.isRegFree(reg)) {
		650	// // Move the instruction that was previously there to a
		651	// // stack allocation.
		652	// const spilled_inst = self.registers[index];
		653	// try self.getFunction().spillInstruction(reg, spilled_inst);
		654	// self.freeReg(reg);
		655	// }
		656	// }
		657	// }
		658
		659	// /// Allocates the specified register with the specified
		660	// /// instruction. Asserts that the register is free and no
		661	// /// spilling is necessary.
		662	// pub fn getRegAssumeFree(self: *Self, reg: Register, inst: Air.Inst.Index) void {
		663	// const index = indexOfRegIntoTracked(reg) orelse return;
		664	// log.debug("getRegAssumeFree {} for inst {}", .{ reg, inst });
		665	// self.markRegAllocated(reg);
		666
		667	// assert(self.isRegFree(reg));
		668	// self.registers[index] = inst;
		669	// self.markRegUsed(reg);
		670	// }
		671
		672	// /// Marks the specified register as free
		673	// pub fn freeReg(self: *Self, reg: Register) void {
		674	// const index = indexOfRegIntoTracked(reg) orelse return;
		675	// log.debug("freeing register {}", .{reg});
		676
		677	// self.registers[index] = undefined;
		678	// self.markRegFree(reg);
		679	// }
		680	// };
		681	//}
		682
335	const MockRegister1 = enum(u2) {	683	const MockRegister1 = enum(u2) {
336	r0,	684	r0,
337	r1,	685	r1,
...	@@ -361,11 +709,15 @@ const MockRegister2 = enum(u2) {	...	@@ -361,11 +709,15 @@ const MockRegister2 = enum(u2) {
361	fn MockFunction(comptime Register: type) type {	709	fn MockFunction(comptime Register: type) type {
362	return struct {	710	return struct {
363	allocator: Allocator,	711	allocator: Allocator,
364	register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{},	712	register_manager: RegisterManagerT = .{},
365	spilled: std.ArrayListUnmanaged(Register) = .{},	713	spilled: std.ArrayListUnmanaged(Register) = .{},
366		714
367	const Self = @This();	715	const Self = @This();
368		716
		717	const RegisterManagerT = RegisterManager(Self, Register, &Register.allocatable_registers);
		718
		719	pub const reg_class: RegisterManagerT.RegisterBitSet = math.maxInt(RegisterManagerT.RegisterBitSet);
		720
369	pub fn deinit(self: *Self) void {	721	pub fn deinit(self: *Self) void {
370	self.spilled.deinit(self.allocator);	722	self.spilled.deinit(self.allocator);
371	}	723	}
...	@@ -410,10 +762,20 @@ test "tryAllocReg: no spilling" {	...	@@ -410,10 +762,20 @@ test "tryAllocReg: no spilling" {
410	defer function.deinit();	762	defer function.deinit();
411		763
412	const mock_instruction: Air.Inst.Index = 1;	764	const mock_instruction: Air.Inst.Index = 1;
413		765	const reg_class = MockFunction1.reg_class;
414	try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction));	766
415	try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction));	767	try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(
416	try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction));	768	mock_instruction,
		769	reg_class,
		770	));
		771	try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(
		772	mock_instruction,
		773	reg_class,
		774	));
		775	try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(
		776	mock_instruction,
		777	reg_class,
		778	));
417		779
418	try expect(function.register_manager.isRegAllocated(.r2));	780	try expect(function.register_manager.isRegAllocated(.r2));
419	try expect(function.register_manager.isRegAllocated(.r3));	781	try expect(function.register_manager.isRegAllocated(.r3));
...	@@ -438,17 +800,30 @@ test "allocReg: spilling" {	...	@@ -438,17 +800,30 @@ test "allocReg: spilling" {
438	defer function.deinit();	800	defer function.deinit();
439		801
440	const mock_instruction: Air.Inst.Index = 1;	802	const mock_instruction: Air.Inst.Index = 1;
		803	const reg_class = MockFunction1.reg_class;
441		804
442	try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));	805	try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(
443	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));	806	mock_instruction,
		807	reg_class,
		808	));
		809	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(
		810	mock_instruction,
		811	reg_class,
		812	));
444		813
445	// Spill a register	814	// Spill a register
446	try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));	815	try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(
		816	mock_instruction,
		817	reg_class,
		818	));
447	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);	819	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
448		820
449	// No spilling necessary	821	// No spilling necessary
450	function.register_manager.freeReg(.r3);	822	function.register_manager.freeReg(.r3);
451	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));	823	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(
		824	mock_instruction,
		825	reg_class,
		826	));
452	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);	827	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
453		828
454	// Locked registers	829	// Locked registers
...	@@ -457,7 +832,10 @@ test "allocReg: spilling" {	...	@@ -457,7 +832,10 @@ test "allocReg: spilling" {
457	const lock = function.register_manager.lockReg(.r2);	832	const lock = function.register_manager.lockReg(.r2);
458	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);	833	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
459		834
460	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));	835	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(
		836	mock_instruction,
		837	reg_class,
		838	));
461	}	839	}
462	try expect(!function.register_manager.lockedRegsExist());	840	try expect(!function.register_manager.lockedRegsExist());
463	}	841	}
...	@@ -470,7 +848,13 @@ test "tryAllocRegs" {	...	@@ -470,7 +848,13 @@ test "tryAllocRegs" {
470	};	848	};
471	defer function.deinit();	849	defer function.deinit();
472		850
473	try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);	851	const reg_class = MockFunction2.reg_class;
		852
		853	try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(
		854	3,
		855	.{ null, null, null },
		856	reg_class,
		857	).?);
474		858
475	try expect(function.register_manager.isRegAllocated(.r0));	859	try expect(function.register_manager.isRegAllocated(.r0));
476	try expect(function.register_manager.isRegAllocated(.r1));	860	try expect(function.register_manager.isRegAllocated(.r1));
...	@@ -485,7 +869,11 @@ test "tryAllocRegs" {	...	@@ -485,7 +869,11 @@ test "tryAllocRegs" {
485	const lock = function.register_manager.lockReg(.r1);	869	const lock = function.register_manager.lockReg(.r1);
486	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);	870	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
487		871
488	try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);	872	try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(
		873	3,
		874	.{ null, null, null },
		875	reg_class,
		876	).?);
489	}	877	}
490	try expect(!function.register_manager.lockedRegsExist());	878	try expect(!function.register_manager.lockedRegsExist());
491		879
...	@@ -505,6 +893,8 @@ test "allocRegs: normal usage" {	...	@@ -505,6 +893,8 @@ test "allocRegs: normal usage" {
505	};	893	};
506	defer function.deinit();	894	defer function.deinit();
507		895
		896	const reg_class = MockFunction2.reg_class;
		897
508	{	898	{
509	const result_reg: MockRegister2 = .r1;	899	const result_reg: MockRegister2 = .r1;
510		900
...	@@ -524,7 +914,7 @@ test "allocRegs: normal usage" {	...	@@ -524,7 +914,7 @@ test "allocRegs: normal usage" {
524	const lock = function.register_manager.lockReg(result_reg);	914	const lock = function.register_manager.lockReg(result_reg);
525	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);	915	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
526		916
527	const regs = try function.register_manager.allocRegs(2, .{ null, null });	917	const regs = try function.register_manager.allocRegs(2, .{ null, null }, reg_class);
528	try function.genAdd(result_reg, regs[0], regs[1]);	918	try function.genAdd(result_reg, regs[0], regs[1]);
529	}	919	}
530	}	920	}
...	@@ -539,6 +929,8 @@ test "allocRegs: selectively reducing register pressure" {	...	@@ -539,6 +929,8 @@ test "allocRegs: selectively reducing register pressure" {
539	};	929	};
540	defer function.deinit();	930	defer function.deinit();
541		931
		932	const reg_class = MockFunction2.reg_class;
		933
542	{	934	{
543	const result_reg: MockRegister2 = .r1;	935	const result_reg: MockRegister2 = .r1;
544		936
...	@@ -546,12 +938,12 @@ test "allocRegs: selectively reducing register pressure" {	...	@@ -546,12 +938,12 @@ test "allocRegs: selectively reducing register pressure" {
546		938
547	// Here, we don't defer unlock because we manually unlock	939	// Here, we don't defer unlock because we manually unlock
548	// after genAdd	940	// after genAdd
549	const regs = try function.register_manager.allocRegs(2, .{ null, null });	941	const regs = try function.register_manager.allocRegs(2, .{ null, null }, reg_class);
550		942
551	try function.genAdd(result_reg, regs[0], regs[1]);	943	try function.genAdd(result_reg, regs[0], regs[1]);
552	function.register_manager.unlockReg(lock.?);	944	function.register_manager.unlockReg(lock.?);
553		945
554	const extra_summand_reg = try function.register_manager.allocReg(null);	946	const extra_summand_reg = try function.register_manager.allocReg(null, reg_class);
555	try function.genAdd(result_reg, result_reg, extra_summand_reg);	947	try function.genAdd(result_reg, result_reg, extra_summand_reg);
556	}	948	}
557	}	949	}

test/behavior/basic.zig+3

test/behavior/math.zig-1

...	@@ -198,7 +198,6 @@ test "const number literal" {	...	@@ -198,7 +198,6 @@ test "const number literal" {
198	const ten = 10;	198	const ten = 10;
199		199
200	test "float equality" {	200	test "float equality" {
201	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
202	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	201	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
203	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	202	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
204		203

test/behavior/union.zig+1

...	@@ -882,6 +882,7 @@ test "extern union doesn't trigger field check at comptime" {	...	@@ -882,6 +882,7 @@ test "extern union doesn't trigger field check at comptime" {
882	test "anonymous union literal syntax" {	882	test "anonymous union literal syntax" {
883	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	883	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
884	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	884	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
		885	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
885		886
886	const S = struct {	887	const S = struct {
887	const Number = union {	888	const Number = union {

...	@@ -21,9 +21,6 @@ const DW = std.dwarf;	...	@@ -21,9 +21,6 @@ const DW = std.dwarf;
21	const leb128 = std.leb;	21	const leb128 = std.leb;
22	const log = std.log.scoped(.codegen);	22	const log = std.log.scoped(.codegen);
23	const build_options = @import("build_options");	23	const build_options = @import("build_options");
24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25	const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
26	const RegisterLock = RegisterManager.RegisterLock;
27		24
28	const FnResult = @import("../../codegen.zig").FnResult;	25	const FnResult = @import("../../codegen.zig").FnResult;
29	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;	26	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
...	@@ -32,8 +29,11 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;	...	@@ -32,8 +29,11 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
32	const bits = @import("bits.zig");	29	const bits = @import("bits.zig");
33	const abi = @import("abi.zig");	30	const abi = @import("abi.zig");
34	const Register = bits.Register;	31	const Register = bits.Register;
		32	const RegisterManager = abi.RegisterManager;
		33	const RegisterLock = RegisterManager.RegisterLock;
35	const Instruction = abi.Instruction;	34	const Instruction = abi.Instruction;
36	const callee_preserved_regs = abi.callee_preserved_regs;	35	const callee_preserved_regs = abi.callee_preserved_regs;
		36	const gp = abi.RegisterClass.gp;
37		37
38	const InnerError = error{	38	const InnerError = error{
39	OutOfMemory,	39	OutOfMemory,
...	@@ -803,7 +803,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {	...	@@ -803,7 +803,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
803	const ptr_bits = self.target.cpu.arch.ptrBitWidth();	803	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
804	const ptr_bytes: u64 = @divExact(ptr_bits, 8);	804	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
805	if (abi_size <= ptr_bytes) {	805	if (abi_size <= ptr_bytes) {
806	if (self.register_manager.tryAllocReg(inst)) \|reg\| {	806	if (self.register_manager.tryAllocReg(inst, gp)) \|reg\| {
807	return MCValue{ .register = reg };	807	return MCValue{ .register = reg };
808	}	808	}
809	}	809	}
...	@@ -826,7 +826,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void	...	@@ -826,7 +826,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
826	/// allocated. A second call to `copyToTmpRegister` may return the same register.	826	/// allocated. A second call to `copyToTmpRegister` may return the same register.
827	/// This can have a side effect of spilling instructions to the stack to free up a register.	827	/// This can have a side effect of spilling instructions to the stack to free up a register.
828	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {	828	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
829	const reg = try self.register_manager.allocReg(null);	829	const reg = try self.register_manager.allocReg(null, gp);
830	try self.genSetReg(ty, reg, mcv);	830	try self.genSetReg(ty, reg, mcv);
831	return reg;	831	return reg;
832	}	832	}
...	@@ -835,7 +835,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {	...	@@ -835,7 +835,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
835	/// `reg_owner` is the instruction that gets associated with the register in the register table.	835	/// `reg_owner` is the instruction that gets associated with the register in the register table.
836	/// This can have a side effect of spilling instructions to the stack to free up a register.	836	/// This can have a side effect of spilling instructions to the stack to free up a register.
837	fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {	837	fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
838	const reg = try self.register_manager.allocReg(reg_owner);	838	const reg = try self.register_manager.allocReg(reg_owner, gp);
839	try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);	839	try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
840	return MCValue{ .register = reg };	840	return MCValue{ .register = reg };
841	}	841	}
...	@@ -958,7 +958,7 @@ fn binOpRegister(	...	@@ -958,7 +958,7 @@ fn binOpRegister(
958	break :inst Air.refToIndex(bin_op.lhs).?;	958	break :inst Air.refToIndex(bin_op.lhs).?;
959	} else null;	959	} else null;
960		960
961	const reg = try self.register_manager.allocReg(track_inst);	961	const reg = try self.register_manager.allocReg(track_inst, gp);
962		962
963	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	963	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
964		964
...	@@ -973,7 +973,7 @@ fn binOpRegister(	...	@@ -973,7 +973,7 @@ fn binOpRegister(
973	break :inst Air.refToIndex(bin_op.rhs).?;	973	break :inst Air.refToIndex(bin_op.rhs).?;
974	} else null;	974	} else null;
975		975
976	const reg = try self.register_manager.allocReg(track_inst);	976	const reg = try self.register_manager.allocReg(track_inst, gp);
977		977
978	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	978	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
979		979
...	@@ -990,9 +990,9 @@ fn binOpRegister(	...	@@ -990,9 +990,9 @@ fn binOpRegister(
990	} else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {	990	} else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
991	break :blk rhs_reg;	991	break :blk rhs_reg;
992	} else {	992	} else {
993	break :blk try self.register_manager.allocReg(inst);	993	break :blk try self.register_manager.allocReg(inst, gp);
994	}	994	}
995	} else try self.register_manager.allocReg(null);	995	} else try self.register_manager.allocReg(null, gp);
996		996
997	if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);	997	if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
998	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);	998	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
...	@@ -1482,7 +1482,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -1482,7 +1482,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
1482	.memory,	1482	.memory,
1483	.stack_offset,	1483	.stack_offset,
1484	=> {	1484	=> {
1485	const reg = try self.register_manager.allocReg(null);	1485	const reg = try self.register_manager.allocReg(null, gp);
1486	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);	1486	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
1487	defer self.register_manager.unlockReg(reg_lock);	1487	defer self.register_manager.unlockReg(reg_lock);
1488		1488

...	@@ -21,9 +21,6 @@ const Type = @import("../../type.zig").Type;	...	@@ -21,9 +21,6 @@ const Type = @import("../../type.zig").Type;
21	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;	21	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
22	const FnResult = @import("../../codegen.zig").FnResult;	22	const FnResult = @import("../../codegen.zig").FnResult;
23	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;	23	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25	const RegisterManager = RegisterManagerFn(Self, Register, &abi.allocatable_regs);
26	const RegisterLock = RegisterManager.RegisterLock;
27		24
28	const build_options = @import("build_options");	25	const build_options = @import("build_options");
29		26
...	@@ -31,7 +28,10 @@ const bits = @import("bits.zig");	...	@@ -31,7 +28,10 @@ const bits = @import("bits.zig");
31	const abi = @import("abi.zig");	28	const abi = @import("abi.zig");
32	const Instruction = bits.Instruction;	29	const Instruction = bits.Instruction;
33	const ShiftWidth = Instruction.ShiftWidth;	30	const ShiftWidth = Instruction.ShiftWidth;
		31	const RegisterManager = abi.RegisterManager;
		32	const RegisterLock = RegisterManager.RegisterLock;
34	const Register = bits.Register;	33	const Register = bits.Register;
		34	const gp = abi.RegisterClass.gp;
35		35
36	const Self = @This();	36	const Self = @This();
37		37
...	@@ -1613,7 +1613,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {	...	@@ -1613,7 +1613,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
1613	if (reg_ok) {	1613	if (reg_ok) {
1614	// Make sure the type can fit in a register before we try to allocate one.	1614	// Make sure the type can fit in a register before we try to allocate one.
1615	if (abi_size <= 8) {	1615	if (abi_size <= 8) {
1616	if (self.register_manager.tryAllocReg(inst)) \|reg\| {	1616	if (self.register_manager.tryAllocReg(inst, gp)) \|reg\| {
1617	return MCValue{ .register = reg };	1617	return MCValue{ .register = reg };
1618	}	1618	}
1619	}	1619	}
...	@@ -1854,7 +1854,7 @@ fn binOpImmediate(	...	@@ -1854,7 +1854,7 @@ fn binOpImmediate(
1854	).?;	1854	).?;
1855	} else null;	1855	} else null;
1856		1856
1857	const reg = try self.register_manager.allocReg(track_inst);	1857	const reg = try self.register_manager.allocReg(track_inst, gp);
1858		1858
1859	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	1859	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
1860		1860
...	@@ -1873,10 +1873,10 @@ fn binOpImmediate(	...	@@ -1873,10 +1873,10 @@ fn binOpImmediate(
1873	)) {	1873	)) {
1874	break :blk lhs_reg;	1874	break :blk lhs_reg;
1875	} else {	1875	} else {
1876	break :blk try self.register_manager.allocReg(md.inst);	1876	break :blk try self.register_manager.allocReg(md.inst, gp);
1877	}	1877	}
1878	} else blk: {	1878	} else blk: {
1879	break :blk try self.register_manager.allocReg(null);	1879	break :blk try self.register_manager.allocReg(null, gp);
1880	},	1880	},
1881	};	1881	};
1882		1882
...	@@ -1953,7 +1953,7 @@ fn binOpRegister(	...	@@ -1953,7 +1953,7 @@ fn binOpRegister(
1953	break :inst Air.refToIndex(md.lhs).?;	1953	break :inst Air.refToIndex(md.lhs).?;
1954	} else null;	1954	} else null;
1955		1955
1956	const reg = try self.register_manager.allocReg(track_inst);	1956	const reg = try self.register_manager.allocReg(track_inst, gp);
1957	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	1957	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
1958		1958
1959	break :blk reg;	1959	break :blk reg;
...	@@ -1966,7 +1966,7 @@ fn binOpRegister(	...	@@ -1966,7 +1966,7 @@ fn binOpRegister(
1966	break :inst Air.refToIndex(md.rhs).?;	1966	break :inst Air.refToIndex(md.rhs).?;
1967	} else null;	1967	} else null;
1968		1968
1969	const reg = try self.register_manager.allocReg(track_inst);	1969	const reg = try self.register_manager.allocReg(track_inst, gp);
1970	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	1970	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
1971		1971
1972	break :blk reg;	1972	break :blk reg;
...	@@ -1981,10 +1981,10 @@ fn binOpRegister(	...	@@ -1981,10 +1981,10 @@ fn binOpRegister(
1981	} else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) {	1981	} else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) {
1982	break :blk rhs_reg;	1982	break :blk rhs_reg;
1983	} else {	1983	} else {
1984	break :blk try self.register_manager.allocReg(md.inst);	1984	break :blk try self.register_manager.allocReg(md.inst, gp);
1985	}	1985	}
1986	} else blk: {	1986	} else blk: {
1987	break :blk try self.register_manager.allocReg(null);	1987	break :blk try self.register_manager.allocReg(null, gp);
1988	},	1988	},
1989	};	1989	};
1990		1990
...	@@ -2077,7 +2077,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {	...	@@ -2077,7 +2077,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
2077	/// allocated. A second call to `copyToTmpRegister` may return the same register.	2077	/// allocated. A second call to `copyToTmpRegister` may return the same register.
2078	/// This can have a side effect of spilling instructions to the stack to free up a register.	2078	/// This can have a side effect of spilling instructions to the stack to free up a register.
2079	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {	2079	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
2080	const reg = try self.register_manager.allocReg(null);	2080	const reg = try self.register_manager.allocReg(null, gp);
2081	try self.genSetReg(ty, reg, mcv);	2081	try self.genSetReg(ty, reg, mcv);
2082	return reg;	2082	return reg;
2083	}	2083	}
...	@@ -2364,7 +2364,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -2364,7 +2364,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
2364	});	2364	});
2365	} else {	2365	} else {
2366	// Need to allocate a temporary register to load 64-bit immediates.	2366	// Need to allocate a temporary register to load 64-bit immediates.
2367	const tmp_reg = try self.register_manager.allocReg(null);	2367	const tmp_reg = try self.register_manager.allocReg(null, gp);
2368		2368
2369	try self.genSetReg(ty, tmp_reg, .{ .immediate = @truncate(u32, x) });	2369	try self.genSetReg(ty, tmp_reg, .{ .immediate = @truncate(u32, x) });
2370	try self.genSetReg(ty, reg, .{ .immediate = @truncate(u32, x >> 32) });	2370	try self.genSetReg(ty, reg, .{ .immediate = @truncate(u32, x >> 32) });
...	@@ -2478,7 +2478,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro	...	@@ -2478,7 +2478,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2478	};	2478	};
2479	const ptr_ty = Type.initPayload(&ptr_ty_payload.base);	2479	const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
2480		2480
2481	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });	2481	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
2482	const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);	2482	const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2483	defer for (regs_locks) \|reg\| {	2483	defer for (regs_locks) \|reg\| {
2484	self.register_manager.unlockReg(reg);	2484	self.register_manager.unlockReg(reg);
...	@@ -2717,14 +2717,14 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -2717,14 +2717,14 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2717	},	2717	},
2718	.stack_offset => \|off\| {	2718	.stack_offset => \|off\| {
2719	if (elem_size <= 8) {	2719	if (elem_size <= 8) {
2720	const tmp_reg = try self.register_manager.allocReg(null);	2720	const tmp_reg = try self.register_manager.allocReg(null, gp);
2721	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	2721	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2722	defer self.register_manager.unlockReg(tmp_reg_lock);	2722	defer self.register_manager.unlockReg(tmp_reg_lock);
2723		2723
2724	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);	2724	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
2725	try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });	2725	try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
2726	} else {	2726	} else {
2727	const regs = try self.register_manager.allocRegs(3, .{ null, null, null });	2727	const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
2728	const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs);	2728	const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs);
2729	defer for (regs_locks) \|reg\| {	2729	defer for (regs_locks) \|reg\| {
2730	self.register_manager.unlockReg(reg);	2730	self.register_manager.unlockReg(reg);

...	@@ -402,6 +402,7 @@ fn testPointerToVoidReturnType2() *const void {	...	@@ -402,6 +402,7 @@ fn testPointerToVoidReturnType2() *const void {
402	test "array 2D const double ptr" {	402	test "array 2D const double ptr" {
403	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	403	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
404	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	404	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
		405	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
405	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	406	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
406		407
407	const rect_2d_vertexes = [_][1]f32{	408	const rect_2d_vertexes = [_][1]f32{
...	@@ -414,6 +415,7 @@ test "array 2D const double ptr" {	...	@@ -414,6 +415,7 @@ test "array 2D const double ptr" {
414	test "array 2D const double ptr with offset" {	415	test "array 2D const double ptr with offset" {
415	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	416	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
416	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	417	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
		418	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
417	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;	419	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
418		420
419	const rect_2d_vertexes = [_][2]f32{	421	const rect_2d_vertexes = [_][2]f32{
...	@@ -426,6 +428,7 @@ test "array 2D const double ptr with offset" {	...	@@ -426,6 +428,7 @@ test "array 2D const double ptr with offset" {
426	test "array 3D const double ptr with offset" {	428	test "array 3D const double ptr with offset" {
427	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	429	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
428	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	430	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
		431	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
429	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	432	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
430		433
431	const rect_3d_vertexes = [_][2][2]f32{	434	const rect_3d_vertexes = [_][2][2]f32{