authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-20 17:19:28+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-05-20 17:19:28+02:00
log704d38ba4981d46ac1d4687d8f6dcce72bf4067c
treef80363ac870b07ab61ab41ff927c42785bfd20b9
parent58943fc6276c49b138c98bda5abac8be2806f94f
parent274654d73e36df29fdc017a009c8607fb46a15e4
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11681 from ziglang/x64-floats-tmp

x64: add prelim support for SSE/AVX-based floating-point ops

17 files changed, 2407 insertions(+), 848 deletions(-)

src/arch/aarch64/CodeGen.zig+31-31
......@@ -21,9 +21,6 @@ const DW = std.dwarf;
2121const leb128 = std.leb;
2222const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
24const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
26const RegisterLock = RegisterManager.RegisterLock;
2724
2825const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
2926const FnResult = @import("../../codegen.zig").FnResult;
......@@ -31,11 +28,14 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
3128
3229const bits = @import("bits.zig");
3330const abi = @import("abi.zig");
31const RegisterManager = abi.RegisterManager;
32const RegisterLock = RegisterManager.RegisterLock;
3433const Register = bits.Register;
3534const Instruction = bits.Instruction;
3635const callee_preserved_regs = abi.callee_preserved_regs;
3736const c_abi_int_param_regs = abi.c_abi_int_param_regs;
3837const c_abi_int_return_regs = abi.c_abi_int_return_regs;
38const gp = abi.RegisterClass.gp;
3939
4040const InnerError = error{
4141 OutOfMemory,
......@@ -888,7 +888,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
888888 if (reg_ok) {
889889 // Make sure the type can fit in a register before we try to allocate one.
890890 if (abi_size <= 8) {
891 if (self.register_manager.tryAllocReg(inst)) |reg| {
891 if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
892892 return MCValue{ .register = registerAlias(reg, abi_size) };
893893 }
894894 }
......@@ -951,7 +951,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
951951/// allocated. A second call to `copyToTmpRegister` may return the same register.
952952/// This can have a side effect of spilling instructions to the stack to free up a register.
953953fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
954 const raw_reg = try self.register_manager.allocReg(null);
954 const raw_reg = try self.register_manager.allocReg(null, gp);
955955 const reg = registerAlias(raw_reg, ty.abiSize(self.target.*));
956956 try self.genSetReg(ty, reg, mcv);
957957 return reg;
......@@ -961,7 +961,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
961961/// `reg_owner` is the instruction that gets associated with the register in the register table.
962962/// This can have a side effect of spilling instructions to the stack to free up a register.
963963fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
964 const raw_reg = try self.register_manager.allocReg(reg_owner);
964 const raw_reg = try self.register_manager.allocReg(reg_owner, gp);
965965 const ty = self.air.typeOfIndex(reg_owner);
966966 const reg = registerAlias(raw_reg, ty.abiSize(self.target.*));
967967 try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
......@@ -1074,11 +1074,11 @@ fn trunc(
10741074 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
10751075 break :blk registerAlias(operand_reg, dest_ty.abiSize(self.target.*));
10761076 } else {
1077 const raw_reg = try self.register_manager.allocReg(inst);
1077 const raw_reg = try self.register_manager.allocReg(inst, gp);
10781078 break :blk registerAlias(raw_reg, dest_ty.abiSize(self.target.*));
10791079 }
10801080 } else blk: {
1081 const raw_reg = try self.register_manager.allocReg(null);
1081 const raw_reg = try self.register_manager.allocReg(null, gp);
10821082 break :blk registerAlias(raw_reg, dest_ty.abiSize(self.target.*));
10831083 };
10841084
......@@ -1160,7 +1160,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11601160 break :blk op_reg;
11611161 }
11621162
1163 const raw_reg = try self.register_manager.allocReg(null);
1163 const raw_reg = try self.register_manager.allocReg(null, gp);
11641164 break :blk raw_reg.to32();
11651165 };
11661166
......@@ -1193,7 +1193,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11931193 break :blk op_reg;
11941194 }
11951195
1196 const raw_reg = try self.register_manager.allocReg(null);
1196 const raw_reg = try self.register_manager.allocReg(null, gp);
11971197 break :blk registerAlias(raw_reg, operand_ty.abiSize(self.target.*));
11981198 };
11991199
......@@ -1293,7 +1293,7 @@ fn binOpRegister(
12931293 break :inst Air.refToIndex(md.lhs).?;
12941294 } else null;
12951295
1296 const raw_reg = try self.register_manager.allocReg(track_inst);
1296 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
12971297 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
12981298
12991299 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
......@@ -1308,7 +1308,7 @@ fn binOpRegister(
13081308 break :inst Air.refToIndex(md.rhs).?;
13091309 } else null;
13101310
1311 const raw_reg = try self.register_manager.allocReg(track_inst);
1311 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
13121312 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
13131313
13141314 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
......@@ -1326,11 +1326,11 @@ fn binOpRegister(
13261326 } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) {
13271327 break :blk rhs_reg;
13281328 } else {
1329 const raw_reg = try self.register_manager.allocReg(md.inst);
1329 const raw_reg = try self.register_manager.allocReg(md.inst, gp);
13301330 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
13311331 }
13321332 } else blk: {
1333 const raw_reg = try self.register_manager.allocReg(null);
1333 const raw_reg = try self.register_manager.allocReg(null, gp);
13341334 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
13351335 },
13361336 };
......@@ -1431,7 +1431,7 @@ fn binOpImmediate(
14311431 ).?;
14321432 } else null;
14331433
1434 const raw_reg = try self.register_manager.allocReg(track_inst);
1434 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
14351435 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
14361436
14371437 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
......@@ -1452,11 +1452,11 @@ fn binOpImmediate(
14521452 )) {
14531453 break :blk lhs_reg;
14541454 } else {
1455 const raw_reg = try self.register_manager.allocReg(md.inst);
1455 const raw_reg = try self.register_manager.allocReg(md.inst, gp);
14561456 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
14571457 }
14581458 } else blk: {
1459 const raw_reg = try self.register_manager.allocReg(null);
1459 const raw_reg = try self.register_manager.allocReg(null, gp);
14601460 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
14611461 },
14621462 };
......@@ -1872,7 +1872,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
18721872 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
18731873 defer self.register_manager.unlockReg(dest_reg_lock);
18741874
1875 const raw_truncated_reg = try self.register_manager.allocReg(null);
1875 const raw_truncated_reg = try self.register_manager.allocReg(null, gp);
18761876 const truncated_reg = registerAlias(raw_truncated_reg, lhs_ty.abiSize(self.target.*));
18771877 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
18781878 defer self.register_manager.unlockReg(truncated_reg_lock);
......@@ -1983,7 +1983,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
19831983 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
19841984 defer self.register_manager.unlockReg(dest_reg_lock);
19851985
1986 const truncated_reg = try self.register_manager.allocReg(null);
1986 const truncated_reg = try self.register_manager.allocReg(null, gp);
19871987 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
19881988 defer self.register_manager.unlockReg(truncated_reg_lock);
19891989
......@@ -2048,7 +2048,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20482048 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
20492049
20502050 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
2051 const raw_reg = try self.register_manager.allocReg(null);
2051 const raw_reg = try self.register_manager.allocReg(null, gp);
20522052 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
20532053 break :blk reg;
20542054 };
......@@ -2056,7 +2056,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20562056 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
20572057
20582058 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
2059 const raw_reg = try self.register_manager.allocReg(null);
2059 const raw_reg = try self.register_manager.allocReg(null, gp);
20602060 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
20612061 break :blk reg;
20622062 };
......@@ -2067,7 +2067,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20672067 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
20682068
20692069 const dest_reg = blk: {
2070 const raw_reg = try self.register_manager.allocReg(null);
2070 const raw_reg = try self.register_manager.allocReg(null, gp);
20712071 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
20722072 break :blk reg;
20732073 };
......@@ -2086,7 +2086,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20862086 } },
20872087 });
20882088
2089 const dest_high_reg = try self.register_manager.allocReg(null);
2089 const dest_high_reg = try self.register_manager.allocReg(null, gp);
20902090 const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg);
20912091 defer self.register_manager.unlockReg(dest_high_reg_lock);
20922092
......@@ -2136,7 +2136,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21362136 }
21372137 },
21382138 .unsigned => {
2139 const dest_high_reg = try self.register_manager.allocReg(null);
2139 const dest_high_reg = try self.register_manager.allocReg(null, gp);
21402140 const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg);
21412141 defer self.register_manager.unlockReg(dest_high_reg_lock);
21422142
......@@ -2192,7 +2192,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21922192 },
21932193 }
21942194
2195 const truncated_reg = try self.register_manager.allocReg(null);
2195 const truncated_reg = try self.register_manager.allocReg(null, gp);
21962196 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
21972197 defer self.register_manager.unlockReg(truncated_reg_lock);
21982198
......@@ -2663,7 +2663,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
26632663 },
26642664 .stack_offset => |off| {
26652665 if (elem_size <= 8) {
2666 const raw_tmp_reg = try self.register_manager.allocReg(null);
2666 const raw_tmp_reg = try self.register_manager.allocReg(null, gp);
26672667 const tmp_reg = registerAlias(raw_tmp_reg, elem_size);
26682668 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
26692669 defer self.register_manager.unlockReg(tmp_reg_lock);
......@@ -2672,7 +2672,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
26722672 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
26732673 } else {
26742674 // TODO optimize the register allocation
2675 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2675 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
26762676 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
26772677 defer for (regs_locks) |reg| {
26782678 self.register_manager.unlockReg(reg);
......@@ -2887,7 +2887,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
28872887 },
28882888 else => {
28892889 if (abi_size <= 8) {
2890 const raw_tmp_reg = try self.register_manager.allocReg(null);
2890 const raw_tmp_reg = try self.register_manager.allocReg(null, gp);
28912891 const tmp_reg = registerAlias(raw_tmp_reg, abi_size);
28922892 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
28932893 defer self.register_manager.unlockReg(tmp_reg_lock);
......@@ -3002,7 +3002,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
30023002 // TODO return special MCValue condition flags
30033003 // get overflow bit: set register to C flag
30043004 // resp. V flag
3005 const raw_dest_reg = try self.register_manager.allocReg(null);
3005 const raw_dest_reg = try self.register_manager.allocReg(null, gp);
30063006 const dest_reg = raw_dest_reg.to32();
30073007
30083008 // C flag: cset reg, cs
......@@ -4065,7 +4065,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
40654065
40664066 const overflow_bit_ty = ty.structFieldType(1);
40674067 const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*));
4068 const raw_cond_reg = try self.register_manager.allocReg(null);
4068 const raw_cond_reg = try self.register_manager.allocReg(null, gp);
40694069 const cond_reg = registerAlias(
40704070 raw_cond_reg,
40714071 @intCast(u32, overflow_bit_ty.abiSize(self.target.*)),
......@@ -4113,7 +4113,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
41134113 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
41144114
41154115 // TODO call extern memcpy
4116 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null });
4116 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
41174117 const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);
41184118 defer for (regs_locks) |reg| {
41194119 self.register_manager.unlockReg(reg);
src/arch/aarch64/abi.zig+20
......@@ -1,6 +1,8 @@
1const std = @import("std");
12const builtin = @import("builtin");
23const bits = @import("bits.zig");
34const Register = bits.Register;
5const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
46
57const callee_preserved_regs_impl = if (builtin.os.tag.isDarwin()) struct {
68 pub const callee_preserved_regs = [_]Register{
......@@ -18,3 +20,21 @@ pub const callee_preserved_regs = callee_preserved_regs_impl.callee_preserved_re
1820
1921pub const c_abi_int_param_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
2022pub const c_abi_int_return_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
23
24const allocatable_registers = callee_preserved_regs;
25pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
26
27// Register classes
28const RegisterBitSet = RegisterManager.RegisterBitSet;
29pub const RegisterClass = struct {
30 pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
31 // TODO uncomment once #11680 is fixed.
32 // pub const gp: RegisterBitSet = blk: {
33 // var set = RegisterBitSet.initEmpty();
34 // set.setRangeValue(.{
35 // .start = 0,
36 // .end = callee_preserved_regs.len,
37 // }, true);
38 // break :blk set;
39 // };
40};
src/arch/arm/CodeGen.zig+35-36
......@@ -21,9 +21,6 @@ const DW = std.dwarf;
2121const leb128 = std.leb;
2222const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
24const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
26const RegisterLock = RegisterManager.RegisterLock;
2724
2825const FnResult = @import("../../codegen.zig").FnResult;
2926const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
......@@ -31,14 +28,16 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
3128
3229const bits = @import("bits.zig");
3330const abi = @import("abi.zig");
31const RegisterManager = abi.RegisterManager;
32const RegisterLock = RegisterManager.RegisterLock;
3433const Register = bits.Register;
3534const Instruction = bits.Instruction;
3635const Condition = bits.Condition;
3736const callee_preserved_regs = abi.callee_preserved_regs;
3837const caller_preserved_regs = abi.caller_preserved_regs;
39const allocatable_registers = abi.allocatable_registers;
4038const c_abi_int_param_regs = abi.c_abi_int_param_regs;
4139const c_abi_int_return_regs = abi.c_abi_int_return_regs;
40const gp = abi.RegisterClass.gp;
4241
4342const InnerError = error{
4443 OutOfMemory,
......@@ -874,7 +873,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
874873 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
875874 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
876875 if (abi_size <= ptr_bytes) {
877 if (self.register_manager.tryAllocReg(inst)) |reg| {
876 if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
878877 return MCValue{ .register = reg };
879878 }
880879 }
......@@ -939,7 +938,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
939938/// allocated. A second call to `copyToTmpRegister` may return the same register.
940939/// This can have a side effect of spilling instructions to the stack to free up a register.
941940fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
942 const reg = try self.register_manager.allocReg(null);
941 const reg = try self.register_manager.allocReg(null, gp);
943942 try self.genSetReg(ty, reg, mcv);
944943 return reg;
945944}
......@@ -948,7 +947,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
948947/// `reg_owner` is the instruction that gets associated with the register in the register table.
949948/// This can have a side effect of spilling instructions to the stack to free up a register.
950949fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
951 const reg = try self.register_manager.allocReg(reg_owner);
950 const reg = try self.register_manager.allocReg(reg_owner, gp);
952951 try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
953952 return MCValue{ .register = reg };
954953}
......@@ -1065,9 +1064,9 @@ fn trunc(
10651064 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
10661065 break :blk operand_reg;
10671066 } else {
1068 break :blk try self.register_manager.allocReg(inst);
1067 break :blk try self.register_manager.allocReg(inst, gp);
10691068 }
1070 } else try self.register_manager.allocReg(null);
1069 } else try self.register_manager.allocReg(null, gp);
10711070
10721071 switch (info_b.bits) {
10731072 32 => {
......@@ -1153,7 +1152,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11531152 break :blk op_reg;
11541153 }
11551154
1156 break :blk try self.register_manager.allocReg(null);
1155 break :blk try self.register_manager.allocReg(null, gp);
11571156 };
11581157
11591158 _ = try self.addInst(.{
......@@ -1183,7 +1182,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11831182 break :blk op_reg;
11841183 }
11851184
1186 break :blk try self.register_manager.allocReg(null);
1185 break :blk try self.register_manager.allocReg(null, gp);
11871186 };
11881187
11891188 _ = try self.addInst(.{
......@@ -1254,9 +1253,9 @@ fn minMax(
12541253 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
12551254 break :blk rhs_reg;
12561255 } else {
1257 break :blk try self.register_manager.allocReg(inst);
1256 break :blk try self.register_manager.allocReg(inst, gp);
12581257 }
1259 } else try self.register_manager.allocReg(null);
1258 } else try self.register_manager.allocReg(null, gp);
12601259
12611260 // lhs == reg should have been checked by airMinMax
12621261 //
......@@ -1438,7 +1437,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
14381437 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
14391438 defer self.register_manager.unlockReg(dest_reg_lock);
14401439
1441 const truncated_reg = try self.register_manager.allocReg(null);
1440 const truncated_reg = try self.register_manager.allocReg(null, gp);
14421441 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
14431442 defer self.register_manager.unlockReg(truncated_reg_lock);
14441443
......@@ -1543,7 +1542,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15431542 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
15441543 defer self.register_manager.unlockReg(dest_reg_lock);
15451544
1546 const truncated_reg = try self.register_manager.allocReg(null);
1545 const truncated_reg = try self.register_manager.allocReg(null, gp);
15471546 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
15481547 defer self.register_manager.unlockReg(truncated_reg_lock);
15491548
......@@ -1582,18 +1581,18 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15821581 const lhs_reg = if (lhs_is_register)
15831582 lhs.register
15841583 else
1585 try self.register_manager.allocReg(null);
1584 try self.register_manager.allocReg(null, gp);
15861585 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
15871586 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
15881587
15891588 const rhs_reg = if (rhs_is_register)
15901589 rhs.register
15911590 else
1592 try self.register_manager.allocReg(null);
1591 try self.register_manager.allocReg(null, gp);
15931592 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
15941593 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
15951594
1596 const dest_regs = try self.register_manager.allocRegs(2, .{ null, null });
1595 const dest_regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
15971596 const dest_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dest_regs);
15981597 defer for (dest_regs_locks) |reg| {
15991598 self.register_manager.unlockReg(reg);
......@@ -1604,7 +1603,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
16041603 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
16051604 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
16061605
1607 const truncated_reg = try self.register_manager.allocReg(null);
1606 const truncated_reg = try self.register_manager.allocReg(null, gp);
16081607 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
16091608 defer self.register_manager.unlockReg(truncated_reg_lock);
16101609
......@@ -2026,7 +2025,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
20262025 const base_reg_lock = self.register_manager.lockRegAssumeUnused(base_reg);
20272026 defer self.register_manager.unlockReg(base_reg_lock);
20282027
2029 const dst_reg = try self.register_manager.allocReg(inst);
2028 const dst_reg = try self.register_manager.allocReg(inst, gp);
20302029 const dst_mcv = MCValue{ .register = dst_reg };
20312030 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
20322031 defer self.register_manager.unlockReg(dst_reg_lock);
......@@ -2234,7 +2233,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
22342233 },
22352234 .stack_offset => |off| {
22362235 if (elem_size <= 4) {
2237 const tmp_reg = try self.register_manager.allocReg(null);
2236 const tmp_reg = try self.register_manager.allocReg(null, gp);
22382237 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
22392238 defer self.register_manager.unlockReg(tmp_reg_lock);
22402239
......@@ -2242,7 +2241,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
22422241 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
22432242 } else {
22442243 // TODO optimize the register allocation
2245 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2244 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
22462245 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
22472246 defer for (regs_locks) |reg_locked| {
22482247 self.register_manager.unlockReg(reg_locked);
......@@ -2271,7 +2270,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
22712270 .stack_offset,
22722271 .stack_argument_offset,
22732272 => {
2274 const reg = try self.register_manager.allocReg(null);
2273 const reg = try self.register_manager.allocReg(null, gp);
22752274 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
22762275 defer self.register_manager.unlockReg(reg_lock);
22772276
......@@ -2338,14 +2337,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
23382337 },
23392338 else => {
23402339 if (elem_size <= 4) {
2341 const tmp_reg = try self.register_manager.allocReg(null);
2340 const tmp_reg = try self.register_manager.allocReg(null, gp);
23422341 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
23432342 defer self.register_manager.unlockReg(tmp_reg_lock);
23442343
23452344 try self.genSetReg(value_ty, tmp_reg, value);
23462345 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
23472346 } else {
2348 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2347 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
23492348 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
23502349 defer for (regs_locks) |reg| {
23512350 self.register_manager.unlockReg(reg);
......@@ -2487,7 +2486,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
24872486 1 => {
24882487 // get overflow bit: set register to C flag
24892488 // resp. V flag
2490 const dest_reg = try self.register_manager.allocReg(null);
2489 const dest_reg = try self.register_manager.allocReg(null, gp);
24912490
24922491 // mov reg, #0
24932492 _ = try self.addInst(.{
......@@ -2567,7 +2566,7 @@ fn binOpRegister(
25672566 break :inst Air.refToIndex(md.lhs).?;
25682567 } else null;
25692568
2570 const reg = try self.register_manager.allocReg(track_inst);
2569 const reg = try self.register_manager.allocReg(track_inst, gp);
25712570
25722571 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
25732572
......@@ -2581,7 +2580,7 @@ fn binOpRegister(
25812580 break :inst Air.refToIndex(md.rhs).?;
25822581 } else null;
25832582
2584 const reg = try self.register_manager.allocReg(track_inst);
2583 const reg = try self.register_manager.allocReg(track_inst, gp);
25852584
25862585 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
25872586
......@@ -2598,9 +2597,9 @@ fn binOpRegister(
25982597 } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) {
25992598 break :blk rhs_reg;
26002599 } else {
2601 break :blk try self.register_manager.allocReg(md.inst);
2600 break :blk try self.register_manager.allocReg(md.inst, gp);
26022601 }
2603 } else try self.register_manager.allocReg(null),
2602 } else try self.register_manager.allocReg(null, gp),
26042603 };
26052604
26062605 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
......@@ -2684,7 +2683,7 @@ fn binOpImmediate(
26842683 ).?;
26852684 } else null;
26862685
2687 const reg = try self.register_manager.allocReg(track_inst);
2686 const reg = try self.register_manager.allocReg(track_inst, gp);
26882687
26892688 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
26902689
......@@ -2704,9 +2703,9 @@ fn binOpImmediate(
27042703 )) {
27052704 break :blk lhs_reg;
27062705 } else {
2707 break :blk try self.register_manager.allocReg(md.inst);
2706 break :blk try self.register_manager.allocReg(md.inst, gp);
27082707 }
2709 } else try self.register_manager.allocReg(null),
2708 } else try self.register_manager.allocReg(null, gp),
27102709 };
27112710
27122711 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
......@@ -4363,7 +4362,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
43634362
43644363 const overflow_bit_ty = ty.structFieldType(1);
43654364 const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*));
4366 const cond_reg = try self.register_manager.allocReg(null);
4365 const cond_reg = try self.register_manager.allocReg(null, gp);
43674366
43684367 // C flag: movcs reg, #1
43694368 // V flag: movvs reg, #1
......@@ -4408,7 +4407,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
44084407 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
44094408
44104409 // TODO call extern memcpy
4411 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null });
4410 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
44124411 const src_reg = regs[0];
44134412 const dst_reg = regs[1];
44144413 const len_reg = regs[2];
......@@ -4782,7 +4781,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I
47824781 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
47834782
47844783 // TODO call extern memcpy
4785 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null });
4784 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
47864785 const src_reg = regs[0];
47874786 const dst_reg = regs[1];
47884787 const len_reg = regs[2];
src/arch/arm/abi.zig+20-1
......@@ -1,9 +1,28 @@
1const std = @import("std");
12const bits = @import("bits.zig");
23const Register = bits.Register;
4const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
35
46pub const callee_preserved_regs = [_]Register{ .r4, .r5, .r6, .r7, .r8, .r10 };
57pub const caller_preserved_regs = [_]Register{ .r0, .r1, .r2, .r3 };
6pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
78
89pub const c_abi_int_param_regs = [_]Register{ .r0, .r1, .r2, .r3 };
910pub const c_abi_int_return_regs = [_]Register{ .r0, .r1 };
11
12const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
13pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
14
15// Register classes
16const RegisterBitSet = RegisterManager.RegisterBitSet;
17pub const RegisterClass = struct {
18 pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
19 // TODO uncomment once #11680 is fixed.
20 // pub const gp: RegisterBitSet = blk: {
21 // var set = RegisterBitSet.initEmpty();
22 // set.setRangeValue(.{
23 // .start = 0,
24 // .end = caller_preserved_regs.len + callee_preserved_regs.len,
25 // }, true);
26 // break :blk set;
27 // };
28};
src/arch/riscv64/CodeGen.zig+11-11
......@@ -21,9 +21,6 @@ const DW = std.dwarf;
2121const leb128 = std.leb;
2222const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
24const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
26const RegisterLock = RegisterManager.RegisterLock;
2724
2825const FnResult = @import("../../codegen.zig").FnResult;
2926const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
......@@ -32,8 +29,11 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
3229const bits = @import("bits.zig");
3330const abi = @import("abi.zig");
3431const Register = bits.Register;
32const RegisterManager = abi.RegisterManager;
33const RegisterLock = RegisterManager.RegisterLock;
3534const Instruction = abi.Instruction;
3635const callee_preserved_regs = abi.callee_preserved_regs;
36const gp = abi.RegisterClass.gp;
3737
3838const InnerError = error{
3939 OutOfMemory,
......@@ -803,7 +803,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
803803 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
804804 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
805805 if (abi_size <= ptr_bytes) {
806 if (self.register_manager.tryAllocReg(inst)) |reg| {
806 if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
807807 return MCValue{ .register = reg };
808808 }
809809 }
......@@ -826,7 +826,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
826826/// allocated. A second call to `copyToTmpRegister` may return the same register.
827827/// This can have a side effect of spilling instructions to the stack to free up a register.
828828fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
829 const reg = try self.register_manager.allocReg(null);
829 const reg = try self.register_manager.allocReg(null, gp);
830830 try self.genSetReg(ty, reg, mcv);
831831 return reg;
832832}
......@@ -835,7 +835,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
835835/// `reg_owner` is the instruction that gets associated with the register in the register table.
836836/// This can have a side effect of spilling instructions to the stack to free up a register.
837837fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
838 const reg = try self.register_manager.allocReg(reg_owner);
838 const reg = try self.register_manager.allocReg(reg_owner, gp);
839839 try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
840840 return MCValue{ .register = reg };
841841}
......@@ -958,7 +958,7 @@ fn binOpRegister(
958958 break :inst Air.refToIndex(bin_op.lhs).?;
959959 } else null;
960960
961 const reg = try self.register_manager.allocReg(track_inst);
961 const reg = try self.register_manager.allocReg(track_inst, gp);
962962
963963 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
964964
......@@ -973,7 +973,7 @@ fn binOpRegister(
973973 break :inst Air.refToIndex(bin_op.rhs).?;
974974 } else null;
975975
976 const reg = try self.register_manager.allocReg(track_inst);
976 const reg = try self.register_manager.allocReg(track_inst, gp);
977977
978978 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
979979
......@@ -990,9 +990,9 @@ fn binOpRegister(
990990 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
991991 break :blk rhs_reg;
992992 } else {
993 break :blk try self.register_manager.allocReg(inst);
993 break :blk try self.register_manager.allocReg(inst, gp);
994994 }
995 } else try self.register_manager.allocReg(null);
995 } else try self.register_manager.allocReg(null, gp);
996996
997997 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
998998 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
......@@ -1482,7 +1482,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
14821482 .memory,
14831483 .stack_offset,
14841484 => {
1485 const reg = try self.register_manager.allocReg(null);
1485 const reg = try self.register_manager.allocReg(null, gp);
14861486 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
14871487 defer self.register_manager.unlockReg(reg_lock);
14881488
src/arch/riscv64/abi.zig+20
......@@ -1,6 +1,26 @@
1const std = @import("std");
12const bits = @import("bits.zig");
23const Register = bits.Register;
4const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
35
46pub const callee_preserved_regs = [_]Register{
57 .s0, .s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11,
68};
9
10const allocatable_registers = callee_preserved_regs;
11pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
12
13// Register classes
14const RegisterBitSet = RegisterManager.RegisterBitSet;
15pub const RegisterClass = struct {
16 pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
17 // TODO uncomment once #11680 is fixed.
18 // pub const gp: RegisterBitSet = blk: {
19 // var set = RegisterBitSet.initEmpty();
20 // set.setRangeValue(.{
21 // .start = 0,
22 // .end = callee_preserved_regs.len,
23 // }, true);
24 // break :blk set;
25 // };
26};
src/arch/sparc64/CodeGen.zig+16-16
......@@ -21,9 +21,6 @@ const Type = @import("../../type.zig").Type;
2121const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
2222const FnResult = @import("../../codegen.zig").FnResult;
2323const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
24const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25const RegisterManager = RegisterManagerFn(Self, Register, &abi.allocatable_regs);
26const RegisterLock = RegisterManager.RegisterLock;
2724
2825const build_options = @import("build_options");
2926
......@@ -31,7 +28,10 @@ const bits = @import("bits.zig");
3128const abi = @import("abi.zig");
3229const Instruction = bits.Instruction;
3330const ShiftWidth = Instruction.ShiftWidth;
31const RegisterManager = abi.RegisterManager;
32const RegisterLock = RegisterManager.RegisterLock;
3433const Register = bits.Register;
34const gp = abi.RegisterClass.gp;
3535
3636const Self = @This();
3737
......@@ -1613,7 +1613,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
16131613 if (reg_ok) {
16141614 // Make sure the type can fit in a register before we try to allocate one.
16151615 if (abi_size <= 8) {
1616 if (self.register_manager.tryAllocReg(inst)) |reg| {
1616 if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
16171617 return MCValue{ .register = reg };
16181618 }
16191619 }
......@@ -1854,7 +1854,7 @@ fn binOpImmediate(
18541854 ).?;
18551855 } else null;
18561856
1857 const reg = try self.register_manager.allocReg(track_inst);
1857 const reg = try self.register_manager.allocReg(track_inst, gp);
18581858
18591859 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
18601860
......@@ -1873,10 +1873,10 @@ fn binOpImmediate(
18731873 )) {
18741874 break :blk lhs_reg;
18751875 } else {
1876 break :blk try self.register_manager.allocReg(md.inst);
1876 break :blk try self.register_manager.allocReg(md.inst, gp);
18771877 }
18781878 } else blk: {
1879 break :blk try self.register_manager.allocReg(null);
1879 break :blk try self.register_manager.allocReg(null, gp);
18801880 },
18811881 };
18821882
......@@ -1953,7 +1953,7 @@ fn binOpRegister(
19531953 break :inst Air.refToIndex(md.lhs).?;
19541954 } else null;
19551955
1956 const reg = try self.register_manager.allocReg(track_inst);
1956 const reg = try self.register_manager.allocReg(track_inst, gp);
19571957 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
19581958
19591959 break :blk reg;
......@@ -1966,7 +1966,7 @@ fn binOpRegister(
19661966 break :inst Air.refToIndex(md.rhs).?;
19671967 } else null;
19681968
1969 const reg = try self.register_manager.allocReg(track_inst);
1969 const reg = try self.register_manager.allocReg(track_inst, gp);
19701970 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
19711971
19721972 break :blk reg;
......@@ -1981,10 +1981,10 @@ fn binOpRegister(
19811981 } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) {
19821982 break :blk rhs_reg;
19831983 } else {
1984 break :blk try self.register_manager.allocReg(md.inst);
1984 break :blk try self.register_manager.allocReg(md.inst, gp);
19851985 }
19861986 } else blk: {
1987 break :blk try self.register_manager.allocReg(null);
1987 break :blk try self.register_manager.allocReg(null, gp);
19881988 },
19891989 };
19901990
......@@ -2077,7 +2077,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
20772077/// allocated. A second call to `copyToTmpRegister` may return the same register.
20782078/// This can have a side effect of spilling instructions to the stack to free up a register.
20792079fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
2080 const reg = try self.register_manager.allocReg(null);
2080 const reg = try self.register_manager.allocReg(null, gp);
20812081 try self.genSetReg(ty, reg, mcv);
20822082 return reg;
20832083}
......@@ -2364,7 +2364,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
23642364 });
23652365 } else {
23662366 // Need to allocate a temporary register to load 64-bit immediates.
2367 const tmp_reg = try self.register_manager.allocReg(null);
2367 const tmp_reg = try self.register_manager.allocReg(null, gp);
23682368
23692369 try self.genSetReg(ty, tmp_reg, .{ .immediate = @truncate(u32, x) });
23702370 try self.genSetReg(ty, reg, .{ .immediate = @truncate(u32, x >> 32) });
......@@ -2478,7 +2478,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
24782478 };
24792479 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
24802480
2481 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2481 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
24822482 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
24832483 defer for (regs_locks) |reg| {
24842484 self.register_manager.unlockReg(reg);
......@@ -2717,14 +2717,14 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
27172717 },
27182718 .stack_offset => |off| {
27192719 if (elem_size <= 8) {
2720 const tmp_reg = try self.register_manager.allocReg(null);
2720 const tmp_reg = try self.register_manager.allocReg(null, gp);
27212721 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
27222722 defer self.register_manager.unlockReg(tmp_reg_lock);
27232723
27242724 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
27252725 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
27262726 } else {
2727 const regs = try self.register_manager.allocRegs(3, .{ null, null, null });
2727 const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
27282728 const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs);
27292729 defer for (regs_locks) |reg| {
27302730 self.register_manager.unlockReg(reg);
src/arch/sparc64/abi.zig+20-1
......@@ -1,5 +1,7 @@
1const std = @import("std");
12const bits = @import("bits.zig");
23const Register = bits.Register;
4const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
35
46// SPARCv9 stack constants.
57// See: Registers and the Stack Frame, page 3P-8, SCD 2.4.1.
......@@ -21,7 +23,7 @@ pub const stack_save_area = 176;
2123pub const caller_preserved_regs = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5, .g1, .g4, .g5 };
2224
2325// Try to allocate i, l, o, then g sets of registers, in order of priority.
24pub const allocatable_regs = [_]Register{
26const allocatable_regs = [_]Register{
2527 // zig fmt: off
2628 .@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5",
2729 .l0, .l1, .l2, .l3, .l4, .l5, .l6, .l7,
......@@ -35,3 +37,20 @@ pub const c_abi_int_param_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2"
3537
3638pub const c_abi_int_return_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3 };
3739pub const c_abi_int_return_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3" };
40
41pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_regs);
42
43// Register classes
44const RegisterBitSet = RegisterManager.RegisterBitSet;
45pub const RegisterClass = struct {
46 pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet);
47 // TODO uncomment once #11680 is fixed.
48 // pub const gp: RegisterBitSet = blk: {
49 // var set = RegisterBitSet.initEmpty();
50 // set.setRangeValue(.{
51 // .start = 0,
52 // .end = allocatable_regs.len,
53 // }, true);
54 // break :blk set;
55 // };
56};
src/arch/x86_64/CodeGen.zig+673-434
......@@ -21,9 +21,6 @@ const Emit = @import("Emit.zig");
2121const Liveness = @import("../../Liveness.zig");
2222const Mir = @import("Mir.zig");
2323const Module = @import("../../Module.zig");
24const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
26const RegisterLock = RegisterManager.RegisterLock;
2724const Target = std.Target;
2825const Type = @import("../../type.zig").Type;
2926const TypedValue = @import("../../TypedValue.zig");
......@@ -31,13 +28,19 @@ const Value = @import("../../value.zig").Value;
3128
3229const bits = @import("bits.zig");
3330const abi = @import("abi.zig");
34const Register = bits.Register;
31
3532const callee_preserved_regs = abi.callee_preserved_regs;
3633const caller_preserved_regs = abi.caller_preserved_regs;
37const allocatable_registers = abi.allocatable_registers;
3834const c_abi_int_param_regs = abi.c_abi_int_param_regs;
3935const c_abi_int_return_regs = abi.c_abi_int_return_regs;
4036
37const RegisterManager = abi.RegisterManager;
38const RegisterLock = RegisterManager.RegisterLock;
39const Register = bits.Register;
40
41const gp = abi.RegisterClass.gp;
42const sse = abi.RegisterClass.sse;
43
4144const InnerError = error{
4245 OutOfMemory,
4346 CodegenFail,
......@@ -119,12 +122,12 @@ pub const MCValue = union(enum) {
119122 /// A pointer-sized integer that fits in a register.
120123 /// If the type is a pointer, this is the pointer address in virtual address space.
121124 immediate: u64,
122 /// The value is in a target-specific register.
125 /// The value is in a GP register.
123126 register: Register,
124 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the register,
127 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register,
125128 /// and the operation is an unsigned operation.
126129 register_overflow_unsigned: Register,
127 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the register,
130 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register,
128131 /// and the operation is a signed operation.
129132 register_overflow_signed: Register,
130133 /// The value is in memory at a hard-coded address.
......@@ -387,17 +390,15 @@ fn gen(self: *Self) InnerError!void {
387390 if (cc != .Naked) {
388391 _ = try self.addInst(.{
389392 .tag = .push,
390 .ops = (Mir.Ops{
391 .reg1 = .rbp,
392 }).encode(),
393 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
393394 .data = undefined, // unused for push reg,
394395 });
395396 _ = try self.addInst(.{
396397 .tag = .mov,
397 .ops = (Mir.Ops{
398 .ops = Mir.Inst.Ops.encode(.{
398399 .reg1 = .rbp,
399400 .reg2 = .rsp,
400 }).encode(),
401 }),
401402 .data = undefined,
402403 });
403404 // We want to subtract the aligned stack frame size from rsp here, but we don't
......@@ -434,9 +435,7 @@ fn gen(self: *Self) InnerError!void {
434435 // push the callee_preserved_regs that were used
435436 const backpatch_push_callee_preserved_regs_i = try self.addInst(.{
436437 .tag = .push_regs_from_callee_preserved_regs,
437 .ops = (Mir.Ops{
438 .reg1 = .rbp,
439 }).encode(),
438 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
440439 .data = .{ .payload = undefined }, // to be backpatched
441440 });
442441
......@@ -476,9 +475,7 @@ fn gen(self: *Self) InnerError!void {
476475 // pop the callee_preserved_regs
477476 _ = try self.addInst(.{
478477 .tag = .pop_regs_from_callee_preserved_regs,
479 .ops = (Mir.Ops{
480 .reg1 = .rbp,
481 }).encode(),
478 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
482479 .data = .{ .payload = callee_preserved_regs_payload },
483480 });
484481
......@@ -497,17 +494,13 @@ fn gen(self: *Self) InnerError!void {
497494
498495 _ = try self.addInst(.{
499496 .tag = .pop,
500 .ops = (Mir.Ops{
501 .reg1 = .rbp,
502 }).encode(),
497 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
503498 .data = undefined,
504499 });
505500
506501 _ = try self.addInst(.{
507502 .tag = .ret,
508 .ops = (Mir.Ops{
509 .flags = 0b11,
510 }).encode(),
503 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
511504 .data = undefined,
512505 });
513506
......@@ -521,16 +514,12 @@ fn gen(self: *Self) InnerError!void {
521514 if (aligned_stack_end > 0) {
522515 self.mir_instructions.set(backpatch_stack_sub, .{
523516 .tag = .sub,
524 .ops = (Mir.Ops{
525 .reg1 = .rsp,
526 }).encode(),
517 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
527518 .data = .{ .imm = aligned_stack_end },
528519 });
529520 self.mir_instructions.set(backpatch_stack_add, .{
530521 .tag = .add,
531 .ops = (Mir.Ops{
532 .reg1 = .rsp,
533 }).encode(),
522 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
534523 .data = .{ .imm = aligned_stack_end },
535524 });
536525 }
......@@ -889,13 +878,30 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
889878 self.stack_align = abi_align;
890879
891880 if (reg_ok) {
892 // Make sure the type can fit in a register before we try to allocate one.
893 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
894 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
895 if (abi_size <= ptr_bytes) {
896 if (self.register_manager.tryAllocReg(inst)) |reg| {
897 return MCValue{ .register = registerAlias(reg, abi_size) };
898 }
881 switch (elem_ty.zigTypeTag()) {
882 .Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}),
883 .Float => {
884 if (intrinsicsAllowed(self.target.*, elem_ty)) {
885 const ptr_bytes: u64 = 32;
886 if (abi_size <= ptr_bytes) {
887 if (self.register_manager.tryAllocReg(inst, sse)) |reg| {
888 return MCValue{ .register = registerAlias(reg, abi_size) };
889 }
890 }
891 }
892
893 return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{});
894 },
895 else => {
896 // Make sure the type can fit in a register before we try to allocate one.
897 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
898 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
899 if (abi_size <= ptr_bytes) {
900 if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
901 return MCValue{ .register = registerAlias(reg, abi_size) };
902 }
903 }
904 },
899905 }
900906 }
901907 const stack_offset = try self.allocMem(inst, abi_size, abi_align);
......@@ -962,7 +968,14 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou
962968/// allocated. A second call to `copyToTmpRegister` may return the same register.
963969/// This can have a side effect of spilling instructions to the stack to free up a register.
964970fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
965 const reg = try self.register_manager.allocReg(null);
971 const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
972 .Float => blk: {
973 if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
974 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
975 },
976 else => gp,
977 };
978 const reg: Register = try self.register_manager.allocReg(null, reg_class);
966979 try self.genSetReg(ty, reg, mcv);
967980 return reg;
968981}
......@@ -972,7 +985,14 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
972985/// This can have a side effect of spilling instructions to the stack to free up a register.
973986/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
974987fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {
975 const reg = try self.register_manager.allocReg(reg_owner);
988 const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
989 .Float => blk: {
990 if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
991 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
992 },
993 else => gp,
994 };
995 const reg: Register = try self.register_manager.allocReg(reg_owner, reg_class);
976996 try self.genSetReg(ty, reg, mcv);
977997 return MCValue{ .register = reg };
978998}
......@@ -1028,7 +1048,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
10281048 };
10291049 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
10301050
1031 const reg = try self.register_manager.allocReg(inst);
1051 const reg = try self.register_manager.allocReg(inst, gp);
10321052 try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });
10331053 try self.genSetReg(operand_ty, reg, operand);
10341054 break :blk MCValue{ .register = reg };
......@@ -1192,10 +1212,10 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
11921212 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv);
11931213 _ = try self.addInst(.{
11941214 .tag = if (signedness == .signed) .cond_mov_lt else .cond_mov_below,
1195 .ops = (Mir.Ops{
1215 .ops = Mir.Inst.Ops.encode(.{
11961216 .reg1 = dst_mcv.register,
11971217 .reg2 = lhs_reg,
1198 }).encode(),
1218 }),
11991219 .data = undefined,
12001220 });
12011221
......@@ -1383,7 +1403,7 @@ fn genSetStackTruncatedOverflowCompare(
13831403 .unsigned => ty,
13841404 };
13851405
1386 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null });
1406 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
13871407 const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
13881408 defer for (temp_regs_locks) |rreg| {
13891409 self.register_manager.unlockReg(rreg);
......@@ -1396,10 +1416,10 @@ fn genSetStackTruncatedOverflowCompare(
13961416 };
13971417 _ = try self.addInst(.{
13981418 .tag = .cond_set_byte_overflow,
1399 .ops = (Mir.Ops{
1419 .ops = Mir.Inst.Ops.encode(.{
14001420 .reg1 = overflow_reg.to8(),
14011421 .flags = flags,
1402 }).encode(),
1422 }),
14031423 .data = undefined,
14041424 });
14051425
......@@ -1416,10 +1436,7 @@ fn genSetStackTruncatedOverflowCompare(
14161436 const eq_reg = temp_regs[2];
14171437 _ = try self.addInst(.{
14181438 .tag = .cond_set_byte_eq_ne,
1419 .ops = (Mir.Ops{
1420 .reg1 = eq_reg.to8(),
1421 .flags = 0b00,
1422 }).encode(),
1439 .ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }),
14231440 .data = undefined,
14241441 });
14251442
......@@ -1565,19 +1582,17 @@ fn genIntMulDivOpMir(
15651582 .signed => {
15661583 _ = try self.addInst(.{
15671584 .tag = .cwd,
1568 .ops = (Mir.Ops{
1569 .flags = 0b11,
1570 }).encode(),
1585 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
15711586 .data = undefined,
15721587 });
15731588 },
15741589 .unsigned => {
15751590 _ = try self.addInst(.{
15761591 .tag = .xor,
1577 .ops = (Mir.Ops{
1592 .ops = Mir.Inst.Ops.encode(.{
15781593 .reg1 = .rdx,
15791594 .reg2 = .rdx,
1580 }).encode(),
1595 }),
15811596 .data = undefined,
15821597 });
15831598 },
......@@ -1596,16 +1611,14 @@ fn genIntMulDivOpMir(
15961611 .register => |reg| {
15971612 _ = try self.addInst(.{
15981613 .tag = tag,
1599 .ops = (Mir.Ops{
1600 .reg1 = reg,
1601 }).encode(),
1614 .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
16021615 .data = undefined,
16031616 });
16041617 },
16051618 .stack_offset => |off| {
16061619 _ = try self.addInst(.{
16071620 .tag = tag,
1608 .ops = (Mir.Ops{
1621 .ops = Mir.Inst.Ops.encode(.{
16091622 .reg2 = .rbp,
16101623 .flags = switch (abi_size) {
16111624 1 => 0b00,
......@@ -1614,7 +1627,7 @@ fn genIntMulDivOpMir(
16141627 8 => 0b11,
16151628 else => unreachable,
16161629 },
1617 }).encode(),
1630 }),
16181631 .data = .{ .imm = @bitCast(u32, -off) },
16191632 });
16201633 },
......@@ -1647,34 +1660,34 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
16471660
16481661 _ = try self.addInst(.{
16491662 .tag = .xor,
1650 .ops = (Mir.Ops{
1663 .ops = Mir.Inst.Ops.encode(.{
16511664 .reg1 = divisor.to64(),
16521665 .reg2 = dividend.to64(),
1653 }).encode(),
1666 }),
16541667 .data = undefined,
16551668 });
16561669 _ = try self.addInst(.{
16571670 .tag = .sar,
1658 .ops = (Mir.Ops{
1671 .ops = Mir.Inst.Ops.encode(.{
16591672 .reg1 = divisor.to64(),
16601673 .flags = 0b10,
1661 }).encode(),
1674 }),
16621675 .data = .{ .imm = 63 },
16631676 });
16641677 _ = try self.addInst(.{
16651678 .tag = .@"test",
1666 .ops = (Mir.Ops{
1679 .ops = Mir.Inst.Ops.encode(.{
16671680 .reg1 = .rdx,
16681681 .reg2 = .rdx,
1669 }).encode(),
1682 }),
16701683 .data = undefined,
16711684 });
16721685 _ = try self.addInst(.{
16731686 .tag = .cond_mov_eq,
1674 .ops = (Mir.Ops{
1687 .ops = Mir.Inst.Ops.encode(.{
16751688 .reg1 = divisor.to64(),
16761689 .reg2 = .rdx,
1677 }).encode(),
1690 }),
16781691 .data = undefined,
16791692 });
16801693 try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
......@@ -2052,17 +2065,17 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
20522065 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
20532066 defer self.register_manager.unlockReg(offset_reg_lock);
20542067
2055 const addr_reg = try self.register_manager.allocReg(null);
2068 const addr_reg = try self.register_manager.allocReg(null, gp);
20562069 switch (slice_mcv) {
20572070 .stack_offset => |off| {
20582071 // mov reg, [rbp - 8]
20592072 _ = try self.addInst(.{
20602073 .tag = .mov,
2061 .ops = (Mir.Ops{
2074 .ops = Mir.Inst.Ops.encode(.{
20622075 .reg1 = addr_reg.to64(),
20632076 .reg2 = .rbp,
20642077 .flags = 0b01,
2065 }).encode(),
2078 }),
20662079 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
20672080 });
20682081 },
......@@ -2131,7 +2144,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
21312144 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
21322145 defer self.register_manager.unlockReg(offset_reg_lock);
21332146
2134 const addr_reg = try self.register_manager.allocReg(null);
2147 const addr_reg = try self.register_manager.allocReg(null, gp);
21352148 switch (array) {
21362149 .register => {
21372150 const off = @intCast(i32, try self.allocMem(
......@@ -2143,10 +2156,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
21432156 // lea reg, [rbp]
21442157 _ = try self.addInst(.{
21452158 .tag = .lea,
2146 .ops = (Mir.Ops{
2159 .ops = Mir.Inst.Ops.encode(.{
21472160 .reg1 = addr_reg.to64(),
21482161 .reg2 = .rbp,
2149 }).encode(),
2162 }),
21502163 .data = .{ .imm = @bitCast(u32, -off) },
21512164 });
21522165 },
......@@ -2154,10 +2167,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
21542167 // lea reg, [rbp]
21552168 _ = try self.addInst(.{
21562169 .tag = .lea,
2157 .ops = (Mir.Ops{
2170 .ops = Mir.Inst.Ops.encode(.{
21582171 .reg1 = addr_reg.to64(),
21592172 .reg2 = .rbp,
2160 }).encode(),
2173 }),
21612174 .data = .{ .imm = @bitCast(u32, -off) },
21622175 });
21632176 },
......@@ -2222,11 +2235,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
22222235 // mov dst_mcv, [dst_mcv]
22232236 _ = try self.addInst(.{
22242237 .tag = .mov,
2225 .ops = (Mir.Ops{
2226 .flags = 0b01,
2238 .ops = Mir.Inst.Ops.encode(.{
22272239 .reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),
22282240 .reg2 = dst_mcv.register,
2229 }).encode(),
2241 .flags = 0b01,
2242 }),
22302243 .data = .{ .imm = 0 },
22312244 });
22322245 break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
......@@ -2488,17 +2501,17 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
24882501 // mov dst_reg, [reg]
24892502 _ = try self.addInst(.{
24902503 .tag = .mov,
2491 .ops = (Mir.Ops{
2504 .ops = Mir.Inst.Ops.encode(.{
24922505 .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
24932506 .reg2 = reg,
24942507 .flags = 0b01,
2495 }).encode(),
2508 }),
24962509 .data = .{ .imm = 0 },
24972510 });
24982511 },
24992512 .stack_offset => |off| {
25002513 if (abi_size <= 8) {
2501 const tmp_reg = try self.register_manager.allocReg(null);
2514 const tmp_reg = try self.register_manager.allocReg(null, gp);
25022515 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
25032516 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
25042517 }
......@@ -2559,10 +2572,10 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
25592572 const fn_owner_decl = mod.declPtr(self.mod_fn.owner_decl);
25602573 _ = try self.addInst(.{
25612574 .tag = .lea_pie,
2562 .ops = (Mir.Ops{
2575 .ops = Mir.Inst.Ops.encode(.{
25632576 .reg1 = registerAlias(reg, abi_size),
25642577 .flags = flags,
2565 }).encode(),
2578 }),
25662579 .data = .{
25672580 .load_reloc = .{
25682581 .atom_index = fn_owner_decl.link.macho.local_sym_index,
......@@ -2623,7 +2636,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
26232636 });
26242637 _ = try self.addInst(.{
26252638 .tag = .mov_mem_imm,
2626 .ops = (Mir.Ops{
2639 .ops = Mir.Inst.Ops.encode(.{
26272640 .reg1 = reg.to64(),
26282641 .flags = switch (abi_size) {
26292642 1 => 0b00,
......@@ -2631,7 +2644,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
26312644 4 => 0b10,
26322645 else => unreachable,
26332646 },
2634 }).encode(),
2647 }),
26352648 .data = .{ .payload = payload },
26362649 });
26372650 },
......@@ -2645,11 +2658,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
26452658 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
26462659 _ = try self.addInst(.{
26472660 .tag = .mov,
2648 .ops = (Mir.Ops{
2661 .ops = Mir.Inst.Ops.encode(.{
26492662 .reg1 = reg.to64(),
26502663 .reg2 = tmp_reg.to64(),
26512664 .flags = 0b10,
2652 }).encode(),
2665 }),
26532666 .data = .{ .imm = 0 },
26542667 });
26552668 },
......@@ -2661,11 +2674,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
26612674 .register => |src_reg| {
26622675 _ = try self.addInst(.{
26632676 .tag = .mov,
2664 .ops = (Mir.Ops{
2677 .ops = Mir.Inst.Ops.encode(.{
26652678 .reg1 = reg.to64(),
26662679 .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
26672680 .flags = 0b10,
2668 }).encode(),
2681 }),
26692682 .data = .{ .imm = 0 },
26702683 });
26712684 },
......@@ -2699,7 +2712,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
26992712 };
27002713 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
27012714
2702 const addr_reg = try self.register_manager.allocReg(null);
2715 const addr_reg = try self.register_manager.allocReg(null, gp);
27032716 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
27042717 defer self.register_manager.unlockReg(addr_reg_lock);
27052718
......@@ -2709,11 +2722,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27092722 // mov reg, [reg]
27102723 _ = try self.addInst(.{
27112724 .tag = .mov,
2712 .ops = (Mir.Ops{
2725 .ops = Mir.Inst.Ops.encode(.{
27132726 .reg1 = addr_reg.to64(),
27142727 .reg2 = addr_reg.to64(),
27152728 .flags = 0b01,
2716 }).encode(),
2729 }),
27172730 .data = .{ .imm = 0 },
27182731 });
27192732
......@@ -2748,21 +2761,21 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27482761 }
27492762 _ = try self.addInst(.{
27502763 .tag = .mov_mem_imm,
2751 .ops = (Mir.Ops{
2764 .ops = Mir.Inst.Ops.encode(.{
27522765 .reg1 = addr_reg.to64(),
27532766 .flags = flags,
2754 }).encode(),
2767 }),
27552768 .data = .{ .payload = payload },
27562769 });
27572770 },
27582771 .register => |reg| {
27592772 _ = try self.addInst(.{
27602773 .tag = .mov,
2761 .ops = (Mir.Ops{
2774 .ops = Mir.Inst.Ops.encode(.{
27622775 .reg1 = addr_reg.to64(),
27632776 .reg2 = reg,
27642777 .flags = 0b10,
2765 }).encode(),
2778 }),
27662779 .data = .{ .imm = 0 },
27672780 });
27682781 },
......@@ -2771,7 +2784,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27712784 .memory,
27722785 => {
27732786 if (abi_size <= 8) {
2774 const tmp_reg = try self.register_manager.allocReg(null);
2787 const tmp_reg = try self.register_manager.allocReg(null, gp);
27752788 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
27762789 defer self.register_manager.unlockReg(tmp_reg_lock);
27772790
......@@ -2779,20 +2792,20 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27792792
27802793 _ = try self.addInst(.{
27812794 .tag = .mov,
2782 .ops = (Mir.Ops{
2795 .ops = Mir.Inst.Ops.encode(.{
27832796 .reg1 = tmp_reg,
27842797 .reg2 = tmp_reg,
27852798 .flags = 0b01,
2786 }).encode(),
2799 }),
27872800 .data = .{ .imm = 0 },
27882801 });
27892802 _ = try self.addInst(.{
27902803 .tag = .mov,
2791 .ops = (Mir.Ops{
2804 .ops = Mir.Inst.Ops.encode(.{
27922805 .reg1 = addr_reg.to64(),
27932806 .reg2 = tmp_reg,
27942807 .flags = 0b10,
2795 }).encode(),
2808 }),
27962809 .data = .{ .imm = 0 },
27972810 });
27982811 return;
......@@ -2806,11 +2819,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
28062819 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
28072820 _ = try self.addInst(.{
28082821 .tag = .mov,
2809 .ops = (Mir.Ops{
2822 .ops = Mir.Inst.Ops.encode(.{
28102823 .reg1 = addr_reg.to64(),
28112824 .reg2 = tmp_reg,
28122825 .flags = 0b10,
2813 }).encode(),
2826 }),
28142827 .data = .{ .imm = 0 },
28152828 });
28162829 return;
......@@ -2889,7 +2902,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
28892902 if (can_reuse_operand) {
28902903 break :blk reg;
28912904 } else {
2892 const result_reg = try self.register_manager.allocReg(inst);
2905 const result_reg = try self.register_manager.allocReg(inst, gp);
28932906 try self.genSetReg(ptr_ty, result_reg, mcv);
28942907 break :blk result_reg;
28952908 }
......@@ -2967,10 +2980,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
29672980 if (signedness == .signed and field_size < 8) {
29682981 _ = try self.addInst(.{
29692982 .tag = .mov_sign_extend,
2970 .ops = (Mir.Ops{
2983 .ops = Mir.Inst.Ops.encode(.{
29712984 .reg1 = dst_mcv.register,
29722985 .reg2 = registerAlias(dst_mcv.register, field_size),
2973 }).encode(),
2986 }),
29742987 .data = undefined,
29752988 });
29762989 }
......@@ -2990,7 +3003,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
29903003 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
29913004 defer self.register_manager.unlockReg(reg_lock);
29923005
2993 const dst_reg = try self.register_manager.allocReg(inst);
3006 const dst_reg = try self.register_manager.allocReg(inst, gp);
29943007 const flags: u2 = switch (mcv) {
29953008 .register_overflow_unsigned => 0b10,
29963009 .register_overflow_signed => 0b00,
......@@ -2998,10 +3011,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
29983011 };
29993012 _ = try self.addInst(.{
30003013 .tag = .cond_set_byte_overflow,
3001 .ops = (Mir.Ops{
3014 .ops = Mir.Inst.Ops.encode(.{
30023015 .reg1 = dst_reg.to8(),
30033016 .flags = flags,
3004 }).encode(),
3017 }),
30053018 .data = undefined,
30063019 });
30073020 break :result MCValue{ .register = dst_reg.to8() };
......@@ -3042,10 +3055,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
30423055 1 => {
30433056 _ = try self.addInst(.{
30443057 .tag = tag,
3045 .ops = (Mir.Ops{
3046 .reg1 = registerAlias(reg, abi_size),
3047 .flags = 0b00,
3048 }).encode(),
3058 .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
30493059 .data = undefined,
30503060 });
30513061 return;
......@@ -3053,10 +3063,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
30533063 else => {
30543064 _ = try self.addInst(.{
30553065 .tag = tag,
3056 .ops = (Mir.Ops{
3066 .ops = Mir.Inst.Ops.encode(.{
30573067 .reg1 = registerAlias(reg, abi_size),
30583068 .flags = 0b10,
3059 }).encode(),
3069 }),
30603070 .data = .{ .imm = @intCast(u8, imm) },
30613071 });
30623072 return;
......@@ -3074,10 +3084,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
30743084
30753085 _ = try self.addInst(.{
30763086 .tag = tag,
3077 .ops = (Mir.Ops{
3087 .ops = Mir.Inst.Ops.encode(.{
30783088 .reg1 = registerAlias(reg, abi_size),
30793089 .flags = 0b01,
3080 }).encode(),
3090 }),
30813091 .data = undefined,
30823092 });
30833093}
......@@ -3326,7 +3336,7 @@ fn genBinOp(
33263336 const rhs = try self.resolveInst(rhs_air);
33273337 const lhs_ty = self.air.typeOf(lhs_air);
33283338 const rhs_ty = self.air.typeOf(rhs_air);
3329 if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) {
3339 if (lhs_ty.zigTypeTag() == .Vector) {
33303340 return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
33313341 }
33323342 if (lhs_ty.abiSize(self.target.*) > 8) {
......@@ -3450,22 +3460,62 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
34503460 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
34513461 return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
34523462 },
3453 .register => |src_reg| {
3454 _ = try self.addInst(.{
3455 .tag = mir_tag,
3456 .ops = (Mir.Ops{
3457 .reg1 = registerAlias(dst_reg, abi_size),
3458 .reg2 = registerAlias(src_reg, abi_size),
3459 }).encode(),
3460 .data = undefined,
3461 });
3463 .register => |src_reg| switch (dst_ty.zigTypeTag()) {
3464 .Float => {
3465 if (intrinsicsAllowed(self.target.*, dst_ty)) {
3466 const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3467 .f32 => switch (mir_tag) {
3468 .add => if (hasAvxSupport(self.target.*))
3469 Mir.Inst.Tag.add_f32_avx
3470 else
3471 Mir.Inst.Tag.add_f32_sse,
3472 .cmp => if (hasAvxSupport(self.target.*))
3473 Mir.Inst.Tag.cmp_f32_avx
3474 else
3475 Mir.Inst.Tag.cmp_f32_sse,
3476 else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
3477 },
3478 .f64 => switch (mir_tag) {
3479 .add => if (hasAvxSupport(self.target.*))
3480 Mir.Inst.Tag.add_f64_avx
3481 else
3482 Mir.Inst.Tag.add_f64_sse,
3483 .cmp => if (hasAvxSupport(self.target.*))
3484 Mir.Inst.Tag.cmp_f64_avx
3485 else
3486 Mir.Inst.Tag.cmp_f64_sse,
3487 else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
3488 },
3489 else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
3490 };
3491 _ = try self.addInst(.{
3492 .tag = actual_tag,
3493 .ops = Mir.Inst.Ops.encode(.{
3494 .reg1 = dst_reg.to128(),
3495 .reg2 = src_reg.to128(),
3496 }),
3497 .data = undefined,
3498 });
3499 return;
3500 }
3501
3502 return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
3503 },
3504 else => {
3505 _ = try self.addInst(.{
3506 .tag = mir_tag,
3507 .ops = Mir.Inst.Ops.encode(.{
3508 .reg1 = registerAlias(dst_reg, abi_size),
3509 .reg2 = registerAlias(src_reg, abi_size),
3510 }),
3511 .data = undefined,
3512 });
3513 },
34623514 },
34633515 .immediate => |imm| {
34643516 _ = try self.addInst(.{
34653517 .tag = mir_tag,
3466 .ops = (Mir.Ops{
3467 .reg1 = registerAlias(dst_reg, abi_size),
3468 }).encode(),
3518 .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }),
34693519 .data = .{ .imm = @truncate(u32, imm) },
34703520 });
34713521 },
......@@ -3488,11 +3538,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
34883538 }
34893539 _ = try self.addInst(.{
34903540 .tag = mir_tag,
3491 .ops = (Mir.Ops{
3541 .ops = Mir.Inst.Ops.encode(.{
34923542 .reg1 = registerAlias(dst_reg, abi_size),
34933543 .reg2 = .rbp,
34943544 .flags = 0b01,
3495 }).encode(),
3545 }),
34963546 .data = .{ .imm = @bitCast(u32, -off) },
34973547 });
34983548 },
......@@ -3515,11 +3565,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
35153565 .register => |src_reg| {
35163566 _ = try self.addInst(.{
35173567 .tag = mir_tag,
3518 .ops = (Mir.Ops{
3568 .ops = Mir.Inst.Ops.encode(.{
35193569 .reg1 = .rbp,
35203570 .reg2 = registerAlias(src_reg, abi_size),
35213571 .flags = 0b10,
3522 }).encode(),
3572 }),
35233573 .data = .{ .imm = @bitCast(u32, -off) },
35243574 });
35253575 },
......@@ -3546,10 +3596,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
35463596 });
35473597 _ = try self.addInst(.{
35483598 .tag = tag,
3549 .ops = (Mir.Ops{
3599 .ops = Mir.Inst.Ops.encode(.{
35503600 .reg1 = .rbp,
35513601 .flags = flags,
3552 }).encode(),
3602 }),
35533603 .data = .{ .payload = payload },
35543604 });
35553605 },
......@@ -3604,10 +3654,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
36043654 // register, register
36053655 _ = try self.addInst(.{
36063656 .tag = .imul_complex,
3607 .ops = (Mir.Ops{
3657 .ops = Mir.Inst.Ops.encode(.{
36083658 .reg1 = registerAlias(dst_reg, abi_size),
36093659 .reg2 = registerAlias(src_reg, abi_size),
3610 }).encode(),
3660 }),
36113661 .data = undefined,
36123662 });
36133663 },
......@@ -3617,11 +3667,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
36173667 if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
36183668 _ = try self.addInst(.{
36193669 .tag = .imul_complex,
3620 .ops = (Mir.Ops{
3670 .ops = Mir.Inst.Ops.encode(.{
36213671 .reg1 = dst_reg.to32(),
36223672 .reg2 = dst_reg.to32(),
36233673 .flags = 0b10,
3624 }).encode(),
3674 }),
36253675 .data = .{ .imm = @truncate(u32, imm) },
36263676 });
36273677 } else {
......@@ -3633,11 +3683,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
36333683 .stack_offset => |off| {
36343684 _ = try self.addInst(.{
36353685 .tag = .imul_complex,
3636 .ops = (Mir.Ops{
3686 .ops = Mir.Inst.Ops.encode(.{
36373687 .reg1 = registerAlias(dst_reg, abi_size),
36383688 .reg2 = .rbp,
36393689 .flags = 0b01,
3640 }).encode(),
3690 }),
36413691 .data = .{ .imm = @bitCast(u32, -off) },
36423692 });
36433693 },
......@@ -3670,10 +3720,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
36703720 // register, register
36713721 _ = try self.addInst(.{
36723722 .tag = .imul_complex,
3673 .ops = (Mir.Ops{
3723 .ops = Mir.Inst.Ops.encode(.{
36743724 .reg1 = registerAlias(dst_reg, abi_size),
36753725 .reg2 = registerAlias(src_reg, abi_size),
3676 }).encode(),
3726 }),
36773727 .data = undefined,
36783728 });
36793729 // copy dst_reg back out
......@@ -3780,9 +3830,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
37803830fn airBreakpoint(self: *Self) !void {
37813831 _ = try self.addInst(.{
37823832 .tag = .interrupt,
3783 .ops = (Mir.Ops{
3784 .flags = 0b00,
3785 }).encode(),
3833 .ops = Mir.Inst.Ops.encode(.{}),
37863834 .data = undefined,
37873835 });
37883836 return self.finishAirBookkeeping();
......@@ -3883,9 +3931,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
38833931 // Adjust the stack
38843932 _ = try self.addInst(.{
38853933 .tag = .sub,
3886 .ops = (Mir.Ops{
3887 .reg1 = .rsp,
3888 }).encode(),
3934 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
38893935 .data = .{ .imm = info.stack_byte_count },
38903936 });
38913937 }
......@@ -3909,9 +3955,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
39093955 unreachable;
39103956 _ = try self.addInst(.{
39113957 .tag = .call,
3912 .ops = (Mir.Ops{
3913 .flags = 0b01,
3914 }).encode(),
3958 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
39153959 .data = .{ .imm = @truncate(u32, got_addr) },
39163960 });
39173961 } else if (func_value.castTag(.extern_fn)) |_| {
......@@ -3925,10 +3969,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
39253969 try self.genSetReg(Type.initTag(.usize), .rax, mcv);
39263970 _ = try self.addInst(.{
39273971 .tag = .call,
3928 .ops = (Mir.Ops{
3972 .ops = Mir.Inst.Ops.encode(.{
39293973 .reg1 = .rax,
39303974 .flags = 0b01,
3931 }).encode(),
3975 }),
39323976 .data = undefined,
39333977 });
39343978 }
......@@ -3943,10 +3987,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
39433987 // callq *%rax
39443988 _ = try self.addInst(.{
39453989 .tag = .call,
3946 .ops = (Mir.Ops{
3990 .ops = Mir.Inst.Ops.encode(.{
39473991 .reg1 = .rax,
39483992 .flags = 0b01,
3949 }).encode(),
3993 }),
39503994 .data = undefined,
39513995 });
39523996 } else if (func_value.castTag(.extern_fn)) |func_payload| {
......@@ -3978,10 +4022,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
39784022 try self.genSetReg(Type.initTag(.usize), .rax, mcv);
39794023 _ = try self.addInst(.{
39804024 .tag = .call,
3981 .ops = (Mir.Ops{
4025 .ops = Mir.Inst.Ops.encode(.{
39824026 .reg1 = .rax,
39834027 .flags = 0b01,
3984 }).encode(),
4028 }),
39854029 .data = undefined,
39864030 });
39874031 }
......@@ -3996,9 +4040,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
39964040 const fn_got_addr = got_addr + got_index * ptr_bytes;
39974041 _ = try self.addInst(.{
39984042 .tag = .call,
3999 .ops = (Mir.Ops{
4000 .flags = 0b01,
4001 }).encode(),
4043 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
40024044 .data = .{ .imm = @intCast(u32, fn_got_addr) },
40034045 });
40044046 } else return self.fail("TODO implement calling extern fn on plan9", .{});
......@@ -4008,10 +4050,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
40084050 try self.genSetReg(Type.initTag(.usize), .rax, mcv);
40094051 _ = try self.addInst(.{
40104052 .tag = .call,
4011 .ops = (Mir.Ops{
4053 .ops = Mir.Inst.Ops.encode(.{
40124054 .reg1 = .rax,
40134055 .flags = 0b01,
4014 }).encode(),
4056 }),
40154057 .data = undefined,
40164058 });
40174059 }
......@@ -4021,9 +4063,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
40214063 // Readjust the stack
40224064 _ = try self.addInst(.{
40234065 .tag = .add,
4024 .ops = (Mir.Ops{
4025 .reg1 = .rsp,
4026 }).encode(),
4066 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
40274067 .data = .{ .imm = info.stack_byte_count },
40284068 });
40294069 }
......@@ -4081,9 +4121,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
40814121 // which is available if the jump is 127 bytes or less forward.
40824122 const jmp_reloc = try self.addInst(.{
40834123 .tag = .jmp,
4084 .ops = (Mir.Ops{
4085 .flags = 0b00,
4086 }).encode(),
4124 .ops = Mir.Inst.Ops.encode(.{}),
40874125 .data = .{ .inst = undefined },
40884126 });
40894127 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
......@@ -4116,9 +4154,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
41164154 // which is available if the jump is 127 bytes or less forward.
41174155 const jmp_reloc = try self.addInst(.{
41184156 .tag = .jmp,
4119 .ops = (Mir.Ops{
4120 .flags = 0b00,
4121 }).encode(),
4157 .ops = Mir.Inst.Ops.encode(.{}),
41224158 .data = .{ .inst = undefined },
41234159 });
41244160 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
......@@ -4162,8 +4198,28 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
41624198
41634199 const dst_mcv = MCValue{ .register = dst_reg };
41644200
4201 const rhs_ty = self.air.typeOf(bin_op.rhs);
41654202 // This instruction supports only signed 32-bit immediates at most.
4166 const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
4203 const src_mcv: MCValue = blk: {
4204 switch (rhs_ty.zigTypeTag()) {
4205 .Float => {
4206 const rhs = try self.resolveInst(bin_op.rhs);
4207 const rhs_lock: ?RegisterLock = switch (rhs) {
4208 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4209 else => null,
4210 };
4211 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
4212 const src_reg = try self.copyToTmpRegister(rhs_ty, rhs);
4213 break :blk MCValue{ .register = src_reg };
4214 },
4215 else => break :blk try self.limitImmediateType(bin_op.rhs, i32),
4216 }
4217 };
4218 const src_lock: ?RegisterLock = switch (src_mcv) {
4219 .register => |reg| self.register_manager.lockReg(reg),
4220 else => null,
4221 };
4222 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
41674223
41684224 try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
41694225 break :result switch (signedness) {
......@@ -4362,9 +4418,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
43624418 Mir.Inst.Tag.cond_jmp_greater_less;
43634419 return self.addInst(.{
43644420 .tag = tag,
4365 .ops = (Mir.Ops{
4366 .flags = flags,
4367 }).encode(),
4421 .ops = Mir.Inst.Ops.encode(.{ .flags = flags }),
43684422 .data = .{ .inst = undefined },
43694423 });
43704424 },
......@@ -4372,17 +4426,12 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
43724426 try self.spillCompareFlagsIfOccupied();
43734427 _ = try self.addInst(.{
43744428 .tag = .@"test",
4375 .ops = (Mir.Ops{
4376 .reg1 = reg,
4377 .flags = 0b00,
4378 }).encode(),
4429 .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
43794430 .data = .{ .imm = 1 },
43804431 });
43814432 return self.addInst(.{
43824433 .tag = .cond_jmp_eq_ne,
4383 .ops = (Mir.Ops{
4384 .flags = 0b01,
4385 }).encode(),
4434 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
43864435 .data = .{ .inst = undefined },
43874436 });
43884437 },
......@@ -4776,9 +4825,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
47764825 try self.genBody(body);
47774826 _ = try self.addInst(.{
47784827 .tag = .jmp,
4779 .ops = (Mir.Ops{
4780 .flags = 0b00,
4781 }).encode(),
4828 .ops = Mir.Inst.Ops.encode(.{}),
47824829 .data = .{ .inst = jmp_target },
47834830 });
47844831 return self.finishAirBookkeeping();
......@@ -4829,19 +4876,17 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
48294876 .immediate => |imm| {
48304877 _ = try self.addInst(.{
48314878 .tag = .xor,
4832 .ops = (Mir.Ops{
4833 .reg1 = registerAlias(cond_reg, abi_size),
4834 }).encode(),
4879 .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }),
48354880 .data = .{ .imm = @intCast(u32, imm) },
48364881 });
48374882 },
48384883 .register => |reg| {
48394884 _ = try self.addInst(.{
48404885 .tag = .xor,
4841 .ops = (Mir.Ops{
4886 .ops = Mir.Inst.Ops.encode(.{
48424887 .reg1 = registerAlias(cond_reg, abi_size),
48434888 .reg2 = registerAlias(reg, abi_size),
4844 }).encode(),
4889 }),
48454890 .data = undefined,
48464891 });
48474892 },
......@@ -4860,17 +4905,15 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
48604905
48614906 _ = try self.addInst(.{
48624907 .tag = .@"test",
4863 .ops = (Mir.Ops{
4908 .ops = Mir.Inst.Ops.encode(.{
48644909 .reg1 = registerAlias(cond_reg, abi_size),
48654910 .reg2 = registerAlias(cond_reg, abi_size),
4866 }).encode(),
4911 }),
48674912 .data = undefined,
48684913 });
48694914 return self.addInst(.{
48704915 .tag = .cond_jmp_eq_ne,
4871 .ops = (Mir.Ops{
4872 .flags = 0b00,
4873 }).encode(),
4916 .ops = Mir.Inst.Ops.encode(.{}),
48744917 .data = .{ .inst = undefined },
48754918 });
48764919 },
......@@ -5036,9 +5079,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
50365079 // Leave the jump offset undefined
50375080 const jmp_reloc = try self.addInst(.{
50385081 .tag = .jmp,
5039 .ops = (Mir.Ops{
5040 .flags = 0b00,
5041 }).encode(),
5082 .ops = Mir.Inst.Ops.encode(.{}),
50425083 .data = .{ .inst = undefined },
50435084 });
50445085 block_data.relocs.appendAssumeCapacity(jmp_reloc);
......@@ -5126,9 +5167,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
51265167 };
51275168 _ = try self.addInst(.{
51285169 .tag = .push,
5129 .ops = (Mir.Ops{
5130 .flags = 0b10,
5131 }).encode(),
5170 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b10 }),
51325171 .data = .{ .imm = n },
51335172 });
51345173 } else if (mem.indexOf(u8, arg, "%%")) |l| {
......@@ -5137,9 +5176,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
51375176 return self.fail("unrecognized register: '{s}'", .{reg_name});
51385177 _ = try self.addInst(.{
51395178 .tag = .push,
5140 .ops = (Mir.Ops{
5141 .reg1 = reg,
5142 }).encode(),
5179 .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
51435180 .data = undefined,
51445181 });
51455182 } else return self.fail("TODO more push operands", .{});
......@@ -5151,9 +5188,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
51515188 return self.fail("unrecognized register: '{s}'", .{reg_name});
51525189 _ = try self.addInst(.{
51535190 .tag = .pop,
5154 .ops = (Mir.Ops{
5155 .reg1 = reg,
5156 }).encode(),
5191 .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
51575192 .data = undefined,
51585193 });
51595194 } else return self.fail("TODO more pop operands", .{});
......@@ -5265,7 +5300,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
52655300 });
52665301 _ = try self.addInst(.{
52675302 .tag = .mov_mem_imm,
5268 .ops = (Mir.Ops{
5303 .ops = Mir.Inst.Ops.encode(.{
52695304 .reg1 = .rsp,
52705305 .flags = switch (abi_size) {
52715306 1 => 0b00,
......@@ -5273,7 +5308,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
52735308 4 => 0b10,
52745309 else => unreachable,
52755310 },
5276 }).encode(),
5311 }),
52775312 .data = .{ .payload = payload },
52785313 });
52795314 },
......@@ -5299,15 +5334,50 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
52995334 });
53005335 },
53015336 .register => |reg| {
5302 _ = try self.addInst(.{
5303 .tag = .mov,
5304 .ops = (Mir.Ops{
5305 .reg1 = .rsp,
5306 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5307 .flags = 0b10,
5308 }).encode(),
5309 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5310 });
5337 switch (ty.zigTypeTag()) {
5338 .Float => {
5339 if (intrinsicsAllowed(self.target.*, ty)) {
5340 const tag: Mir.Inst.Tag = switch (ty.tag()) {
5341 .f32 => if (hasAvxSupport(self.target.*))
5342 Mir.Inst.Tag.mov_f32_avx
5343 else
5344 Mir.Inst.Tag.mov_f32_sse,
5345 .f64 => if (hasAvxSupport(self.target.*))
5346 Mir.Inst.Tag.mov_f64_avx
5347 else
5348 Mir.Inst.Tag.mov_f64_sse,
5349 else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
5350 };
5351 _ = try self.addInst(.{
5352 .tag = tag,
5353 .ops = Mir.Inst.Ops.encode(.{
5354 .reg1 = switch (ty.tag()) {
5355 .f32 => .esp,
5356 .f64 => .rsp,
5357 else => unreachable,
5358 },
5359 .reg2 = reg.to128(),
5360 .flags = 0b01,
5361 }),
5362 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5363 });
5364 return;
5365 }
5366
5367 return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
5368 },
5369 else => {
5370 _ = try self.addInst(.{
5371 .tag = .mov,
5372 .ops = Mir.Inst.Ops.encode(.{
5373 .reg1 = .rsp,
5374 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5375 .flags = 0b10,
5376 }),
5377 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5378 });
5379 },
5380 }
53115381 },
53125382 .ptr_stack_offset => {
53135383 const reg = try self.copyToTmpRegister(ty, mcv);
......@@ -5360,7 +5430,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
53605430
53615431 const overflow_bit_ty = ty.structFieldType(1);
53625432 const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
5363 const tmp_reg = try self.register_manager.allocReg(null);
5433 const tmp_reg = try self.register_manager.allocReg(null, gp);
53645434 const flags: u2 = switch (mcv) {
53655435 .register_overflow_unsigned => 0b10,
53665436 .register_overflow_signed => 0b00,
......@@ -5368,10 +5438,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
53685438 };
53695439 _ = try self.addInst(.{
53705440 .tag = .cond_set_byte_overflow,
5371 .ops = (Mir.Ops{
5441 .ops = Mir.Inst.Ops.encode(.{
53725442 .reg1 = tmp_reg.to8(),
53735443 .flags = flags,
5374 }).encode(),
5444 }),
53755445 .data = undefined,
53765446 });
53775447
......@@ -5398,7 +5468,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
53985468 });
53995469 _ = try self.addInst(.{
54005470 .tag = .mov_mem_imm,
5401 .ops = (Mir.Ops{
5471 .ops = Mir.Inst.Ops.encode(.{
54025472 .reg1 = base_reg,
54035473 .flags = switch (abi_size) {
54045474 1 => 0b00,
......@@ -5406,7 +5476,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
54065476 4 => 0b10,
54075477 else => unreachable,
54085478 },
5409 }).encode(),
5479 }),
54105480 .data = .{ .payload = payload },
54115481 });
54125482 },
......@@ -5420,10 +5490,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
54205490 });
54215491 _ = try self.addInst(.{
54225492 .tag = .mov_mem_imm,
5423 .ops = (Mir.Ops{
5493 .ops = Mir.Inst.Ops.encode(.{
54245494 .reg1 = base_reg,
54255495 .flags = 0b10,
5426 }).encode(),
5496 }),
54275497 .data = .{ .payload = payload },
54285498 });
54295499 }
......@@ -5434,10 +5504,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
54345504 });
54355505 _ = try self.addInst(.{
54365506 .tag = .mov_mem_imm,
5437 .ops = (Mir.Ops{
5507 .ops = Mir.Inst.Ops.encode(.{
54385508 .reg1 = base_reg,
54395509 .flags = 0b10,
5440 }).encode(),
5510 }),
54415511 .data = .{ .payload = payload },
54425512 });
54435513 }
......@@ -5453,44 +5523,80 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
54535523 }
54545524
54555525 const base_reg = opts.dest_stack_base orelse .rbp;
5456 if (!math.isPowerOfTwo(abi_size)) {
5457 const reg_lock = self.register_manager.lockReg(reg);
5458 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
54595526
5460 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
5527 switch (ty.zigTypeTag()) {
5528 .Float => {
5529 if (intrinsicsAllowed(self.target.*, ty)) {
5530 const tag: Mir.Inst.Tag = switch (ty.tag()) {
5531 .f32 => if (hasAvxSupport(self.target.*))
5532 Mir.Inst.Tag.mov_f32_avx
5533 else
5534 Mir.Inst.Tag.mov_f32_sse,
5535 .f64 => if (hasAvxSupport(self.target.*))
5536 Mir.Inst.Tag.mov_f64_avx
5537 else
5538 Mir.Inst.Tag.mov_f64_sse,
5539 else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
5540 };
5541 _ = try self.addInst(.{
5542 .tag = tag,
5543 .ops = Mir.Inst.Ops.encode(.{
5544 .reg1 = switch (ty.tag()) {
5545 .f32 => base_reg.to32(),
5546 .f64 => base_reg.to64(),
5547 else => unreachable,
5548 },
5549 .reg2 = reg.to128(),
5550 .flags = 0b01,
5551 }),
5552 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5553 });
5554 return;
5555 }
5556
5557 return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
5558 },
5559 else => {
5560 if (!math.isPowerOfTwo(abi_size)) {
5561 const reg_lock = self.register_manager.lockReg(reg);
5562 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
5563
5564 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
54615565
5462 var next_offset = stack_offset;
5463 var remainder = abi_size;
5464 while (remainder > 0) {
5465 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5566 var next_offset = stack_offset;
5567 var remainder = abi_size;
5568 while (remainder > 0) {
5569 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
54665570
5467 _ = try self.addInst(.{
5468 .tag = .mov,
5469 .ops = (Mir.Ops{
5470 .reg1 = base_reg,
5471 .reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5472 .flags = 0b10,
5473 }).encode(),
5474 .data = .{ .imm = @bitCast(u32, -next_offset) },
5475 });
5571 _ = try self.addInst(.{
5572 .tag = .mov,
5573 .ops = Mir.Inst.Ops.encode(.{
5574 .reg1 = base_reg,
5575 .reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5576 .flags = 0b10,
5577 }),
5578 .data = .{ .imm = @bitCast(u32, -next_offset) },
5579 });
54765580
5477 if (nearest_power_of_two > 1) {
5478 try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 });
5479 }
5581 if (nearest_power_of_two > 1) {
5582 try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 });
5583 }
54805584
5481 remainder -= nearest_power_of_two;
5482 next_offset -= nearest_power_of_two;
5483 }
5484 } else {
5485 _ = try self.addInst(.{
5486 .tag = .mov,
5487 .ops = (Mir.Ops{
5488 .reg1 = base_reg,
5489 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5490 .flags = 0b10,
5491 }).encode(),
5492 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5493 });
5585 remainder -= nearest_power_of_two;
5586 next_offset -= nearest_power_of_two;
5587 }
5588 } else {
5589 _ = try self.addInst(.{
5590 .tag = .mov,
5591 .ops = Mir.Inst.Ops.encode(.{
5592 .reg1 = base_reg,
5593 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5594 .flags = 0b10,
5595 }),
5596 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5597 });
5598 }
5599 },
54945600 }
54955601 },
54965602 .memory,
......@@ -5558,7 +5664,7 @@ fn genInlineMemcpy(
55585664 null;
55595665 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
55605666
5561 const dst_addr_reg = try self.register_manager.allocReg(null);
5667 const dst_addr_reg = try self.register_manager.allocReg(null, gp);
55625668 switch (dst_ptr) {
55635669 .memory,
55645670 .got_load,
......@@ -5569,20 +5675,20 @@ fn genInlineMemcpy(
55695675 .ptr_stack_offset, .stack_offset => |off| {
55705676 _ = try self.addInst(.{
55715677 .tag = .lea,
5572 .ops = (Mir.Ops{
5678 .ops = Mir.Inst.Ops.encode(.{
55735679 .reg1 = dst_addr_reg.to64(),
55745680 .reg2 = opts.dest_stack_base orelse .rbp,
5575 }).encode(),
5681 }),
55765682 .data = .{ .imm = @bitCast(u32, -off) },
55775683 });
55785684 },
55795685 .register => |reg| {
55805686 _ = try self.addInst(.{
55815687 .tag = .mov,
5582 .ops = (Mir.Ops{
5688 .ops = Mir.Inst.Ops.encode(.{
55835689 .reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),
55845690 .reg2 = reg,
5585 }).encode(),
5691 }),
55865692 .data = undefined,
55875693 });
55885694 },
......@@ -5593,7 +5699,7 @@ fn genInlineMemcpy(
55935699 const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
55945700 defer self.register_manager.unlockReg(dst_addr_reg_lock);
55955701
5596 const src_addr_reg = try self.register_manager.allocReg(null);
5702 const src_addr_reg = try self.register_manager.allocReg(null, gp);
55975703 switch (src_ptr) {
55985704 .memory,
55995705 .got_load,
......@@ -5604,20 +5710,20 @@ fn genInlineMemcpy(
56045710 .ptr_stack_offset, .stack_offset => |off| {
56055711 _ = try self.addInst(.{
56065712 .tag = .lea,
5607 .ops = (Mir.Ops{
5713 .ops = Mir.Inst.Ops.encode(.{
56085714 .reg1 = src_addr_reg.to64(),
56095715 .reg2 = opts.source_stack_base orelse .rbp,
5610 }).encode(),
5716 }),
56115717 .data = .{ .imm = @bitCast(u32, -off) },
56125718 });
56135719 },
56145720 .register => |reg| {
56155721 _ = try self.addInst(.{
56165722 .tag = .mov,
5617 .ops = (Mir.Ops{
5723 .ops = Mir.Inst.Ops.encode(.{
56185724 .reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),
56195725 .reg2 = reg,
5620 }).encode(),
5726 }),
56215727 .data = undefined,
56225728 });
56235729 },
......@@ -5628,7 +5734,7 @@ fn genInlineMemcpy(
56285734 const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
56295735 defer self.register_manager.unlockReg(src_addr_reg_lock);
56305736
5631 const regs = try self.register_manager.allocRegs(2, .{ null, null });
5737 const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
56325738 const count_reg = regs[0].to64();
56335739 const tmp_reg = regs[1].to8();
56345740
......@@ -5637,18 +5743,14 @@ fn genInlineMemcpy(
56375743 // mov rcx, 0
56385744 _ = try self.addInst(.{
56395745 .tag = .mov,
5640 .ops = (Mir.Ops{
5641 .reg1 = .rcx,
5642 }).encode(),
5746 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }),
56435747 .data = .{ .imm = 0 },
56445748 });
56455749
56465750 // mov rax, 0
56475751 _ = try self.addInst(.{
56485752 .tag = .mov,
5649 .ops = (Mir.Ops{
5650 .reg1 = .rax,
5651 }).encode(),
5753 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
56525754 .data = .{ .imm = 0 },
56535755 });
56545756
......@@ -5656,70 +5758,62 @@ fn genInlineMemcpy(
56565758 // cmp count, 0
56575759 const loop_start = try self.addInst(.{
56585760 .tag = .cmp,
5659 .ops = (Mir.Ops{
5660 .reg1 = count_reg,
5661 }).encode(),
5761 .ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
56625762 .data = .{ .imm = 0 },
56635763 });
56645764
56655765 // je end
56665766 const loop_reloc = try self.addInst(.{
56675767 .tag = .cond_jmp_eq_ne,
5668 .ops = (Mir.Ops{ .flags = 0b01 }).encode(),
5768 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
56695769 .data = .{ .inst = undefined },
56705770 });
56715771
56725772 // mov tmp, [addr + rcx]
56735773 _ = try self.addInst(.{
56745774 .tag = .mov_scale_src,
5675 .ops = (Mir.Ops{
5775 .ops = Mir.Inst.Ops.encode(.{
56765776 .reg1 = tmp_reg.to8(),
56775777 .reg2 = src_addr_reg,
5678 }).encode(),
5778 }),
56795779 .data = .{ .imm = 0 },
56805780 });
56815781
56825782 // mov [stack_offset + rax], tmp
56835783 _ = try self.addInst(.{
56845784 .tag = .mov_scale_dst,
5685 .ops = (Mir.Ops{
5785 .ops = Mir.Inst.Ops.encode(.{
56865786 .reg1 = dst_addr_reg,
56875787 .reg2 = tmp_reg.to8(),
5688 }).encode(),
5788 }),
56895789 .data = .{ .imm = 0 },
56905790 });
56915791
56925792 // add rcx, 1
56935793 _ = try self.addInst(.{
56945794 .tag = .add,
5695 .ops = (Mir.Ops{
5696 .reg1 = .rcx,
5697 }).encode(),
5795 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }),
56985796 .data = .{ .imm = 1 },
56995797 });
57005798
57015799 // add rax, 1
57025800 _ = try self.addInst(.{
57035801 .tag = .add,
5704 .ops = (Mir.Ops{
5705 .reg1 = .rax,
5706 }).encode(),
5802 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
57075803 .data = .{ .imm = 1 },
57085804 });
57095805
57105806 // sub count, 1
57115807 _ = try self.addInst(.{
57125808 .tag = .sub,
5713 .ops = (Mir.Ops{
5714 .reg1 = count_reg,
5715 }).encode(),
5809 .ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
57165810 .data = .{ .imm = 1 },
57175811 });
57185812
57195813 // jmp loop
57205814 _ = try self.addInst(.{
57215815 .tag = .jmp,
5722 .ops = (Mir.Ops{ .flags = 0b00 }).encode(),
5816 .ops = Mir.Inst.Ops.encode(.{}),
57235817 .data = .{ .inst = loop_start },
57245818 });
57255819
......@@ -5740,7 +5834,7 @@ fn genInlineMemset(
57405834 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
57415835 defer self.register_manager.unlockReg(rax_lock);
57425836
5743 const addr_reg = try self.register_manager.allocReg(null);
5837 const addr_reg = try self.register_manager.allocReg(null, gp);
57445838 switch (dst_ptr) {
57455839 .memory,
57465840 .got_load,
......@@ -5751,20 +5845,20 @@ fn genInlineMemset(
57515845 .ptr_stack_offset, .stack_offset => |off| {
57525846 _ = try self.addInst(.{
57535847 .tag = .lea,
5754 .ops = (Mir.Ops{
5848 .ops = Mir.Inst.Ops.encode(.{
57555849 .reg1 = addr_reg.to64(),
57565850 .reg2 = opts.dest_stack_base orelse .rbp,
5757 }).encode(),
5851 }),
57585852 .data = .{ .imm = @bitCast(u32, -off) },
57595853 });
57605854 },
57615855 .register => |reg| {
57625856 _ = try self.addInst(.{
57635857 .tag = .mov,
5764 .ops = (Mir.Ops{
5858 .ops = Mir.Inst.Ops.encode(.{
57655859 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
57665860 .reg2 = reg,
5767 }).encode(),
5861 }),
57685862 .data = undefined,
57695863 });
57705864 },
......@@ -5782,16 +5876,14 @@ fn genInlineMemset(
57825876 // cmp rax, -1
57835877 const loop_start = try self.addInst(.{
57845878 .tag = .cmp,
5785 .ops = (Mir.Ops{
5786 .reg1 = .rax,
5787 }).encode(),
5879 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
57885880 .data = .{ .imm = @bitCast(u32, @as(i32, -1)) },
57895881 });
57905882
57915883 // je end
57925884 const loop_reloc = try self.addInst(.{
57935885 .tag = .cond_jmp_eq_ne,
5794 .ops = (Mir.Ops{ .flags = 0b01 }).encode(),
5886 .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
57955887 .data = .{ .inst = undefined },
57965888 });
57975889
......@@ -5807,9 +5899,7 @@ fn genInlineMemset(
58075899 });
58085900 _ = try self.addInst(.{
58095901 .tag = .mov_mem_index_imm,
5810 .ops = (Mir.Ops{
5811 .reg1 = addr_reg,
5812 }).encode(),
5902 .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg }),
58135903 .data = .{ .payload = payload },
58145904 });
58155905 },
......@@ -5819,16 +5909,14 @@ fn genInlineMemset(
58195909 // sub rax, 1
58205910 _ = try self.addInst(.{
58215911 .tag = .sub,
5822 .ops = (Mir.Ops{
5823 .reg1 = .rax,
5824 }).encode(),
5912 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }),
58255913 .data = .{ .imm = 1 },
58265914 });
58275915
58285916 // jmp loop
58295917 _ = try self.addInst(.{
58305918 .tag = .jmp,
5831 .ops = (Mir.Ops{ .flags = 0b00 }).encode(),
5919 .ops = Mir.Inst.Ops.encode(.{}),
58325920 .data = .{ .inst = loop_start },
58335921 });
58345922
......@@ -5849,10 +5937,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
58495937 }
58505938 _ = try self.addInst(.{
58515939 .tag = .lea,
5852 .ops = (Mir.Ops{
5940 .ops = Mir.Inst.Ops.encode(.{
58535941 .reg1 = registerAlias(reg, abi_size),
58545942 .reg2 = .rbp,
5855 }).encode(),
5943 }),
58565944 .data = .{ .imm = @bitCast(u32, -off) },
58575945 });
58585946 },
......@@ -5889,10 +5977,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
58895977 };
58905978 _ = try self.addInst(.{
58915979 .tag = tag,
5892 .ops = (Mir.Ops{
5980 .ops = Mir.Inst.Ops.encode(.{
58935981 .reg1 = reg.to8(),
58945982 .flags = flags,
5895 }).encode(),
5983 }),
58965984 .data = undefined,
58975985 });
58985986 },
......@@ -5902,10 +5990,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
59025990 if (x == 0) {
59035991 _ = try self.addInst(.{
59045992 .tag = .xor,
5905 .ops = (Mir.Ops{
5993 .ops = Mir.Inst.Ops.encode(.{
59065994 .reg1 = reg.to32(),
59075995 .reg2 = reg.to32(),
5908 }).encode(),
5996 }),
59095997 .data = undefined,
59105998 });
59115999 return;
......@@ -5914,9 +6002,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
59146002 // Next best case: if we set the lower four bytes, the upper four will be zeroed.
59156003 _ = try self.addInst(.{
59166004 .tag = .mov,
5917 .ops = (Mir.Ops{
5918 .reg1 = registerAlias(reg, abi_size),
5919 }).encode(),
6005 .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
59206006 .data = .{ .imm = @truncate(u32, x) },
59216007 });
59226008 return;
......@@ -5931,9 +6017,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
59316017 const payload = try self.addExtra(Mir.Imm64.encode(x));
59326018 _ = try self.addInst(.{
59336019 .tag = .movabs,
5934 .ops = (Mir.Ops{
5935 .reg1 = reg.to64(),
5936 }).encode(),
6020 .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }),
59376021 .data = .{ .payload = payload },
59386022 });
59396023 },
......@@ -5942,150 +6026,290 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
59426026 if (src_reg.id() == reg.id())
59436027 return;
59446028
5945 if (ty.zigTypeTag() == .Int) blk: {
5946 switch (ty.intInfo(self.target.*).signedness) {
6029 switch (ty.zigTypeTag()) {
6030 .Int => switch (ty.intInfo(self.target.*).signedness) {
59476031 .signed => {
5948 if (abi_size > 4) break :blk;
5949 _ = try self.addInst(.{
5950 .tag = .mov_sign_extend,
5951 .ops = (Mir.Ops{
5952 .reg1 = reg.to64(),
5953 .reg2 = registerAlias(src_reg, abi_size),
5954 }).encode(),
5955 .data = undefined,
5956 });
6032 if (abi_size <= 4) {
6033 _ = try self.addInst(.{
6034 .tag = .mov_sign_extend,
6035 .ops = Mir.Inst.Ops.encode(.{
6036 .reg1 = reg.to64(),
6037 .reg2 = registerAlias(src_reg, abi_size),
6038 }),
6039 .data = undefined,
6040 });
6041 return;
6042 }
59576043 },
59586044 .unsigned => {
5959 if (abi_size > 2) break :blk;
6045 if (abi_size <= 2) {
6046 _ = try self.addInst(.{
6047 .tag = .mov_zero_extend,
6048 .ops = Mir.Inst.Ops.encode(.{
6049 .reg1 = reg.to64(),
6050 .reg2 = registerAlias(src_reg, abi_size),
6051 }),
6052 .data = undefined,
6053 });
6054 return;
6055 }
6056 },
6057 },
6058 .Float => {
6059 if (intrinsicsAllowed(self.target.*, ty)) {
6060 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6061 .f32 => if (hasAvxSupport(self.target.*))
6062 Mir.Inst.Tag.mov_f32_avx
6063 else
6064 Mir.Inst.Tag.mov_f32_sse,
6065 .f64 => if (hasAvxSupport(self.target.*))
6066 Mir.Inst.Tag.mov_f64_avx
6067 else
6068 Mir.Inst.Tag.mov_f64_sse,
6069 else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
6070 };
59606071 _ = try self.addInst(.{
5961 .tag = .mov_zero_extend,
5962 .ops = (Mir.Ops{
5963 .reg1 = reg.to64(),
5964 .reg2 = registerAlias(src_reg, abi_size),
5965 }).encode(),
6072 .tag = tag,
6073 .ops = Mir.Inst.Ops.encode(.{
6074 .reg1 = reg.to128(),
6075 .reg2 = src_reg.to128(),
6076 .flags = 0b10,
6077 }),
59666078 .data = undefined,
59676079 });
5968 },
5969 }
5970 return;
6080 return;
6081 }
6082
6083 return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
6084 },
6085 else => {},
59716086 }
59726087
59736088 _ = try self.addInst(.{
59746089 .tag = .mov,
5975 .ops = (Mir.Ops{
6090 .ops = Mir.Inst.Ops.encode(.{
59766091 .reg1 = registerAlias(reg, abi_size),
59776092 .reg2 = registerAlias(src_reg, abi_size),
5978 }).encode(),
6093 }),
59796094 .data = undefined,
59806095 });
59816096 },
59826097 .direct_load,
59836098 .got_load,
59846099 => {
5985 try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
5986 _ = try self.addInst(.{
5987 .tag = .mov,
5988 .ops = (Mir.Ops{
5989 .reg1 = registerAlias(reg, abi_size),
5990 .reg2 = reg.to64(),
5991 .flags = 0b01,
5992 }).encode(),
5993 .data = .{ .imm = 0 },
5994 });
6100 switch (ty.zigTypeTag()) {
6101 .Float => {
6102 const base_reg = try self.register_manager.allocReg(null, gp);
6103 try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
6104
6105 if (intrinsicsAllowed(self.target.*, ty)) {
6106 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6107 .f32 => if (hasAvxSupport(self.target.*))
6108 Mir.Inst.Tag.mov_f32_avx
6109 else
6110 Mir.Inst.Tag.mov_f32_sse,
6111 .f64 => if (hasAvxSupport(self.target.*))
6112 Mir.Inst.Tag.mov_f64_avx
6113 else
6114 Mir.Inst.Tag.mov_f64_sse,
6115 else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6116 };
6117
6118 _ = try self.addInst(.{
6119 .tag = tag,
6120 .ops = Mir.Inst.Ops.encode(.{
6121 .reg1 = reg.to128(),
6122 .reg2 = switch (ty.tag()) {
6123 .f32 => base_reg.to32(),
6124 .f64 => base_reg.to64(),
6125 else => unreachable,
6126 },
6127 }),
6128 .data = .{ .imm = 0 },
6129 });
6130 return;
6131 }
6132
6133 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
6134 },
6135 else => {
6136 try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
6137 _ = try self.addInst(.{
6138 .tag = .mov,
6139 .ops = Mir.Inst.Ops.encode(.{
6140 .reg1 = registerAlias(reg, abi_size),
6141 .reg2 = reg.to64(),
6142 .flags = 0b01,
6143 }),
6144 .data = .{ .imm = 0 },
6145 });
6146 },
6147 }
59956148 },
5996 .memory => |x| {
5997 if (x <= math.maxInt(i32)) {
5998 // mov reg, [ds:imm32]
5999 _ = try self.addInst(.{
6000 .tag = .mov,
6001 .ops = (Mir.Ops{
6002 .reg1 = registerAlias(reg, abi_size),
6003 .flags = 0b01,
6004 }).encode(),
6005 .data = .{ .imm = @truncate(u32, x) },
6006 });
6007 } else {
6008 // If this is RAX, we can use a direct load.
6009 // Otherwise, we need to load the address, then indirectly load the value.
6010 if (reg.id() == 0) {
6011 // movabs rax, ds:moffs64
6012 const payload = try self.addExtra(Mir.Imm64.encode(x));
6149 .memory => |x| switch (ty.zigTypeTag()) {
6150 .Float => {
6151 const base_reg = try self.register_manager.allocReg(null, gp);
6152 try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
6153
6154 if (intrinsicsAllowed(self.target.*, ty)) {
6155 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6156 .f32 => if (hasAvxSupport(self.target.*))
6157 Mir.Inst.Tag.mov_f32_avx
6158 else
6159 Mir.Inst.Tag.mov_f32_sse,
6160 .f64 => if (hasAvxSupport(self.target.*))
6161 Mir.Inst.Tag.mov_f64_avx
6162 else
6163 Mir.Inst.Tag.mov_f64_sse,
6164 else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6165 };
6166
60136167 _ = try self.addInst(.{
6014 .tag = .movabs,
6015 .ops = (Mir.Ops{
6016 .reg1 = .rax,
6017 .flags = 0b01, // imm64 will become moffs64
6018 }).encode(),
6019 .data = .{ .payload = payload },
6168 .tag = tag,
6169 .ops = Mir.Inst.Ops.encode(.{
6170 .reg1 = reg.to128(),
6171 .reg2 = switch (ty.tag()) {
6172 .f32 => base_reg.to32(),
6173 .f64 => base_reg.to64(),
6174 else => unreachable,
6175 },
6176 }),
6177 .data = .{ .imm = 0 },
60206178 });
6021 } else {
6022 // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
6023 try self.genSetReg(ty, reg, MCValue{ .immediate = x });
6179 return;
6180 }
60246181
6025 // mov reg, [reg + 0x0]
6182 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
6183 },
6184 else => {
6185 if (x <= math.maxInt(i32)) {
6186 // mov reg, [ds:imm32]
60266187 _ = try self.addInst(.{
60276188 .tag = .mov,
6028 .ops = (Mir.Ops{
6189 .ops = Mir.Inst.Ops.encode(.{
60296190 .reg1 = registerAlias(reg, abi_size),
6030 .reg2 = reg.to64(),
60316191 .flags = 0b01,
6032 }).encode(),
6033 .data = .{ .imm = 0 },
6192 }),
6193 .data = .{ .imm = @truncate(u32, x) },
60346194 });
6195 } else {
6196 // If this is RAX, we can use a direct load.
6197 // Otherwise, we need to load the address, then indirectly load the value.
6198 if (reg.id() == 0) {
6199 // movabs rax, ds:moffs64
6200 const payload = try self.addExtra(Mir.Imm64.encode(x));
6201 _ = try self.addInst(.{
6202 .tag = .movabs,
6203 .ops = Mir.Inst.Ops.encode(.{
6204 .reg1 = .rax,
6205 .flags = 0b01, // imm64 will become moffs64
6206 }),
6207 .data = .{ .payload = payload },
6208 });
6209 } else {
6210 // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
6211 try self.genSetReg(ty, reg, MCValue{ .immediate = x });
6212
6213 // mov reg, [reg + 0x0]
6214 _ = try self.addInst(.{
6215 .tag = .mov,
6216 .ops = Mir.Inst.Ops.encode(.{
6217 .reg1 = registerAlias(reg, abi_size),
6218 .reg2 = reg.to64(),
6219 .flags = 0b01,
6220 }),
6221 .data = .{ .imm = 0 },
6222 });
6223 }
60356224 }
6036 }
6225 },
60376226 },
60386227 .stack_offset => |off| {
60396228 if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
60406229 return self.fail("stack offset too large", .{});
60416230 }
60426231
6043 if (ty.zigTypeTag() == .Int) blk: {
6044 switch (ty.intInfo(self.target.*).signedness) {
6232 switch (ty.zigTypeTag()) {
6233 .Int => switch (ty.intInfo(self.target.*).signedness) {
60456234 .signed => {
6046 const flags: u2 = switch (abi_size) {
6047 1 => 0b01,
6048 2 => 0b10,
6049 4 => 0b11,
6050 else => break :blk,
6051 };
6052 _ = try self.addInst(.{
6053 .tag = .mov_sign_extend,
6054 .ops = (Mir.Ops{
6055 .reg1 = reg.to64(),
6056 .reg2 = .rbp,
6057 .flags = flags,
6058 }).encode(),
6059 .data = .{ .imm = @bitCast(u32, -off) },
6060 });
6235 if (abi_size <= 4) {
6236 const flags: u2 = switch (abi_size) {
6237 1 => 0b01,
6238 2 => 0b10,
6239 4 => 0b11,
6240 else => unreachable,
6241 };
6242 _ = try self.addInst(.{
6243 .tag = .mov_sign_extend,
6244 .ops = Mir.Inst.Ops.encode(.{
6245 .reg1 = reg.to64(),
6246 .reg2 = .rbp,
6247 .flags = flags,
6248 }),
6249 .data = .{ .imm = @bitCast(u32, -off) },
6250 });
6251 return;
6252 }
60616253 },
60626254 .unsigned => {
6063 const flags: u2 = switch (abi_size) {
6064 1 => 0b01,
6065 2 => 0b10,
6066 else => break :blk,
6255 if (abi_size <= 2) {
6256 const flags: u2 = switch (abi_size) {
6257 1 => 0b01,
6258 2 => 0b10,
6259 else => unreachable,
6260 };
6261 _ = try self.addInst(.{
6262 .tag = .mov_zero_extend,
6263 .ops = Mir.Inst.Ops.encode(.{
6264 .reg1 = reg.to64(),
6265 .reg2 = .rbp,
6266 .flags = flags,
6267 }),
6268 .data = .{ .imm = @bitCast(u32, -off) },
6269 });
6270 return;
6271 }
6272 },
6273 },
6274 .Float => {
6275 if (intrinsicsAllowed(self.target.*, ty)) {
6276 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6277 .f32 => if (hasAvxSupport(self.target.*))
6278 Mir.Inst.Tag.mov_f32_avx
6279 else
6280 Mir.Inst.Tag.mov_f32_sse,
6281 .f64 => if (hasAvxSupport(self.target.*))
6282 Mir.Inst.Tag.mov_f64_avx
6283 else
6284 Mir.Inst.Tag.mov_f64_sse,
6285 else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
60676286 };
60686287 _ = try self.addInst(.{
6069 .tag = .mov_zero_extend,
6070 .ops = (Mir.Ops{
6071 .reg1 = reg.to64(),
6072 .reg2 = .rbp,
6073 .flags = flags,
6074 }).encode(),
6288 .tag = tag,
6289 .ops = Mir.Inst.Ops.encode(.{
6290 .reg1 = reg.to128(),
6291 .reg2 = switch (ty.tag()) {
6292 .f32 => .ebp,
6293 .f64 => .rbp,
6294 else => unreachable,
6295 },
6296 }),
60756297 .data = .{ .imm = @bitCast(u32, -off) },
60766298 });
6077 },
6078 }
6079 return;
6299 return;
6300 }
6301 return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
6302 },
6303 else => {},
60806304 }
60816305
60826306 _ = try self.addInst(.{
60836307 .tag = .mov,
6084 .ops = (Mir.Ops{
6308 .ops = Mir.Inst.Ops.encode(.{
60856309 .reg1 = registerAlias(reg, abi_size),
60866310 .reg2 = .rbp,
60876311 .flags = 0b01,
6088 }).encode(),
6312 }),
60896313 .data = .{ .imm = @bitCast(u32, -off) },
60906314 });
60916315 },
......@@ -6152,14 +6376,14 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
61526376 };
61536377 _ = try self.addInst(.{
61546378 .tag = .fld,
6155 .ops = (Mir.Ops{
6379 .ops = Mir.Inst.Ops.encode(.{
6380 .reg1 = .rbp,
61566381 .flags = switch (src_ty.abiSize(self.target.*)) {
61576382 4 => 0b01,
61586383 8 => 0b10,
61596384 else => |size| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
61606385 },
6161 .reg1 = .rbp,
6162 }).encode(),
6386 }),
61636387 .data = .{ .imm = @bitCast(u32, -stack_offset) },
61646388 });
61656389
......@@ -6167,15 +6391,15 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
61676391 const stack_dst = try self.allocRegOrMem(inst, false);
61686392 _ = try self.addInst(.{
61696393 .tag = .fisttp,
6170 .ops = (Mir.Ops{
6394 .ops = Mir.Inst.Ops.encode(.{
6395 .reg1 = .rbp,
61716396 .flags = switch (dst_ty.abiSize(self.target.*)) {
61726397 1...2 => 0b00,
61736398 3...4 => 0b01,
61746399 5...8 => 0b10,
61756400 else => |size| return self.fail("TODO convert float with abiSize={}", .{size}),
61766401 },
6177 .reg1 = .rbp,
6178 }).encode(),
6402 }),
61796403 .data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) },
61806404 });
61816405
......@@ -6267,15 +6491,15 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
62676491 const src: MCValue = blk: {
62686492 switch (src_ptr) {
62696493 .got_load, .direct_load, .memory => {
6270 const reg = try self.register_manager.allocReg(null);
6494 const reg = try self.register_manager.allocReg(null, gp);
62716495 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
62726496 _ = try self.addInst(.{
62736497 .tag = .mov,
6274 .ops = (Mir.Ops{
6498 .ops = Mir.Inst.Ops.encode(.{
62756499 .reg1 = reg,
62766500 .reg2 = reg,
62776501 .flags = 0b01,
6278 }).encode(),
6502 }),
62796503 .data = .{ .imm = 0 },
62806504 });
62816505 break :blk MCValue{ .register = reg };
......@@ -6839,9 +7063,11 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
68397063 return reg.to32();
68407064 } else if (size_bytes <= 8) {
68417065 return reg.to64();
6842 } else {
6843 unreachable; // TODO handle floating-point registers
6844 }
7066 } else if (size_bytes <= 16) {
7067 return reg.to128();
7068 } else if (size_bytes <= 32) {
7069 return reg.to256();
7070 } else unreachable;
68457071}
68467072
68477073/// Truncates the value in the register in place.
......@@ -6867,3 +7093,16 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
68677093 },
68687094 }
68697095}
7096
7097fn intrinsicsAllowed(target: Target, ty: Type) bool {
7098 return switch (ty.tag()) {
7099 .f32,
7100 .f64,
7101 => Target.x86.featureSetHasAny(target.cpu.features, .{ .sse2, .avx, .avx2 }),
7102 else => unreachable, // TODO finish this off
7103 };
7104}
7105
7106fn hasAvxSupport(target: Target) bool {
7107 return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
7108}
src/arch/x86_64/Emit.zig+785-236
......@@ -25,8 +25,8 @@ const MCValue = @import("CodeGen.zig").MCValue;
2525const Mir = @import("Mir.zig");
2626const Module = @import("../../Module.zig");
2727const Instruction = bits.Instruction;
28const Register = bits.Register;
2928const Type = @import("../../type.zig").Type;
29const Register = bits.Register;
3030
3131mir: Mir,
3232bin_file: *link.File,
......@@ -67,6 +67,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
6767 const inst = @intCast(u32, index);
6868 try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
6969 switch (tag) {
70 // GPR instructions
7071 .adc => try emit.mirArith(.adc, inst),
7172 .add => try emit.mirArith(.add, inst),
7273 .sub => try emit.mirArith(.sub, inst),
......@@ -181,6 +182,27 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
181182 .interrupt => try emit.mirInterrupt(inst),
182183 .nop => try emit.mirNop(),
183184
185 // SSE instructions
186 .mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),
187 .mov_f32_sse => try emit.mirMovFloatSse(.movss, inst),
188
189 .add_f64_sse => try emit.mirAddFloatSse(.addsd, inst),
190 .add_f32_sse => try emit.mirAddFloatSse(.addss, inst),
191
192 .cmp_f64_sse => try emit.mirCmpFloatSse(.ucomisd, inst),
193 .cmp_f32_sse => try emit.mirCmpFloatSse(.ucomiss, inst),
194
195 // AVX instructions
196 .mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst),
197 .mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst),
198
199 .add_f64_avx => try emit.mirAddFloatAvx(.vaddsd, inst),
200 .add_f32_avx => try emit.mirAddFloatAvx(.vaddss, inst),
201
202 .cmp_f64_avx => try emit.mirCmpFloatAvx(.vucomisd, inst),
203 .cmp_f32_avx => try emit.mirCmpFloatAvx(.vucomiss, inst),
204
205 // Pseudo-instructions
184206 .call_extern => try emit.mirCallExtern(inst),
185207
186208 .dbg_line => try emit.mirDbgLine(inst),
......@@ -228,7 +250,7 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
228250fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
229251 const tag = emit.mir.instructions.items(.tag)[inst];
230252 assert(tag == .interrupt);
231 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
253 const ops = emit.mir.instructions.items(.ops)[inst].decode();
232254 switch (ops.flags) {
233255 0b00 => return lowerToZoEnc(.int3, emit.code),
234256 else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}),
......@@ -244,7 +266,7 @@ fn mirSyscall(emit: *Emit) InnerError!void {
244266}
245267
246268fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
247 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
269 const ops = emit.mir.instructions.items(.ops)[inst].decode();
248270 switch (ops.flags) {
249271 0b00 => {
250272 // PUSH/POP reg
......@@ -271,8 +293,9 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
271293 0b11 => unreachable,
272294 }
273295}
296
274297fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
275 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
298 const ops = emit.mir.instructions.items(.ops)[inst].decode();
276299 const payload = emit.mir.instructions.items(.data)[inst].payload;
277300 const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data;
278301 const regs = data.regs;
......@@ -295,7 +318,7 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
295318}
296319
297320fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
298 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
321 const ops = emit.mir.instructions.items(.ops)[inst].decode();
299322 switch (ops.flags) {
300323 0b00 => {
301324 const target = emit.mir.instructions.items(.data)[inst].inst;
......@@ -324,7 +347,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
324347 0b10 => {
325348 // JMP/CALL r/m64
326349 const imm = emit.mir.instructions.items(.data)[inst].imm;
327 return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
350 return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
328351 .disp = imm,
329352 .base = ops.reg1,
330353 }), emit.code);
......@@ -334,7 +357,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
334357}
335358
336359fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
337 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
360 const ops = emit.mir.instructions.items(.ops)[inst].decode();
338361 const target = emit.mir.instructions.items(.data)[inst].inst;
339362 const tag = switch (mir_tag) {
340363 .cond_jmp_greater_less => switch (ops.flags) {
......@@ -366,7 +389,7 @@ fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerErr
366389}
367390
368391fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
369 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
392 const ops = emit.mir.instructions.items(.ops)[inst].decode();
370393 const tag = switch (mir_tag) {
371394 .cond_set_byte_greater_less => switch (ops.flags) {
372395 0b00 => Tag.setge,
......@@ -396,7 +419,7 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne
396419}
397420
398421fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
399 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
422 const ops = emit.mir.instructions.items(.ops)[inst].decode();
400423 if (ops.flags == 0b00) {
401424 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
402425 }
......@@ -416,7 +439,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
416439fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
417440 const tag = emit.mir.instructions.items(.tag)[inst];
418441 assert(tag == .@"test");
419 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
442 const ops = emit.mir.instructions.items(.ops)[inst].decode();
420443 switch (ops.flags) {
421444 0b00 => {
422445 if (ops.reg2 == .none) {
......@@ -440,7 +463,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
440463fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
441464 const tag = emit.mir.instructions.items(.tag)[inst];
442465 assert(tag == .ret);
443 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
466 const ops = emit.mir.instructions.items(.ops)[inst].decode();
444467 switch (ops.flags) {
445468 0b00 => {
446469 // RETF imm16
......@@ -464,7 +487,7 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
464487}
465488
466489fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
467 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
490 const ops = emit.mir.instructions.items(.ops)[inst].decode();
468491 switch (ops.flags) {
469492 0b00 => {
470493 if (ops.reg2 == .none) {
......@@ -481,8 +504,8 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
481504 // mov reg1, [reg2 + imm32]
482505 // RM
483506 const imm = emit.mir.instructions.items(.data)[inst].imm;
484 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
485 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
507 const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
508 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
486509 .disp = imm,
487510 .base = src_reg,
488511 }), emit.code);
......@@ -494,7 +517,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
494517 // mov [reg1 + imm32], reg2
495518 // MR
496519 const imm = emit.mir.instructions.items(.data)[inst].imm;
497 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
520 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
498521 .disp = imm,
499522 .base = ops.reg1,
500523 }), ops.reg2, emit.code);
......@@ -506,7 +529,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
506529}
507530
508531fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
509 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
532 const ops = emit.mir.instructions.items(.ops)[inst].decode();
510533 assert(ops.reg2 == .none);
511534 const payload = emit.mir.instructions.items(.data)[inst].payload;
512535 const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
......@@ -523,14 +546,15 @@ fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
523546}
524547
525548inline fn setRexWRegister(reg: Register) bool {
549 if (reg.size() > 64) return false;
526550 if (reg.size() == 64) return true;
527551 return switch (reg) {
528 .ah, .bh, .ch, .dh => true,
552 .ah, .ch, .dh, .bh => true,
529553 else => false,
530554 };
531555}
532556
533inline fn immOpSize(u_imm: u32) u8 {
557inline fn immOpSize(u_imm: u32) u6 {
534558 const imm = @bitCast(i32, u_imm);
535559 if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
536560 return 8;
......@@ -542,7 +566,7 @@ inline fn immOpSize(u_imm: u32) u8 {
542566}
543567
544568fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
545 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
569 const ops = emit.mir.instructions.items(.ops)[inst].decode();
546570 const scale = ops.flags;
547571 const imm = emit.mir.instructions.items(.data)[inst].imm;
548572 // OP reg1, [reg2 + scale*rcx + imm32]
......@@ -550,7 +574,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
550574 .scale = scale,
551575 .index = .rcx,
552576 };
553 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
577 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
554578 .disp = imm,
555579 .base = ops.reg2,
556580 .scale_index = scale_index,
......@@ -558,7 +582,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
558582}
559583
560584fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
561 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
585 const ops = emit.mir.instructions.items(.ops)[inst].decode();
562586 const scale = ops.flags;
563587 const imm = emit.mir.instructions.items(.data)[inst].imm;
564588 const scale_index = ScaleIndex{
......@@ -574,7 +598,7 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
574598 }), imm, emit.code);
575599 }
576600 // OP [reg1 + scale*rax + imm32], reg2
577 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
601 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
578602 .disp = imm,
579603 .base = ops.reg1,
580604 .scale_index = scale_index,
......@@ -582,7 +606,7 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
582606}
583607
584608fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
585 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
609 const ops = emit.mir.instructions.items(.ops)[inst].decode();
586610 const scale = ops.flags;
587611 const payload = emit.mir.instructions.items(.data)[inst].payload;
588612 const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
......@@ -599,7 +623,7 @@ fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
599623}
600624
601625fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
602 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
626 const ops = emit.mir.instructions.items(.ops)[inst].decode();
603627 assert(ops.reg2 == .none);
604628 const payload = emit.mir.instructions.items(.data)[inst].payload;
605629 const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
......@@ -624,7 +648,7 @@ fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!v
624648fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
625649 const mir_tag = emit.mir.instructions.items(.tag)[inst];
626650 assert(mir_tag == .mov_sign_extend);
627 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
651 const ops = emit.mir.instructions.items(.ops)[inst].decode();
628652 const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
629653 switch (ops.flags) {
630654 0b00 => {
......@@ -655,7 +679,7 @@ fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
655679fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
656680 const mir_tag = emit.mir.instructions.items(.tag)[inst];
657681 assert(mir_tag == .mov_zero_extend);
658 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
682 const ops = emit.mir.instructions.items(.ops)[inst].decode();
659683 const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;
660684 switch (ops.flags) {
661685 0b00 => {
......@@ -682,31 +706,46 @@ fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
682706fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
683707 const tag = emit.mir.instructions.items(.tag)[inst];
684708 assert(tag == .movabs);
685 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
686 const imm: u64 = if (ops.reg1.size() == 64) blk: {
687 const payload = emit.mir.instructions.items(.data)[inst].payload;
688 const imm = emit.mir.extraData(Mir.Imm64, payload).data;
689 break :blk imm.decode();
690 } else emit.mir.instructions.items(.data)[inst].imm;
691 if (ops.flags == 0b00) {
692 // movabs reg, imm64
693 // OI
694 return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
695 }
696 if (ops.reg1 == .none) {
697 // movabs moffs64, rax
698 // TD
699 return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
709 const ops = emit.mir.instructions.items(.ops)[inst].decode();
710 switch (ops.flags) {
711 0b00 => {
712 const imm: u64 = if (ops.reg1.size() == 64) blk: {
713 const payload = emit.mir.instructions.items(.data)[inst].payload;
714 const imm = emit.mir.extraData(Mir.Imm64, payload).data;
715 break :blk imm.decode();
716 } else emit.mir.instructions.items(.data)[inst].imm;
717 // movabs reg, imm64
718 // OI
719 return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
720 },
721 0b01 => {
722 if (ops.reg1 == .none) {
723 const imm: u64 = if (ops.reg2.size() == 64) blk: {
724 const payload = emit.mir.instructions.items(.data)[inst].payload;
725 const imm = emit.mir.extraData(Mir.Imm64, payload).data;
726 break :blk imm.decode();
727 } else emit.mir.instructions.items(.data)[inst].imm;
728 // movabs moffs64, rax
729 // TD
730 return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
731 }
732 const imm: u64 = if (ops.reg1.size() == 64) blk: {
733 const payload = emit.mir.instructions.items(.data)[inst].payload;
734 const imm = emit.mir.extraData(Mir.Imm64, payload).data;
735 break :blk imm.decode();
736 } else emit.mir.instructions.items(.data)[inst].imm;
737 // movabs rax, moffs64
738 // FD
739 return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
740 },
741 else => return emit.fail("TODO unused variant: movabs 0b{b}", .{ops.flags}),
700742 }
701 // movabs rax, moffs64
702 // FD
703 return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
704743}
705744
706745fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
707746 const tag = emit.mir.instructions.items(.tag)[inst];
708747 assert(tag == .fisttp);
709 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
748 const ops = emit.mir.instructions.items(.ops)[inst].decode();
710749
711750 // the selecting between operand sizes for this particular `fisttp` instruction
712751 // is done via opcode instead of the usual prefixes.
......@@ -728,7 +767,7 @@ fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
728767fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
729768 const tag = emit.mir.instructions.items(.tag)[inst];
730769 assert(tag == .fld);
731 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
770 const ops = emit.mir.instructions.items(.ops)[inst].decode();
732771
733772 // the selecting between operand sizes for this particular `fisttp` instruction
734773 // is done via opcode instead of the usual prefixes.
......@@ -745,8 +784,9 @@ fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
745784 };
746785 return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
747786}
787
748788fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
749 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
789 const ops = emit.mir.instructions.items(.ops)[inst].decode();
750790 switch (ops.flags) {
751791 0b00 => {
752792 // sal reg1, 1
......@@ -771,12 +811,11 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
771811}
772812
773813fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
774 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
814 const ops = emit.mir.instructions.items(.ops)[inst].decode();
775815 if (ops.reg1 != .none) {
776816 assert(ops.reg2 == .none);
777817 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
778818 }
779 assert(ops.reg1 == .none);
780819 assert(ops.reg2 != .none);
781820 const imm = emit.mir.instructions.items(.data)[inst].imm;
782821 const ptr_size: Memory.PtrSize = switch (ops.flags) {
......@@ -794,14 +833,14 @@ fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
794833fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
795834 const tag = emit.mir.instructions.items(.tag)[inst];
796835 assert(tag == .imul_complex);
797 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
836 const ops = emit.mir.instructions.items(.ops)[inst].decode();
798837 switch (ops.flags) {
799838 0b00 => {
800839 return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
801840 },
802841 0b01 => {
803842 const imm = emit.mir.instructions.items(.data)[inst].imm;
804 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
843 const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
805844 return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{
806845 .disp = imm,
807846 .base = src_reg,
......@@ -823,7 +862,7 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
823862}
824863
825864fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
826 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
865 const ops = emit.mir.instructions.items(.ops)[inst].decode();
827866 const tag: Tag = switch (ops.flags) {
828867 0b00 => .cbw,
829868 0b01 => .cwd,
......@@ -836,17 +875,17 @@ fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
836875fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
837876 const tag = emit.mir.instructions.items(.tag)[inst];
838877 assert(tag == .lea);
839 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
878 const ops = emit.mir.instructions.items(.ops)[inst].decode();
840879 switch (ops.flags) {
841880 0b00 => {
842881 // lea reg1, [reg2 + imm32]
843882 // RM
844883 const imm = emit.mir.instructions.items(.data)[inst].imm;
845 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
884 const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
846885 return lowerToRmEnc(
847886 .lea,
848887 ops.reg1,
849 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
888 RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
850889 .disp = imm,
851890 .base = src_reg,
852891 }),
......@@ -860,7 +899,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
860899 try lowerToRmEnc(
861900 .lea,
862901 ops.reg1,
863 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
902 RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
864903 emit.code,
865904 );
866905 const end_offset = emit.code.items.len;
......@@ -873,7 +912,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
873912 0b10 => {
874913 // lea reg, [rbp + rcx + imm32]
875914 const imm = emit.mir.instructions.items(.data)[inst].imm;
876 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
915 const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
877916 const scale_index = ScaleIndex{
878917 .scale = 0,
879918 .index = .rcx,
......@@ -881,7 +920,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
881920 return lowerToRmEnc(
882921 .lea,
883922 ops.reg1,
884 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
923 RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
885924 .disp = imm,
886925 .base = src_reg,
887926 .scale_index = scale_index,
......@@ -896,7 +935,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
896935fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
897936 const tag = emit.mir.instructions.items(.tag)[inst];
898937 assert(tag == .lea_pie);
899 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
938 const ops = emit.mir.instructions.items(.ops)[inst].decode();
900939 const load_reloc = emit.mir.instructions.items(.data)[inst].load_reloc;
901940
902941 // lea reg1, [rip + reloc]
......@@ -904,7 +943,7 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
904943 try lowerToRmEnc(
905944 .lea,
906945 ops.reg1,
907 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
946 RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
908947 emit.code,
909948 );
910949
......@@ -935,6 +974,99 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
935974 }
936975}
937976
977// SSE instructions
978
979fn mirMovFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
980 const ops = emit.mir.instructions.items(.ops)[inst].decode();
981 switch (ops.flags) {
982 0b00 => {
983 const imm = emit.mir.instructions.items(.data)[inst].imm;
984 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
985 .disp = imm,
986 .base = ops.reg2,
987 }), emit.code);
988 },
989 0b01 => {
990 const imm = emit.mir.instructions.items(.data)[inst].imm;
991 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
992 .disp = imm,
993 .base = ops.reg1,
994 }), ops.reg2, emit.code);
995 },
996 0b10 => {
997 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
998 },
999 else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1000 }
1001}
1002
1003fn mirAddFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1004 const ops = emit.mir.instructions.items(.ops)[inst].decode();
1005 switch (ops.flags) {
1006 0b00 => {
1007 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1008 },
1009 else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1010 }
1011}
1012
1013fn mirCmpFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1014 const ops = emit.mir.instructions.items(.ops)[inst].decode();
1015 switch (ops.flags) {
1016 0b00 => {
1017 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1018 },
1019 else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1020 }
1021}
1022// AVX instructions
1023
1024fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1025 const ops = emit.mir.instructions.items(.ops)[inst].decode();
1026 switch (ops.flags) {
1027 0b00 => {
1028 const imm = emit.mir.instructions.items(.data)[inst].imm;
1029 return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
1030 .disp = imm,
1031 .base = ops.reg2,
1032 }), emit.code);
1033 },
1034 0b01 => {
1035 const imm = emit.mir.instructions.items(.data)[inst].imm;
1036 return lowerToMvEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
1037 .disp = imm,
1038 .base = ops.reg1,
1039 }), ops.reg2, emit.code);
1040 },
1041 0b10 => {
1042 return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1043 },
1044 else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1045 }
1046}
1047
1048fn mirAddFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1049 const ops = emit.mir.instructions.items(.ops)[inst].decode();
1050 switch (ops.flags) {
1051 0b00 => {
1052 return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1053 },
1054 else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1055 }
1056}
1057
1058fn mirCmpFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1059 const ops = emit.mir.instructions.items(.ops)[inst].decode();
1060 switch (ops.flags) {
1061 0b00 => {
1062 return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1063 },
1064 else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
1065 }
1066}
1067
1068// Pseudo-instructions
1069
9381070fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
9391071 const tag = emit.mir.instructions.items(.tag)[inst];
9401072 assert(tag == .call_extern);
......@@ -1168,6 +1300,54 @@ const Tag = enum {
11681300 cmovng,
11691301 cmovb,
11701302 cmovnae,
1303 movsd,
1304 movss,
1305 addsd,
1306 addss,
1307 cmpsd,
1308 cmpss,
1309 ucomisd,
1310 ucomiss,
1311 vmovsd,
1312 vmovss,
1313 vaddsd,
1314 vaddss,
1315 vcmpsd,
1316 vcmpss,
1317 vucomisd,
1318 vucomiss,
1319
1320 fn isSse(tag: Tag) bool {
1321 return switch (tag) {
1322 .movsd,
1323 .movss,
1324 .addsd,
1325 .addss,
1326 .cmpsd,
1327 .cmpss,
1328 .ucomisd,
1329 .ucomiss,
1330 => true,
1331
1332 else => false,
1333 };
1334 }
1335
1336 fn isAvx(tag: Tag) bool {
1337 return switch (tag) {
1338 .vmovsd,
1339 .vmovss,
1340 .vaddsd,
1341 .vaddss,
1342 .vcmpsd,
1343 .vcmpss,
1344 .vucomisd,
1345 .vucomiss,
1346 => true,
1347
1348 else => false,
1349 };
1350 }
11711351
11721352 fn isSetCC(tag: Tag) bool {
11731353 return switch (tag) {
......@@ -1252,177 +1432,273 @@ const Encoding = enum {
12521432
12531433 /// OP r64, r/m64, imm32
12541434 rmi,
1255};
12561435
1257const OpCode = union(enum) {
1258 one_byte: u8,
1259 two_byte: struct { _1: u8, _2: u8 },
1436 /// OP xmm1, xmm2/m64
1437 vm,
12601438
1261 fn oneByte(opc: u8) OpCode {
1262 return .{ .one_byte = opc };
1263 }
1439 /// OP m64, xmm1
1440 mv,
12641441
1265 fn twoByte(opc1: u8, opc2: u8) OpCode {
1266 return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
1442 /// OP xmm1, xmm2, xmm3/m64
1443 rvm,
1444
1445 /// OP xmm1, xmm2, xmm3/m64, imm8
1446 rvmi,
1447};
1448
1449const OpCode = struct {
1450 bytes: [3]u8,
1451 count: usize,
1452
1453 fn init(comptime in_bytes: []const u8) OpCode {
1454 comptime assert(in_bytes.len <= 3);
1455 comptime var bytes: [3]u8 = undefined;
1456 inline for (in_bytes) |x, i| {
1457 bytes[i] = x;
1458 }
1459 return .{ .bytes = bytes, .count = in_bytes.len };
12671460 }
12681461
12691462 fn encode(opc: OpCode, encoder: Encoder) void {
1270 switch (opc) {
1271 .one_byte => |v| encoder.opcode_1byte(v),
1272 .two_byte => |v| encoder.opcode_2byte(v._1, v._2),
1463 switch (opc.count) {
1464 1 => encoder.opcode_1byte(opc.bytes[0]),
1465 2 => encoder.opcode_2byte(opc.bytes[0], opc.bytes[1]),
1466 3 => encoder.opcode_3byte(opc.bytes[0], opc.bytes[1], opc.bytes[2]),
1467 else => unreachable,
12731468 }
12741469 }
12751470
12761471 fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1277 assert(opc == .one_byte);
1278 encoder.opcode_withReg(opc.one_byte, reg.lowId());
1472 assert(opc.count == 1);
1473 encoder.opcode_withReg(opc.bytes[0], reg.lowEnc());
12791474 }
12801475};
12811476
1282inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1477inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) OpCode {
1478 // zig fmt: off
12831479 switch (enc) {
12841480 .zo => return switch (tag) {
1285 .ret_near => OpCode.oneByte(0xc3),
1286 .ret_far => OpCode.oneByte(0xcb),
1287 .int3 => OpCode.oneByte(0xcc),
1288 .nop => OpCode.oneByte(0x90),
1289 .syscall => OpCode.twoByte(0x0f, 0x05),
1290 .cbw => OpCode.oneByte(0x98),
1291 .cwd, .cdq, .cqo => OpCode.oneByte(0x99),
1292 else => null,
1481 .ret_near => OpCode.init(&.{0xc3}),
1482 .ret_far => OpCode.init(&.{0xcb}),
1483 .int3 => OpCode.init(&.{0xcc}),
1484 .nop => OpCode.init(&.{0x90}),
1485 .syscall => OpCode.init(&.{ 0x0f, 0x05 }),
1486 .cbw => OpCode.init(&.{0x98}),
1487 .cwd,
1488 .cdq,
1489 .cqo => OpCode.init(&.{0x99}),
1490 else => unreachable,
12931491 },
12941492 .d => return switch (tag) {
1295 .jmp_near => OpCode.oneByte(0xe9),
1296 .call_near => OpCode.oneByte(0xe8),
1297 .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
1298 .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
1299 .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
1300 .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
1301 .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
1302 .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
1303 .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
1304 .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
1305 .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
1306 .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
1307 .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
1308 .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
1309 .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
1310 .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
1311 .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
1312 .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
1313 else => null,
1493 .jmp_near => OpCode.init(&.{0xe9}),
1494 .call_near => OpCode.init(&.{0xe8}),
1495 .jo => if (is_one_byte) OpCode.init(&.{0x70}) else OpCode.init(&.{0x0f,0x80}),
1496 .jno => if (is_one_byte) OpCode.init(&.{0x71}) else OpCode.init(&.{0x0f,0x81}),
1497 .jb,
1498 .jc,
1499 .jnae => if (is_one_byte) OpCode.init(&.{0x72}) else OpCode.init(&.{0x0f,0x82}),
1500 .jnb,
1501 .jnc,
1502 .jae => if (is_one_byte) OpCode.init(&.{0x73}) else OpCode.init(&.{0x0f,0x83}),
1503 .je,
1504 .jz => if (is_one_byte) OpCode.init(&.{0x74}) else OpCode.init(&.{0x0f,0x84}),
1505 .jne,
1506 .jnz => if (is_one_byte) OpCode.init(&.{0x75}) else OpCode.init(&.{0x0f,0x85}),
1507 .jna,
1508 .jbe => if (is_one_byte) OpCode.init(&.{0x76}) else OpCode.init(&.{0x0f,0x86}),
1509 .jnbe,
1510 .ja => if (is_one_byte) OpCode.init(&.{0x77}) else OpCode.init(&.{0x0f,0x87}),
1511 .js => if (is_one_byte) OpCode.init(&.{0x78}) else OpCode.init(&.{0x0f,0x88}),
1512 .jns => if (is_one_byte) OpCode.init(&.{0x79}) else OpCode.init(&.{0x0f,0x89}),
1513 .jpe,
1514 .jp => if (is_one_byte) OpCode.init(&.{0x7a}) else OpCode.init(&.{0x0f,0x8a}),
1515 .jpo,
1516 .jnp => if (is_one_byte) OpCode.init(&.{0x7b}) else OpCode.init(&.{0x0f,0x8b}),
1517 .jnge,
1518 .jl => if (is_one_byte) OpCode.init(&.{0x7c}) else OpCode.init(&.{0x0f,0x8c}),
1519 .jge,
1520 .jnl => if (is_one_byte) OpCode.init(&.{0x7d}) else OpCode.init(&.{0x0f,0x8d}),
1521 .jle,
1522 .jng => if (is_one_byte) OpCode.init(&.{0x7e}) else OpCode.init(&.{0x0f,0x8e}),
1523 .jg,
1524 .jnle => if (is_one_byte) OpCode.init(&.{0x7f}) else OpCode.init(&.{0x0f,0x8f}),
1525 else => unreachable,
13141526 },
13151527 .m => return switch (tag) {
1316 .jmp_near, .call_near, .push => OpCode.oneByte(0xff),
1317 .pop => OpCode.oneByte(0x8f),
1318 .seto => OpCode.twoByte(0x0f, 0x90),
1319 .setno => OpCode.twoByte(0x0f, 0x91),
1320 .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
1321 .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
1322 .sete, .setz => OpCode.twoByte(0x0f, 0x94),
1323 .setne, .setnz => OpCode.twoByte(0x0f, 0x95),
1324 .setbe, .setna => OpCode.twoByte(0x0f, 0x96),
1325 .seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
1326 .sets => OpCode.twoByte(0x0f, 0x98),
1327 .setns => OpCode.twoByte(0x0f, 0x99),
1328 .setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
1329 .setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
1330 .setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
1331 .setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
1332 .setle, .setng => OpCode.twoByte(0x0f, 0x9e),
1333 .setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1334 .idiv, .div, .imul, .mul => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1335 .fisttp16 => OpCode.oneByte(0xdf),
1336 .fisttp32 => OpCode.oneByte(0xdb),
1337 .fisttp64 => OpCode.oneByte(0xdd),
1338 .fld32 => OpCode.oneByte(0xd9),
1339 .fld64 => OpCode.oneByte(0xdd),
1340 else => null,
1528 .jmp_near,
1529 .call_near,
1530 .push => OpCode.init(&.{0xff}),
1531 .pop => OpCode.init(&.{0x8f}),
1532 .seto => OpCode.init(&.{0x0f,0x90}),
1533 .setno => OpCode.init(&.{0x0f,0x91}),
1534 .setb,
1535 .setc,
1536 .setnae => OpCode.init(&.{0x0f,0x92}),
1537 .setnb,
1538 .setnc,
1539 .setae => OpCode.init(&.{0x0f,0x93}),
1540 .sete,
1541 .setz => OpCode.init(&.{0x0f,0x94}),
1542 .setne,
1543 .setnz => OpCode.init(&.{0x0f,0x95}),
1544 .setbe,
1545 .setna => OpCode.init(&.{0x0f,0x96}),
1546 .seta,
1547 .setnbe => OpCode.init(&.{0x0f,0x97}),
1548 .sets => OpCode.init(&.{0x0f,0x98}),
1549 .setns => OpCode.init(&.{0x0f,0x99}),
1550 .setp,
1551 .setpe => OpCode.init(&.{0x0f,0x9a}),
1552 .setnp,
1553 .setop => OpCode.init(&.{0x0f,0x9b}),
1554 .setl,
1555 .setnge => OpCode.init(&.{0x0f,0x9c}),
1556 .setnl,
1557 .setge => OpCode.init(&.{0x0f,0x9d}),
1558 .setle,
1559 .setng => OpCode.init(&.{0x0f,0x9e}),
1560 .setnle,
1561 .setg => OpCode.init(&.{0x0f,0x9f}),
1562 .idiv,
1563 .div,
1564 .imul,
1565 .mul => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1566 .fisttp16 => OpCode.init(&.{0xdf}),
1567 .fisttp32 => OpCode.init(&.{0xdb}),
1568 .fisttp64 => OpCode.init(&.{0xdd}),
1569 .fld32 => OpCode.init(&.{0xd9}),
1570 .fld64 => OpCode.init(&.{0xdd}),
1571 else => unreachable,
13411572 },
13421573 .o => return switch (tag) {
1343 .push => OpCode.oneByte(0x50),
1344 .pop => OpCode.oneByte(0x58),
1345 else => null,
1574 .push => OpCode.init(&.{0x50}),
1575 .pop => OpCode.init(&.{0x58}),
1576 else => unreachable,
13461577 },
13471578 .i => return switch (tag) {
1348 .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
1349 .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
1350 .ret_near => OpCode.oneByte(0xc2),
1351 .ret_far => OpCode.oneByte(0xca),
1352 else => null,
1579 .push => if (is_one_byte) OpCode.init(&.{0x6a}) else OpCode.init(&.{0x68}),
1580 .@"test" => if (is_one_byte) OpCode.init(&.{0xa8}) else OpCode.init(&.{0xa9}),
1581 .ret_near => OpCode.init(&.{0xc2}),
1582 .ret_far => OpCode.init(&.{0xca}),
1583 else => unreachable,
13531584 },
13541585 .m1 => return switch (tag) {
1355 .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd0 else 0xd1),
1356 else => null,
1586 .shl, .sal,
1587 .shr, .sar => if (is_one_byte) OpCode.init(&.{0xd0}) else OpCode.init(&.{0xd1}),
1588 else => unreachable,
13571589 },
13581590 .mc => return switch (tag) {
1359 .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd2 else 0xd3),
1360 else => null,
1591 .shl, .sal,
1592 .shr, .sar => if (is_one_byte) OpCode.init(&.{0xd2}) else OpCode.init(&.{0xd3}),
1593 else => unreachable,
13611594 },
13621595 .mi => return switch (tag) {
1363 .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
1364 .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
1365 .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1366 else => null,
1596 .adc, .add,
1597 .sub, .xor,
1598 .@"and", .@"or",
1599 .sbb, .cmp => if (is_one_byte) OpCode.init(&.{0x80}) else OpCode.init(&.{0x81}),
1600 .mov => if (is_one_byte) OpCode.init(&.{0xc6}) else OpCode.init(&.{0xc7}),
1601 .@"test" => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1602 else => unreachable,
13671603 },
13681604 .mi8 => return switch (tag) {
1369 .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(0x83),
1370 .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xc0 else 0xc1),
1371 else => null,
1605 .adc, .add,
1606 .sub, .xor,
1607 .@"and", .@"or",
1608 .sbb, .cmp => OpCode.init(&.{0x83}),
1609 .shl, .sal,
1610 .shr, .sar => if (is_one_byte) OpCode.init(&.{0xc0}) else OpCode.init(&.{0xc1}),
1611 else => unreachable,
13721612 },
13731613 .mr => return switch (tag) {
1374 .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
1375 .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
1376 .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
1377 .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
1378 .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
1379 .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
1380 .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1381 .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1382 .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1383 .@"test" => OpCode.oneByte(if (is_one_byte) 0x84 else 0x85),
1384 else => null,
1614 .adc => if (is_one_byte) OpCode.init(&.{0x10}) else OpCode.init(&.{0x11}),
1615 .add => if (is_one_byte) OpCode.init(&.{0x00}) else OpCode.init(&.{0x01}),
1616 .sub => if (is_one_byte) OpCode.init(&.{0x28}) else OpCode.init(&.{0x29}),
1617 .xor => if (is_one_byte) OpCode.init(&.{0x30}) else OpCode.init(&.{0x31}),
1618 .@"and" => if (is_one_byte) OpCode.init(&.{0x20}) else OpCode.init(&.{0x21}),
1619 .@"or" => if (is_one_byte) OpCode.init(&.{0x08}) else OpCode.init(&.{0x09}),
1620 .sbb => if (is_one_byte) OpCode.init(&.{0x18}) else OpCode.init(&.{0x19}),
1621 .cmp => if (is_one_byte) OpCode.init(&.{0x38}) else OpCode.init(&.{0x39}),
1622 .mov => if (is_one_byte) OpCode.init(&.{0x88}) else OpCode.init(&.{0x89}),
1623 .@"test" => if (is_one_byte) OpCode.init(&.{0x84}) else OpCode.init(&.{0x85}),
1624 .movsd => OpCode.init(&.{0xf2,0x0f,0x11}),
1625 .movss => OpCode.init(&.{0xf3,0x0f,0x11}),
1626 else => unreachable,
13851627 },
13861628 .rm => return switch (tag) {
1387 .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
1388 .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
1389 .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
1390 .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
1391 .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1392 .@"or" => OpCode.oneByte(if (is_one_byte) 0x0a else 0x0b),
1393 .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
1394 .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
1395 .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
1396 .movsx => OpCode.twoByte(0x0f, if (is_one_byte) 0xbe else 0xbf),
1397 .movsxd => OpCode.oneByte(0x63),
1398 .movzx => OpCode.twoByte(0x0f, if (is_one_byte) 0xb6 else 0xb7),
1399 .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1400 .imul => OpCode.twoByte(0x0f, 0xaf),
1401 .cmove, .cmovz => OpCode.twoByte(0x0f, 0x44),
1402 .cmovb, .cmovnae => OpCode.twoByte(0x0f, 0x42),
1403 .cmovl, .cmovng => OpCode.twoByte(0x0f, 0x4c),
1404 else => null,
1629 .adc => if (is_one_byte) OpCode.init(&.{0x12}) else OpCode.init(&.{0x13}),
1630 .add => if (is_one_byte) OpCode.init(&.{0x02}) else OpCode.init(&.{0x03}),
1631 .sub => if (is_one_byte) OpCode.init(&.{0x2a}) else OpCode.init(&.{0x2b}),
1632 .xor => if (is_one_byte) OpCode.init(&.{0x32}) else OpCode.init(&.{0x33}),
1633 .@"and" => if (is_one_byte) OpCode.init(&.{0x22}) else OpCode.init(&.{0x23}),
1634 .@"or" => if (is_one_byte) OpCode.init(&.{0x0a}) else OpCode.init(&.{0x0b}),
1635 .sbb => if (is_one_byte) OpCode.init(&.{0x1a}) else OpCode.init(&.{0x1b}),
1636 .cmp => if (is_one_byte) OpCode.init(&.{0x3a}) else OpCode.init(&.{0x3b}),
1637 .mov => if (is_one_byte) OpCode.init(&.{0x8a}) else OpCode.init(&.{0x8b}),
1638 .movsx => if (is_one_byte) OpCode.init(&.{0x0f,0xbe}) else OpCode.init(&.{0x0f,0xbf}),
1639 .movsxd => OpCode.init(&.{0x63}),
1640 .movzx => if (is_one_byte) OpCode.init(&.{0x0f,0xb6}) else OpCode.init(&.{0x0f,0xb7}),
1641 .lea => if (is_one_byte) OpCode.init(&.{0x8c}) else OpCode.init(&.{0x8d}),
1642 .imul => OpCode.init(&.{0x0f,0xaf}),
1643 .cmove,
1644 .cmovz => OpCode.init(&.{0x0f,0x44}),
1645 .cmovb,
1646 .cmovnae => OpCode.init(&.{0x0f,0x42}),
1647 .cmovl,
1648 .cmovng => OpCode.init(&.{0x0f,0x4c}),
1649 .movsd => OpCode.init(&.{0xf2,0x0f,0x10}),
1650 .movss => OpCode.init(&.{0xf3,0x0f,0x10}),
1651 .addsd => OpCode.init(&.{0xf2,0x0f,0x58}),
1652 .addss => OpCode.init(&.{0xf3,0x0f,0x58}),
1653 .ucomisd => OpCode.init(&.{0x66,0x0f,0x2e}),
1654 .ucomiss => OpCode.init(&.{0x0f,0x2e}),
1655 else => unreachable,
14051656 },
14061657 .oi => return switch (tag) {
1407 .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
1408 else => null,
1658 .mov => if (is_one_byte) OpCode.init(&.{0xb0}) else OpCode.init(&.{0xb8}),
1659 else => unreachable,
14091660 },
14101661 .fd => return switch (tag) {
1411 .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
1412 else => null,
1662 .mov => if (is_one_byte) OpCode.init(&.{0xa0}) else OpCode.init(&.{0xa1}),
1663 else => unreachable,
14131664 },
14141665 .td => return switch (tag) {
1415 .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
1416 else => null,
1666 .mov => if (is_one_byte) OpCode.init(&.{0xa2}) else OpCode.init(&.{0xa3}),
1667 else => unreachable,
14171668 },
14181669 .rmi => return switch (tag) {
1419 .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1420 else => null,
1670 .imul => if (is_one_byte) OpCode.init(&.{0x6b}) else OpCode.init(&.{0x69}),
1671 else => unreachable,
1672 },
1673 .mv => return switch (tag) {
1674 .vmovsd,
1675 .vmovss => OpCode.init(&.{0x11}),
1676 else => unreachable,
1677 },
1678 .vm => return switch (tag) {
1679 .vmovsd,
1680 .vmovss => OpCode.init(&.{0x10}),
1681 .vucomisd,
1682 .vucomiss => OpCode.init(&.{0x2e}),
1683 else => unreachable,
1684 },
1685 .rvm => return switch (tag) {
1686 .vaddsd,
1687 .vaddss => OpCode.init(&.{0x58}),
1688 .vmovsd,
1689 .vmovss => OpCode.init(&.{0x10}),
1690 else => unreachable,
1691 },
1692 .rvmi => return switch (tag) {
1693 .vcmpsd,
1694 .vcmpss => OpCode.init(&.{0xc2}),
1695 else => unreachable,
14211696 },
14221697 }
1698 // zig fmt: on
14231699}
14241700
1425inline fn getModRmExt(tag: Tag) ?u3 {
1701inline fn getModRmExt(tag: Tag) u3 {
14261702 return switch (tag) {
14271703 .adc => 0x2,
14281704 .add => 0x0,
......@@ -1483,11 +1759,101 @@ inline fn getModRmExt(tag: Tag) ?u3 {
14831759 .fisttp64 => 0x1,
14841760 .fld32 => 0x0,
14851761 .fld64 => 0x0,
1486 else => null,
1762 else => unreachable,
14871763 };
14881764}
14891765
1490const ScaleIndex = struct {
1766const VexEncoding = struct {
1767 prefix: Encoder.Vex,
1768 reg: ?enum {
1769 ndd,
1770 nds,
1771 dds,
1772 },
1773};
1774
1775inline fn getVexEncoding(tag: Tag, enc: Encoding) VexEncoding {
1776 const desc: struct {
1777 reg: enum {
1778 none,
1779 ndd,
1780 nds,
1781 dds,
1782 } = .none,
1783 len_256: bool = false,
1784 wig: bool = false,
1785 lig: bool = false,
1786 lz: bool = false,
1787 lead_opc: enum {
1788 l_0f,
1789 l_0f_3a,
1790 l_0f_38,
1791 } = .l_0f,
1792 simd_prefix: enum {
1793 none,
1794 p_66,
1795 p_f2,
1796 p_f3,
1797 } = .none,
1798 } = blk: {
1799 switch (enc) {
1800 .mv => switch (tag) {
1801 .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1802 .vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1803 else => unreachable,
1804 },
1805 .vm => switch (tag) {
1806 .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1807 .vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1808 .vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true },
1809 .vucomiss => break :blk .{ .lig = true, .wig = true },
1810 else => unreachable,
1811 },
1812 .rvm => switch (tag) {
1813 .vaddsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1814 .vaddss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1815 .vmovsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1816 .vmovss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1817 else => unreachable,
1818 },
1819 .rvmi => switch (tag) {
1820 .vcmpsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1821 .vcmpss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1822 else => unreachable,
1823 },
1824 else => unreachable,
1825 }
1826 };
1827
1828 var vex: Encoder.Vex = .{};
1829
1830 if (desc.len_256) vex.len_256();
1831 if (desc.wig) vex.wig();
1832 if (desc.lig) vex.lig();
1833 if (desc.lz) vex.lz();
1834
1835 switch (desc.lead_opc) {
1836 .l_0f => {},
1837 .l_0f_3a => vex.lead_opc_0f_3a(),
1838 .l_0f_38 => vex.lead_opc_0f_38(),
1839 }
1840
1841 switch (desc.simd_prefix) {
1842 .none => {},
1843 .p_66 => vex.simd_prefix_66(),
1844 .p_f2 => vex.simd_prefix_f2(),
1845 .p_f3 => vex.simd_prefix_f3(),
1846 }
1847
1848 return VexEncoding{ .prefix = vex, .reg = switch (desc.reg) {
1849 .none => null,
1850 .nds => .nds,
1851 .dds => .dds,
1852 .ndd => .ndd,
1853 } };
1854}
1855
1856const ScaleIndex = packed struct {
14911857 scale: u2,
14921858 index: Register,
14931859};
......@@ -1499,49 +1865,38 @@ const Memory = struct {
14991865 ptr_size: PtrSize,
15001866 scale_index: ?ScaleIndex = null,
15011867
1502 const PtrSize = enum {
1503 byte_ptr,
1504 word_ptr,
1505 dword_ptr,
1506 qword_ptr,
1868 const PtrSize = enum(u2) {
1869 byte_ptr = 0b00,
1870 word_ptr = 0b01,
1871 dword_ptr = 0b10,
1872 qword_ptr = 0b11,
15071873
1508 fn fromBits(in_bits: u64) PtrSize {
1509 return switch (in_bits) {
1510 8 => .byte_ptr,
1511 16 => .word_ptr,
1512 32 => .dword_ptr,
1513 64 => .qword_ptr,
1514 else => unreachable,
1515 };
1874 fn new(bit_size: u64) PtrSize {
1875 return @intToEnum(PtrSize, math.log2_int(u4, @intCast(u4, @divExact(bit_size, 8))));
15161876 }
15171877
15181878 /// Returns size in bits.
15191879 fn size(ptr_size: PtrSize) u64 {
1520 return switch (ptr_size) {
1521 .byte_ptr => 8,
1522 .word_ptr => 16,
1523 .dword_ptr => 32,
1524 .qword_ptr => 64,
1525 };
1880 return 8 * (math.powi(u8, 2, @enumToInt(ptr_size)) catch unreachable);
15261881 }
15271882 };
15281883
15291884 fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
15301885 if (mem_op.base) |base| {
1531 const dst = base.lowId();
1886 const dst = base.lowEnc();
15321887 const src = operand;
15331888 if (dst == 4 or mem_op.scale_index != null) {
15341889 if (mem_op.disp == 0 and dst != 5) {
15351890 encoder.modRm_SIBDisp0(src);
15361891 if (mem_op.scale_index) |si| {
1537 encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
1892 encoder.sib_scaleIndexBase(si.scale, si.index.lowEnc(), dst);
15381893 } else {
15391894 encoder.sib_base(dst);
15401895 }
15411896 } else if (immOpSize(mem_op.disp) == 8) {
15421897 encoder.modRm_SIBDisp8(src);
15431898 if (mem_op.scale_index) |si| {
1544 encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
1899 encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowEnc(), dst);
15451900 } else {
15461901 encoder.sib_baseDisp8(dst);
15471902 }
......@@ -1549,7 +1904,7 @@ const Memory = struct {
15491904 } else {
15501905 encoder.modRm_SIBDisp32(src);
15511906 if (mem_op.scale_index) |si| {
1552 encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
1907 encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowEnc(), dst);
15531908 } else {
15541909 encoder.sib_baseDisp32(dst);
15551910 }
......@@ -1572,7 +1927,7 @@ const Memory = struct {
15721927 } else {
15731928 encoder.modRm_SIBDisp0(operand);
15741929 if (mem_op.scale_index) |si| {
1575 encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
1930 encoder.sib_scaleIndexDisp32(si.scale, si.index.lowEnc());
15761931 } else {
15771932 encoder.sib_disp32();
15781933 }
......@@ -1581,6 +1936,7 @@ const Memory = struct {
15811936 }
15821937 }
15831938
1939 /// Returns size in bits.
15841940 fn size(memory: Memory) u64 {
15851941 return memory.ptr_size.size();
15861942 }
......@@ -1629,6 +1985,7 @@ const RegisterOrMemory = union(enum) {
16291985 };
16301986 }
16311987
1988 /// Returns size in bits.
16321989 fn size(reg_or_mem: RegisterOrMemory) u64 {
16331990 return switch (reg_or_mem) {
16341991 .register => |reg| reg.size(),
......@@ -1638,7 +1995,8 @@ const RegisterOrMemory = union(enum) {
16381995};
16391996
16401997fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
1641 const opc = getOpCode(tag, .zo, false).?;
1998 assert(!tag.isAvx());
1999 const opc = getOpCode(tag, .zo, false);
16422000 const encoder = try Encoder.init(code, 2);
16432001 switch (tag) {
16442002 .cqo => {
......@@ -1652,14 +2010,15 @@ fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
16522010}
16532011
16542012fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2013 assert(!tag.isAvx());
16552014 if (tag == .ret_far or tag == .ret_near) {
16562015 const encoder = try Encoder.init(code, 3);
1657 const opc = getOpCode(tag, .i, false).?;
2016 const opc = getOpCode(tag, .i, false);
16582017 opc.encode(encoder);
16592018 encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
16602019 return;
16612020 }
1662 const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
2021 const opc = getOpCode(tag, .i, immOpSize(imm) == 8);
16632022 const encoder = try Encoder.init(code, 5);
16642023 if (immOpSize(imm) == 16) {
16652024 encoder.prefix16BitMode();
......@@ -1669,7 +2028,8 @@ fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
16692028}
16702029
16712030fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
1672 const opc = getOpCode(tag, .o, false).?;
2031 assert(!tag.isAvx());
2032 const opc = getOpCode(tag, .o, false);
16732033 const encoder = try Encoder.init(code, 3);
16742034 if (reg.size() == 16) {
16752035 encoder.prefix16BitMode();
......@@ -1682,15 +2042,17 @@ fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!voi
16822042}
16832043
16842044fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
1685 const opc = getOpCode(tag, .d, false).?;
2045 assert(!tag.isAvx());
2046 const opc = getOpCode(tag, .d, false);
16862047 const encoder = try Encoder.init(code, 6);
16872048 opc.encode(encoder);
16882049 encoder.imm32(@bitCast(i32, imm));
16892050}
16902051
16912052fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void {
1692 const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?;
1693 const modrm_ext = getModRmExt(tag).?;
2053 assert(!tag.isAvx());
2054 const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
2055 const modrm_ext = getModRmExt(tag);
16942056 switch (reg_or_mem) {
16952057 .register => |reg| {
16962058 const encoder = try Encoder.init(code, 4);
......@@ -1703,7 +2065,7 @@ fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *st
17032065 .b = reg.isExtended(),
17042066 });
17052067 opc.encode(encoder);
1706 encoder.modRm_direct(modrm_ext, reg.lowId());
2068 encoder.modRm_direct(modrm_ext, reg.lowEnc());
17072069 },
17082070 .memory => |mem_op| {
17092071 const encoder = try Encoder.init(code, 8);
......@@ -1744,10 +2106,8 @@ fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) I
17442106}
17452107
17462108fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void {
1747 const opc = if (td)
1748 getOpCode(tag, .td, reg.size() == 8).?
1749 else
1750 getOpCode(tag, .fd, reg.size() == 8).?;
2109 assert(!tag.isAvx());
2110 const opc = if (td) getOpCode(tag, .td, reg.size() == 8) else getOpCode(tag, .fd, reg.size() == 8);
17512111 const encoder = try Encoder.init(code, 10);
17522112 if (reg.size() == 16) {
17532113 encoder.prefix16BitMode();
......@@ -1766,7 +2126,8 @@ fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8),
17662126}
17672127
17682128fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void {
1769 const opc = getOpCode(tag, .oi, reg.size() == 8).?;
2129 assert(!tag.isAvx());
2130 const opc = getOpCode(tag, .oi, reg.size() == 8);
17702131 const encoder = try Encoder.init(code, 10);
17712132 if (reg.size() == 16) {
17722133 encoder.prefix16BitMode();
......@@ -1792,8 +2153,9 @@ fn lowerToMiXEnc(
17922153 enc: Encoding,
17932154 code: *std.ArrayList(u8),
17942155) InnerError!void {
1795 const modrm_ext = getModRmExt(tag).?;
1796 const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?;
2156 assert(!tag.isAvx());
2157 const modrm_ext = getModRmExt(tag);
2158 const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
17972159 switch (reg_or_mem) {
17982160 .register => |dst_reg| {
17992161 const encoder = try Encoder.init(code, 7);
......@@ -1808,7 +2170,7 @@ fn lowerToMiXEnc(
18082170 .b = dst_reg.isExtended(),
18092171 });
18102172 opc.encode(encoder);
1811 encoder.modRm_direct(modrm_ext, dst_reg.lowId());
2173 encoder.modRm_direct(modrm_ext, dst_reg.lowEnc());
18122174 encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_reg.size());
18132175 },
18142176 .memory => |dst_mem| {
......@@ -1847,10 +2209,11 @@ fn lowerToRmEnc(
18472209 reg_or_mem: RegisterOrMemory,
18482210 code: *std.ArrayList(u8),
18492211) InnerError!void {
1850 const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8).?;
2212 assert(!tag.isAvx());
2213 const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8);
18512214 switch (reg_or_mem) {
18522215 .register => |src_reg| {
1853 const encoder = try Encoder.init(code, 4);
2216 const encoder = try Encoder.init(code, 5);
18542217 if (reg.size() == 16) {
18552218 encoder.prefix16BitMode();
18562219 }
......@@ -1860,7 +2223,7 @@ fn lowerToRmEnc(
18602223 .b = src_reg.isExtended(),
18612224 });
18622225 opc.encode(encoder);
1863 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
2226 encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
18642227 },
18652228 .memory => |src_mem| {
18662229 const encoder = try Encoder.init(code, 9);
......@@ -1882,7 +2245,7 @@ fn lowerToRmEnc(
18822245 });
18832246 }
18842247 opc.encode(encoder);
1885 src_mem.encode(encoder, reg.lowId());
2248 src_mem.encode(encoder, reg.lowEnc());
18862249 },
18872250 }
18882251}
......@@ -1893,7 +2256,8 @@ fn lowerToMrEnc(
18932256 reg: Register,
18942257 code: *std.ArrayList(u8),
18952258) InnerError!void {
1896 const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8).?;
2259 assert(!tag.isAvx());
2260 const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8);
18972261 switch (reg_or_mem) {
18982262 .register => |dst_reg| {
18992263 const encoder = try Encoder.init(code, 4);
......@@ -1906,7 +2270,7 @@ fn lowerToMrEnc(
19062270 .b = dst_reg.isExtended(),
19072271 });
19082272 opc.encode(encoder);
1909 encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
2273 encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
19102274 },
19112275 .memory => |dst_mem| {
19122276 const encoder = try Encoder.init(code, 9);
......@@ -1926,7 +2290,7 @@ fn lowerToMrEnc(
19262290 });
19272291 }
19282292 opc.encode(encoder);
1929 dst_mem.encode(encoder, reg.lowId());
2293 dst_mem.encode(encoder, reg.lowEnc());
19302294 },
19312295 }
19322296}
......@@ -1938,7 +2302,8 @@ fn lowerToRmiEnc(
19382302 imm: u32,
19392303 code: *std.ArrayList(u8),
19402304) InnerError!void {
1941 const opc = getOpCode(tag, .rmi, false).?;
2305 assert(!tag.isAvx());
2306 const opc = getOpCode(tag, .rmi, false);
19422307 const encoder = try Encoder.init(code, 13);
19432308 if (reg.size() == 16) {
19442309 encoder.prefix16BitMode();
......@@ -1951,7 +2316,7 @@ fn lowerToRmiEnc(
19512316 .b = src_reg.isExtended(),
19522317 });
19532318 opc.encode(encoder);
1954 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
2319 encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
19552320 },
19562321 .memory => |src_mem| {
19572322 if (src_mem.base) |base| {
......@@ -1969,12 +2334,165 @@ fn lowerToRmiEnc(
19692334 });
19702335 }
19712336 opc.encode(encoder);
1972 src_mem.encode(encoder, reg.lowId());
2337 src_mem.encode(encoder, reg.lowEnc());
19732338 },
19742339 }
19752340 encodeImm(encoder, imm, reg.size());
19762341}
19772342
2343/// Also referred to as XM encoding in Intel manual.
2344fn lowerToVmEnc(
2345 tag: Tag,
2346 reg: Register,
2347 reg_or_mem: RegisterOrMemory,
2348 code: *std.ArrayList(u8),
2349) InnerError!void {
2350 const opc = getOpCode(tag, .vm, false);
2351 var enc = getVexEncoding(tag, .vm);
2352 const vex = &enc.prefix;
2353 switch (reg_or_mem) {
2354 .register => |src_reg| {
2355 const encoder = try Encoder.init(code, 5);
2356 vex.rex(.{
2357 .r = reg.isExtended(),
2358 .b = src_reg.isExtended(),
2359 });
2360 encoder.vex(enc.prefix);
2361 opc.encode(encoder);
2362 encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2363 },
2364 .memory => |src_mem| {
2365 const encoder = try Encoder.init(code, 10);
2366 if (src_mem.base) |base| {
2367 vex.rex(.{
2368 .r = reg.isExtended(),
2369 .b = base.isExtended(),
2370 });
2371 } else {
2372 vex.rex(.{
2373 .r = reg.isExtended(),
2374 });
2375 }
2376 encoder.vex(enc.prefix);
2377 opc.encode(encoder);
2378 src_mem.encode(encoder, reg.lowEnc());
2379 },
2380 }
2381}
2382
2383/// Usually referred to as MR encoding with V/V in Intel manual.
2384fn lowerToMvEnc(
2385 tag: Tag,
2386 reg_or_mem: RegisterOrMemory,
2387 reg: Register,
2388 code: *std.ArrayList(u8),
2389) InnerError!void {
2390 const opc = getOpCode(tag, .mv, false);
2391 var enc = getVexEncoding(tag, .mv);
2392 const vex = &enc.prefix;
2393 switch (reg_or_mem) {
2394 .register => |dst_reg| {
2395 const encoder = try Encoder.init(code, 4);
2396 vex.rex(.{
2397 .r = reg.isExtended(),
2398 .b = dst_reg.isExtended(),
2399 });
2400 encoder.vex(enc.prefix);
2401 opc.encode(encoder);
2402 encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2403 },
2404 .memory => |dst_mem| {
2405 const encoder = try Encoder.init(code, 10);
2406 if (dst_mem.base) |base| {
2407 vex.rex(.{
2408 .r = reg.isExtended(),
2409 .b = base.isExtended(),
2410 });
2411 } else {
2412 vex.rex(.{
2413 .r = reg.isExtended(),
2414 });
2415 }
2416 encoder.vex(enc.prefix);
2417 opc.encode(encoder);
2418 dst_mem.encode(encoder, reg.lowEnc());
2419 },
2420 }
2421}
2422
2423fn lowerToRvmEnc(
2424 tag: Tag,
2425 reg1: Register,
2426 reg2: Register,
2427 reg_or_mem: RegisterOrMemory,
2428 code: *std.ArrayList(u8),
2429) InnerError!void {
2430 const opc = getOpCode(tag, .rvm, false);
2431 var enc = getVexEncoding(tag, .rvm);
2432 const vex = &enc.prefix;
2433 switch (reg_or_mem) {
2434 .register => |reg3| {
2435 if (enc.reg) |vvvv| {
2436 switch (vvvv) {
2437 .nds => vex.reg(reg2.enc()),
2438 else => unreachable, // TODO
2439 }
2440 }
2441 const encoder = try Encoder.init(code, 5);
2442 vex.rex(.{
2443 .r = reg1.isExtended(),
2444 .b = reg3.isExtended(),
2445 });
2446 encoder.vex(enc.prefix);
2447 opc.encode(encoder);
2448 encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2449 },
2450 .memory => |dst_mem| {
2451 _ = dst_mem;
2452 unreachable; // TODO
2453 },
2454 }
2455}
2456
2457fn lowerToRvmiEnc(
2458 tag: Tag,
2459 reg1: Register,
2460 reg2: Register,
2461 reg_or_mem: RegisterOrMemory,
2462 imm: u32,
2463 code: *std.ArrayList(u8),
2464) InnerError!void {
2465 const opc = getOpCode(tag, .rvmi, false);
2466 var enc = getVexEncoding(tag, .rvmi);
2467 const vex = &enc.prefix;
2468 const encoder: Encoder = blk: {
2469 switch (reg_or_mem) {
2470 .register => |reg3| {
2471 if (enc.reg) |vvvv| {
2472 switch (vvvv) {
2473 .nds => vex.reg(reg2.enc()),
2474 else => unreachable, // TODO
2475 }
2476 }
2477 const encoder = try Encoder.init(code, 5);
2478 vex.rex(.{
2479 .r = reg1.isExtended(),
2480 .b = reg3.isExtended(),
2481 });
2482 encoder.vex(enc.prefix);
2483 opc.encode(encoder);
2484 encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2485 break :blk encoder;
2486 },
2487 .memory => |dst_mem| {
2488 _ = dst_mem;
2489 unreachable; // TODO
2490 },
2491 }
2492 };
2493 encodeImm(encoder, imm, 8); // TODO
2494}
2495
19782496fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
19792497 assert(expected.len > 0);
19802498 if (mem.eql(u8, expected, given)) return;
......@@ -2369,3 +2887,34 @@ test "lower RMI encoding" {
23692887 try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
23702888 try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
23712889}
2890
2891test "lower MV encoding" {
2892 var emit = TestEmit.init();
2893 defer emit.deinit();
2894 try lowerToMvEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), .xmm1, emit.code());
2895 try expectEqualHexStrings(
2896 "\xC5\xFB\x11\x0D\x10\x00\x00\x00",
2897 emit.lowered(),
2898 "vmovsd qword ptr [rip + 0x10], xmm1",
2899 );
2900}
2901
2902test "lower VM encoding" {
2903 var emit = TestEmit.init();
2904 defer emit.deinit();
2905 try lowerToVmEnc(.vmovsd, .xmm1, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2906 try expectEqualHexStrings(
2907 "\xC5\xFB\x10\x0D\x10\x00\x00\x00",
2908 emit.lowered(),
2909 "vmovsd xmm1, qword ptr [rip + 0x10]",
2910 );
2911}
2912
2913test "lower to RVM encoding" {
2914 var emit = TestEmit.init();
2915 defer emit.deinit();
2916 try lowerToRvmEnc(.vaddsd, .xmm0, .xmm1, RegisterOrMemory.reg(.xmm2), emit.code());
2917 try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2");
2918 try lowerToRvmEnc(.vaddsd, .xmm0, .xmm0, RegisterOrMemory.reg(.xmm1), emit.code());
2919 try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1");
2920}
src/arch/x86_64/Mir.zig+67-30
......@@ -22,11 +22,7 @@ extra: []const u32,
2222
2323pub const Inst = struct {
2424 tag: Tag,
25 /// This is 3 fields, and the meaning of each depends on `tag`.
26 /// reg1: Register
27 /// reg2: Register
28 /// flags: u2
29 ops: u16,
25 ops: Ops,
3026 /// The meaning of this depends on `tag` and `ops`.
3127 data: Data,
3228
......@@ -349,6 +345,42 @@ pub const Inst = struct {
349345 /// Nop
350346 nop,
351347
348 /// SSE instructions
349 /// ops flags: form:
350 /// 0b00 reg1, qword ptr [reg2 + imm32]
351 /// 0b01 qword ptr [reg1 + imm32], reg2
352 /// 0b10 reg1, reg2
353 mov_f64_sse,
354 mov_f32_sse,
355
356 /// ops flags: form:
357 /// 0b00 reg1, reg2
358 add_f64_sse,
359 add_f32_sse,
360
361 /// ops flags: form:
362 /// 0b00 reg1, reg2
363 cmp_f64_sse,
364 cmp_f32_sse,
365
366 /// AVX instructions
367 /// ops flags: form:
368 /// 0b00 reg1, qword ptr [reg2 + imm32]
369 /// 0b01 qword ptr [reg1 + imm32], reg2
370 /// 0b10 reg1, reg1, reg2
371 mov_f64_avx,
372 mov_f32_avx,
373
374 /// ops flags: form:
375 /// 0b00 reg1, reg1, reg2
376 add_f64_avx,
377 add_f32_avx,
378
379 /// ops flags: form:
380 /// 0b00 reg1, reg1, reg2
381 cmp_f64_avx,
382 cmp_f32_avx,
383
352384 /// Pseudo-instructions
353385 /// call extern function
354386 /// Notes:
......@@ -381,6 +413,36 @@ pub const Inst = struct {
381413 /// The position of an MIR instruction within the `Mir` instructions array.
382414 pub const Index = u32;
383415
416 pub const Ops = packed struct {
417 reg1: u7,
418 reg2: u7,
419 flags: u2,
420
421 pub fn encode(vals: struct {
422 reg1: Register = .none,
423 reg2: Register = .none,
424 flags: u2 = 0b00,
425 }) Ops {
426 return .{
427 .reg1 = @enumToInt(vals.reg1),
428 .reg2 = @enumToInt(vals.reg2),
429 .flags = vals.flags,
430 };
431 }
432
433 pub fn decode(ops: Ops) struct {
434 reg1: Register,
435 reg2: Register,
436 flags: u2,
437 } {
438 return .{
439 .reg1 = @intToEnum(Register, ops.reg1),
440 .reg2 = @intToEnum(Register, ops.reg2),
441 .flags = ops.flags,
442 };
443 }
444 };
445
384446 /// All instructions have a 4-byte payload, which is contained within
385447 /// this union. `Tag` determines which union field is active, as well as
386448 /// how to interpret the data within.
......@@ -450,31 +512,6 @@ pub const DbgLineColumn = struct {
450512 column: u32,
451513};
452514
453pub const Ops = struct {
454 reg1: Register = .none,
455 reg2: Register = .none,
456 flags: u2 = 0b00,
457
458 pub fn encode(self: Ops) u16 {
459 var ops: u16 = 0;
460 ops |= @intCast(u16, @enumToInt(self.reg1)) << 9;
461 ops |= @intCast(u16, @enumToInt(self.reg2)) << 2;
462 ops |= self.flags;
463 return ops;
464 }
465
466 pub fn decode(ops: u16) Ops {
467 const reg1 = @intToEnum(Register, @truncate(u7, ops >> 9));
468 const reg2 = @intToEnum(Register, @truncate(u7, ops >> 2));
469 const flags = @truncate(u2, ops);
470 return .{
471 .reg1 = reg1,
472 .reg2 = reg2,
473 .flags = flags,
474 };
475 }
476};
477
478515pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
479516 mir.instructions.deinit(gpa);
480517 gpa.free(mir.extra);
src/arch/x86_64/abi.zig+36-1
......@@ -3,6 +3,7 @@ const Type = @import("../../type.zig").Type;
33const Target = std.Target;
44const assert = std.debug.assert;
55const Register = @import("bits.zig").Register;
6const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
67
78pub const Class = enum { integer, sse, sseup, x87, x87up, complex_x87, memory, none };
89
......@@ -378,6 +379,40 @@ pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 };
378379/// the caller relinquishes control to a subroutine via call instruction (or similar).
379380/// In other words, these registers are free to use by the callee.
380381pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 };
381pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
382
382383pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
383384pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
385
386const sse_avx_regs = [_]Register{
387 .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7,
388 .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15,
389};
390const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ sse_avx_regs;
391pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
392
393// Register classes
394const RegisterBitSet = RegisterManager.RegisterBitSet;
395pub const RegisterClass = struct {
396 pub const gp: RegisterBitSet = @as(RegisterBitSet, std.math.maxInt(std.meta.Int(
397 .unsigned,
398 caller_preserved_regs.len + callee_preserved_regs.len,
399 )));
400 pub const sse: RegisterBitSet = std.math.maxInt(RegisterBitSet) - gp;
401 // TODO uncomment once #11680 is fixed.
402 // pub const gp: RegisterBitSet = blk: {
403 // var set = RegisterBitSet.initEmpty();
404 // set.setRangeValue(.{
405 // .start = 0,
406 // .end = caller_preserved_regs.len + callee_preserved_regs.len,
407 // }, true);
408 // break :blk set;
409 // };
410 // pub const sse: RegisterBitSet = blk: {
411 // var set = RegisterBitSet.initEmpty();
412 // set.setRangeValue(.{
413 // .start = caller_preserved_regs.len + callee_preserved_regs.len,
414 // .end = allocatable_registers.len,
415 // }, true);
416 // break :blk set;
417 // };
418};
src/arch/x86_64/bits.zig+245-18
......@@ -8,7 +8,7 @@ const DW = std.dwarf;
88
99// zig fmt: off
1010
11/// Definitions of all of the x64 registers. The order is semantically meaningful.
11/// Definitions of all of the general purpose x64 registers. The order is semantically meaningful.
1212/// The registers are defined such that IDs go in descending order of 64-bit,
1313/// 32-bit, 16-bit, and then 8-bit, and each set contains exactly sixteen
1414/// registers. This results in some useful properties:
......@@ -43,17 +43,36 @@ pub const Register = enum(u7) {
4343 al, cl, dl, bl, ah, ch, dh, bh,
4444 r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,
4545
46 // Pseudo, used only for MIR to signify that the
47 // operand is not a register but an immediate, etc.
46 // 64-79, 256-bit registers.
47 // id is int value - 64.
48 ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,
49 ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
50
51 // 80-95, 128-bit registers.
52 // id is int value - 80.
53 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
54 xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
55
56 // Pseudo-value for MIR instructions.
4857 none,
4958
59 pub fn id(self: Register) u7 {
60 return switch (@enumToInt(self)) {
61 0...63 => @as(u7, @truncate(u4, @enumToInt(self))),
62 64...79 => @enumToInt(self),
63 else => unreachable,
64 };
65 }
66
5067 /// Returns the bit-width of the register.
51 pub fn size(self: Register) u7 {
68 pub fn size(self: Register) u9 {
5269 return switch (@enumToInt(self)) {
5370 0...15 => 64,
5471 16...31 => 32,
5572 32...47 => 16,
56 48...64 => 8,
73 48...63 => 8,
74 64...79 => 256,
75 80...95 => 128,
5776 else => unreachable,
5877 };
5978 }
......@@ -72,33 +91,41 @@ pub const Register = enum(u7) {
7291 /// an instruction (@see isExtended), and requires special handling. The
7392 /// lower three bits are often embedded directly in instructions (such as
7493 /// the B8 variant of moves), or used in R/M bytes.
75 pub fn id(self: Register) u4 {
94 pub fn enc(self: Register) u4 {
7695 return @truncate(u4, @enumToInt(self));
7796 }
7897
79 /// Like id, but only returns the lower 3 bits.
80 pub fn lowId(self: Register) u3 {
98 /// Like enc, but only returns the lower 3 bits.
99 pub fn lowEnc(self: Register) u3 {
81100 return @truncate(u3, @enumToInt(self));
82101 }
83102
103 pub fn to256(self: Register) Register {
104 return @intToEnum(Register, @as(u8, self.enc()) + 64);
105 }
106
107 pub fn to128(self: Register) Register {
108 return @intToEnum(Register, @as(u8, self.enc()) + 80);
109 }
110
84111 /// Convert from any register to its 64 bit alias.
85112 pub fn to64(self: Register) Register {
86 return @intToEnum(Register, self.id());
113 return @intToEnum(Register, self.enc());
87114 }
88115
89116 /// Convert from any register to its 32 bit alias.
90117 pub fn to32(self: Register) Register {
91 return @intToEnum(Register, @as(u8, self.id()) + 16);
118 return @intToEnum(Register, @as(u8, self.enc()) + 16);
92119 }
93120
94121 /// Convert from any register to its 16 bit alias.
95122 pub fn to16(self: Register) Register {
96 return @intToEnum(Register, @as(u8, self.id()) + 32);
123 return @intToEnum(Register, @as(u8, self.enc()) + 32);
97124 }
98125
99126 /// Convert from any register to its 8 bit alias.
100127 pub fn to8(self: Register) Register {
101 return @intToEnum(Register, @as(u8, self.id()) + 48);
128 return @intToEnum(Register, @as(u8, self.enc()) + 48);
102129 }
103130
104131 pub fn dwarfLocOp(self: Register) u8 {
......@@ -251,6 +278,115 @@ pub const Encoder = struct {
251278 self.code.appendAssumeCapacity(0x66);
252279 }
253280
281 pub const Vex = struct {
282 rex_prefix: Rex = .{},
283 lead_opc: u5 = 0b0_0001,
284 register: u4 = 0b1111,
285 length: u1 = 0b0,
286 simd_prefix: u2 = 0b00,
287 wig_desc: bool = false,
288 lig_desc: bool = false,
289 lz_desc: bool = false,
290
291 pub fn rex(self: *Vex, r: Rex) void {
292 self.rex_prefix = r;
293 }
294
295 pub fn lead_opc_0f(self: *Vex) void {
296 self.lead_opc = 0b0_0001;
297 }
298
299 pub fn lead_opc_0f_38(self: *Vex) void {
300 self.lead_opc = 0b0_0010;
301 }
302
303 pub fn lead_opc_0f_3a(self: *Vex) void {
304 self.lead_opc = 0b0_0011;
305 }
306
307 pub fn reg(self: *Vex, register: u4) void {
308 self.register = ~register;
309 }
310
311 pub fn len_128(self: *Vex) void {
312 self.length = 0;
313 }
314
315 pub fn len_256(self: *Vex) void {
316 assert(!self.lz_desc);
317 self.length = 1;
318 }
319
320 pub fn simd_prefix_66(self: *Vex) void {
321 self.simd_prefix = 0b01;
322 }
323
324 pub fn simd_prefix_f3(self: *Vex) void {
325 self.simd_prefix = 0b10;
326 }
327
328 pub fn simd_prefix_f2(self: *Vex) void {
329 self.simd_prefix = 0b11;
330 }
331
332 pub fn wig(self: *Vex) void {
333 self.wig_desc = true;
334 }
335
336 pub fn lig(self: *Vex) void {
337 self.lig_desc = true;
338 }
339
340 pub fn lz(self: *Vex) void {
341 self.lz_desc = true;
342 }
343
344 pub fn write(self: Vex, writer: anytype) usize {
345 var buf: [3]u8 = .{0} ** 3;
346 const form_3byte: bool = blk: {
347 if (self.rex_prefix.w and !self.wig_desc) break :blk true;
348 if (self.rex_prefix.x or self.rex_prefix.b) break :blk true;
349 break :blk self.lead_opc != 0b0_0001;
350 };
351
352 if (self.lz_desc) {
353 assert(self.length == 0);
354 }
355
356 if (form_3byte) {
357 // First byte
358 buf[0] = 0xc4;
359 // Second byte
360 const rxb_mask: u3 = @intCast(u3, @boolToInt(!self.rex_prefix.r)) << 2 |
361 @intCast(u2, @boolToInt(!self.rex_prefix.x)) << 1 |
362 @boolToInt(!self.rex_prefix.b);
363 buf[1] |= @intCast(u8, rxb_mask) << 5;
364 buf[1] |= self.lead_opc;
365 // Third byte
366 buf[2] |= @intCast(u8, @boolToInt(!self.rex_prefix.w)) << 7;
367 buf[2] |= @intCast(u7, self.register) << 3;
368 buf[2] |= @intCast(u3, self.length) << 2;
369 buf[2] |= self.simd_prefix;
370 } else {
371 // First byte
372 buf[0] = 0xc5;
373 // Second byte
374 buf[1] |= @intCast(u8, @boolToInt(!self.rex_prefix.r)) << 7;
375 buf[1] |= @intCast(u7, self.register) << 3;
376 buf[1] |= @intCast(u3, self.length) << 2;
377 buf[1] |= self.simd_prefix;
378 }
379
380 const count: usize = if (form_3byte) 3 else 2;
381 _ = writer.writeAll(buf[0..count]) catch unreachable;
382 return count;
383 }
384 };
385
386 pub fn vex(self: Self, prefix: Vex) void {
387 _ = prefix.write(self.code.writer());
388 }
389
254390 /// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB
255391 pub const Rex = struct {
256392 /// Wide, enables 64-bit operation
......@@ -305,6 +441,17 @@ pub const Encoder = struct {
305441 self.code.appendAssumeCapacity(opcode);
306442 }
307443
444 /// Encodes a 3 byte opcode
445 ///
446 /// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
447 ///
448 /// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
449 pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) void {
450 self.code.appendAssumeCapacity(prefix_1);
451 self.code.appendAssumeCapacity(prefix_2);
452 self.code.appendAssumeCapacity(opcode);
453 }
454
308455 /// Encodes a 1 byte opcode with a reg field
309456 ///
310457 /// Remember to add a REX prefix byte if reg is extended!
......@@ -543,7 +690,7 @@ pub const Encoder = struct {
543690 }
544691};
545692
546test "x86_64 Encoder helpers" {
693test "Encoder helpers - general purpose registers" {
547694 var code = ArrayList(u8).init(testing.allocator);
548695 defer code.deinit();
549696
......@@ -560,8 +707,8 @@ test "x86_64 Encoder helpers" {
560707 });
561708 encoder.opcode_2byte(0x0f, 0xaf);
562709 encoder.modRm_direct(
563 Register.eax.lowId(),
564 Register.edi.lowId(),
710 Register.eax.lowEnc(),
711 Register.edi.lowEnc(),
565712 );
566713
567714 try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);
......@@ -580,8 +727,8 @@ test "x86_64 Encoder helpers" {
580727 });
581728 encoder.opcode_1byte(0x89);
582729 encoder.modRm_direct(
583 Register.edi.lowId(),
584 Register.eax.lowId(),
730 Register.edi.lowEnc(),
731 Register.eax.lowEnc(),
585732 );
586733
587734 try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);
......@@ -607,7 +754,7 @@ test "x86_64 Encoder helpers" {
607754 encoder.opcode_1byte(0x81);
608755 encoder.modRm_direct(
609756 0,
610 Register.rcx.lowId(),
757 Register.rcx.lowEnc(),
611758 );
612759 encoder.imm32(2147483647);
613760
......@@ -615,6 +762,86 @@ test "x86_64 Encoder helpers" {
615762 }
616763}
617764
765test "Encoder helpers - Vex prefix" {
766 var buf: [3]u8 = undefined;
767 var stream = std.io.fixedBufferStream(&buf);
768 const writer = stream.writer();
769
770 {
771 var vex_prefix = Encoder.Vex{};
772 vex_prefix.rex(.{
773 .r = true,
774 });
775 const nwritten = vex_prefix.write(writer);
776 try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, buf[0..nwritten]);
777 }
778
779 {
780 stream.reset();
781 var vex_prefix = Encoder.Vex{};
782 vex_prefix.reg(Register.xmm15.enc());
783 const nwritten = vex_prefix.write(writer);
784 try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]);
785 }
786
787 {
788 stream.reset();
789 var vex_prefix = Encoder.Vex{};
790 vex_prefix.rex(.{
791 .w = true,
792 .x = true,
793 });
794 const nwritten = vex_prefix.write(writer);
795 try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, buf[0..nwritten]);
796 }
797
798 {
799 stream.reset();
800 var vex_prefix = Encoder.Vex{};
801 vex_prefix.rex(.{
802 .w = true,
803 .r = true,
804 });
805 vex_prefix.len_256();
806 vex_prefix.lead_opc_0f();
807 vex_prefix.simd_prefix_66();
808 const nwritten = vex_prefix.write(writer);
809 try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, buf[0..nwritten]);
810 }
811
812 var code = ArrayList(u8).init(testing.allocator);
813 defer code.deinit();
814
815 {
816 // vmovapd xmm1, xmm2
817 const encoder = try Encoder.init(&code, 4);
818 var vex = Encoder.Vex{};
819 vex.simd_prefix_66();
820 encoder.vex(vex); // use 64 bit operation
821 encoder.opcode_1byte(0x28);
822 encoder.modRm_direct(0, Register.xmm1.lowEnc());
823 try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items);
824 }
825
826 {
827 try code.resize(0);
828
829 // vmovhpd xmm13, xmm1, qword ptr [rip]
830 const encoder = try Encoder.init(&code, 9);
831 var vex = Encoder.Vex{};
832 vex.len_128();
833 vex.simd_prefix_66();
834 vex.lead_opc_0f();
835 vex.rex(.{ .r = true });
836 vex.reg(Register.xmm1.enc());
837 encoder.vex(vex);
838 encoder.opcode_1byte(0x16);
839 encoder.modRm_RIPDisp32(Register.xmm13.lowEnc());
840 encoder.disp32(0);
841 try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items);
842 }
843}
844
618845// TODO add these registers to the enum and populate dwarfLocOp
619846// // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register.
620847// RA = (16, "RA"),
src/register_manager.zig+424-32
......@@ -41,28 +41,33 @@ pub fn RegisterManager(
4141 registers: [tracked_registers.len]Air.Inst.Index = undefined,
4242 /// Tracks which registers are free (in which case the
4343 /// corresponding bit is set to 1)
44 free_registers: FreeRegInt = math.maxInt(FreeRegInt),
44 free_registers: RegisterBitSet = math.maxInt(RegisterBitSet),
4545 /// Tracks all registers allocated in the course of this
4646 /// function
47 allocated_registers: FreeRegInt = 0,
47 allocated_registers: RegisterBitSet = 0,
4848 /// Tracks registers which are locked from being allocated
49 locked_registers: FreeRegInt = 0,
49 locked_registers: RegisterBitSet = 0,
5050
5151 const Self = @This();
5252
5353 /// An integer whose bits represent all the registers and
5454 /// whether they are free.
55 const FreeRegInt = std.meta.Int(.unsigned, tracked_registers.len);
56 const ShiftInt = math.Log2Int(FreeRegInt);
55 pub const RegisterBitSet = std.meta.Int(.unsigned, tracked_registers.len);
56 const ShiftInt = math.Log2Int(RegisterBitSet);
5757
5858 fn getFunction(self: *Self) *Function {
5959 return @fieldParentPtr(Function, "register_manager", self);
6060 }
6161
62 fn getRegisterMask(reg: Register) ?FreeRegInt {
62 fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool {
63 const mask = getRegisterMask(reg) orelse return true;
64 return mask & register_class == 0;
65 }
66
67 fn getRegisterMask(reg: Register) ?RegisterBitSet {
6368 const index = indexOfRegIntoTracked(reg) orelse return null;
6469 const shift = @intCast(ShiftInt, index);
65 const mask = @as(FreeRegInt, 1) << shift;
70 const mask = @as(RegisterBitSet, 1) << shift;
6671 return mask;
6772 }
6873
......@@ -81,7 +86,10 @@ pub fn RegisterManager(
8186 self.free_registers |= mask;
8287 }
8388
84 pub fn indexOfReg(comptime registers: []const Register, reg: Register) ?std.math.IntFittingRange(0, registers.len - 1) {
89 pub fn indexOfReg(
90 comptime registers: []const Register,
91 reg: Register,
92 ) ?std.math.IntFittingRange(0, registers.len - 1) {
8593 inline for (tracked_registers) |cpreg, i| {
8694 if (reg.id() == cpreg.id()) return i;
8795 }
......@@ -180,17 +188,20 @@ pub fn RegisterManager(
180188 self: *Self,
181189 comptime count: comptime_int,
182190 insts: [count]?Air.Inst.Index,
191 register_class: RegisterBitSet,
183192 ) ?[count]Register {
184193 comptime assert(count > 0 and count <= tracked_registers.len);
185194
186 const free_and_not_locked_registers = self.free_registers & ~self.locked_registers;
187 const free_and_not_locked_registers_count = @popCount(FreeRegInt, free_and_not_locked_registers);
195 const free_registers = self.free_registers & register_class;
196 const free_and_not_locked_registers = free_registers & ~self.locked_registers;
197 const free_and_not_locked_registers_count = @popCount(RegisterBitSet, free_and_not_locked_registers);
188198 if (free_and_not_locked_registers_count < count) return null;
189199
190200 var regs: [count]Register = undefined;
191201 var i: usize = 0;
192202 for (tracked_registers) |reg| {
193203 if (i >= count) break;
204 if (excludeRegister(reg, register_class)) continue;
194205 if (self.isRegLocked(reg)) continue;
195206 if (!self.isRegFree(reg)) continue;
196207
......@@ -216,8 +227,8 @@ pub fn RegisterManager(
216227 /// Allocates a register and optionally tracks it with a
217228 /// corresponding instruction. Returns `null` if all registers
218229 /// are allocated.
219 pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index) ?Register {
220 return if (tryAllocRegs(self, 1, .{inst})) |regs| regs[0] else null;
230 pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, register_class: RegisterBitSet) ?Register {
231 return if (tryAllocRegs(self, 1, .{inst}, register_class)) |regs| regs[0] else null;
221232 }
222233
223234 /// Allocates a specified number of registers, optionally
......@@ -227,12 +238,16 @@ pub fn RegisterManager(
227238 self: *Self,
228239 comptime count: comptime_int,
229240 insts: [count]?Air.Inst.Index,
241 register_class: RegisterBitSet,
230242 ) AllocateRegistersError![count]Register {
231243 comptime assert(count > 0 and count <= tracked_registers.len);
232 const locked_registers_count = @popCount(FreeRegInt, self.locked_registers);
233 if (count > tracked_registers.len - locked_registers_count) return error.OutOfRegisters;
234244
235 const result = self.tryAllocRegs(count, insts) orelse blk: {
245 const available_registers_count = @popCount(RegisterBitSet, register_class);
246 const locked_registers = self.locked_registers & register_class;
247 const locked_registers_count = @popCount(RegisterBitSet, locked_registers);
248 if (count > available_registers_count - locked_registers_count) return error.OutOfRegisters;
249
250 const result = self.tryAllocRegs(count, insts, register_class) orelse blk: {
236251 // We'll take over the first count registers. Spill
237252 // the instructions that were previously there to a
238253 // stack allocations.
......@@ -240,6 +255,7 @@ pub fn RegisterManager(
240255 var i: usize = 0;
241256 for (tracked_registers) |reg| {
242257 if (i >= count) break;
258 if (excludeRegister(reg, register_class)) continue;
243259 if (self.isRegLocked(reg)) continue;
244260
245261 regs[i] = reg;
......@@ -275,8 +291,12 @@ pub fn RegisterManager(
275291
276292 /// Allocates a register and optionally tracks it with a
277293 /// corresponding instruction.
278 pub fn allocReg(self: *Self, inst: ?Air.Inst.Index) AllocateRegistersError!Register {
279 return (try self.allocRegs(1, .{inst}))[0];
294 pub fn allocReg(
295 self: *Self,
296 inst: ?Air.Inst.Index,
297 register_class: RegisterBitSet,
298 ) AllocateRegistersError!Register {
299 return (try self.allocRegs(1, .{inst}, register_class))[0];
280300 }
281301
282302 /// Spills the register if it is currently allocated. If a
......@@ -332,6 +352,334 @@ pub fn RegisterManager(
332352 };
333353}
334354
355// TODO delete current implementation of RegisterManager above, and uncomment the one
356// below once #11680 is fixed:
357// https://github.com/ziglang/zig/issues/11680
358
359//pub fn RegisterManager(
360// comptime Function: type,
361// comptime Register: type,
362// comptime tracked_registers: []const Register,
363//) type {
364// // architectures which do not have a concept of registers should
365// // refrain from using RegisterManager
366// assert(tracked_registers.len > 0); // see note above
367
368// return struct {
369// /// Tracks the AIR instruction allocated to every register. If
370// /// no instruction is allocated to a register (i.e. the
371// /// register is free), the value in that slot is undefined.
372// ///
373// /// The key must be canonical register.
374// registers: [tracked_registers.len]Air.Inst.Index = undefined,
375// /// Tracks which registers are free (in which case the
376// /// corresponding bit is set to 1)
377// free_registers: RegisterBitSet = RegisterBitSet.initFull(),
378// /// Tracks all registers allocated in the course of this
379// /// function
380// allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
381// /// Tracks registers which are locked from being allocated
382// locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(),
383
384// const Self = @This();
385
386// pub const RegisterBitSet = StaticBitSet(tracked_registers.len);
387
388// fn getFunction(self: *Self) *Function {
389// return @fieldParentPtr(Function, "register_manager", self);
390// }
391
392// fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool {
393// const index = indexOfRegIntoTracked(reg) orelse return true;
394// return !register_class.isSet(index);
395// }
396
397// fn markRegAllocated(self: *Self, reg: Register) void {
398// const index = indexOfRegIntoTracked(reg) orelse return;
399// self.allocated_registers.set(index);
400// }
401
402// fn markRegUsed(self: *Self, reg: Register) void {
403// const index = indexOfRegIntoTracked(reg) orelse return;
404// self.free_registers.unset(index);
405// }
406
407// fn markRegFree(self: *Self, reg: Register) void {
408// const index = indexOfRegIntoTracked(reg) orelse return;
409// self.free_registers.set(index);
410// }
411
412// pub fn indexOfReg(
413// comptime registers: []const Register,
414// reg: Register,
415// ) ?std.math.IntFittingRange(0, registers.len - 1) {
416// inline for (tracked_registers) |cpreg, i| {
417// if (reg.id() == cpreg.id()) return i;
418// }
419// return null;
420// }
421
422// pub fn indexOfRegIntoTracked(reg: Register) ?RegisterBitSet.ShiftInt {
423// return indexOfReg(tracked_registers, reg);
424// }
425
426// /// Returns true when this register is not tracked
427// pub fn isRegFree(self: Self, reg: Register) bool {
428// const index = indexOfRegIntoTracked(reg) orelse return true;
429// return self.free_registers.isSet(index);
430// }
431
432// /// Returns whether this register was allocated in the course
433// /// of this function.
434// ///
435// /// Returns false when this register is not tracked
436// pub fn isRegAllocated(self: Self, reg: Register) bool {
437// const index = indexOfRegIntoTracked(reg) orelse return false;
438// return self.allocated_registers.isSet(index);
439// }
440
441// /// Returns whether this register is locked
442// ///
443// /// Returns false when this register is not tracked
444// pub fn isRegLocked(self: Self, reg: Register) bool {
445// const index = indexOfRegIntoTracked(reg) orelse return false;
446// return self.locked_registers.isSet(index);
447// }
448
449// pub const RegisterLock = struct {
450// register: Register,
451// };
452
453// /// Prevents the register from being allocated until they are
454// /// unlocked again.
455// /// Returns `RegisterLock` if the register was not already
456// /// locked, or `null` otherwise.
457// /// Only the owner of the `RegisterLock` can unlock the
458// /// register later.
459// pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
460// log.debug("locking {}", .{reg});
461// if (self.isRegLocked(reg)) {
462// log.debug(" register already locked", .{});
463// return null;
464// }
465// const index = indexOfRegIntoTracked(reg) orelse return null;
466// self.locked_registers.set(index);
467// return RegisterLock{ .register = reg };
468// }
469
470// /// Like `lockReg` but asserts the register was unused always
471// /// returning a valid lock.
472// pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
473// log.debug("locking asserting free {}", .{reg});
474// assert(!self.isRegLocked(reg));
475// const index = indexOfRegIntoTracked(reg) orelse unreachable;
476// self.locked_registers.set(index);
477// return RegisterLock{ .register = reg };
478// }
479
480// /// Like `lockRegAssumeUnused` but locks multiple registers.
481// pub fn lockRegsAssumeUnused(
482// self: *Self,
483// comptime count: comptime_int,
484// regs: [count]Register,
485// ) [count]RegisterLock {
486// var buf: [count]RegisterLock = undefined;
487// for (regs) |reg, i| {
488// buf[i] = self.lockRegAssumeUnused(reg);
489// }
490// return buf;
491// }
492
493// /// Unlocks the register allowing its re-allocation and re-use.
494// /// Requires `RegisterLock` to unlock a register.
495// /// Call `lockReg` to obtain the lock first.
496// pub fn unlockReg(self: *Self, lock: RegisterLock) void {
497// log.debug("unlocking {}", .{lock.register});
498// const index = indexOfRegIntoTracked(lock.register) orelse return;
499// self.locked_registers.unset(index);
500// }
501
502// /// Returns true when at least one register is locked
503// pub fn lockedRegsExist(self: Self) bool {
504// return self.locked_registers.count() > 0;
505// }
506
507// /// Allocates a specified number of registers, optionally
508// /// tracking them. Returns `null` if not enough registers are
509// /// free.
510// pub fn tryAllocRegs(
511// self: *Self,
512// comptime count: comptime_int,
513// insts: [count]?Air.Inst.Index,
514// register_class: RegisterBitSet,
515// ) ?[count]Register {
516// comptime assert(count > 0 and count <= tracked_registers.len);
517
518// var free_and_not_locked_registers = self.free_registers;
519// free_and_not_locked_registers.setIntersection(register_class);
520
521// var unlocked_registers = self.locked_registers;
522// unlocked_registers.toggleAll();
523
524// free_and_not_locked_registers.setIntersection(unlocked_registers);
525
526// if (free_and_not_locked_registers.count() < count) return null;
527
528// var regs: [count]Register = undefined;
529// var i: usize = 0;
530// for (tracked_registers) |reg| {
531// if (i >= count) break;
532// if (excludeRegister(reg, register_class)) continue;
533// if (self.isRegLocked(reg)) continue;
534// if (!self.isRegFree(reg)) continue;
535
536// regs[i] = reg;
537// i += 1;
538// }
539// assert(i == count);
540
541// for (regs) |reg, j| {
542// self.markRegAllocated(reg);
543
544// if (insts[j]) |inst| {
545// // Track the register
546// const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null
547// self.registers[index] = inst;
548// self.markRegUsed(reg);
549// }
550// }
551
552// return regs;
553// }
554
555// /// Allocates a register and optionally tracks it with a
556// /// corresponding instruction. Returns `null` if all registers
557// /// are allocated.
558// pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, register_class: RegisterBitSet) ?Register {
559// return if (tryAllocRegs(self, 1, .{inst}, register_class)) |regs| regs[0] else null;
560// }
561
562// /// Allocates a specified number of registers, optionally
563// /// tracking them. Asserts that count is not
564// /// larger than the total number of registers available.
565// pub fn allocRegs(
566// self: *Self,
567// comptime count: comptime_int,
568// insts: [count]?Air.Inst.Index,
569// register_class: RegisterBitSet,
570// ) AllocateRegistersError![count]Register {
571// comptime assert(count > 0 and count <= tracked_registers.len);
572
573// var locked_registers = self.locked_registers;
574// locked_registers.setIntersection(register_class);
575
576// if (count > register_class.count() - locked_registers.count()) return error.OutOfRegisters;
577
578// const result = self.tryAllocRegs(count, insts, register_class) orelse blk: {
579// // We'll take over the first count registers. Spill
580// // the instructions that were previously there to a
581// // stack allocations.
582// var regs: [count]Register = undefined;
583// var i: usize = 0;
584// for (tracked_registers) |reg| {
585// if (i >= count) break;
586// if (excludeRegister(reg, register_class)) break;
587// if (self.isRegLocked(reg)) continue;
588
589// regs[i] = reg;
590// self.markRegAllocated(reg);
591// const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null
592// if (insts[i]) |inst| {
593// // Track the register
594// if (self.isRegFree(reg)) {
595// self.markRegUsed(reg);
596// } else {
597// const spilled_inst = self.registers[index];
598// try self.getFunction().spillInstruction(reg, spilled_inst);
599// }
600// self.registers[index] = inst;
601// } else {
602// // Don't track the register
603// if (!self.isRegFree(reg)) {
604// const spilled_inst = self.registers[index];
605// try self.getFunction().spillInstruction(reg, spilled_inst);
606// self.freeReg(reg);
607// }
608// }
609
610// i += 1;
611// }
612
613// break :blk regs;
614// };
615
616// log.debug("allocated registers {any} for insts {any}", .{ result, insts });
617// return result;
618// }
619
620// /// Allocates a register and optionally tracks it with a
621// /// corresponding instruction.
622// pub fn allocReg(
623// self: *Self,
624// inst: ?Air.Inst.Index,
625// register_class: RegisterBitSet,
626// ) AllocateRegistersError!Register {
627// return (try self.allocRegs(1, .{inst}, register_class))[0];
628// }
629
630// /// Spills the register if it is currently allocated. If a
631// /// corresponding instruction is passed, will also track this
632// /// register.
633// pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
634// const index = indexOfRegIntoTracked(reg) orelse return;
635// log.debug("getReg {} for inst {}", .{ reg, inst });
636// self.markRegAllocated(reg);
637
638// if (inst) |tracked_inst|
639// if (!self.isRegFree(reg)) {
640// // Move the instruction that was previously there to a
641// // stack allocation.
642// const spilled_inst = self.registers[index];
643// self.registers[index] = tracked_inst;
644// try self.getFunction().spillInstruction(reg, spilled_inst);
645// } else {
646// self.getRegAssumeFree(reg, tracked_inst);
647// }
648// else {
649// if (!self.isRegFree(reg)) {
650// // Move the instruction that was previously there to a
651// // stack allocation.
652// const spilled_inst = self.registers[index];
653// try self.getFunction().spillInstruction(reg, spilled_inst);
654// self.freeReg(reg);
655// }
656// }
657// }
658
659// /// Allocates the specified register with the specified
660// /// instruction. Asserts that the register is free and no
661// /// spilling is necessary.
662// pub fn getRegAssumeFree(self: *Self, reg: Register, inst: Air.Inst.Index) void {
663// const index = indexOfRegIntoTracked(reg) orelse return;
664// log.debug("getRegAssumeFree {} for inst {}", .{ reg, inst });
665// self.markRegAllocated(reg);
666
667// assert(self.isRegFree(reg));
668// self.registers[index] = inst;
669// self.markRegUsed(reg);
670// }
671
672// /// Marks the specified register as free
673// pub fn freeReg(self: *Self, reg: Register) void {
674// const index = indexOfRegIntoTracked(reg) orelse return;
675// log.debug("freeing register {}", .{reg});
676
677// self.registers[index] = undefined;
678// self.markRegFree(reg);
679// }
680// };
681//}
682
335683const MockRegister1 = enum(u2) {
336684 r0,
337685 r1,
......@@ -361,11 +709,15 @@ const MockRegister2 = enum(u2) {
361709fn MockFunction(comptime Register: type) type {
362710 return struct {
363711 allocator: Allocator,
364 register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{},
712 register_manager: RegisterManagerT = .{},
365713 spilled: std.ArrayListUnmanaged(Register) = .{},
366714
367715 const Self = @This();
368716
717 const RegisterManagerT = RegisterManager(Self, Register, &Register.allocatable_registers);
718
719 pub const reg_class: RegisterManagerT.RegisterBitSet = math.maxInt(RegisterManagerT.RegisterBitSet);
720
369721 pub fn deinit(self: *Self) void {
370722 self.spilled.deinit(self.allocator);
371723 }
......@@ -410,10 +762,20 @@ test "tryAllocReg: no spilling" {
410762 defer function.deinit();
411763
412764 const mock_instruction: Air.Inst.Index = 1;
413
414 try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction));
415 try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction));
416 try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction));
765 const reg_class = MockFunction1.reg_class;
766
767 try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(
768 mock_instruction,
769 reg_class,
770 ));
771 try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(
772 mock_instruction,
773 reg_class,
774 ));
775 try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(
776 mock_instruction,
777 reg_class,
778 ));
417779
418780 try expect(function.register_manager.isRegAllocated(.r2));
419781 try expect(function.register_manager.isRegAllocated(.r3));
......@@ -438,17 +800,30 @@ test "allocReg: spilling" {
438800 defer function.deinit();
439801
440802 const mock_instruction: Air.Inst.Index = 1;
803 const reg_class = MockFunction1.reg_class;
441804
442 try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));
443 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
805 try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(
806 mock_instruction,
807 reg_class,
808 ));
809 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(
810 mock_instruction,
811 reg_class,
812 ));
444813
445814 // Spill a register
446 try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction));
815 try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(
816 mock_instruction,
817 reg_class,
818 ));
447819 try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
448820
449821 // No spilling necessary
450822 function.register_manager.freeReg(.r3);
451 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
823 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(
824 mock_instruction,
825 reg_class,
826 ));
452827 try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
453828
454829 // Locked registers
......@@ -457,7 +832,10 @@ test "allocReg: spilling" {
457832 const lock = function.register_manager.lockReg(.r2);
458833 defer if (lock) |reg| function.register_manager.unlockReg(reg);
459834
460 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
835 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(
836 mock_instruction,
837 reg_class,
838 ));
461839 }
462840 try expect(!function.register_manager.lockedRegsExist());
463841}
......@@ -470,7 +848,13 @@ test "tryAllocRegs" {
470848 };
471849 defer function.deinit();
472850
473 try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
851 const reg_class = MockFunction2.reg_class;
852
853 try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(
854 3,
855 .{ null, null, null },
856 reg_class,
857 ).?);
474858
475859 try expect(function.register_manager.isRegAllocated(.r0));
476860 try expect(function.register_manager.isRegAllocated(.r1));
......@@ -485,7 +869,11 @@ test "tryAllocRegs" {
485869 const lock = function.register_manager.lockReg(.r1);
486870 defer if (lock) |reg| function.register_manager.unlockReg(reg);
487871
488 try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
872 try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(
873 3,
874 .{ null, null, null },
875 reg_class,
876 ).?);
489877 }
490878 try expect(!function.register_manager.lockedRegsExist());
491879
......@@ -505,6 +893,8 @@ test "allocRegs: normal usage" {
505893 };
506894 defer function.deinit();
507895
896 const reg_class = MockFunction2.reg_class;
897
508898 {
509899 const result_reg: MockRegister2 = .r1;
510900
......@@ -524,7 +914,7 @@ test "allocRegs: normal usage" {
524914 const lock = function.register_manager.lockReg(result_reg);
525915 defer if (lock) |reg| function.register_manager.unlockReg(reg);
526916
527 const regs = try function.register_manager.allocRegs(2, .{ null, null });
917 const regs = try function.register_manager.allocRegs(2, .{ null, null }, reg_class);
528918 try function.genAdd(result_reg, regs[0], regs[1]);
529919 }
530920}
......@@ -539,6 +929,8 @@ test "allocRegs: selectively reducing register pressure" {
539929 };
540930 defer function.deinit();
541931
932 const reg_class = MockFunction2.reg_class;
933
542934 {
543935 const result_reg: MockRegister2 = .r1;
544936
......@@ -546,12 +938,12 @@ test "allocRegs: selectively reducing register pressure" {
546938
547939 // Here, we don't defer unlock because we manually unlock
548940 // after genAdd
549 const regs = try function.register_manager.allocRegs(2, .{ null, null });
941 const regs = try function.register_manager.allocRegs(2, .{ null, null }, reg_class);
550942
551943 try function.genAdd(result_reg, regs[0], regs[1]);
552944 function.register_manager.unlockReg(lock.?);
553945
554 const extra_summand_reg = try function.register_manager.allocReg(null);
946 const extra_summand_reg = try function.register_manager.allocReg(null, reg_class);
555947 try function.genAdd(result_reg, result_reg, extra_summand_reg);
556948 }
557949}
test/behavior/basic.zig+3
......@@ -402,6 +402,7 @@ fn testPointerToVoidReturnType2() *const void {
402402test "array 2D const double ptr" {
403403 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
404404 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
405 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
405406 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
406407
407408 const rect_2d_vertexes = [_][1]f32{
......@@ -414,6 +415,7 @@ test "array 2D const double ptr" {
414415test "array 2D const double ptr with offset" {
415416 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
416417 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
418 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
417419 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
418420
419421 const rect_2d_vertexes = [_][2]f32{
......@@ -426,6 +428,7 @@ test "array 2D const double ptr with offset" {
426428test "array 3D const double ptr with offset" {
427429 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
428430 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
431 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
429432 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
430433
431434 const rect_3d_vertexes = [_][2][2]f32{
test/behavior/math.zig-1
......@@ -198,7 +198,6 @@ test "const number literal" {
198198const ten = 10;
199199
200200test "float equality" {
201 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
202201 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
203202 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
204203
test/behavior/union.zig+1
......@@ -882,6 +882,7 @@ test "extern union doesn't trigger field check at comptime" {
882882test "anonymous union literal syntax" {
883883 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
884884 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
885 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
885886
886887 const S = struct {
887888 const Number = union {