authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-18 12:23:07+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-19 19:39:34+02:00
logf34615082058549d8945a39195cf0dc826b688e1
tree8134a2b559441b8b583e37454146e4a2b41ca9aa
parent549174f743d3b56544e2d3d6993f4d43bba1e7fd

x64: use register classes mask to select between gp and avx

4 files changed, 81 insertions(+), 39 deletions(-)

src/arch/x86_64/CodeGen.zig+49-20
......@@ -38,6 +38,9 @@ const c_abi_int_return_regs = abi.c_abi_int_return_regs;
3838const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
3939const RegisterLock = RegisterManager.RegisterLock;
4040const Register = bits.Register;
41const RegisterClass = abi.RegisterClass;
42const gp = RegisterClass.gp;
43const avx = RegisterClass.avx;
4144
4245const InnerError = error{
4346 OutOfMemory,
......@@ -882,7 +885,9 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
882885 // TODO check if AVX available
883886 const ptr_bytes: u64 = 32;
884887 if (abi_size <= ptr_bytes) {
885 if (self.register_manager.tryAllocReg(inst, .{})) |reg| {
888 if (self.register_manager.tryAllocReg(inst, .{
889 .selector_mask = avx,
890 })) |reg| {
886891 return MCValue{ .register = registerAlias(reg, abi_size) };
887892 }
888893 }
......@@ -892,7 +897,9 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
892897 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
893898 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
894899 if (abi_size <= ptr_bytes) {
895 if (self.register_manager.tryAllocReg(inst, .{})) |reg| {
900 if (self.register_manager.tryAllocReg(inst, .{
901 .selector_mask = gp,
902 })) |reg| {
896903 return MCValue{ .register = registerAlias(reg, abi_size) };
897904 }
898905 }
......@@ -963,7 +970,13 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou
963970/// allocated. A second call to `copyToTmpRegister` may return the same register.
964971/// This can have a side effect of spilling instructions to the stack to free up a register.
965972fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
966 const reg = try self.register_manager.allocReg(null, .{});
973 const mask = switch (ty.zigTypeTag()) {
974 .Float => avx,
975 else => gp,
976 };
977 const reg: Register = try self.register_manager.allocReg(null, .{
978 .selector_mask = mask,
979 });
967980 try self.genSetReg(ty, reg, mcv);
968981 return reg;
969982}
......@@ -973,7 +986,13 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
973986/// This can have a side effect of spilling instructions to the stack to free up a register.
974987/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
975988fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {
976 const reg = try self.register_manager.allocReg(reg_owner, .{});
989 const mask = switch (ty.zigTypeTag()) {
990 .Float => avx,
991 else => gp,
992 };
993 const reg: Register = try self.register_manager.allocReg(reg_owner, .{
994 .selector_mask = mask,
995 });
977996 try self.genSetReg(ty, reg, mcv);
978997 return MCValue{ .register = reg };
979998}
......@@ -1029,7 +1048,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
10291048 };
10301049 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
10311050
1032 const reg = try self.register_manager.allocReg(inst, .{});
1051 const reg = try self.register_manager.allocReg(inst, .{
1052 .selector_mask = gp,
1053 });
10331054 try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });
10341055 try self.genSetReg(operand_ty, reg, operand);
10351056 break :blk MCValue{ .register = reg };
......@@ -1384,7 +1405,9 @@ fn genSetStackTruncatedOverflowCompare(
13841405 .unsigned => ty,
13851406 };
13861407
1387 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{});
1408 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{
1409 .selector_mask = gp,
1410 });
13881411 const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
13891412 defer for (temp_regs_locks) |rreg| {
13901413 self.register_manager.unlockReg(rreg);
......@@ -2046,7 +2069,9 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
20462069 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
20472070 defer self.register_manager.unlockReg(offset_reg_lock);
20482071
2049 const addr_reg = try self.register_manager.allocReg(null, .{});
2072 const addr_reg = try self.register_manager.allocReg(null, .{
2073 .selector_mask = gp,
2074 });
20502075 switch (slice_mcv) {
20512076 .stack_offset => |off| {
20522077 // mov reg, [rbp - 8]
......@@ -2125,7 +2150,9 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
21252150 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
21262151 defer self.register_manager.unlockReg(offset_reg_lock);
21272152
2128 const addr_reg = try self.register_manager.allocReg(null, .{});
2153 const addr_reg = try self.register_manager.allocReg(null, .{
2154 .selector_mask = gp,
2155 });
21292156 switch (array) {
21302157 .register => {
21312158 const off = @intCast(i32, try self.allocMem(
......@@ -2492,7 +2519,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
24922519 },
24932520 .stack_offset => |off| {
24942521 if (abi_size <= 8) {
2495 const tmp_reg = try self.register_manager.allocReg(null, .{});
2522 const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
24962523 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
24972524 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
24982525 }
......@@ -2693,7 +2720,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
26932720 };
26942721 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
26952722
2696 const addr_reg = try self.register_manager.allocReg(null, .{});
2723 const addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
26972724 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
26982725 defer self.register_manager.unlockReg(addr_reg_lock);
26992726
......@@ -2765,7 +2792,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27652792 .memory,
27662793 => {
27672794 if (abi_size <= 8) {
2768 const tmp_reg = try self.register_manager.allocReg(null, .{});
2795 const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
27692796 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
27702797 defer self.register_manager.unlockReg(tmp_reg_lock);
27712798
......@@ -2883,7 +2910,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
28832910 if (can_reuse_operand) {
28842911 break :blk reg;
28852912 } else {
2886 const result_reg = try self.register_manager.allocReg(inst, .{});
2913 const result_reg = try self.register_manager.allocReg(inst, .{ .selector_mask = gp });
28872914 try self.genSetReg(ptr_ty, result_reg, mcv);
28882915 break :blk result_reg;
28892916 }
......@@ -2984,7 +3011,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
29843011 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
29853012 defer self.register_manager.unlockReg(reg_lock);
29863013
2987 const dst_reg = try self.register_manager.allocReg(inst, .{});
3014 const dst_reg = try self.register_manager.allocReg(inst, .{ .selector_mask = gp });
29883015 const flags: u2 = switch (mcv) {
29893016 .register_overflow_unsigned => 0b10,
29903017 .register_overflow_signed => 0b00,
......@@ -5362,7 +5389,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
53625389
53635390 const overflow_bit_ty = ty.structFieldType(1);
53645391 const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
5365 const tmp_reg = try self.register_manager.allocReg(null, .{});
5392 const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
53665393 const flags: u2 = switch (mcv) {
53675394 .register_overflow_unsigned => 0b10,
53685395 .register_overflow_signed => 0b00,
......@@ -5580,7 +5607,7 @@ fn genInlineMemcpy(
55805607 null;
55815608 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
55825609
5583 const dst_addr_reg = try self.register_manager.allocReg(null, .{});
5610 const dst_addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
55845611 switch (dst_ptr) {
55855612 .memory,
55865613 .got_load,
......@@ -5615,7 +5642,7 @@ fn genInlineMemcpy(
56155642 const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
56165643 defer self.register_manager.unlockReg(dst_addr_reg_lock);
56175644
5618 const src_addr_reg = try self.register_manager.allocReg(null, .{});
5645 const src_addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
56195646 switch (src_ptr) {
56205647 .memory,
56215648 .got_load,
......@@ -5650,7 +5677,9 @@ fn genInlineMemcpy(
56505677 const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
56515678 defer self.register_manager.unlockReg(src_addr_reg_lock);
56525679
5653 const regs = try self.register_manager.allocRegs(2, .{ null, null }, .{});
5680 const regs = try self.register_manager.allocRegs(2, .{ null, null }, .{
5681 .selector_mask = gp,
5682 });
56545683 const count_reg = regs[0].to64();
56555684 const tmp_reg = regs[1].to8();
56565685
......@@ -5750,7 +5779,7 @@ fn genInlineMemset(
57505779 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
57515780 defer self.register_manager.unlockReg(rax_lock);
57525781
5753 const addr_reg = try self.register_manager.allocReg(null, .{});
5782 const addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
57545783 switch (dst_ptr) {
57555784 .memory,
57565785 .got_load,
......@@ -6018,7 +6047,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
60186047 switch (ty.tag()) {
60196048 .f32 => return self.fail("TODO genSetReg from memory for f32", .{}),
60206049 .f64 => {
6021 const base_reg = try self.register_manager.allocReg(null, .{});
6050 const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
60226051 try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
60236052 _ = try self.addInst(.{
60246053 .tag = .mov_f64,
......@@ -6328,7 +6357,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
63286357 const src: MCValue = blk: {
63296358 switch (src_ptr) {
63306359 .got_load, .direct_load, .memory => {
6331 const reg = try self.register_manager.allocReg(null, .{});
6360 const reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
63326361 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
63336362 _ = try self.addInst(.{
63346363 .tag = .mov,
src/arch/x86_64/abi.zig+7-6
......@@ -384,11 +384,12 @@ pub const avx_regs = [_]Register{
384384};
385385pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs;
386386
387// Masks for register manager
388const FreeRegInt = std.meta.Int(.unsigned, allocatable_registers.len);
389// TODO
390pub const gp_mask: FreeRegInt = 0x3fff;
391pub const avx_mask: FreeRegInt = 0x3fff_c000;
392
393387pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
394388pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
389
390// Masks for register manager
391const FreeRegInt = std.meta.Int(.unsigned, allocatable_registers.len);
392pub const RegisterClass = struct {
393 pub const gp: FreeRegInt = 0x3fff;
394 pub const avx: FreeRegInt = 0x3fff_c000;
395};
src/arch/x86_64/bits.zig+9-9
......@@ -56,10 +56,10 @@ pub const Register = enum(u7) {
5656 // Pseudo-value for MIR instructions.
5757 none,
5858
59 pub fn id(self: Register) u5 {
59 pub fn id(self: Register) u7 {
6060 return switch (@enumToInt(self)) {
61 0...63 => @as(u5, @truncate(u4, @enumToInt(self))),
62 64...79 => @truncate(u5, @enumToInt(self)),
61 0...63 => @as(u7, @truncate(u4, @enumToInt(self))),
62 64...79 => @enumToInt(self),
6363 else => unreachable,
6464 };
6565 }
......@@ -101,31 +101,31 @@ pub const Register = enum(u7) {
101101 }
102102
103103 pub fn to256(self: Register) Register {
104 return @intToEnum(Register, @as(u8, self.id()) + 64);
104 return @intToEnum(Register, @as(u8, self.enc()) + 64);
105105 }
106106
107107 pub fn to128(self: Register) Register {
108 return @intToEnum(Register, @as(u8, self.id()) + 80);
108 return @intToEnum(Register, @as(u8, self.enc()) + 80);
109109 }
110110
111111 /// Convert from any register to its 64 bit alias.
112112 pub fn to64(self: Register) Register {
113 return @intToEnum(Register, self.id());
113 return @intToEnum(Register, self.enc());
114114 }
115115
116116 /// Convert from any register to its 32 bit alias.
117117 pub fn to32(self: Register) Register {
118 return @intToEnum(Register, @as(u8, self.id()) + 16);
118 return @intToEnum(Register, @as(u8, self.enc()) + 16);
119119 }
120120
121121 /// Convert from any register to its 16 bit alias.
122122 pub fn to16(self: Register) Register {
123 return @intToEnum(Register, @as(u8, self.id()) + 32);
123 return @intToEnum(Register, @as(u8, self.enc()) + 32);
124124 }
125125
126126 /// Convert from any register to its 8 bit alias.
127127 pub fn to8(self: Register) Register {
128 return @intToEnum(Register, @as(u8, self.id()) + 48);
128 return @intToEnum(Register, @as(u8, self.enc()) + 48);
129129 }
130130
131131 pub fn dwarfLocOp(self: Register) u8 {
src/register_manager.zig+16-4
......@@ -66,6 +66,11 @@ pub fn RegisterManager(
6666 return mask;
6767 }
6868
69 fn excludeRegister(reg: Register, mask: FreeRegInt) bool {
70 const reg_mask = getRegisterMask(reg) orelse return true;
71 return reg_mask & mask == 0;
72 }
73
6974 fn markRegAllocated(self: *Self, reg: Register) void {
7075 const mask = getRegisterMask(reg) orelse return;
7176 self.allocated_registers |= mask;
......@@ -186,10 +191,11 @@ pub fn RegisterManager(
186191 insts: [count]?Air.Inst.Index,
187192 opts: AllocOpts,
188193 ) ?[count]Register {
189 _ = opts;
190194 comptime assert(count > 0 and count <= tracked_registers.len);
191195
192 const free_and_not_locked_registers = self.free_registers & ~self.locked_registers;
196 const selector_mask = if (opts.selector_mask) |mask| mask else ~@as(FreeRegInt, 0);
197 const free_registers = self.free_registers & selector_mask;
198 const free_and_not_locked_registers = free_registers & ~self.locked_registers;
193199 const free_and_not_locked_registers_count = @popCount(FreeRegInt, free_and_not_locked_registers);
194200 if (free_and_not_locked_registers_count < count) return null;
195201
......@@ -197,6 +203,7 @@ pub fn RegisterManager(
197203 var i: usize = 0;
198204 for (tracked_registers) |reg| {
199205 if (i >= count) break;
206 if (excludeRegister(reg, selector_mask)) continue;
200207 if (self.isRegLocked(reg)) continue;
201208 if (!self.isRegFree(reg)) continue;
202209
......@@ -236,8 +243,12 @@ pub fn RegisterManager(
236243 opts: AllocOpts,
237244 ) AllocateRegistersError![count]Register {
238245 comptime assert(count > 0 and count <= tracked_registers.len);
239 const locked_registers_count = @popCount(FreeRegInt, self.locked_registers);
240 if (count > tracked_registers.len - locked_registers_count) return error.OutOfRegisters;
246
247 const selector_mask = if (opts.selector_mask) |mask| mask else ~@as(FreeRegInt, 0);
248 const available_registers_count = @popCount(FreeRegInt, selector_mask);
249 const locked_registers = self.locked_registers & selector_mask;
250 const locked_registers_count = @popCount(FreeRegInt, locked_registers);
251 if (count > available_registers_count - locked_registers_count) return error.OutOfRegisters;
241252
242253 const result = self.tryAllocRegs(count, insts, opts) orelse blk: {
243254 // We'll take over the first count registers. Spill
......@@ -247,6 +258,7 @@ pub fn RegisterManager(
247258 var i: usize = 0;
248259 for (tracked_registers) |reg| {
249260 if (i >= count) break;
261 if (excludeRegister(reg, selector_mask)) continue;
250262 if (self.isRegLocked(reg)) continue;
251263
252264 regs[i] = reg;