authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-03 07:55:09-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-03 23:18:23-04:00
log46d3e5bb04393c6be196b10e262d122e008403aa
tree295250ac18b7151352bd64cac40c2e97a278534f
parent509be7cf1f10c5d329d2b0524f2af6bfcabd52de

x86_64: reduce `RegisterManager` performance regression

This reduces the regression from 0.11.0 by 95%. Closes #17678

3 files changed, 123 insertions(+), 71 deletions(-)

src/arch/x86_64/CodeGen.zig+20-10
......@@ -2608,8 +2608,17 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
26082608 }
26092609}
26102610
2611pub fn spillRegisters(self: *Self, registers: []const Register) !void {
2612 for (registers) |reg| try self.register_manager.getReg(reg, null);
2611pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention) !void {
2612 switch (cc) {
2613 inline .SysV, .Win64 => |known_cc| try self.spillRegisters(
2614 comptime abi.getCallerPreservedRegs(known_cc),
2615 ),
2616 else => unreachable,
2617 }
2618}
2619
2620pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
2621 inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null);
26132622}
26142623
26152624/// Copies a value to a register without tracking the register. The register is not considered
......@@ -10639,30 +10648,30 @@ fn genCall(self: *Self, info: union(enum) {
1063910648 }
1064010649
1064110650 try self.spillEflagsIfOccupied();
10642 try self.spillRegisters(abi.getCallerPreservedRegs(resolved_cc));
10651 try self.spillCallerPreservedRegs(resolved_cc);
1064310652
1064410653 // set stack arguments first because this can clobber registers
1064510654 // also clobber spill arguments as we go
1064610655 switch (call_info.return_value.long) {
1064710656 .none, .unreach => {},
10648 .indirect => |reg_off| try self.spillRegisters(&.{reg_off.reg}),
10657 .indirect => |reg_off| try self.register_manager.getReg(reg_off.reg, null),
1064910658 else => unreachable,
1065010659 }
1065110660 for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, *frame_index|
1065210661 switch (dst_arg) {
1065310662 .none => {},
1065410663 .register => |reg| {
10655 try self.spillRegisters(&.{reg});
10664 try self.register_manager.getReg(reg, null);
1065610665 try reg_locks.append(self.register_manager.lockReg(reg));
1065710666 },
1065810667 .register_pair => |regs| {
10659 try self.spillRegisters(&regs);
10668 for (regs) |reg| try self.register_manager.getReg(reg, null);
1066010669 try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
1066110670 },
1066210671 .indirect => |reg_off| {
1066310672 frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, mod));
1066410673 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg);
10665 try self.spillRegisters(&.{reg_off.reg});
10674 try self.register_manager.getReg(reg_off.reg, null);
1066610675 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
1066710676 },
1066810677 .load_frame => {
......@@ -11990,8 +11999,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1199011999 var args = std.ArrayList(MCValue).init(self.gpa);
1199112000 try args.ensureTotalCapacity(outputs.len + inputs.len);
1199212001 defer {
11993 for (args.items) |arg| if (arg.getReg()) |reg|
11994 self.register_manager.unlockReg(.{ .register = reg });
12002 for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{
12003 .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue,
12004 });
1199512005 args.deinit();
1199612006 }
1199712007 var arg_map = std.StringHashMap(u8).init(self.gpa);
......@@ -14557,7 +14567,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
1455714567 }
1455814568
1455914569 try self.spillEflagsIfOccupied();
14560 try self.spillRegisters(abi.getCallerPreservedRegs(resolved_cc));
14570 try self.spillCallerPreservedRegs(resolved_cc);
1456114571
1456214572 const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
1456314573
src/arch/x86_64/bits.zig+1-1
......@@ -222,7 +222,7 @@ pub const Register = enum(u7) {
222222 @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax),
223223 @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax),
224224 @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al),
225 @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah) - 4,
225 @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah),
226226
227227 @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0) - 16,
228228 @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0) - 16,
src/register_manager.zig+102-60
......@@ -55,6 +55,7 @@ pub fn RegisterManager(
5555 const Self = @This();
5656
5757 pub const TrackedRegisters = [tracked_registers.len]Air.Inst.Index;
58 pub const TrackedIndex = std.math.IntFittingRange(0, tracked_registers.len - 1);
5859 pub const RegisterBitSet = StaticBitSet(tracked_registers.len);
5960
6061 fn getFunction(self: *Self) *Function {
......@@ -66,45 +67,64 @@ pub fn RegisterManager(
6667 return !register_class.isSet(index);
6768 }
6869
70 fn markRegIndexAllocated(self: *Self, tracked_index: TrackedIndex) void {
71 self.allocated_registers.set(tracked_index);
72 }
6973 fn markRegAllocated(self: *Self, reg: Register) void {
70 const index = indexOfRegIntoTracked(reg) orelse return;
71 self.allocated_registers.set(index);
74 self.markRegIndexAllocated(indexOfRegIntoTracked(reg) orelse return);
7275 }
7376
77 fn markRegIndexUsed(self: *Self, tracked_index: TrackedIndex) void {
78 self.free_registers.unset(tracked_index);
79 }
7480 fn markRegUsed(self: *Self, reg: Register) void {
75 const index = indexOfRegIntoTracked(reg) orelse return;
76 self.free_registers.unset(index);
81 self.markRegIndexUsed(indexOfRegIntoTracked(reg) orelse return);
7782 }
7883
84 fn markRegIndexFree(self: *Self, tracked_index: TrackedIndex) void {
85 self.free_registers.set(tracked_index);
86 }
7987 fn markRegFree(self: *Self, reg: Register) void {
80 const index = indexOfRegIntoTracked(reg) orelse return;
81 self.free_registers.set(index);
88 self.markRegIndexFree(indexOfRegIntoTracked(reg) orelse return);
8289 }
8390
8491 pub fn indexOfReg(
85 comptime registers: []const Register,
92 comptime set: []const Register,
8693 reg: Register,
87 ) ?std.math.IntFittingRange(0, registers.len - 1) {
88 inline for (tracked_registers, 0..) |cpreg, i| {
89 if (reg.id() == cpreg.id()) return i;
94 ) ?std.math.IntFittingRange(0, set.len - 1) {
95 const Id = @TypeOf(reg.id());
96 comptime var min_id: Id = std.math.maxInt(Id);
97 comptime var max_id: Id = std.math.minInt(Id);
98 inline for (set) |elem| {
99 const elem_id = comptime elem.id();
100 min_id = @min(elem_id, min_id);
101 max_id = @max(elem_id, max_id);
90102 }
91 return null;
103
104 const OptionalIndex = std.math.IntFittingRange(0, set.len);
105 comptime var map = [1]OptionalIndex{set.len} ** (max_id + 1 - min_id);
106 inline for (set, 0..) |elem, elem_index| map[comptime elem.id() - min_id] = elem_index;
107
108 const id_index = reg.id() -% min_id;
109 if (id_index >= map.len) return null;
110 const set_index = map[id_index];
111 return if (set_index < set.len) @intCast(set_index) else null;
92112 }
93113
94 pub fn indexOfRegIntoTracked(
95 reg: Register,
96 ) ?std.math.IntFittingRange(0, tracked_registers.len) {
114 pub fn indexOfRegIntoTracked(reg: Register) ?TrackedIndex {
97115 return indexOfReg(tracked_registers, reg);
98116 }
99117
100 pub fn regAtTrackedIndex(index: std.math.IntFittingRange(0, tracked_registers.len)) Register {
101 return tracked_registers[index];
118 pub fn regAtTrackedIndex(tracked_index: TrackedIndex) Register {
119 return tracked_registers[tracked_index];
102120 }
103121
104122 /// Returns true when this register is not tracked
123 pub fn isRegIndexFree(self: Self, tracked_index: TrackedIndex) bool {
124 return self.free_registers.isSet(tracked_index);
125 }
105126 pub fn isRegFree(self: Self, reg: Register) bool {
106 const index = indexOfRegIntoTracked(reg) orelse return true;
107 return self.free_registers.isSet(index);
127 return self.isRegIndexFree(indexOfRegIntoTracked(reg) orelse return true);
108128 }
109129
110130 /// Returns whether this register was allocated in the course
......@@ -119,14 +139,14 @@ pub fn RegisterManager(
119139 /// Returns whether this register is locked
120140 ///
121141 /// Returns false when this register is not tracked
142 fn isRegIndexLocked(self: Self, tracked_index: TrackedIndex) bool {
143 return self.locked_registers.isSet(tracked_index);
144 }
122145 pub fn isRegLocked(self: Self, reg: Register) bool {
123 const index = indexOfRegIntoTracked(reg) orelse return false;
124 return self.locked_registers.isSet(index);
146 return self.isRegIndexLocked(indexOfRegIntoTracked(reg) orelse return false);
125147 }
126148
127 pub const RegisterLock = struct {
128 register: Register,
129 };
149 pub const RegisterLock = struct { tracked_index: TrackedIndex };
130150
131151 /// Prevents the register from being allocated until they are
132152 /// unlocked again.
......@@ -134,25 +154,29 @@ pub fn RegisterManager(
134154 /// locked, or `null` otherwise.
135155 /// Only the owner of the `RegisterLock` can unlock the
136156 /// register later.
137 pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
138 log.debug("locking {}", .{reg});
139 if (self.isRegLocked(reg)) {
157 pub fn lockRegIndex(self: *Self, tracked_index: TrackedIndex) ?RegisterLock {
158 log.debug("locking {}", .{regAtTrackedIndex(tracked_index)});
159 if (self.isRegIndexLocked(tracked_index)) {
140160 log.debug(" register already locked", .{});
141161 return null;
142162 }
143 const index = indexOfRegIntoTracked(reg) orelse return null;
144 self.locked_registers.set(index);
145 return RegisterLock{ .register = reg };
163 self.locked_registers.set(tracked_index);
164 return RegisterLock{ .tracked_index = tracked_index };
165 }
166 pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
167 return self.lockRegIndex(indexOfRegIntoTracked(reg) orelse return null);
146168 }
147169
148170 /// Like `lockReg` but asserts the register was unused always
149171 /// returning a valid lock.
172 pub fn lockRegIndexAssumeUnused(self: *Self, tracked_index: TrackedIndex) RegisterLock {
173 log.debug("locking asserting free {}", .{regAtTrackedIndex(tracked_index)});
174 assert(!self.isRegIndexLocked(tracked_index));
175 self.locked_registers.set(tracked_index);
176 return RegisterLock{ .tracked_index = tracked_index };
177 }
150178 pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
151 log.debug("locking asserting free {}", .{reg});
152 assert(!self.isRegLocked(reg));
153 const index = indexOfRegIntoTracked(reg) orelse unreachable;
154 self.locked_registers.set(index);
155 return RegisterLock{ .register = reg };
179 return self.lockRegIndexAssumeUnused(indexOfRegIntoTracked(reg) orelse unreachable);
156180 }
157181
158182 /// Like `lockReg` but locks multiple registers.
......@@ -181,9 +205,8 @@ pub fn RegisterManager(
181205 /// Requires `RegisterLock` to unlock a register.
182206 /// Call `lockReg` to obtain the lock first.
183207 pub fn unlockReg(self: *Self, lock: RegisterLock) void {
184 log.debug("unlocking {}", .{lock.register});
185 const index = indexOfRegIntoTracked(lock.register) orelse return;
186 self.locked_registers.unset(index);
208 log.debug("unlocking {}", .{regAtTrackedIndex(lock.tracked_index)});
209 self.locked_registers.unset(lock.tracked_index);
187210 }
188211
189212 /// Returns true when at least one register is locked
......@@ -319,44 +342,63 @@ pub fn RegisterManager(
319342 /// Spills the register if it is currently allocated. If a
320343 /// corresponding instruction is passed, will also track this
321344 /// register.
322 pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
323 const index = indexOfRegIntoTracked(reg) orelse return;
324 log.debug("getReg {} for inst {?}", .{ reg, inst });
325
326 if (!self.isRegFree(reg)) {
327 self.markRegAllocated(reg);
345 fn getRegIndex(
346 self: *Self,
347 tracked_index: TrackedIndex,
348 inst: ?Air.Inst.Index,
349 ) AllocateRegistersError!void {
350 log.debug("getReg {} for inst {?}", .{ regAtTrackedIndex(tracked_index), inst });
351 if (!self.isRegIndexFree(tracked_index)) {
352 self.markRegIndexAllocated(tracked_index);
328353
329354 // Move the instruction that was previously there to a
330355 // stack allocation.
331 const spilled_inst = self.registers[index];
332 if (inst) |tracked_inst| self.registers[index] = tracked_inst;
333 try self.getFunction().spillInstruction(reg, spilled_inst);
334 if (inst == null) self.freeReg(reg);
335 } else self.getRegAssumeFree(reg, inst);
356 const spilled_inst = self.registers[tracked_index];
357 if (inst) |tracked_inst| self.registers[tracked_index] = tracked_inst;
358 try self.getFunction().spillInstruction(regAtTrackedIndex(tracked_index), spilled_inst);
359 if (inst == null) self.freeRegIndex(tracked_index);
360 } else self.getRegIndexAssumeFree(tracked_index, inst);
361 }
362 pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
363 return self.getRegIndex(indexOfRegIntoTracked(reg) orelse return, inst);
364 }
365 pub fn getKnownReg(
366 self: *Self,
367 comptime reg: Register,
368 inst: ?Air.Inst.Index,
369 ) AllocateRegistersError!void {
370 return self.getRegIndex((comptime indexOfRegIntoTracked(reg)) orelse return, inst);
336371 }
337372
338373 /// Allocates the specified register with the specified
339374 /// instruction. Asserts that the register is free and no
340375 /// spilling is necessary.
341 pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
342 const index = indexOfRegIntoTracked(reg) orelse return;
343 log.debug("getRegAssumeFree {} for inst {?}", .{ reg, inst });
344 self.markRegAllocated(reg);
376 fn getRegIndexAssumeFree(
377 self: *Self,
378 tracked_index: TrackedIndex,
379 inst: ?Air.Inst.Index,
380 ) void {
381 log.debug("getRegAssumeFree {} for inst {?}", .{ regAtTrackedIndex(tracked_index), inst });
382 self.markRegIndexAllocated(tracked_index);
345383
346 assert(self.isRegFree(reg));
384 assert(self.isRegIndexFree(tracked_index));
347385 if (inst) |tracked_inst| {
348 self.registers[index] = tracked_inst;
349 self.markRegUsed(reg);
386 self.registers[tracked_index] = tracked_inst;
387 self.markRegIndexUsed(tracked_index);
350388 }
351389 }
390 pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
391 self.getRegIndexAssumeFree(indexOfRegIntoTracked(reg) orelse return, inst);
392 }
352393
353394 /// Marks the specified register as free
395 fn freeRegIndex(self: *Self, tracked_index: TrackedIndex) void {
396 log.debug("freeing register {}", .{regAtTrackedIndex(tracked_index)});
397 self.registers[tracked_index] = undefined;
398 self.markRegIndexFree(tracked_index);
399 }
354400 pub fn freeReg(self: *Self, reg: Register) void {
355 const index = indexOfRegIntoTracked(reg) orelse return;
356 log.debug("freeing register {}", .{reg});
357
358 self.registers[index] = undefined;
359 self.markRegFree(reg);
401 self.freeRegIndex(indexOfRegIntoTracked(reg) orelse return);
360402 }
361403 };
362404}