authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-06-07 11:16:08+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-06-07 19:33:40+02:00
log117f9f69e79a628347c9fdb22e7ee8618de143c5
treed9b478936cee22ade6ee75731393cbabf487ae01
parente9fc58eab77d60dfb02155ff17178b496d75d035

x64: simplify saving registers to stack in prologue

4 files changed, 155 insertions(+), 106 deletions(-)

src/arch/x86_64/CodeGen.zig+79-65
......@@ -409,13 +409,11 @@ fn gen(self: *Self) InnerError!void {
409409 // The address where to store the return value for the caller is in `.rdi`
410410 // register which the callee is free to clobber. Therefore, we purposely
411411 // spill it to stack immediately.
412 const ptr_ty = Type.usize;
413 const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*));
414 const abi_align = ptr_ty.abiAlignment(self.target.*);
415 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);
412 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + 8, 8);
416413 self.next_stack_offset = stack_offset;
417414 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
418 try self.genSetStack(ptr_ty, @intCast(i32, stack_offset), MCValue{ .register = .rdi }, .{});
415
416 try self.genSetStack(Type.usize, @intCast(i32, stack_offset), MCValue{ .register = .rdi }, .{});
419417 self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };
420418 log.debug("gen: spilling .rdi to stack at offset {}", .{stack_offset});
421419 }
......@@ -426,11 +424,11 @@ fn gen(self: *Self) InnerError!void {
426424 .data = undefined,
427425 });
428426
429 // push the callee_preserved_regs that were used
430 const backpatch_push_callee_preserved_regs_i = try self.addInst(.{
431 .tag = .push_regs_from_callee_preserved_regs,
432 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
433 .data = .{ .payload = undefined }, // to be backpatched
427 // Push callee-preserved regs that were used actually in use.
428 const backpatch_push_callee_preserved_regs = try self.addInst(.{
429 .tag = .nop,
430 .ops = undefined,
431 .data = undefined,
434432 });
435433
436434 try self.genBody(self.air.getMainBody());
......@@ -446,31 +444,21 @@ fn gen(self: *Self) InnerError!void {
446444 self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);
447445 }
448446
449 // calculate the data for callee_preserved_regs to be pushed and popped
450 const callee_preserved_regs_payload = blk: {
451 var data = Mir.RegsToPushOrPop{
452 .regs = 0,
453 .disp = mem.alignForwardGeneric(u32, self.next_stack_offset, 8),
454 };
455 var disp = data.disp + 8;
456 inline for (callee_preserved_regs) |reg, i| {
457 if (self.register_manager.isRegAllocated(reg)) {
458 data.regs |= 1 << @intCast(u5, i);
459 self.max_end_stack += 8;
460 disp += 8;
461 }
447 // Create list of registers to save in the prologue.
448 // TODO handle register classes
449 var reg_list: Mir.RegisterList(Register, &callee_preserved_regs) = .{};
450 inline for (callee_preserved_regs) |reg| {
451 if (self.register_manager.isRegAllocated(reg)) {
452 reg_list.push(reg);
462453 }
463 break :blk try self.addExtra(data);
464 };
454 }
455 const saved_regs_stack_space: u32 = reg_list.count() * 8;
465456
466 const data = self.mir_instructions.items(.data);
467 // backpatch the push instruction
468 data[backpatch_push_callee_preserved_regs_i].payload = callee_preserved_regs_payload;
469 // pop the callee_preserved_regs
470 _ = try self.addInst(.{
471 .tag = .pop_regs_from_callee_preserved_regs,
472 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
473 .data = .{ .payload = callee_preserved_regs_payload },
457 // Pop saved callee-preserved regs.
458 const backpatch_pop_callee_preserved_regs = try self.addInst(.{
459 .tag = .nop,
460 .ops = undefined,
461 .data = undefined,
474462 });
475463
476464 _ = try self.addInst(.{
......@@ -502,9 +490,11 @@ fn gen(self: *Self) InnerError!void {
502490 if (self.max_end_stack > math.maxInt(i32)) {
503491 return self.failSymbol("too much stack used in call parameters", .{});
504492 }
505 // TODO we should reuse this mechanism to align the stack when calling any function even if
506 // we do not pass any args on the stack BUT we still push regs to stack with `push` inst.
507 const aligned_stack_end = @intCast(u32, mem.alignForward(self.max_end_stack, self.stack_align));
493
494 const aligned_stack_end = @intCast(
495 u32,
496 mem.alignForward(self.max_end_stack + saved_regs_stack_space, self.stack_align),
497 );
508498 if (aligned_stack_end > 0) {
509499 self.mir_instructions.set(backpatch_stack_sub, .{
510500 .tag = .sub,
......@@ -516,6 +506,21 @@ fn gen(self: *Self) InnerError!void {
516506 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
517507 .data = .{ .imm = aligned_stack_end },
518508 });
509
510 const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
511 .register_list = reg_list.asInt(),
512 .stack_end = aligned_stack_end,
513 });
514 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
515 .tag = .push_regs,
516 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
517 .data = .{ .payload = save_reg_list },
518 });
519 self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
520 .tag = .pop_regs,
521 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
522 .data = .{ .payload = save_reg_list },
523 });
519524 }
520525 } else {
521526 _ = try self.addInst(.{
......@@ -907,6 +912,39 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
907912 return MCValue{ .stack_offset = @intCast(i32, stack_offset) };
908913}
909914
915const State = struct {
916 next_stack_offset: u32,
917 registers: abi.RegisterManager.TrackedRegisters,
918 free_registers: abi.RegisterManager.RegisterBitSet,
919 eflags_inst: ?Air.Inst.Index,
920 stack: std.AutoHashMapUnmanaged(u32, StackAllocation),
921
922 fn deinit(state: *State, gpa: Allocator) void {
923 state.stack.deinit(gpa);
924 }
925};
926
927fn captureState(self: *Self) !State {
928 return State{
929 .next_stack_offset = self.next_stack_offset,
930 .registers = self.register_manager.registers,
931 .free_registers = self.register_manager.free_registers,
932 .eflags_inst = self.eflags_inst,
933 .stack = try self.stack.clone(self.gpa),
934 };
935}
936
937fn revertState(self: *Self, state: State) void {
938 self.register_manager.registers = state.registers;
939 self.eflags_inst = state.eflags_inst;
940
941 self.stack.deinit(self.gpa);
942 self.stack = state.stack;
943
944 self.next_stack_offset = state.next_stack_offset;
945 self.register_manager.free_registers = state.free_registers;
946}
947
910948pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
911949 const stack_mcv = try self.allocRegOrMem(inst, false);
912950 log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
......@@ -4503,12 +4541,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
45034541 }
45044542
45054543 // Capture the state of register and stack allocation state so that we can revert to it.
4506 const parent_next_stack_offset = self.next_stack_offset;
4507 const parent_free_registers = self.register_manager.free_registers;
4508 const parent_eflags_inst = self.eflags_inst;
4509 var parent_stack = try self.stack.clone(self.gpa);
4510 defer parent_stack.deinit(self.gpa);
4511 const parent_registers = self.register_manager.registers;
4544 const saved_state = try self.captureState();
45124545
45134546 try self.branch_stack.append(.{});
45144547 errdefer {
......@@ -4526,17 +4559,10 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
45264559 var saved_then_branch = self.branch_stack.pop();
45274560 defer saved_then_branch.deinit(self.gpa);
45284561
4529 self.register_manager.registers = parent_registers;
4530 self.eflags_inst = parent_eflags_inst;
4531
4532 self.stack.deinit(self.gpa);
4533 self.stack = parent_stack;
4534 parent_stack = .{};
4535
4536 self.next_stack_offset = parent_next_stack_offset;
4537 self.register_manager.free_registers = parent_free_registers;
4562 self.revertState(saved_state);
45384563
45394564 try self.performReloc(reloc);
4565
45404566 const else_branch = self.branch_stack.addOneAssumeCapacity();
45414567 else_branch.* = .{};
45424568
......@@ -5021,12 +5047,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
50215047 }
50225048
50235049 // Capture the state of register and stack allocation state so that we can revert to it.
5024 const parent_next_stack_offset = self.next_stack_offset;
5025 const parent_free_registers = self.register_manager.free_registers;
5026 const parent_eflags_inst = self.eflags_inst;
5027 var parent_stack = try self.stack.clone(self.gpa);
5028 defer parent_stack.deinit(self.gpa);
5029 const parent_registers = self.register_manager.registers;
5050 const saved_state = try self.captureState();
50305051
50315052 try self.branch_stack.append(.{});
50325053 errdefer {
......@@ -5044,14 +5065,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
50445065 var saved_case_branch = self.branch_stack.pop();
50455066 defer saved_case_branch.deinit(self.gpa);
50465067
5047 self.register_manager.registers = parent_registers;
5048 self.eflags_inst = parent_eflags_inst;
5049 self.stack.deinit(self.gpa);
5050 self.stack = parent_stack;
5051 parent_stack = .{};
5052
5053 self.next_stack_offset = parent_next_stack_offset;
5054 self.register_manager.free_registers = parent_free_registers;
5068 self.revertState(saved_state);
50555069
50565070 for (relocs) |reloc| {
50575071 try self.performReloc(reloc);
src/arch/x86_64/Emit.zig+21-24
......@@ -169,7 +169,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
169169 .@"test" => try emit.mirTest(inst),
170170
171171 .interrupt => try emit.mirInterrupt(inst),
172 .nop => try emit.mirNop(),
172 .nop => {}, // just skip it
173173
174174 // SSE instructions
175175 .mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),
......@@ -198,8 +198,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
198198 .dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
199199 .dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
200200
201 .push_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
202 .pop_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
201 .push_regs => try emit.mirPushPopRegisterList(.push, inst),
202 .pop_regs => try emit.mirPushPopRegisterList(.pop, inst),
203203
204204 else => {
205205 return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {s}", .{tag});
......@@ -246,10 +246,6 @@ fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
246246 }
247247}
248248
249fn mirNop(emit: *Emit) InnerError!void {
250 return lowerToZoEnc(.nop, emit.code);
251}
252
253249fn mirSyscall(emit: *Emit) InnerError!void {
254250 return lowerToZoEnc(.syscall, emit.code);
255251}
......@@ -283,26 +279,27 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
283279 }
284280}
285281
286fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
282fn mirPushPopRegisterList(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
287283 const ops = emit.mir.instructions.items(.ops)[inst].decode();
288284 const payload = emit.mir.instructions.items(.data)[inst].payload;
289 const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data;
290 const regs = data.regs;
291 var disp: u32 = data.disp + 8;
292 for (abi.callee_preserved_regs) |reg, i| {
293 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
294 if (tag == .push) {
295 try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
296 .disp = @bitCast(u32, -@intCast(i32, disp)),
297 .base = ops.reg1,
298 }), reg.to64(), emit.code);
299 } else {
300 try lowerToRmEnc(.mov, reg.to64(), RegisterOrMemory.mem(.qword_ptr, .{
301 .disp = @bitCast(u32, -@intCast(i32, disp)),
302 .base = ops.reg1,
303 }), emit.code);
285 const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;
286 const reg_list = Mir.RegisterList(Register, &abi.callee_preserved_regs).fromInt(save_reg_list.register_list);
287 var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
288 inline for (abi.callee_preserved_regs) |reg| {
289 if (reg_list.isSet(reg)) {
290 switch (tag) {
291 .push => try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
292 .disp = @bitCast(u32, disp),
293 .base = ops.reg1,
294 }), reg, emit.code),
295 .pop => try lowerToRmEnc(.mov, reg, RegisterOrMemory.mem(.qword_ptr, .{
296 .disp = @bitCast(u32, disp),
297 .base = ops.reg1,
298 }), emit.code),
299 else => unreachable,
300 }
301 disp += 8;
304302 }
305 disp += 8;
306303 }
307304}
308305
src/arch/x86_64/Mir.zig+53-16
......@@ -14,6 +14,7 @@ const assert = std.debug.assert;
1414const bits = @import("bits.zig");
1515const Air = @import("../../Air.zig");
1616const CodeGen = @import("CodeGen.zig");
17const IntegerBitSet = std.bit_set.IntegerBitSet;
1718const Register = bits.Register;
1819
1920instructions: std.MultiArrayList(Inst).Slice,
......@@ -379,19 +380,13 @@ pub const Inst = struct {
379380 /// update debug line
380381 dbg_line,
381382
382 /// push registers from the callee_preserved_regs
383 /// data is the bitfield of which regs to push
384 /// for example on x86_64, the callee_preserved_regs are [_]Register{ .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11 }; };
385 /// so to push rcx and r8 one would make data 0b00000000_00000000_00000000_00001001 (the first and fourth bits are set)
386 /// ops is unused
387 push_regs_from_callee_preserved_regs,
388
389 /// pop registers from the callee_preserved_regs
390 /// data is the bitfield of which regs to pop
391 /// for example on x86_64, the callee_preserved_regs are [_]Register{ .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11 }; };
392 /// so to pop rcx and r8 one would make data 0b00000000_00000000_00000000_00001001 (the first and fourth bits are set)
393 /// ops is unused
394 pop_regs_from_callee_preserved_regs,
383 /// push registers
384 /// Uses `payload` field with `SaveRegisterList` as payload.
385 push_regs,
386
387 /// pop registers
388 /// Uses `payload` field with `SaveRegisterList` as payload.
389 pop_regs,
395390 };
396391 /// The position of an MIR instruction within the `Mir` instructions array.
397392 pub const Index = u32;
......@@ -471,9 +466,51 @@ pub const Inst = struct {
471466 }
472467};
473468
474pub const RegsToPushOrPop = struct {
475 regs: u32,
476 disp: u32,
469pub fn RegisterList(comptime Reg: type, comptime registers: []const Reg) type {
470 assert(registers.len <= @bitSizeOf(u32));
471 return struct {
472 bitset: RegBitSet = RegBitSet.initEmpty(),
473
474 const RegBitSet = IntegerBitSet(registers.len);
475 const Self = @This();
476
477 fn getIndexForReg(reg: Reg) RegBitSet.MaskInt {
478 inline for (registers) |cpreg, i| {
479 if (reg.id() == cpreg.id()) return i;
480 }
481 unreachable; // register not in input register list!
482 }
483
484 pub fn push(self: *Self, reg: Reg) void {
485 const index = getIndexForReg(reg);
486 self.bitset.set(index);
487 }
488
489 pub fn isSet(self: Self, reg: Reg) bool {
490 const index = getIndexForReg(reg);
491 return self.bitset.isSet(index);
492 }
493
494 pub fn asInt(self: Self) u32 {
495 return self.bitset.mask;
496 }
497
498 pub fn fromInt(mask: u32) Self {
499 return .{
500 .bitset = RegBitSet{ .mask = @intCast(RegBitSet.MaskInt, mask) },
501 };
502 }
503
504 pub fn count(self: Self) u32 {
505 return @intCast(u32, self.bitset.count());
506 }
507 };
508}
509
510pub const SaveRegisterList = struct {
511 /// Use `RegisterList` to populate.
512 register_list: u32,
513 stack_end: u32,
477514};
478515
479516pub const ImmPair = struct {
src/register_manager.zig+2-1
......@@ -39,7 +39,7 @@ pub fn RegisterManager(
3939 /// register is free), the value in that slot is undefined.
4040 ///
4141 /// The key must be canonical register.
42 registers: [tracked_registers.len]Air.Inst.Index = undefined,
42 registers: TrackedRegisters = undefined,
4343 /// Tracks which registers are free (in which case the
4444 /// corresponding bit is set to 1)
4545 free_registers: RegisterBitSet = RegisterBitSet.initFull(),
......@@ -51,6 +51,7 @@ pub fn RegisterManager(
5151
5252 const Self = @This();
5353
54 pub const TrackedRegisters = [tracked_registers.len]Air.Inst.Index;
5455 pub const RegisterBitSet = StaticBitSet(tracked_registers.len);
5556
5657 fn getFunction(self: *Self) *Function {