authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-22 13:37:26+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-22 21:56:34+01:00
log3be6c79ca2414044600426204ac19ea31774420f
treeb5cd25a04f0207d6be728fccb90243daf56d8202
parentb9b4e4671f4377024efb076edd9cc2da9e889a5a

x64: spill compare flags between blocks, extern calls, and cmp insts

This is just the first step towards the final solution as to get here I had omit a safety assert check in `getResolvedInst` helper.

1 files changed, 102 insertions(+), 38 deletions(-)

src/arch/x86_64/CodeGen.zig+102-38
......@@ -50,6 +50,7 @@ src_loc: Module.SrcLoc,
5050stack_align: u32,
5151
5252ret_backpatch: ?Mir.Inst.Index = null,
53compare_flags_inst: ?Air.Inst.Index = null,
5354
5455/// MIR Instructions
5556mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
......@@ -774,6 +775,9 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
774775 const canon_reg = reg.to64();
775776 self.register_manager.freeReg(canon_reg);
776777 },
778 .compare_flags_signed, .compare_flags_unsigned => {
779 self.compare_flags_inst = null;
780 },
777781 else => {}, // TODO process stack allocation death
778782 }
779783}
......@@ -889,6 +893,22 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
889893 try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
890894}
891895
896pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
897 if (self.compare_flags_inst) |inst_to_save| {
898 const mcv = self.getResolvedInstValue(inst_to_save);
899 assert(mcv == .compare_flags_signed or mcv == .compare_flags_unsigned);
900
901 const new_mcv = try self.allocRegOrMem(inst_to_save, true);
902 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
903 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
904
905 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
906 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
907
908 self.compare_flags_inst = null;
909 }
910}
911
892912/// Copies a value to a register without tracking the register. The register is not considered
893913/// allocated. A second call to `copyToTmpRegister` may return the same register.
894914/// This can have a side effect of spilling instructions to the stack to free up a register.
......@@ -2763,6 +2783,8 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
27632783 .memory,
27642784 .got_load,
27652785 .direct_load,
2786 .compare_flags_signed,
2787 .compare_flags_unsigned,
27662788 => {
27672789 assert(abi_size <= 8);
27682790 self.register_manager.freezeRegs(&.{dst_reg});
......@@ -2784,12 +2806,6 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
27842806 .data = .{ .imm = @bitCast(u32, -off) },
27852807 });
27862808 },
2787 .compare_flags_unsigned => {
2788 return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
2789 },
2790 .compare_flags_signed => {
2791 return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (signed)", .{});
2792 },
27932809 }
27942810 },
27952811 .stack_offset => |off| {
......@@ -3061,6 +3077,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
30613077 var info = try self.resolveCallingConventionValues(fn_ty);
30623078 defer info.deinit(self);
30633079
3080 try self.spillCompareFlagsIfOccupied();
3081
30643082 if (info.return_value == .stack_offset) {
30653083 const ret_ty = fn_ty.fnReturnType();
30663084 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
......@@ -3384,17 +3402,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
33843402 break :blk ty.intInfo(self.target.*).signedness;
33853403 };
33863404
3387 const lhs = try self.resolveInst(bin_op.lhs);
3388 const rhs = try self.resolveInst(bin_op.rhs);
3405 try self.spillCompareFlagsIfOccupied();
3406 self.compare_flags_inst = inst;
3407
33893408 const result: MCValue = result: {
33903409 // There are 2 operands, destination and source.
33913410 // Either one, but not both, can be a memory operand.
33923411 // Source operand can be an immediate, 8 bits or 32 bits.
3393 // TODO this looks wrong. Why do simply reuse lhs without checking if it is dead or alive?
3394 const dst_mcv = if (lhs.isImmediate() or (lhs.isMemory() and rhs.isMemory()))
3395 MCValue{ .register = try self.copyToTmpRegister(ty, lhs) }
3396 else
3397 lhs;
3412 // TODO look into reusing the operand
3413 const lhs = try self.resolveInst(bin_op.lhs);
3414 lhs.freezeIfRegister(&self.register_manager);
3415 defer lhs.unfreezeIfRegister(&self.register_manager);
3416
3417 const dst_reg = try self.copyToTmpRegister(ty, lhs);
3418 self.register_manager.freezeRegs(&.{dst_reg});
3419 defer self.register_manager.unfreezeRegs(&.{dst_reg});
3420
3421 const dst_mcv = MCValue{ .register = dst_reg };
3422
33983423 // This instruction supports only signed 32-bit immediates at most.
33993424 const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
34003425
......@@ -3404,6 +3429,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
34043429 .unsigned => MCValue{ .compare_flags_unsigned = op },
34053430 };
34063431 };
3432
34073433 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
34083434}
34093435
......@@ -3451,6 +3477,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
34513477 });
34523478 },
34533479 .register => |reg| {
3480 try self.spillCompareFlagsIfOccupied();
34543481 _ = try self.addInst(.{
34553482 .tag = .@"test",
34563483 .ops = (Mir.Ops{
......@@ -3467,19 +3494,15 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
34673494 .data = .{ .inst = undefined },
34683495 });
34693496 },
3470 .immediate => {
3471 if (abi_size <= 8) {
3472 const reg = try self.copyToTmpRegister(ty, mcv);
3473 return self.genCondBrMir(ty, .{ .register = reg });
3474 }
3475 return self.fail("TODO implement condbr when condition is immediate larger than 4 bytes", .{});
3476 },
3477 .stack_offset => {
3497 .immediate,
3498 .stack_offset,
3499 => {
3500 try self.spillCompareFlagsIfOccupied();
34783501 if (abi_size <= 8) {
34793502 const reg = try self.copyToTmpRegister(ty, mcv);
34803503 return self.genCondBrMir(ty, .{ .register = reg });
34813504 }
3482 return self.fail("TODO implement condbr when condition is stack offset with abi larger than 8 bytes", .{});
3505 return self.fail("TODO implement condbr when condition is {} with abi larger than 8 bytes", .{mcv});
34833506 },
34843507 else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}),
34853508 }
......@@ -3496,9 +3519,23 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
34963519
34973520 const reloc = try self.genCondBrMir(cond_ty, cond);
34983521
3522 // If the condition dies here in this condbr instruction, process
3523 // that death now instead of later as this has an effect on
3524 // whether it needs to be spilled in the branches
3525 // TODO I need investigate how to make this work without removing
3526 // an assertion from getResolvedInstValue()
3527 if (self.liveness.operandDies(inst, 0)) {
3528 const op_int = @enumToInt(pl_op.operand);
3529 if (op_int >= Air.Inst.Ref.typed_value_map.len) {
3530 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
3531 self.processDeath(op_index);
3532 }
3533 }
3534
34993535 // Capture the state of register and stack allocation state so that we can revert to it.
35003536 const parent_next_stack_offset = self.next_stack_offset;
35013537 const parent_free_registers = self.register_manager.free_registers;
3538 const parent_compare_flags_inst = self.compare_flags_inst;
35023539 var parent_stack = try self.stack.clone(self.gpa);
35033540 defer parent_stack.deinit(self.gpa);
35043541 const parent_registers = self.register_manager.registers;
......@@ -3520,6 +3557,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
35203557 defer saved_then_branch.deinit(self.gpa);
35213558
35223559 self.register_manager.registers = parent_registers;
3560 self.compare_flags_inst = parent_compare_flags_inst;
35233561
35243562 self.stack.deinit(self.gpa);
35253563 self.stack = parent_stack;
......@@ -3590,6 +3628,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
35903628 // We already deleted the items from this table that matched the else_branch.
35913629 // So these are all instructions that are only overridden in the then branch.
35923630 parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
3631 log.debug("then_value = {}", .{then_value});
35933632 if (then_value == .dead)
35943633 continue;
35953634 const parent_mcv = blk: {
......@@ -3614,23 +3653,31 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
36143653 return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
36153654}
36163655
3617fn isNull(self: *Self, ty: Type, operand: MCValue) !MCValue {
3656fn isNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
3657 try self.spillCompareFlagsIfOccupied();
3658 self.compare_flags_inst = inst;
3659
36183660 try self.genBinMathOpMir(.cmp, ty, operand, MCValue{ .immediate = 0 });
36193661 return MCValue{ .compare_flags_unsigned = .eq };
36203662}
36213663
3622fn isNonNull(self: *Self, ty: Type, operand: MCValue) !MCValue {
3623 const is_null_res = try self.isNull(ty, operand);
3664fn isNonNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
3665 const is_null_res = try self.isNull(inst, ty, operand);
36243666 assert(is_null_res.compare_flags_unsigned == .eq);
36253667 return MCValue{ .compare_flags_unsigned = .neq };
36263668}
36273669
3628fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
3670fn isErr(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
36293671 const err_type = ty.errorUnionSet();
36303672 const payload_type = ty.errorUnionPayload();
36313673 if (!err_type.hasRuntimeBits()) {
36323674 return MCValue{ .immediate = 0 }; // always false
3633 } else if (!payload_type.hasRuntimeBits()) {
3675 }
3676
3677 try self.spillCompareFlagsIfOccupied();
3678 self.compare_flags_inst = inst;
3679
3680 if (!payload_type.hasRuntimeBits()) {
36343681 if (err_type.abiSize(self.target.*) <= 8) {
36353682 try self.genBinMathOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 });
36363683 return MCValue{ .compare_flags_unsigned = .gt };
......@@ -3643,8 +3690,8 @@ fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
36433690 }
36443691}
36453692
3646fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
3647 const is_err_res = try self.isErr(ty, operand);
3693fn isNonErr(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
3694 const is_err_res = try self.isErr(inst, ty, operand);
36483695 switch (is_err_res) {
36493696 .compare_flags_unsigned => |op| {
36503697 assert(op == .gt);
......@@ -3663,7 +3710,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
36633710 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
36643711 const operand = try self.resolveInst(un_op);
36653712 const ty = self.air.typeOf(un_op);
3666 break :result try self.isNull(ty, operand);
3713 break :result try self.isNull(inst, ty, operand);
36673714 };
36683715 return self.finishAir(inst, result, .{ un_op, .none, .none });
36693716}
......@@ -3684,7 +3731,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
36843731 };
36853732 const ptr_ty = self.air.typeOf(un_op);
36863733 try self.load(operand, operand_ptr, ptr_ty);
3687 break :result try self.isNull(ptr_ty.elemType(), operand);
3734 break :result try self.isNull(inst, ptr_ty.elemType(), operand);
36883735 };
36893736 return self.finishAir(inst, result, .{ un_op, .none, .none });
36903737}
......@@ -3694,7 +3741,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
36943741 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
36953742 const operand = try self.resolveInst(un_op);
36963743 const ty = self.air.typeOf(un_op);
3697 break :result try self.isNonNull(ty, operand);
3744 break :result try self.isNonNull(inst, ty, operand);
36983745 };
36993746 return self.finishAir(inst, result, .{ un_op, .none, .none });
37003747}
......@@ -3715,7 +3762,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
37153762 };
37163763 const ptr_ty = self.air.typeOf(un_op);
37173764 try self.load(operand, operand_ptr, ptr_ty);
3718 break :result try self.isNonNull(ptr_ty.elemType(), operand);
3765 break :result try self.isNonNull(inst, ptr_ty.elemType(), operand);
37193766 };
37203767 return self.finishAir(inst, result, .{ un_op, .none, .none });
37213768}
......@@ -3725,7 +3772,7 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
37253772 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
37263773 const operand = try self.resolveInst(un_op);
37273774 const ty = self.air.typeOf(un_op);
3728 break :result try self.isErr(ty, operand);
3775 break :result try self.isErr(inst, ty, operand);
37293776 };
37303777 return self.finishAir(inst, result, .{ un_op, .none, .none });
37313778}
......@@ -3746,7 +3793,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
37463793 };
37473794 const ptr_ty = self.air.typeOf(un_op);
37483795 try self.load(operand, operand_ptr, ptr_ty);
3749 break :result try self.isErr(ptr_ty.elemType(), operand);
3796 break :result try self.isErr(inst, ptr_ty.elemType(), operand);
37503797 };
37513798 return self.finishAir(inst, result, .{ un_op, .none, .none });
37523799}
......@@ -3756,7 +3803,7 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
37563803 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
37573804 const operand = try self.resolveInst(un_op);
37583805 const ty = self.air.typeOf(un_op);
3759 break :result try self.isNonErr(ty, operand);
3806 break :result try self.isNonErr(inst, ty, operand);
37603807 };
37613808 return self.finishAir(inst, result, .{ un_op, .none, .none });
37623809}
......@@ -3777,7 +3824,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
37773824 };
37783825 const ptr_ty = self.air.typeOf(un_op);
37793826 try self.load(operand, operand_ptr, ptr_ty);
3780 break :result try self.isNonErr(ptr_ty.elemType(), operand);
3827 break :result try self.isNonErr(inst, ptr_ty.elemType(), operand);
37813828 };
37823829 return self.finishAir(inst, result, .{ un_op, .none, .none });
37833830}
......@@ -3832,6 +3879,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
38323879 .compare_flags_signed => unreachable,
38333880 .compare_flags_unsigned => unreachable,
38343881 .register => |cond_reg| {
3882 try self.spillCompareFlagsIfOccupied();
38353883 switch (case) {
38363884 .none => unreachable,
38373885 .undef => unreachable,
......@@ -3916,9 +3964,23 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
39163964 relocs[item_i] = try self.genCondSwitchMir(condition_ty, condition, item_mcv);
39173965 }
39183966
3967 // If the condition dies here in this condbr instruction, process
3968 // that death now instead of later as this has an effect on
3969 // whether it needs to be spilled in the branches
3970 // TODO I need investigate how to make this work without removing
3971 // an assertion from getResolvedInstValue()
3972 if (self.liveness.operandDies(inst, 0)) {
3973 const op_int = @enumToInt(pl_op.operand);
3974 if (op_int >= Air.Inst.Ref.typed_value_map.len) {
3975 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
3976 self.processDeath(op_index);
3977 }
3978 }
3979
39193980 // Capture the state of register and stack allocation state so that we can revert to it.
39203981 const parent_next_stack_offset = self.next_stack_offset;
39213982 const parent_free_registers = self.register_manager.free_registers;
3983 const parent_compare_flags_inst = self.compare_flags_inst;
39223984 var parent_stack = try self.stack.clone(self.gpa);
39233985 defer parent_stack.deinit(self.gpa);
39243986 const parent_registers = self.register_manager.registers;
......@@ -3940,6 +4002,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
39404002 defer saved_case_branch.deinit(self.gpa);
39414003
39424004 self.register_manager.registers = parent_registers;
4005 self.compare_flags_inst = parent_compare_flags_inst;
39434006 self.stack.deinit(self.gpa);
39444007 self.stack = parent_stack;
39454008 parent_stack = .{};
......@@ -5208,7 +5271,8 @@ fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
52085271 while (true) {
52095272 i -= 1;
52105273 if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| {
5211 assert(mcv != .dead);
5274 // TODO see comment in `airCondBr` and `airSwitch`
5275 // assert(mcv != .dead);
52125276 return mcv;
52135277 }
52145278 }