authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-25 16:47:44-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-26 19:05:17-04:00
logeff0e6a726e46901bf8dededab9ef8db78398811
treef44c0baf4094900de7356c93b0add66d503b13a1
parenta8842b6cbf4242c2cfec203b21149872a6d2e246

x86_64: fix rem/mod behavior and hazards

1 files changed, 49 insertions(+), 10 deletions(-)

src/arch/x86_64/CodeGen.zig+49-10
...@@ -2601,7 +2601,13 @@ fn genIntMulDivOpMir(...@@ -2601,7 +2601,13 @@ fn genIntMulDivOpMir(
2601 else => unreachable,2601 else => unreachable,
2602 .mul, .imul => {},2602 .mul, .imul => {},
2603 .div => try self.asmRegisterRegister(.xor, .edx, .edx),2603 .div => try self.asmRegisterRegister(.xor, .edx, .edx),
2604 .idiv => try self.asmOpOnly(.cqo),2604 .idiv => switch (self.regBitSize(ty)) {
2605 8 => try self.asmOpOnly(.cbw),
2606 16 => try self.asmOpOnly(.cwd),
2607 32 => try self.asmOpOnly(.cdq),
2608 64 => try self.asmOpOnly(.cqo),
2609 else => unreachable,
2610 },
2605 }2611 }
26062612
2607 const mat_rhs: MCValue = switch (rhs) {2613 const mat_rhs: MCValue = switch (rhs) {
...@@ -2631,7 +2637,8 @@ fn genIntMulDivOpMir(...@@ -2631,7 +2637,8 @@ fn genIntMulDivOpMir(
2631/// Always returns a register.2637/// Always returns a register.
2632/// Clobbers .rax and .rdx registers.2638/// Clobbers .rax and .rdx registers.
2633fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {2639fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
2634 const signedness = ty.intInfo(self.target.*).signedness;2640 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
2641 const int_info = ty.intInfo(self.target.*);
2635 const dividend: Register = switch (lhs) {2642 const dividend: Register = switch (lhs) {
2636 .register => |reg| reg,2643 .register => |reg| reg,
2637 else => try self.copyToTmpRegister(ty, lhs),2644 else => try self.copyToTmpRegister(ty, lhs),
...@@ -2646,16 +2653,32 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -2646,16 +2653,32 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
2646 const divisor_lock = self.register_manager.lockReg(divisor);2653 const divisor_lock = self.register_manager.lockReg(divisor);
2647 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);2654 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);
26482655
2649 try self.genIntMulDivOpMir(switch (signedness) {2656 try self.genIntMulDivOpMir(switch (int_info.signedness) {
2650 .signed => .idiv,2657 .signed => .idiv,
2651 .unsigned => .div,2658 .unsigned => .div,
2652 }, Type.isize, .{ .register = dividend }, .{ .register = divisor });2659 }, ty, .{ .register = dividend }, .{ .register = divisor });
26532660
2654 try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64());2661 try self.asmRegisterRegister(
2655 try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63));2662 .xor,
2656 try self.asmRegisterRegister(.@"test", .rdx, .rdx);2663 registerAlias(divisor, abi_size),
2657 try self.asmCmovccRegisterRegister(divisor.to64(), .rdx, .e);2664 registerAlias(dividend, abi_size),
2658 try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });2665 );
2666 try self.asmRegisterImmediate(
2667 .sar,
2668 registerAlias(divisor, abi_size),
2669 Immediate.u(int_info.bits - 1),
2670 );
2671 try self.asmRegisterRegister(
2672 .@"test",
2673 registerAlias(.rdx, abi_size),
2674 registerAlias(.rdx, abi_size),
2675 );
2676 try self.asmCmovccRegisterRegister(
2677 registerAlias(divisor, abi_size),
2678 registerAlias(.rdx, abi_size),
2679 .z,
2680 );
2681 try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });
2659 return MCValue{ .register = divisor };2682 return MCValue{ .register = divisor };
2660}2683}
26612684
...@@ -4928,8 +4951,24 @@ fn genMulDivBinOp(...@@ -4928,8 +4951,24 @@ fn genMulDivBinOp(
49284951
4929 switch (signedness) {4952 switch (signedness) {
4930 .signed => {4953 .signed => {
4954 const lhs_lock = switch (lhs) {
4955 .register => |reg| self.register_manager.lockReg(reg),
4956 else => null,
4957 };
4958 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
4959 const rhs_lock = switch (rhs) {
4960 .register => |reg| self.register_manager.lockReg(reg),
4961 else => null,
4962 };
4963 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
4964
4965 // hack around hazard between rhs and div_floor by copying rhs to another register
4966 const rhs_copy = try self.copyToTmpRegister(ty, rhs);
4967 const rhs_copy_lock = self.register_manager.lockRegAssumeUnused(rhs_copy);
4968 defer self.register_manager.unlockReg(rhs_copy_lock);
4969
4931 const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);4970 const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);
4932 try self.genIntMulComplexOpMir(ty, div_floor, rhs);4971 try self.genIntMulComplexOpMir(ty, div_floor, .{ .register = rhs_copy });
4933 const div_floor_lock = self.register_manager.lockReg(div_floor.register);4972 const div_floor_lock = self.register_manager.lockReg(div_floor.register);
4934 defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock);4973 defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock);
49354974