authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-06 20:21:54-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-13 04:17:47-04:00
log3b22ce82643f2e547c85bf4b17477bc27133eb42
tree33b0ad3f77b572fc3e726199554fb44eb5f5a178
parent0165187cd07b26f15c2ca7e021747d0989d1956b

x86_64: fix atomic loop implementation

3 files changed, 79 insertions(+), 60 deletions(-)

src/arch/x86_64/CodeGen.zig+78-59
......@@ -1734,10 +1734,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
17341734 };
17351735 try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
17361736
1737 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1737 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
17381738 try self.asmCmovccRegisterRegister(
1739 registerAlias(dst_reg, abi_size),
1740 registerAlias(limit_reg, abi_size),
1739 registerAlias(dst_reg, cmov_abi_size),
1740 registerAlias(limit_reg, cmov_abi_size),
17411741 cc,
17421742 );
17431743 break :result dst_mcv;
......@@ -1785,10 +1785,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
17851785 };
17861786 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
17871787
1788 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1788 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
17891789 try self.asmCmovccRegisterRegister(
1790 registerAlias(dst_reg, abi_size),
1791 registerAlias(limit_reg, abi_size),
1790 registerAlias(dst_reg, cmov_abi_size),
1791 registerAlias(limit_reg, cmov_abi_size),
17921792 cc,
17931793 );
17941794 break :result dst_mcv;
......@@ -1841,10 +1841,10 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
18411841 };
18421842
18431843 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);
1844 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1844 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
18451845 try self.asmCmovccRegisterRegister(
1846 registerAlias(dst_mcv.register, abi_size),
1847 registerAlias(limit_reg, abi_size),
1846 registerAlias(dst_mcv.register, cmov_abi_size),
1847 registerAlias(limit_reg, cmov_abi_size),
18481848 cc,
18491849 );
18501850 break :result dst_mcv;
......@@ -3102,10 +3102,10 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
31023102 try self.copyToRegisterWithInstTracking(inst, dst_ty, .{ .immediate = src_bits });
31033103 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
31043104
3105 const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2));
3105 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
31063106 try self.asmCmovccRegisterRegister(
3107 registerAlias(dst_reg, dst_abi_size),
3108 registerAlias(width_mcv.register, dst_abi_size),
3107 registerAlias(dst_reg, cmov_abi_size),
3108 registerAlias(width_mcv.register, cmov_abi_size),
31093109 .z,
31103110 );
31113111
......@@ -3162,10 +3162,10 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
31623162 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
31633163 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
31643164
3165 const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3165 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
31663166 try self.asmCmovccRegisterRegister(
3167 registerAlias(dst_reg, abi_size),
3168 registerAlias(width_reg, abi_size),
3167 registerAlias(dst_reg, cmov_abi_size),
3168 registerAlias(width_reg, cmov_abi_size),
31693169 .z,
31703170 );
31713171 break :result dst_mcv;
......@@ -4766,7 +4766,7 @@ fn genBinOp(
47664766 },
47674767 };
47684768
4769 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
4769 const cmov_abi_size = @max(@intCast(u32, lhs_ty.abiSize(self.target.*)), 2);
47704770 const tmp_reg = switch (dst_mcv) {
47714771 .register => |reg| reg,
47724772 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),
......@@ -4784,13 +4784,13 @@ fn genBinOp(
47844784 .ptr_stack_offset,
47854785 => unreachable,
47864786 .register => |src_reg| try self.asmCmovccRegisterRegister(
4787 registerAlias(tmp_reg, abi_size),
4788 registerAlias(src_reg, abi_size),
4787 registerAlias(tmp_reg, cmov_abi_size),
4788 registerAlias(src_reg, cmov_abi_size),
47894789 cc,
47904790 ),
47914791 .stack_offset => |off| try self.asmCmovccRegisterMemory(
4792 registerAlias(tmp_reg, abi_size),
4793 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4792 registerAlias(tmp_reg, cmov_abi_size),
4793 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
47944794 .base = .rbp,
47954795 .disp = -off,
47964796 }),
......@@ -4803,8 +4803,8 @@ fn genBinOp(
48034803
48044804 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mat_src_mcv);
48054805 try self.asmCmovccRegisterMemory(
4806 registerAlias(tmp_reg, abi_size),
4807 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
4806 registerAlias(tmp_reg, cmov_abi_size),
4807 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }),
48084808 cc,
48094809 );
48104810 },
......@@ -7727,7 +7727,6 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
77277727
77287728fn atomicOp(
77297729 self: *Self,
7730 dst_reg: Register,
77317730 ptr_mcv: MCValue,
77327731 val_mcv: MCValue,
77337732 ptr_ty: Type,
......@@ -7735,11 +7734,7 @@ fn atomicOp(
77357734 unused: bool,
77367735 rmw_op: ?std.builtin.AtomicRmwOp,
77377736 order: std.builtin.AtomicOrder,
7738) InnerError!void {
7739 const dst_mcv = MCValue{ .register = dst_reg };
7740 const dst_lock = self.register_manager.lockReg(dst_reg);
7741 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
7742
7737) InnerError!MCValue {
77437738 const ptr_lock = switch (ptr_mcv) {
77447739 .register => |reg| self.register_manager.lockReg(reg),
77457740 else => null,
......@@ -7794,9 +7789,14 @@ fn atomicOp(
77947789 .SeqCst => .xchg,
77957790 };
77967791
7792 const dst_reg = try self.register_manager.allocReg(null, gp);
7793 const dst_mcv = MCValue{ .register = dst_reg };
7794 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
7795 defer self.register_manager.unlockReg(dst_lock);
7796
77977797 try self.genSetReg(val_ty, dst_reg, val_mcv);
77987798 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
7799 try self.genUnOpMir(.neg, val_ty, .{ .register = dst_reg });
7799 try self.genUnOpMir(.neg, val_ty, dst_mcv);
78007800 }
78017801 _ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
78027802 .mov, .xchg => .mr_sib,
......@@ -7806,25 +7806,31 @@ fn atomicOp(
78067806 .r = registerAlias(dst_reg, val_abi_size),
78077807 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
78087808 } } });
7809
7810 return if (unused) .none else dst_mcv;
78097811 },
7810 .loop => _ = try self.asmJccReloc(if (val_abi_size <= 8) loop: {
7811 try self.genSetReg(val_ty, dst_reg, val_mcv);
7812 .loop => _ = if (val_abi_size <= 8) {
7813 const tmp_reg = try self.register_manager.allocReg(null, gp);
7814 const tmp_mcv = MCValue{ .register = tmp_reg };
7815 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7816 defer self.register_manager.unlockReg(tmp_lock);
7817
78127818 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
78137819 const loop = @intCast(u32, self.mir_instructions.len);
78147820 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
7815 try self.genSetReg(val_ty, dst_reg, .{ .register = .rax });
7821 try self.genSetReg(val_ty, tmp_reg, .{ .register = .rax });
78167822 }
78177823 if (rmw_op) |op| switch (op) {
7818 .Xchg => try self.genSetReg(val_ty, dst_reg, val_mcv),
7819 .Add => try self.genBinOpMir(.add, val_ty, dst_mcv, val_mcv),
7820 .Sub => try self.genBinOpMir(.sub, val_ty, dst_mcv, val_mcv),
7821 .And => try self.genBinOpMir(.@"and", val_ty, dst_mcv, val_mcv),
7824 .Xchg => try self.genSetReg(val_ty, tmp_reg, val_mcv),
7825 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
7826 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
7827 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
78227828 .Nand => {
7823 try self.genBinOpMir(.@"and", val_ty, dst_mcv, val_mcv);
7824 try self.genUnOpMir(.not, val_ty, dst_mcv);
7829 try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);
7830 try self.genUnOpMir(.not, val_ty, tmp_mcv);
78257831 },
7826 .Or => try self.genBinOpMir(.@"or", val_ty, dst_mcv, val_mcv),
7827 .Xor => try self.genBinOpMir(.xor, val_ty, dst_mcv, val_mcv),
7832 .Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),
7833 .Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),
78287834 .Min, .Max => {
78297835 const cc: Condition = switch (if (val_ty.isAbiInt())
78307836 val_ty.intInfo(self.target.*).signedness
......@@ -7842,17 +7848,18 @@ fn atomicOp(
78427848 },
78437849 };
78447850
7845 try self.genBinOpMir(.cmp, val_ty, dst_mcv, val_mcv);
7851 try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);
7852 const cmov_abi_size = @max(val_abi_size, 2);
78467853 switch (val_mcv) {
78477854 .register => |val_reg| try self.asmCmovccRegisterRegister(
7848 registerAlias(dst_reg, val_abi_size),
7849 registerAlias(val_reg, val_abi_size),
7855 registerAlias(tmp_reg, cmov_abi_size),
7856 registerAlias(val_reg, cmov_abi_size),
78507857 cc,
78517858 ),
78527859 .stack_offset => |val_off| try self.asmCmovccRegisterMemory(
7853 registerAlias(dst_reg, val_abi_size),
7860 registerAlias(tmp_reg, cmov_abi_size),
78547861 Memory.sib(
7855 Memory.PtrSize.fromSize(val_abi_size),
7862 Memory.PtrSize.fromSize(cmov_abi_size),
78567863 .{ .base = .rbp, .disp = -val_off },
78577864 ),
78587865 cc,
......@@ -7860,8 +7867,8 @@ fn atomicOp(
78607867 else => {
78617868 const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
78627869 try self.asmCmovccRegisterRegister(
7863 registerAlias(dst_reg, val_abi_size),
7864 registerAlias(val_reg, val_abi_size),
7870 registerAlias(tmp_reg, cmov_abi_size),
7871 registerAlias(val_reg, cmov_abi_size),
78657872 cc,
78667873 );
78677874 },
......@@ -7869,11 +7876,12 @@ fn atomicOp(
78697876 },
78707877 };
78717878 _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
7872 .r = registerAlias(dst_reg, val_abi_size),
7879 .r = registerAlias(tmp_reg, val_abi_size),
78737880 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
78747881 } } });
7875 break :loop loop;
7876 } else loop: {
7882 _ = try self.asmJccReloc(loop, .ne);
7883 return if (unused) .none else .{ .register = .rax };
7884 } else {
78777885 try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
78787886 .base = ptr_mem.sib.base,
78797887 .scale_index = ptr_mem.sib.scale_index,
......@@ -7939,8 +7947,22 @@ fn atomicOp(
79397947 _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
79407948 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
79417949 } });
7942 break :loop loop;
7943 }, .ne),
7950 _ = try self.asmJccReloc(loop, .ne);
7951
7952 if (unused) return .none;
7953 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
7954 try self.asmMemoryRegister(
7955 .mov,
7956 Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - dst_mcv.stack_offset }),
7957 .rax,
7958 );
7959 try self.asmMemoryRegister(
7960 .mov,
7961 Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - dst_mcv.stack_offset }),
7962 .rdx,
7963 );
7964 return dst_mcv;
7965 },
79447966 .libcall => return self.fail("TODO implement x86 atomic libcall", .{}),
79457967 }
79467968}
......@@ -7954,7 +7976,6 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
79547976 defer for (regs_lock) |lock| self.register_manager.unlockReg(lock);
79557977
79567978 const unused = self.liveness.isUnused(inst);
7957 const dst_reg = try self.register_manager.allocReg(if (unused) null else inst, gp);
79587979
79597980 const ptr_ty = self.air.typeOf(pl_op.operand);
79607981 const ptr_mcv = try self.resolveInst(pl_op.operand);
......@@ -7962,8 +7983,8 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
79627983 const val_ty = self.air.typeOf(extra.operand);
79637984 const val_mcv = try self.resolveInst(extra.operand);
79647985
7965 try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
7966 const result: MCValue = if (unused) .dead else .{ .register = dst_reg };
7986 const result =
7987 try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
79677988 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
79687989}
79697990
......@@ -7996,16 +8017,14 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
79968017fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
79978018 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
79988019
7999 const dst_reg = try self.register_manager.allocReg(null, gp);
8000
80018020 const ptr_ty = self.air.typeOf(bin_op.lhs);
80028021 const ptr_mcv = try self.resolveInst(bin_op.lhs);
80038022
80048023 const val_ty = self.air.typeOf(bin_op.rhs);
80058024 const val_mcv = try self.resolveInst(bin_op.rhs);
80068025
8007 try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
8008 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
8026 const result = try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
8027 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
80098028}
80108029
80118030fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
test/behavior/atomics.zig-1
......@@ -232,7 +232,6 @@ fn testAtomicRmwFloat() !void {
232232
233233test "atomicrmw with ints" {
234234 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
235 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
236235 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
237236 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
238237 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
test/behavior/pointers.zig+1
......@@ -506,6 +506,7 @@ test "ptrToInt on a generic function" {
506506 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
507507 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
508508 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
509 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
509510
510511 const S = struct {
511512 fn generic(i: anytype) @TypeOf(i) {