| author |  Alex Rønne Petersen <alex@alexrp.com> 2024-10-31 09:00:02+01:00 |
| committer | Alex Rønne Petersen <alex@alexrp.com> 2025-05-14 05:43:28+02:00 |
| log | 59f92bff69adb8e3e22bb3a8b4d75658dafcb626 |
| tree | 34c674886c44a2f5dcfb86e27dcc3cd51a0bf34b |
| parent | 199782edd16ca1fe08cee57ee2219c0a0c385fdc |
| signature |  Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM |
Air: Fix mustLower() for atomic_load with inter-thread ordering.
3 files changed, 29 insertions(+), 18 deletions(-)
src- Air.zig
- arch
- riscv64
- x86_64
src/Air.zig+4-1
| ... | @@ -1871,7 +1871,10 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { | ... | @@ -1871,7 +1871,10 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { |
| 1871 | }, | 1871 | }, |
| 1872 | .load => air.typeOf(data.ty_op.operand, ip).isVolatilePtrIp(ip), | 1872 | .load => air.typeOf(data.ty_op.operand, ip).isVolatilePtrIp(ip), |
| 1873 | .slice_elem_val, .ptr_elem_val => air.typeOf(data.bin_op.lhs, ip).isVolatilePtrIp(ip), | 1873 | .slice_elem_val, .ptr_elem_val => air.typeOf(data.bin_op.lhs, ip).isVolatilePtrIp(ip), |
| 1874 | .atomic_load => air.typeOf(data.atomic_load.ptr, ip).isVolatilePtrIp(ip), | 1874 | .atomic_load => switch (data.atomic_load.order) { |
| 1875 | .unordered, .monotonic => air.typeOf(data.atomic_load.ptr, ip).isVolatilePtrIp(ip), | ||
| 1876 | else => true, // Stronger memory orderings have inter-thread side effects. | ||
| 1877 | }, | ||
| 1875 | }; | 1878 | }; |
| 1876 | } | 1879 | } |
| 1877 | 1880 |
src/arch/riscv64/CodeGen.zig+8-6
| ... | @@ -7737,7 +7737,10 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -7737,7 +7737,10 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { |
| 7737 | const bit_size = elem_ty.bitSize(zcu); | 7737 | const bit_size = elem_ty.bitSize(zcu); |
| 7738 | if (bit_size > 64) return func.fail("TODO: airAtomicLoad > 64 bits", .{}); | 7738 | if (bit_size > 64) return func.fail("TODO: airAtomicLoad > 64 bits", .{}); |
| 7739 | 7739 | ||
| 7740 | const result_mcv = try func.allocRegOrMem(elem_ty, inst, true); | 7740 | const result_mcv: MCValue = if (func.liveness.isUnused(inst)) |
| 7741 | .{ .register = .zero } | ||
| 7742 | else | ||
| 7743 | try func.allocRegOrMem(elem_ty, inst, true); | ||
| 7741 | assert(result_mcv == .register); // should be less than 8 bytes | 7744 | assert(result_mcv == .register); // should be less than 8 bytes |
| 7742 | 7745 | ||
| 7743 | if (order == .seq_cst) { | 7746 | if (order == .seq_cst) { |
| ... | @@ -7753,11 +7756,10 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -7753,11 +7756,10 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { |
| 7753 | try func.load(result_mcv, ptr_mcv, ptr_ty); | 7756 | try func.load(result_mcv, ptr_mcv, ptr_ty); |
| 7754 | 7757 | ||
| 7755 | switch (order) { | 7758 | switch (order) { |
| 7756 | // Don't guarnetee other memory operations to be ordered after the load. | 7759 | // Don't guarantee other memory operations to be ordered after the load. |
| 7757 | .unordered => {}, | 7760 | .unordered, .monotonic => {}, |
| 7758 | .monotonic => {}, | 7761 | // Make sure all previous reads happen before any reading or writing occurs. |
| 7759 | // Make sure all previous reads happen before any reading or writing accurs. | 7762 | .acquire, .seq_cst => { |
| 7760 | .seq_cst, .acquire => { | ||
| 7761 | _ = try func.addInst(.{ | 7763 | _ = try func.addInst(.{ |
| 7762 | .tag = .fence, | 7764 | .tag = .fence, |
| 7763 | .data = .{ .fence = .{ | 7765 | .data = .{ .fence = .{ |
src/arch/x86_64/CodeGen.zig+17-11
| ... | @@ -97117,23 +97117,29 @@ fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -97117,23 +97117,29 @@ fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 97117 | 97117 | ||
| 97118 | fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void { | 97118 | fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 97119 | const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; | 97119 | const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; |
| 97120 | const result: MCValue = result: { | ||
| 97121 | const ptr_ty = self.typeOf(atomic_load.ptr); | ||
| 97122 | const ptr_mcv = try self.resolveInst(atomic_load.ptr); | ||
| 97123 | const ptr_lock = switch (ptr_mcv) { | ||
| 97124 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | ||
| 97125 | else => null, | ||
| 97126 | }; | ||
| 97127 | defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); | ||
| 97120 | 97128 | ||
| 97121 | const ptr_ty = self.typeOf(atomic_load.ptr); | 97129 | const unused = self.liveness.isUnused(inst); |
| 97122 | const ptr_mcv = try self.resolveInst(atomic_load.ptr); | ||
| 97123 | const ptr_lock = switch (ptr_mcv) { | ||
| 97124 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | ||
| 97125 | else => null, | ||
| 97126 | }; | ||
| 97127 | defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); | ||
| 97128 | 97130 | ||
| 97129 | const dst_mcv = | 97131 | const dst_mcv: MCValue = if (unused) |
| 97130 | if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv)) | 97132 | .{ .register = try self.register_manager.allocReg(null, self.regSetForType(ptr_ty.childType(self.pt.zcu))) } |
| 97133 | else if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv)) | ||
| 97131 | ptr_mcv | 97134 | ptr_mcv |
| 97132 | else | 97135 | else |
| 97133 | try self.allocRegOrMem(inst, true); | 97136 | try self.allocRegOrMem(inst, true); |
| 97134 | 97137 | ||
| 97135 | try self.load(dst_mcv, ptr_ty, ptr_mcv); | 97138 | try self.load(dst_mcv, ptr_ty, ptr_mcv); |
| 97136 | return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none }); | 97139 | |
| 97140 | break :result if (unused) .unreach else dst_mcv; | ||
| 97141 | }; | ||
| 97142 | return self.finishAir(inst, result, .{ atomic_load.ptr, .none, .none }); | ||
| 97137 | } | 97143 | } |
| 97138 | 97144 | ||
| 97139 | fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void { | 97145 | fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void { |