authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-12-01 20:34:26-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-12-03 10:22:06-05:00
log917b4ad5e0b21d37e3b0f69dc0db02a62cf2e75b
treed43107bfd255c1c351fa1acef1adb91b0fcfbd9d
parent8a8fd47d217eadb8a45ec6b9325acfe1d42704d7

x86_64: implement more atomic ops

2 files changed, 96 insertions(+), 11 deletions(-)

src/arch/x86_64/CodeGen.zig+96-9
...@@ -14162,9 +14162,8 @@ fn atomicOp(...@@ -14162,9 +14162,8 @@ fn atomicOp(
14162 };14162 };
14163 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);14163 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
1416414164
14165 const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())14165 const use_sse = rmw_op orelse .Xchg != .Xchg and val_ty.isRuntimeFloat();
14166 .loop14166 const strat: enum { lock, loop, libcall } = if (use_sse) .loop else switch (rmw_op orelse .Xchg) {
14167 else switch (rmw_op orelse .Xchg) {
14168 .Xchg,14167 .Xchg,
14169 .Add,14168 .Add,
14170 .Sub,14169 .Sub,
...@@ -14178,7 +14177,7 @@ fn atomicOp(...@@ -14178,7 +14177,7 @@ fn atomicOp(
14178 .Min,14177 .Min,
14179 => if (val_abi_size <= 16) .loop else .libcall,14178 => if (val_abi_size <= 16) .loop else .libcall,
14180 };14179 };
14181 switch (method) {14180 switch (strat) {
14182 .lock => {14181 .lock => {
14183 const tag: Mir.Inst.Tag = if (rmw_op) |op| switch (op) {14182 const tag: Mir.Inst.Tag = if (rmw_op) |op| switch (op) {
14184 .Xchg => if (unused) .mov else .xchg,14183 .Xchg => if (unused) .mov else .xchg,
...@@ -14216,6 +14215,14 @@ fn atomicOp(...@@ -14216,6 +14215,14 @@ fn atomicOp(
14216 return if (unused) .unreach else dst_mcv;14215 return if (unused) .unreach else dst_mcv;
14217 },14216 },
14218 .loop => _ = if (val_abi_size <= 8) {14217 .loop => _ = if (val_abi_size <= 8) {
14218 const sse_reg: Register = if (use_sse)
14219 try self.register_manager.allocReg(null, abi.RegisterClass.sse)
14220 else
14221 undefined;
14222 const sse_lock =
14223 if (use_sse) self.register_manager.lockRegAssumeUnused(sse_reg) else undefined;
14224 defer if (use_sse) self.register_manager.unlockReg(sse_lock);
14225
14219 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);14226 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
14220 const tmp_mcv = MCValue{ .register = tmp_reg };14227 const tmp_mcv = MCValue{ .register = tmp_reg };
14221 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);14228 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...@@ -14223,10 +14230,67 @@ fn atomicOp(...@@ -14223,10 +14230,67 @@ fn atomicOp(
1422314230
14224 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);14231 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
14225 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);14232 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
14226 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {14233 if (!use_sse and rmw_op orelse .Xchg != .Xchg) {
14227 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });14234 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
14228 }14235 }
14229 if (rmw_op) |op| switch (op) {14236 if (rmw_op) |op| if (use_sse) {
14237 const mir_tag = @as(?Mir.Inst.FixedTag, switch (op) {
14238 .Add => switch (val_ty.floatBits(self.target.*)) {
14239 32 => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
14240 64 => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
14241 else => null,
14242 },
14243 .Sub => switch (val_ty.floatBits(self.target.*)) {
14244 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
14245 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
14246 else => null,
14247 },
14248 .Min => switch (val_ty.floatBits(self.target.*)) {
14249 32 => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
14250 64 => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
14251 else => null,
14252 },
14253 .Max => switch (val_ty.floatBits(self.target.*)) {
14254 32 => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
14255 64 => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
14256 else => null,
14257 },
14258 else => unreachable,
14259 }) orelse return self.fail("TODO implement atomicOp of {s} for {}", .{
14260 @tagName(op), val_ty.fmt(mod),
14261 });
14262 try self.genSetReg(sse_reg, val_ty, .{ .register = .rax });
14263 switch (mir_tag[0]) {
14264 .v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory(
14265 mir_tag,
14266 sse_reg.to128(),
14267 sse_reg.to128(),
14268 try val_mcv.mem(self, self.memSize(val_ty)),
14269 ) else try self.asmRegisterRegisterRegister(
14270 mir_tag,
14271 sse_reg.to128(),
14272 sse_reg.to128(),
14273 (if (val_mcv.isRegister())
14274 val_mcv.getReg().?
14275 else
14276 try self.copyToTmpRegister(val_ty, val_mcv)).to128(),
14277 ),
14278 ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory(
14279 mir_tag,
14280 sse_reg.to128(),
14281 try val_mcv.mem(self, self.memSize(val_ty)),
14282 ) else try self.asmRegisterRegister(
14283 mir_tag,
14284 sse_reg.to128(),
14285 (if (val_mcv.isRegister())
14286 val_mcv.getReg().?
14287 else
14288 try self.copyToTmpRegister(val_ty, val_mcv)).to128(),
14289 ),
14290 else => unreachable,
14291 }
14292 try self.genSetReg(tmp_reg, val_ty, .{ .register = sse_reg });
14293 } else switch (op) {
14230 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),14294 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
14231 .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),14295 .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
14232 .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),14296 .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
...@@ -14362,9 +14426,32 @@ fn atomicOp(...@@ -14362,9 +14426,32 @@ fn atomicOp(
14362 try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);14426 try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);
14363 try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);14427 try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
14364 },14428 },
14365 else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{14429 .Min, .Max => {
14366 val_ty.fmt(mod), @tagName(op),14430 const cc: Condition = switch (if (val_ty.isAbiInt(mod))
14367 }),14431 val_ty.intInfo(mod).signedness
14432 else
14433 .unsigned) {
14434 .unsigned => switch (op) {
14435 .Min => .a,
14436 .Max => .b,
14437 else => unreachable,
14438 },
14439 .signed => switch (op) {
14440 .Min => .g,
14441 .Max => .l,
14442 else => unreachable,
14443 },
14444 };
14445
14446 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .register = .rcx });
14447 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
14448 defer self.register_manager.unlockReg(tmp_lock);
14449
14450 try self.asmRegisterMemory(.{ ._, .cmp }, .rbx, val_lo_mem);
14451 try self.asmRegisterMemory(.{ ._, .sbb }, tmp_reg, val_hi_mem);
14452 try self.asmCmovccRegisterMemory(cc, .rbx, val_lo_mem);
14453 try self.asmCmovccRegisterMemory(cc, .rcx, val_hi_mem);
14454 },
14368 };14455 };
14369 try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);14456 try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
14370 _ = try self.asmJccReloc(.ne, loop);14457 _ = try self.asmJccReloc(.ne, loop);
test/behavior/atomics.zig-2
...@@ -208,7 +208,6 @@ fn testAtomicStore() !void {...@@ -208,7 +208,6 @@ fn testAtomicStore() !void {
208}208}
209209
210test "atomicrmw with floats" {210test "atomicrmw with floats" {
211 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
212 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO211 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
213 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO212 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
214 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO213 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...@@ -316,7 +315,6 @@ test "atomicrmw with 128-bit ints" {...@@ -316,7 +315,6 @@ test "atomicrmw with 128-bit ints" {
316 if (!supports_128_bit_atomics) return error.SkipZigTest;315 if (!supports_128_bit_atomics) return error.SkipZigTest;
317316
318 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO317 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
319 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
320318
321 // TODO "ld.lld: undefined symbol: __sync_lock_test_and_set_16" on -mcpu x86_64319 // TODO "ld.lld: undefined symbol: __sync_lock_test_and_set_16" on -mcpu x86_64
322 if (builtin.cpu.arch == .x86_64 and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;320 if (builtin.cpu.arch == .x86_64 and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;