authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-12-01 20:34:26-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-12-03 10:22:06-05:00
log917b4ad5e0b21d37e3b0f69dc0db02a62cf2e75b
treed43107bfd255c1c351fa1acef1adb91b0fcfbd9d
parent8a8fd47d217eadb8a45ec6b9325acfe1d42704d7

x86_64: implement more atomic ops

2 files changed, 96 insertions(+), 11 deletions(-)

src/arch/x86_64/CodeGen.zig+96-9
......@@ -14162,9 +14162,8 @@ fn atomicOp(
1416214162 };
1416314163 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
1416414164
14165 const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())
14166 .loop
14167 else switch (rmw_op orelse .Xchg) {
14165 const use_sse = rmw_op orelse .Xchg != .Xchg and val_ty.isRuntimeFloat();
14166 const strat: enum { lock, loop, libcall } = if (use_sse) .loop else switch (rmw_op orelse .Xchg) {
1416814167 .Xchg,
1416914168 .Add,
1417014169 .Sub,
......@@ -14178,7 +14177,7 @@ fn atomicOp(
1417814177 .Min,
1417914178 => if (val_abi_size <= 16) .loop else .libcall,
1418014179 };
14181 switch (method) {
14180 switch (strat) {
1418214181 .lock => {
1418314182 const tag: Mir.Inst.Tag = if (rmw_op) |op| switch (op) {
1418414183 .Xchg => if (unused) .mov else .xchg,
......@@ -14216,6 +14215,14 @@ fn atomicOp(
1421614215 return if (unused) .unreach else dst_mcv;
1421714216 },
1421814217 .loop => _ = if (val_abi_size <= 8) {
14218 const sse_reg: Register = if (use_sse)
14219 try self.register_manager.allocReg(null, abi.RegisterClass.sse)
14220 else
14221 undefined;
14222 const sse_lock =
14223 if (use_sse) self.register_manager.lockRegAssumeUnused(sse_reg) else undefined;
14224 defer if (use_sse) self.register_manager.unlockReg(sse_lock);
14225
1421914226 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
1422014227 const tmp_mcv = MCValue{ .register = tmp_reg };
1422114228 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
......@@ -14223,10 +14230,67 @@ fn atomicOp(
1422314230
1422414231 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
1422514232 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
14226 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
14233 if (!use_sse and rmw_op orelse .Xchg != .Xchg) {
1422714234 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
1422814235 }
14229 if (rmw_op) |op| switch (op) {
14236 if (rmw_op) |op| if (use_sse) {
14237 const mir_tag = @as(?Mir.Inst.FixedTag, switch (op) {
14238 .Add => switch (val_ty.floatBits(self.target.*)) {
14239 32 => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
14240 64 => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
14241 else => null,
14242 },
14243 .Sub => switch (val_ty.floatBits(self.target.*)) {
14244 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
14245 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
14246 else => null,
14247 },
14248 .Min => switch (val_ty.floatBits(self.target.*)) {
14249 32 => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
14250 64 => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
14251 else => null,
14252 },
14253 .Max => switch (val_ty.floatBits(self.target.*)) {
14254 32 => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
14255 64 => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
14256 else => null,
14257 },
14258 else => unreachable,
14259 }) orelse return self.fail("TODO implement atomicOp of {s} for {}", .{
14260 @tagName(op), val_ty.fmt(mod),
14261 });
14262 try self.genSetReg(sse_reg, val_ty, .{ .register = .rax });
14263 switch (mir_tag[0]) {
14264 .v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory(
14265 mir_tag,
14266 sse_reg.to128(),
14267 sse_reg.to128(),
14268 try val_mcv.mem(self, self.memSize(val_ty)),
14269 ) else try self.asmRegisterRegisterRegister(
14270 mir_tag,
14271 sse_reg.to128(),
14272 sse_reg.to128(),
14273 (if (val_mcv.isRegister())
14274 val_mcv.getReg().?
14275 else
14276 try self.copyToTmpRegister(val_ty, val_mcv)).to128(),
14277 ),
14278 ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory(
14279 mir_tag,
14280 sse_reg.to128(),
14281 try val_mcv.mem(self, self.memSize(val_ty)),
14282 ) else try self.asmRegisterRegister(
14283 mir_tag,
14284 sse_reg.to128(),
14285 (if (val_mcv.isRegister())
14286 val_mcv.getReg().?
14287 else
14288 try self.copyToTmpRegister(val_ty, val_mcv)).to128(),
14289 ),
14290 else => unreachable,
14291 }
14292 try self.genSetReg(tmp_reg, val_ty, .{ .register = sse_reg });
14293 } else switch (op) {
1423014294 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
1423114295 .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
1423214296 .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
......@@ -14362,9 +14426,32 @@ fn atomicOp(
1436214426 try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);
1436314427 try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
1436414428 },
14365 else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
14366 val_ty.fmt(mod), @tagName(op),
14367 }),
14429 .Min, .Max => {
14430 const cc: Condition = switch (if (val_ty.isAbiInt(mod))
14431 val_ty.intInfo(mod).signedness
14432 else
14433 .unsigned) {
14434 .unsigned => switch (op) {
14435 .Min => .a,
14436 .Max => .b,
14437 else => unreachable,
14438 },
14439 .signed => switch (op) {
14440 .Min => .g,
14441 .Max => .l,
14442 else => unreachable,
14443 },
14444 };
14445
14446 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .register = .rcx });
14447 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
14448 defer self.register_manager.unlockReg(tmp_lock);
14449
14450 try self.asmRegisterMemory(.{ ._, .cmp }, .rbx, val_lo_mem);
14451 try self.asmRegisterMemory(.{ ._, .sbb }, tmp_reg, val_hi_mem);
14452 try self.asmCmovccRegisterMemory(cc, .rbx, val_lo_mem);
14453 try self.asmCmovccRegisterMemory(cc, .rcx, val_hi_mem);
14454 },
1436814455 };
1436914456 try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
1437014457 _ = try self.asmJccReloc(.ne, loop);
test/behavior/atomics.zig-2
......@@ -208,7 +208,6 @@ fn testAtomicStore() !void {
208208}
209209
210210test "atomicrmw with floats" {
211 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
212211 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
213212 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
214213 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
......@@ -316,7 +315,6 @@ test "atomicrmw with 128-bit ints" {
316315 if (!supports_128_bit_atomics) return error.SkipZigTest;
317316
318317 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
319 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
320318
321319 // TODO "ld.lld: undefined symbol: __sync_lock_test_and_set_16" on -mcpu x86_64
322320 if (builtin.cpu.arch == .x86_64 and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;