authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2026-10-23 17:04:46-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-24 22:59:09+02:00
log16126cf14165bf97ed3c588fbc10f13c1970f22f
treec93db09058e4d0c1a06d3d92a5694a8d0ee6d115
parentdef373ed782112e19132be11b413a6f44608bc60

x86_64: rewrite cmpxchg_{weak,strong} implementation

2 files changed, 223 insertions(+), 125 deletions(-)

src/codegen/x86_64/CodeGen.zig+216-125
......@@ -210,6 +210,8 @@ pub const MCValue = union(enum) {
210210 register_offset: bits.RegisterOffset,
211211 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
212212 register_overflow: struct { reg: Register, eflags: Condition },
213 /// The value is a tuple { wrapped, overflow } where wrapped value is stored across two registers.
214 register_pair_overflow: struct { pair: [2]Register, eflags: Condition },
213215 /// The value is a bool vector stored in a vector register with a different scalar type.
214216 register_mask: Mask,
215217 /// The value is in memory at a hard-coded address.
......@@ -265,6 +267,7 @@ pub const MCValue = union(enum) {
265267 .register_mask,
266268 .eflags,
267269 .register_overflow,
270 .register_pair_overflow,
268271 .indirect_load_frame,
269272 .lea_frame,
270273 .lea_nav,
......@@ -363,6 +366,7 @@ pub const MCValue = union(enum) {
363366 .indirect,
364367 .indirect_mask,
365368 => |*pl| (&pl.reg)[0..1],
369 .register_pair_overflow => |*rpo| &rpo.pair,
366370 else => &.{},
367371 };
368372 }
......@@ -371,6 +375,7 @@ pub const MCValue = union(enum) {
371375 return switch (mcv) {
372376 .eflags => |cc| cc,
373377 .register_overflow => |reg_ov| reg_ov.eflags,
378 .register_pair_overflow => |rpo| rpo.eflags,
374379 else => null,
375380 };
376381 }
......@@ -396,6 +401,7 @@ pub const MCValue = union(enum) {
396401 .register_quadruple,
397402 .register_offset,
398403 .register_overflow,
404 .register_pair_overflow,
399405 .register_mask,
400406 .indirect_mask,
401407 .lea_frame,
......@@ -437,6 +443,7 @@ pub const MCValue = union(enum) {
437443 .register_triple,
438444 .register_quadruple,
439445 .register_overflow,
446 .register_pair_overflow,
440447 .register_mask,
441448 .memory,
442449 .indirect,
......@@ -487,6 +494,7 @@ pub const MCValue = union(enum) {
487494 .register_triple,
488495 .register_quadruple,
489496 .register_overflow,
497 .register_pair_overflow,
490498 .register_mask,
491499 .memory,
492500 .indirect,
......@@ -530,6 +538,7 @@ pub const MCValue = union(enum) {
530538 .register_quadruple,
531539 .register_offset,
532540 .register_overflow,
541 .register_pair_overflow,
533542 .register_mask,
534543 .indirect_load_frame,
535544 .lea_frame,
......@@ -604,6 +613,11 @@ pub const MCValue = union(enum) {
604613 @tagName(pl.eflags),
605614 @tagName(pl.reg),
606615 }),
616 .register_pair_overflow => |pl| try w.print("{t}:({t}:{t})", .{
617 pl.eflags,
618 pl.pair[0],
619 pl.pair[1],
620 }),
607621 .register_mask => |pl| try w.print("mask({s},{f}):{s}{s}", .{
608622 @tagName(pl.info.kind),
609623 pl.info.scalar,
......@@ -693,6 +707,7 @@ const InstTracking = struct {
693707 .register_quadruple,
694708 .register_offset,
695709 .register_overflow,
710 .register_pair_overflow,
696711 .register_mask,
697712 .indirect,
698713 .indirect_mask,
......@@ -750,6 +765,10 @@ const InstTracking = struct {
750765 function.register_manager.freeReg(reg_ov.reg);
751766 function.eflags_inst = null;
752767 },
768 .register_pair_overflow => |rpo| {
769 for (rpo.pair) |r| function.register_manager.freeReg(r);
770 function.eflags_inst = null;
771 },
753772 .register_mask => |reg_mask| function.register_manager.freeReg(reg_mask.reg),
754773 .eflags => function.eflags_inst = null,
755774 else => {}, // TODO process stack allocation death
......@@ -793,6 +812,7 @@ const InstTracking = struct {
793812 .register_quadruple,
794813 .register_offset,
795814 .register_overflow,
815 .register_pair_overflow,
796816 .register_mask,
797817 .indirect,
798818 .indirect_mask,
......@@ -51941,7 +51961,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5194151961 }),
5194251962 else => |e| return e,
5194351963 };
51944 try res[0].withOverflow(&res[1], cg);
51964 try res[0].withOverflow(&res[1], ty_pl.ty, cg);
5194551965 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
5194651966 },
5194751967 .sub_with_overflow => |air_tag| {
......@@ -52846,7 +52866,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5284652866 }),
5284752867 else => |e| return e,
5284852868 };
52849 try res[0].withOverflow(&res[1], cg);
52869 try res[0].withOverflow(&res[1], ty_pl.ty, cg);
5285052870 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
5285152871 },
5285252872 .mul_with_overflow => |air_tag| {
......@@ -56443,7 +56463,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5644356463 }),
5644456464 else => |e| return e,
5644556465 };
56446 try res[0].withOverflow(&res[1], cg);
56466 try res[0].withOverflow(&res[1], ty_pl.ty, cg);
5644756467 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
5644856468 },
5644956469 .shl_with_overflow => |air_tag| {
......@@ -59788,7 +59808,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5978859808 }),
5978959809 else => |e| return e,
5979059810 };
59791 try res[0].withOverflow(&res[1], cg);
59811 try res[0].withOverflow(&res[1], ty_pl.ty, cg);
5979259812 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
5979359813 },
5979459814 .alloc => {
......@@ -103057,7 +103077,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103057103077 const ty_pl = air_datas[@backingInt(inst)].ty_pl;
103058103078 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
103059103079 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
103060 try ops[0].toPair(&ops[1], cg);
103080 try ops[0].toPair(&ops[1], ty_pl.ty, cg);
103061103081 try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
103062103082 },
103063103083 .slice_len => {
......@@ -103474,7 +103494,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103474103494 var len = try cg.tempInit(.usize, .{
103475103495 .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu),
103476103496 });
103477 try ops[0].toPair(&len, cg);
103497 try ops[0].toPair(&len, ty_op.ty, cg);
103478103498 try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg);
103479103499 },
103480103500 .int_from_float, .int_from_float_optimized => |air_tag| {
......@@ -170236,7 +170256,113 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
170236170256 };
170237170257 for (ops) |op| try op.die(cg);
170238170258 },
170239 .cmpxchg_weak, .cmpxchg_strong => try cg.airCmpxchg(inst),
170259 .cmpxchg_weak, .cmpxchg_strong => |air_tag| {
170260 const ty_pl = air_datas[@backingInt(inst)].ty_pl;
170261 const extra = cg.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
170262 var ops = try cg.tempsFromOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value });
170263 var res: [2]Temp = undefined;
170264 (if (ty_pl.ty.optionalReprIsPayload(zcu))
170265 cg.select(res[0..1], &.{ty_pl.ty}, &ops, comptime &.{.{
170266 .required_features = .{ .@"64bit", .cmov, null, null },
170267 .src_constraints = .{ .any, .{ .size = .qword }, .{ .size = .qword } },
170268 .patterns = &.{
170269 .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } },
170270 },
170271 .extra_temps = .{
170272 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
170273 .unused,
170274 .unused,
170275 .unused,
170276 .unused,
170277 .unused,
170278 .unused,
170279 .unused,
170280 .unused,
170281 .unused,
170282 .unused,
170283 },
170284 .dst_temps = .{ .{ .ref = .src1 }, .unused },
170285 .clobbers = .{ .eflags = true },
170286 .each = .{ .once = &.{
170287 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
170288 .{ ._, .@"lock _", .cmpxchg, .lea(.src0q), .src2q, ._, ._ },
170289 .{ ._, ._e, .cmov, .dst0q, .tmp0q, ._, ._ },
170290 } },
170291 }})
170292 else
170293 cg.select(&res, &.{ ty_pl.ty.childType(zcu), .u1 }, &ops, comptime &.{ .{
170294 .required_features = .{ .@"64bit", null, null, null },
170295 .src_constraints = .{ .any, .{ .int = .byte }, .{ .int = .byte } },
170296 .patterns = &.{
170297 .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } },
170298 },
170299 .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } },
170300 .clobbers = .{ .eflags = true },
170301 .each = .{ .once = &.{
170302 .{ ._, .@"lock _", .cmpxchg, .lea(.src0b), .src2b, ._, ._ },
170303 } },
170304 }, .{
170305 .required_features = .{ .@"64bit", null, null, null },
170306 .src_constraints = .{ .any, .{ .int = .word }, .{ .int = .word } },
170307 .patterns = &.{
170308 .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } },
170309 },
170310 .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } },
170311 .clobbers = .{ .eflags = true },
170312 .each = .{ .once = &.{
170313 .{ ._, .@"lock _", .cmpxchg, .lea(.src0w), .src2w, ._, ._ },
170314 } },
170315 }, .{
170316 .required_features = .{ .@"64bit", null, null, null },
170317 .src_constraints = .{ .any, .{ .int = .dword }, .{ .int = .dword } },
170318 .patterns = &.{
170319 .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } },
170320 },
170321 .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } },
170322 .clobbers = .{ .eflags = true },
170323 .each = .{ .once = &.{
170324 .{ ._, .@"lock _", .cmpxchg, .lea(.src0d), .src2d, ._, ._ },
170325 } },
170326 }, .{
170327 .required_features = .{ .@"64bit", null, null, null },
170328 .src_constraints = .{ .any, .{ .int = .qword }, .{ .int = .qword } },
170329 .patterns = &.{
170330 .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } },
170331 },
170332 .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } },
170333 .clobbers = .{ .eflags = true },
170334 .each = .{ .once = &.{
170335 .{ ._, .@"lock _", .cmpxchg, .lea(.src0q), .src2q, ._, ._ },
170336 } },
170337 }, .{
170338 .required_features = .{ .@"64bit", .cx16, null, null },
170339 .src_constraints = .{ .any, .{ .int = .xword }, .{ .int = .xword } },
170340 .patterns = &.{.{
170341 .src = .{
170342 .to_gpr,
170343 .{ .to_reg_pair = .{ .rax, .rdx } },
170344 .{ .to_reg_pair = .{ .rbx, .rcx } },
170345 },
170346 }},
170347 .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } },
170348 .clobbers = .{ .eflags = true },
170349 .each = .{ .once = &.{
170350 .{ ._, .@"lock _16b", .cmpxchg, .lea(.src0x), ._, ._, ._ },
170351 } },
170352 } })) catch |err| switch (err) {
170353 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{
170354 @tagName(air_tag),
170355 ty_pl.ty.fmt(zcu),
170356 ops[0].tracking(cg),
170357 ops[1].tracking(cg),
170358 ops[2].tracking(cg),
170359 }),
170360 else => |e| return e,
170361 };
170362 if (!ty_pl.ty.optionalReprIsPayload(zcu))
170363 try res[0].withOverflow(&res[1], ty_pl.ty, cg);
170364 try res[0].finish(inst, &.{ extra.ptr, extra.expected_value, extra.new_value }, &ops, cg);
170365 },
170240170366 .atomic_load => try cg.airAtomicLoad(inst),
170241170367 .atomic_store_unordered => try cg.airAtomicStore(inst, .unordered),
170242170368 .atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic),
......@@ -170578,7 +170704,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
170578170704 }),
170579170705 else => |e| return e,
170580170706 };
170581 if (res[0].tracking(cg).short != .register_pair) try ops[0].toPair(&res[0], cg);
170707 if (res[0].tracking(cg).short != .register_pair) try ops[0].toPair(&res[0], .slice_const_u8_sentinel_0, cg);
170582170708 try res[0].finish(inst, &.{un_op}, &ops, cg);
170583170709 },
170584170710 .error_name => |air_tag| {
......@@ -170681,7 +170807,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
170681170807 else => |e| return e,
170682170808 };
170683170809 const orig_res = res;
170684 try res[0].toPair(&res[1], cg);
170810 try res[0].toPair(&res[1], .slice_const_u8_sentinel_0, cg);
170685170811 for (&ops) |*op| for (orig_res) |orig_r| {
170686170812 if (op.index != orig_r.index) continue;
170687170813 op.* = res[0];
......@@ -173190,7 +173316,10 @@ fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void {
173190173316 for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst);
173191173317 switch (value) {
173192173318 else => {},
173193 .eflags, .register_overflow => self.eflags_inst = inst,
173319 .eflags,
173320 .register_overflow,
173321 .register_pair_overflow,
173322 => self.eflags_inst = inst,
173194173323 }
173195173324}
173196173325
......@@ -173218,6 +173347,10 @@ fn freeValue(self: *CodeGen, value: MCValue, comptime opts: FreeOptions) !void {
173218173347 try self.freeReg(reg_ov.reg, opts);
173219173348 self.eflags_inst = null;
173220173349 },
173350 .register_pair_overflow => |rpo| {
173351 for (rpo.pair) |r| try self.freeReg(r, opts);
173352 self.eflags_inst = null;
173353 },
173221173354 .register_mask => |reg_mask| try self.freeReg(reg_mask.reg, opts),
173222173355 .eflags => self.eflags_inst = null,
173223173356 else => {}, // TODO process stack allocation death
......@@ -173760,7 +173893,12 @@ fn reuseOperandAdvanced(
173760173893 return false;
173761173894
173762173895 switch (mcv) {
173763 .register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) |reg| {
173896 .register,
173897 .register_pair,
173898 .register_overflow,
173899 .register_pair_overflow,
173900 .register_mask,
173901 => for (mcv.getRegs()) |reg| {
173764173902 // If it's in the registers table, need to associate the register(s) with the
173765173903 // new instruction.
173766173904 if (maybe_tracked_inst) |tracked_inst| {
......@@ -173775,7 +173913,10 @@ fn reuseOperandAdvanced(
173775173913 else => return false,
173776173914 }
173777173915 switch (mcv) {
173778 .eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst,
173916 .eflags,
173917 .register_overflow,
173918 .register_pair_overflow,
173919 => self.eflags_inst = maybe_tracked_inst,
173779173920 else => {},
173780173921 }
173781173922
......@@ -173802,6 +173943,7 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE
173802173943 .register_triple,
173803173944 .register_quadruple,
173804173945 .register_overflow,
173946 .register_pair_overflow,
173805173947 .register_mask,
173806173948 .indirect_load_frame,
173807173949 .indirect_mask,
......@@ -173861,6 +174003,7 @@ fn store(
173861174003 .register_triple,
173862174004 .register_quadruple,
173863174005 .register_overflow,
174006 .register_pair_overflow,
173864174007 .register_mask,
173865174008 .indirect_load_frame,
173866174009 .indirect_mask,
......@@ -173912,6 +174055,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv:
173912174055 .register_offset,
173913174056 .eflags,
173914174057 .register_overflow,
174058 .register_pair_overflow,
173915174059 .register_mask,
173916174060 .indirect_load_frame,
173917174061 .indirect_mask,
......@@ -174608,6 +174752,7 @@ fn genBinOpMir(
174608174752 .immediate,
174609174753 .eflags,
174610174754 .register_overflow,
174755 .register_pair_overflow,
174611174756 .register_mask,
174612174757 .lea_frame,
174613174758 .lea_nav,
......@@ -174656,6 +174801,7 @@ fn genBinOpMir(
174656174801 .dead,
174657174802 .undef,
174658174803 .register_overflow,
174804 .register_pair_overflow,
174659174805 .register_mask,
174660174806 .indirect_load_frame,
174661174807 .indirect_mask,
......@@ -174834,6 +174980,7 @@ fn genBinOpMir(
174834174980 .dead,
174835174981 .undef,
174836174982 .register_overflow,
174983 .register_pair_overflow,
174837174984 .register_mask,
174838174985 .indirect_load_frame,
174839174986 .indirect_mask,
......@@ -174944,6 +175091,7 @@ fn genBinOpMir(
174944175091 .dead,
174945175092 .undef,
174946175093 .register_overflow,
175094 .register_pair_overflow,
174947175095 .register_mask,
174948175096 .indirect_load_frame,
174949175097 .indirect_mask,
......@@ -178053,6 +178201,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
178053178201 .immediate,
178054178202 .eflags,
178055178203 .register_overflow,
178204 .register_pair_overflow,
178056178205 .register_mask,
178057178206 .indirect_load_frame,
178058178207 .indirect_mask,
......@@ -178077,6 +178226,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
178077178226 .dead,
178078178227 .undef,
178079178228 .register_overflow,
178229 .register_pair_overflow,
178080178230 .register_tee,
178081178231 .elementwise_gpr,
178082178232 .elementwise_sse,
......@@ -178280,6 +178430,7 @@ fn genSetReg(
178280178430 .ywordwise_sse,
178281178431 .zwordwise_sse,
178282178432 .reserved_frame,
178433 .register_pair_overflow,
178283178434 => unreachable,
178284178435 .undef => if (opts.safety) switch (dst_reg.class()) {
178285178436 .general_purpose, .gphi => switch (abi_size) {
......@@ -179083,6 +179234,23 @@ fn genSetMem(
179083179234 @tagName(src_mcv), ty.fmt(zcu),
179084179235 }),
179085179236 },
179237 .register_pair_overflow => |rpo| switch (ty.zigTypeTag(zcu)) {
179238 .optional => {
179239 assert(!ty.optionalReprIsPayload(zcu));
179240 const child_ty = ty.optionalChild(zcu);
179241 try cg.genSetMem(base, disp, child_ty, .{ .register_pair = rpo.pair }, opts);
179242 try cg.genSetMem(
179243 base,
179244 disp + @as(i32, @intCast(child_ty.abiSize(zcu))),
179245 .bool,
179246 .{ .eflags = rpo.eflags },
179247 opts,
179248 );
179249 },
179250 else => return cg.fail("TODO implement genSetMem for {s} of {f}", .{
179251 @tagName(src_mcv), ty.fmt(zcu),
179252 }),
179253 },
179086179254 .register_offset => |reg_off| {
179087179255 const src_reg = cg.copyToTmpRegister(ty, src_mcv) catch |err| switch (err) {
179088179256 error.OutOfRegisters => {
......@@ -179417,111 +179585,6 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
179417179585 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
179418179586}
179419179587
179420fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
179421 const pt = self.pt;
179422 const zcu = pt.zcu;
179423 const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
179424 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
179425
179426 const dst_ty = self.typeOfIndex(inst);
179427 const ptr_ty = self.typeOf(extra.ptr);
179428 const val_ty = self.typeOf(extra.expected_value);
179429 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
179430
179431 try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx });
179432 const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx });
179433 defer for (regs_lock) |lock| self.register_manager.unlockReg(lock);
179434
179435 const exp_mcv = try self.resolveInst(extra.expected_value);
179436 if (val_abi_size > 8) {
179437 const exp_addr_mcv: MCValue = switch (exp_mcv) {
179438 .memory, .indirect, .load_frame => exp_mcv.address(),
179439 else => .{ .register = try self.copyToTmpRegister(.usize, exp_mcv.address()) },
179440 };
179441 const exp_addr_lock =
179442 if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
179443 defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock);
179444
179445 try self.genSetReg(.rax, .usize, exp_addr_mcv.deref(), .{});
179446 try self.genSetReg(.rdx, .usize, exp_addr_mcv.offset(8).deref(), .{});
179447 } else try self.genSetReg(.rax, val_ty, exp_mcv, .{});
179448
179449 const new_mcv = try self.resolveInst(extra.new_value);
179450 const new_reg = if (val_abi_size > 8) new: {
179451 const new_addr_mcv: MCValue = switch (new_mcv) {
179452 .memory, .indirect, .load_frame => new_mcv.address(),
179453 else => .{ .register = try self.copyToTmpRegister(.usize, new_mcv.address()) },
179454 };
179455 const new_addr_lock =
179456 if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
179457 defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock);
179458
179459 try self.genSetReg(.rbx, .usize, new_addr_mcv.deref(), .{});
179460 try self.genSetReg(.rcx, .usize, new_addr_mcv.offset(8).deref(), .{});
179461 break :new null;
179462 } else try self.copyToTmpRegister(val_ty, new_mcv);
179463 const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
179464 defer if (new_lock) |lock| self.register_manager.unlockReg(lock);
179465
179466 const ptr_mcv = try self.resolveInst(extra.ptr);
179467 const mem_size: Memory.Size = .fromSize(val_abi_size);
179468 const ptr_mem: Memory = switch (ptr_mcv) {
179469 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
179470 else => .{
179471 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
179472 .mod = .{ .rm = .{ .size = mem_size } },
179473 },
179474 };
179475 switch (ptr_mem.mod) {
179476 .rm => {},
179477 .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
179478 }
179479 const ptr_lock = switch (ptr_mem.base) {
179480 .none, .frame, .nav, .uav => null,
179481 .reg => |reg| self.register_manager.lockReg(reg),
179482 .table, .rip_inst, .lazy_sym, .extern_func => unreachable,
179483 };
179484 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
179485
179486 try self.spillEflagsIfOccupied();
179487 const null_reg = if (dst_ty.optionalReprIsPayload(zcu) and !self.liveness.isUnused(inst)) null_reg: {
179488 const null_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
179489 try self.asmRegisterRegister(.{ ._, .xor }, null_reg.to32(), null_reg.to32());
179490 break :null_reg null_reg;
179491 } else null;
179492 const null_lock = if (null_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
179493 defer if (null_lock) |lock| self.register_manager.unlockReg(lock);
179494
179495 if (val_abi_size <= 8) try self.asmMemoryRegister(
179496 .{ .@"lock _", .cmpxchg },
179497 ptr_mem,
179498 registerAlias(new_reg.?, val_abi_size),
179499 ) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
179500
179501 const result: MCValue = result: {
179502 if (self.liveness.isUnused(inst)) break :result .unreach;
179503
179504 if (null_reg) |reg| try self.asmCmovccRegisterRegister(
179505 .e,
179506 registerAlias(.rax, val_abi_size),
179507 registerAlias(reg, val_abi_size),
179508 );
179509
179510 if (val_abi_size <= 8) {
179511 if (null_reg) |_| break :result .{ .register = .rax };
179512 self.eflags_inst = inst;
179513 break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } };
179514 }
179515
179516 const dst_mcv = try self.allocRegOrMem(inst, false);
179517 try self.genCopy(.usize, dst_mcv, .{ .register = .rax }, .{});
179518 try self.genCopy(.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});
179519 if (null_reg == null) try self.genCopy(.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});
179520 break :result dst_mcv;
179521 };
179522 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
179523}
179524
179525179588fn atomicOp(
179526179589 self: *CodeGen,
179527179590 ptr_mcv: MCValue,
......@@ -179557,7 +179620,7 @@ fn atomicOp(
179557179620 };
179558179621 switch (ptr_mem.mod) {
179559179622 .rm => {},
179560 .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
179623 .off => return self.fail("TODO atomicOp with {s}", .{@tagName(ptr_mcv)}),
179561179624 }
179562179625 const mem_lock = switch (ptr_mem.base) {
179563179626 .none, .frame, .nav, .uav => null,
......@@ -181936,6 +181999,7 @@ const Temp = struct {
181936181999 .register_triple,
181937182000 .register_quadruple,
181938182001 .register_overflow,
182002 .register_pair_overflow,
181939182003 => true,
181940182004 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
181941182005 },
......@@ -182316,7 +182380,7 @@ const Temp = struct {
182316182380 return true;
182317182381 }
182318182382
182319 fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) InnerError!void {
182383 fn toPair(first_temp: *Temp, second_temp: *Temp, result_ty: Type, cg: *CodeGen) InnerError!void {
182320182384 while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {
182321182385 if (try part_temp.toRegClass(true, .general_purpose, cg)) break;
182322182386 } else break;
......@@ -182331,13 +182395,13 @@ const Temp = struct {
182331182395 assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
182332182396 assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
182333182397 result_temp_index.tracking(cg).* = .init(result);
182334 cg.temp_type[@backingInt(result_temp_index)] = .slice_const_u8;
182398 cg.temp_type[@backingInt(result_temp_index)] = result_ty;
182335182399 cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1));
182336182400 first_temp.* = result_temp;
182337182401 second_temp.* = result_temp;
182338182402 }
182339182403
182340 fn withOverflow(temp: *Temp, overflow_temp: *Temp, cg: *CodeGen) InnerError!void {
182404 fn withOverflow(temp: *Temp, overflow_temp: *Temp, result_ty: Type, cg: *CodeGen) InnerError!void {
182341182405 const temp_index = temp.unwrap(cg).temp;
182342182406 const temp_tracking = temp_index.tracking(cg);
182343182407 const overflow_temp_tracking = overflow_temp.unwrap(cg).temp.tracking(cg);
......@@ -182349,7 +182413,7 @@ const Temp = struct {
182349182413 assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking));
182350182414 assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking));
182351182415 result_temp_index.tracking(cg).* = .init(.{ .register_overflow = .{ .reg = reg, .eflags = overflow_cc } });
182352 cg.temp_type[@backingInt(result_temp_index)] = .slice_const_u8;
182416 cg.temp_type[@backingInt(result_temp_index)] = result_ty;
182353182417 cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1));
182354182418 temp.* = result_temp;
182355182419 overflow_temp.* = result_temp;
......@@ -182361,7 +182425,22 @@ const Temp = struct {
182361182425 assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking));
182362182426 assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking));
182363182427 result_temp_index.tracking(cg).* = .init(.{ .register_pair = .{ reg, overflow_reg } });
182364 cg.temp_type[@backingInt(result_temp_index)] = .slice_const_u8;
182428 cg.temp_type[@backingInt(result_temp_index)] = result_ty;
182429 cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1));
182430 temp.* = result_temp;
182431 overflow_temp.* = result_temp;
182432 return;
182433 },
182434 else => {},
182435 },
182436 .register_pair => |pair| switch (overflow_temp_tracking.short) {
182437 .eflags => |overflow_cc| {
182438 const result_temp_index = cg.next_temp_index;
182439 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
182440 assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking));
182441 assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking));
182442 result_temp_index.tracking(cg).* = .init(.{ .register_pair_overflow = .{ .pair = pair, .eflags = overflow_cc } });
182443 cg.temp_type[@backingInt(result_temp_index)] = result_ty;
182365182444 cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1));
182366182445 temp.* = result_temp;
182367182446 overflow_temp.* = result_temp;
......@@ -182401,6 +182480,7 @@ const Temp = struct {
182401182480 .register_triple,
182402182481 .register_quadruple,
182403182482 .register_overflow,
182483 .register_pair_overflow,
182404182484 .register_mask,
182405182485 .indirect_load_frame,
182406182486 .indirect_mask,
......@@ -188883,6 +188963,7 @@ const Temp = struct {
188883188963 .register_quadruple,
188884188964 .register_offset,
188885188965 .register_overflow,
188966 .register_pair_overflow,
188886188967 .register_mask,
188887188968 .memory,
188888188969 .indirect,
......@@ -189001,6 +189082,7 @@ fn reuseTemp(
189001189082 .register_pair,
189002189083 .register_offset,
189003189084 .register_overflow,
189085 .register_pair_overflow,
189004189086 .register_mask,
189005189087 .indirect,
189006189088 => for (tracking.short.getRegs()) |tracked_reg| {
......@@ -189012,7 +189094,10 @@ fn reuseTemp(
189012189094 else => {},
189013189095 }
189014189096 switch (tracking.short) {
189015 .eflags, .register_overflow => cg.eflags_inst = new_inst,
189097 .eflags,
189098 .register_overflow,
189099 .register_pair_overflow,
189100 => cg.eflags_inst = new_inst,
189016189101 else => {},
189017189102 }
189018189103 tracking.reuse(cg, new_inst, old_inst);
......@@ -191592,7 +191677,13 @@ const Select = struct {
191592191677 } },
191593191678 .eflags => unreachable,
191594191679 .register => |reg| .{ .reg = s.lowerReg(reg.toSize(op.flags.base.size, s.cg.target)) },
191595 .register_pair, .register_triple, .register_quadruple, .register_offset, .register_overflow => unreachable,
191680 .register_pair,
191681 .register_triple,
191682 .register_quadruple,
191683 .register_offset,
191684 .register_overflow,
191685 .register_pair_overflow,
191686 => unreachable,
191596191687 .register_mask => |reg_mask| .{ .reg = s.lowerReg(reg_mask.reg.toSize(op.flags.base.size, s.cg.target)) },
191597191688 .lea_nav => |nav| .{ .imm = .{ .nav = .{ .index = nav } } },
191598191689 .lea_uav => |uav| .{ .imm = .{ .uav = uav } },
test/behavior/atomics.zig+7
......@@ -133,6 +133,13 @@ fn test_u128_cmpxchg() !void {
133133
134134 try expect(@cmpxchgStrong(u128, &x, 5678, 42, .seq_cst, .seq_cst) == null);
135135 try expect(x == 42);
136
137 var a: u128 align(16) = 40;
138 var b: u128 align(16) = 2;
139 _ = &a;
140 _ = &b;
141 try expect(@cmpxchgStrong(u128, &x, a | b, a & b, .seq_cst, .seq_cst) == null);
142 try expect(x == 0);
136143}
137144
138145var a_global_variable = @as(u32, 1234);