authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-28 17:35:17-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-01 19:22:52-04:00
logaaef5288f886c86e4257c5c67122a0be8a64c134
tree8cabea0a1b431accb40b84529790ed122d4c28ea
parent50f96c2949bd7d396eaaf391ffd5fd55e59ebac1

x86_64: fix 128-bit cmpxchg

1 files changed, 67 insertions(+), 67 deletions(-)

src/arch/x86_64/CodeGen.zig+67-67
...@@ -219,9 +219,9 @@ pub const MCValue = union(enum) {...@@ -219,9 +219,9 @@ pub const MCValue = union(enum) {
219 .dead,219 .dead,
220 .undef,220 .undef,
221 .immediate,221 .immediate,
222 .eflags,
222 .register,223 .register,
223 .register_offset,224 .register_offset,
224 .eflags,
225 .register_overflow,225 .register_overflow,
226 .lea_direct,226 .lea_direct,
227 .lea_got,227 .lea_got,
...@@ -297,6 +297,41 @@ pub const MCValue = union(enum) {...@@ -297,6 +297,41 @@ pub const MCValue = union(enum) {
297 };297 };
298 }298 }
299299
300 fn mem(mcv: MCValue, ptr_size: Memory.PtrSize) Memory {
301 return switch (mcv) {
302 .none,
303 .unreach,
304 .dead,
305 .undef,
306 .immediate,
307 .eflags,
308 .register,
309 .register_offset,
310 .register_overflow,
311 .load_direct,
312 .lea_direct,
313 .load_got,
314 .lea_got,
315 .load_tlv,
316 .lea_tlv,
317 .lea_frame,
318 .reserved_frame,
319 => unreachable,
320 .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr|
321 Memory.sib(ptr_size, .{ .base = .{ .reg = .ds }, .disp = small_addr })
322 else
323 Memory.moffs(.ds, addr),
324 .indirect => |reg_off| Memory.sib(ptr_size, .{
325 .base = .{ .reg = reg_off.reg },
326 .disp = reg_off.off,
327 }),
328 .load_frame => |frame_addr| Memory.sib(ptr_size, .{
329 .base = .{ .frame = frame_addr.index },
330 .disp = frame_addr.off,
331 }),
332 };
333 }
334
300 pub fn format(335 pub fn format(
301 mcv: MCValue,336 mcv: MCValue,
302 comptime _: []const u8,337 comptime _: []const u8,
...@@ -7936,70 +7971,50 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7936,70 +7971,50 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7936 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;7971 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
79377972
7938 const ptr_ty = self.air.typeOf(extra.ptr);7973 const ptr_ty = self.air.typeOf(extra.ptr);
7939 const ptr_mcv = try self.resolveInst(extra.ptr);
7940 const val_ty = self.air.typeOf(extra.expected_value);7974 const val_ty = self.air.typeOf(extra.expected_value);
7941 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));7975 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
79427976
7943 try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx });7977 try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx });
7944 const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx });7978 const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx });
7945 for (regs_lock) |lock| self.register_manager.unlockReg(lock);7979 defer for (regs_lock) |lock| self.register_manager.unlockReg(lock);
79467980
7947 const exp_mcv = try self.resolveInst(extra.expected_value);7981 const exp_mcv = try self.resolveInst(extra.expected_value);
7948 if (val_abi_size > 8) switch (exp_mcv) {7982 if (val_abi_size > 8) {
7949 .load_frame => |frame_addr| {7983 try self.genSetReg(.rax, Type.usize, exp_mcv);
7950 try self.genSetReg(.rax, Type.usize, .{ .load_frame = .{7984 try self.genSetReg(.rdx, Type.usize, exp_mcv.address().offset(8).deref());
7951 .index = frame_addr.index,
7952 .off = frame_addr.off + 0,
7953 } });
7954 try self.genSetReg(.rdx, Type.usize, .{ .load_frame = .{
7955 .index = frame_addr.index,
7956 .off = frame_addr.off + 8,
7957 } });
7958 },
7959 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
7960 } else try self.genSetReg(.rax, val_ty, exp_mcv);7985 } else try self.genSetReg(.rax, val_ty, exp_mcv);
7961 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
7962 defer self.register_manager.unlockReg(rax_lock);
79637986
7964 const new_mcv = try self.resolveInst(extra.new_value);7987 const new_mcv = try self.resolveInst(extra.new_value);
7965 const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {7988 const new_reg = if (val_abi_size > 8) new: {
7966 .load_frame => |frame_addr| new: {7989 try self.genSetReg(.rbx, Type.usize, new_mcv);
7967 try self.genSetReg(.rbx, Type.usize, .{ .load_frame = .{7990 try self.genSetReg(.rcx, Type.usize, new_mcv.address().offset(8).deref());
7968 .index = frame_addr.index,7991 break :new null;
7969 .off = frame_addr.off + 0,
7970 } });
7971 try self.genSetReg(.rcx, Type.usize, .{ .load_frame = .{
7972 .index = frame_addr.index,
7973 .off = frame_addr.off + 8,
7974 } });
7975 break :new undefined;
7976 },
7977 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
7978 } else try self.copyToTmpRegister(val_ty, new_mcv);7992 } else try self.copyToTmpRegister(val_ty, new_mcv);
7979 const new_lock = self.register_manager.lockRegAssumeUnused(new_reg);7993 const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
7980 defer self.register_manager.unlockReg(new_lock);7994 defer if (new_lock) |lock| self.register_manager.unlockReg(lock);
79817995
7996 const ptr_mcv = try self.resolveInst(extra.ptr);
7982 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);7997 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7983 const ptr_mem = switch (ptr_mcv) {7998 const ptr_mem = switch (ptr_mcv) {
7984 .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),7999 .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
7985 .lea_frame => |frame_addr| Memory.sib(ptr_size, .{8000 else => Memory.sib(ptr_size, .{
7986 .base = .{ .frame = frame_addr.index },8001 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
7987 .disp = frame_addr.off,
7988 }),8002 }),
7989 else => Memory.sib(ptr_size, .{ .base = .{
7990 .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
7991 } }),
7992 };8003 };
7993 const mem_lock = switch (ptr_mem.base()) {8004 switch (ptr_mem) {
8005 .sib, .rip => {},
8006 .moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
8007 }
8008 const ptr_lock = switch (ptr_mem.base()) {
7994 .none, .frame => null,8009 .none, .frame => null,
7995 .reg => |reg| self.register_manager.lockReg(reg),8010 .reg => |reg| self.register_manager.lockReg(reg),
7996 };8011 };
7997 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);8012 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
79988013
7999 try self.spillEflagsIfOccupied();8014 try self.spillEflagsIfOccupied();
8000 if (val_abi_size <= 8) {8015 if (val_abi_size <= 8) {
8001 _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{8016 _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
8002 .r = registerAlias(new_reg, val_abi_size),8017 .r = registerAlias(new_reg.?, val_abi_size),
8003 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),8018 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8004 } } });8019 } } });
8005 } else {8020 } else {
...@@ -8017,24 +8032,9 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -8017,24 +8032,9 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
8017 }8032 }
80188033
8019 const dst_mcv = try self.allocRegOrMem(inst, false);8034 const dst_mcv = try self.allocRegOrMem(inst, false);
8020 try self.genSetMem(8035 try self.genCopy(Type.usize, dst_mcv, .{ .register = .rax });
8021 .{ .frame = dst_mcv.load_frame.index },8036 try self.genCopy(Type.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx });
8022 dst_mcv.load_frame.off + 16,8037 try self.genCopy(Type.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne });
8023 Type.bool,
8024 .{ .eflags = .ne },
8025 );
8026 try self.genSetMem(
8027 .{ .frame = dst_mcv.load_frame.index },
8028 dst_mcv.load_frame.off + 8,
8029 Type.usize,
8030 .{ .register = .rdx },
8031 );
8032 try self.genSetMem(
8033 .{ .frame = dst_mcv.load_frame.index },
8034 dst_mcv.load_frame.off + 0,
8035 Type.usize,
8036 .{ .register = .rax },
8037 );
8038 break :result dst_mcv;8038 break :result dst_mcv;
8039 };8039 };
8040 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });8040 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
...@@ -8065,15 +8065,15 @@ fn atomicOp(...@@ -8065,15 +8065,15 @@ fn atomicOp(
8065 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));8065 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
8066 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);8066 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
8067 const ptr_mem = switch (ptr_mcv) {8067 const ptr_mem = switch (ptr_mcv) {
8068 .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),8068 .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
8069 .lea_frame => |frame_addr| Memory.sib(ptr_size, .{8069 else => Memory.sib(ptr_size, .{
8070 .base = .{ .frame = frame_addr.index },8070 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
8071 .disp = frame_addr.off,
8072 }),8071 }),
8073 else => Memory.sib(ptr_size, .{ .base = .{
8074 .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
8075 } }),
8076 };8072 };
8073 switch (ptr_mem) {
8074 .sib, .rip => {},
8075 .moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
8076 }
8077 const mem_lock = switch (ptr_mem.base()) {8077 const mem_lock = switch (ptr_mem.base()) {
8078 .none, .frame => null,8078 .none, .frame => null,
8079 .reg => |reg| self.register_manager.lockReg(reg),8079 .reg => |reg| self.register_manager.lockReg(reg),