authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-10-09 02:45:03-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-10-09 03:11:41-04:00
logd559c61902cce2966e9ef9d080dccf6d7e41140e
tree9c4bbe3367ba604b4e3640a466e7636bb94feac1
parent21948d754002d932f27f4f07f5f61061eb49ae1c

x86_64: implement `@min` and `@max` for 128-bit integers

1 files changed, 199 insertions(+), 123 deletions(-)

src/arch/x86_64/CodeGen.zig+199-123
......@@ -1120,7 +1120,7 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
11201120}
11211121
11221122/// A `cc` of `.z_and_np` clobbers `reg2`!
1123fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
1123fn asmCmovccRegisterRegister(self: *Self, cc: bits.Condition, reg1: Register, reg2: Register) !void {
11241124 _ = try self.addInst(.{
11251125 .tag = switch (cc) {
11261126 else => .cmov,
......@@ -1143,7 +1143,7 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi
11431143}
11441144
11451145/// A `cc` of `.z_and_np` is not supported by this encoding!
1146fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
1146fn asmCmovccRegisterMemory(self: *Self, cc: bits.Condition, reg: Register, m: Memory) !void {
11471147 _ = try self.addInst(.{
11481148 .tag = switch (cc) {
11491149 else => .cmov,
......@@ -3232,9 +3232,9 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
32323232
32333233 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
32343234 try self.asmCmovccRegisterRegister(
3235 cc,
32353236 registerAlias(dst_reg, cmov_abi_size),
32363237 registerAlias(limit_reg, cmov_abi_size),
3237 cc,
32383238 );
32393239
32403240 if (reg_extra_bits > 0 and ty.isSignedInt(mod)) {
......@@ -3308,9 +3308,9 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
33083308
33093309 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
33103310 try self.asmCmovccRegisterRegister(
3311 cc,
33113312 registerAlias(dst_reg, cmov_abi_size),
33123313 registerAlias(limit_reg, cmov_abi_size),
3313 cc,
33143314 );
33153315
33163316 if (reg_extra_bits > 0 and ty.isSignedInt(mod)) {
......@@ -3371,9 +3371,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
33713371 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);
33723372 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
33733373 try self.asmCmovccRegisterRegister(
3374 cc,
33743375 registerAlias(dst_mcv.register, cmov_abi_size),
33753376 registerAlias(limit_reg, cmov_abi_size),
3376 cc,
33773377 );
33783378
33793379 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -3743,9 +3743,9 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
37433743 registerAlias(.rdx, abi_size),
37443744 );
37453745 try self.asmCmovccRegisterRegister(
3746 .z,
37463747 registerAlias(divisor, abi_size),
37473748 registerAlias(.rdx, abi_size),
3748 .z,
37493749 );
37503750 try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
37513751 return MCValue{ .register = divisor };
......@@ -4626,7 +4626,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
46264626 tmp_mcv,
46274627 mat_src_mcv.address().offset(8).deref(),
46284628 );
4629 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
4629 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
46304630
46314631 if (src_bits < 128) {
46324632 try self.genBinOpMir(
......@@ -4660,9 +4660,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
46604660
46614661 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
46624662 try self.asmCmovccRegisterRegister(
4663 .z,
46634664 registerAlias(dst_reg, cmov_abi_size),
46644665 registerAlias(imm_reg, cmov_abi_size),
4665 .z,
46664666 );
46674667
46684668 try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
......@@ -4687,9 +4687,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
46874687
46884688 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
46894689 try self.asmCmovccRegisterRegister(
4690 .nz,
46904691 registerAlias(imm_reg, cmov_abi_size),
46914692 registerAlias(dst_reg, cmov_abi_size),
4692 .nz,
46934693 );
46944694
46954695 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
......@@ -4764,7 +4764,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
47644764 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
47654765 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
47664766 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv);
4767 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
4767 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
47684768 } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)});
47694769 break :result dst_mcv;
47704770 }
......@@ -4786,9 +4786,9 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
47864786
47874787 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
47884788 try self.asmCmovccRegisterRegister(
4789 .z,
47894790 registerAlias(dst_reg, cmov_abi_size),
47904791 registerAlias(width_reg, cmov_abi_size),
4791 .z,
47924792 );
47934793 break :result dst_mcv;
47944794 };
......@@ -5400,21 +5400,21 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
54005400 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2);
54015401 switch (src_mcv) {
54025402 .register => |val_reg| try self.asmCmovccRegisterRegister(
5403 .l,
54035404 registerAlias(dst_mcv.register, cmov_abi_size),
54045405 registerAlias(val_reg, cmov_abi_size),
5405 .l,
54065406 ),
54075407 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
5408 .l,
54085409 registerAlias(dst_mcv.register, cmov_abi_size),
54095410 src_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
5410 .l,
54115411 ),
54125412 else => {
54135413 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
54145414 try self.asmCmovccRegisterRegister(
5415 .l,
54155416 registerAlias(dst_mcv.register, cmov_abi_size),
54165417 registerAlias(val_reg, cmov_abi_size),
5417 .l,
54185418 );
54195419 },
54205420 }
......@@ -6605,11 +6605,11 @@ fn genShiftBinOpMir(
66056605 Immediate.u(64),
66066606 );
66076607 try self.asmCmovccRegisterRegister(
6608 .ae,
66086609 lhs_regs[info.indices[1]],
66096610 lhs_regs[info.indices[0]],
6610 .ae,
66116611 );
6612 try self.asmCmovccRegisterRegister(lhs_regs[info.indices[0]], tmp_reg, .ae);
6612 try self.asmCmovccRegisterRegister(.ae, lhs_regs[info.indices[0]], tmp_reg);
66136613 },
66146614 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
66156615 @tagName(lhs_mcv),
......@@ -6737,8 +6737,8 @@ fn genShiftBinOpMir(
67376737 registerAlias(rhs_reg, 1),
67386738 Immediate.u(64),
67396739 );
6740 try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
6741 try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
6740 try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg);
6741 try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);
67426742 try self.asmMemoryRegister(
67436743 .{ ._, .mov },
67446744 Memory.sib(.qword, .{
......@@ -7138,17 +7138,22 @@ fn genBinOp(
71387138 .Float, .Vector => true,
71397139 };
71407140
7141 const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
7142 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
7143 else => null,
7141 const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
7142 .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
7143 .register_pair => |lhs_regs| locks: {
7144 const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
7145 break :locks .{ locks[0], locks[1] };
7146 },
7147 else => .{ null, null },
71447148 };
7145 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
7149 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
71467150
7147 const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
7148 .register => |reg| self.register_manager.lockReg(reg),
7149 else => null,
7151 const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
7152 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
7153 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
7154 else => .{ null, null },
71507155 };
7151 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
7156 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
71527157
71537158 var flipped = false;
71547159 var copied_to_dst = true;
......@@ -7171,11 +7176,12 @@ fn genBinOp(
71717176 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
71727177 break :dst dst_mcv;
71737178 };
7174 const dst_lock: ?RegisterLock = switch (dst_mcv) {
7175 .register => |reg| self.register_manager.lockReg(reg),
7176 else => null,
7179 const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
7180 .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null },
7181 .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs),
7182 else => .{ null, null },
71777183 };
7178 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
7184 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
71797185
71807186 const unmat_src_mcv = if (flipped) lhs_mcv else rhs_mcv;
71817187 const src_mcv: MCValue = if (maybe_mask_reg) |mask_reg|
......@@ -7186,6 +7192,12 @@ fn genBinOp(
71867192 }
71877193 else
71887194 unmat_src_mcv;
7195 const src_locks: [2]?RegisterLock = switch (src_mcv) {
7196 .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null },
7197 .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs),
7198 else => .{ null, null },
7199 };
7200 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
71897201
71907202 if (!vec_op) {
71917203 switch (air_tag) {
......@@ -7236,102 +7248,166 @@ fn genBinOp(
72367248 else => src_mcv,
72377249 .air_ref => |src_ref| try self.resolveInst(src_ref),
72387250 };
7239 const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) {
7240 .immediate,
7241 .eflags,
7242 .register_offset,
7243 .load_direct,
7244 .lea_direct,
7245 .load_got,
7246 .lea_got,
7247 .load_tlv,
7248 .lea_tlv,
7249 .lea_frame,
7250 => true,
7251 .memory => |addr| math.cast(i32, @as(i64, @bitCast(addr))) == null,
7252 else => false,
7253 .register_pair,
7254 .register_overflow,
7255 => unreachable,
7256 })
7257 .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) }
7258 else
7259 resolved_src_mcv;
7260 const mat_mcv_lock = switch (mat_src_mcv) {
7261 .register => |reg| self.register_manager.lockReg(reg),
7262 else => null,
7263 };
7264 defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock);
72657251
7266 try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
7252 if (abi_size > 8) {
7253 const dst_regs = switch (dst_mcv) {
7254 .register_pair => |dst_regs| dst_regs,
7255 else => dst: {
7256 const dst_regs = try self.register_manager.allocRegs(
7257 2,
7258 .{ null, null },
7259 abi.RegisterClass.gp,
7260 );
7261 const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs);
7262 for (dst_regs_locks) |dst_lock| if (dst_lock) |lock|
7263 self.register_manager.unlockReg(lock);
72677264
7268 const int_info = lhs_ty.intInfo(mod);
7269 const cc: Condition = switch (int_info.signedness) {
7270 .unsigned => switch (air_tag) {
7271 .min => .a,
7272 .max => .b,
7273 else => unreachable,
7274 },
7275 .signed => switch (air_tag) {
7276 .min => .g,
7277 .max => .l,
7265 try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv);
7266 break :dst dst_regs;
7267 },
7268 };
7269 const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs);
7270 for (dst_regs_locks) |dst_lock| if (dst_lock) |lock|
7271 self.register_manager.unlockReg(lock);
7272
7273 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
7274 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7275 defer self.register_manager.unlockReg(tmp_lock);
7276
7277 const signed = lhs_ty.isSignedInt(mod);
7278 const cc: Condition = switch (air_tag) {
7279 .min => if (signed) .nl else .nb,
7280 .max => if (signed) .nge else .nae,
72787281 else => unreachable,
7279 },
7280 };
7282 };
72817283
7282 const cmov_abi_size = @max(@as(u32, @intCast(lhs_ty.abiSize(mod))), 2);
7283 const tmp_reg = switch (dst_mcv) {
7284 .register => |reg| reg,
7285 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),
7286 };
7287 const tmp_lock = self.register_manager.lockReg(tmp_reg);
7288 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
7289 switch (mat_src_mcv) {
7290 .none,
7291 .unreach,
7292 .dead,
7293 .undef,
7294 .immediate,
7295 .eflags,
7296 .register_pair,
7297 .register_offset,
7298 .register_overflow,
7299 .load_direct,
7300 .lea_direct,
7301 .load_got,
7302 .lea_got,
7303 .load_tlv,
7304 .lea_tlv,
7305 .lea_frame,
7306 .reserved_frame,
7307 .air_ref,
7308 => unreachable,
7309 .register => |src_reg| try self.asmCmovccRegisterRegister(
7310 registerAlias(tmp_reg, cmov_abi_size),
7311 registerAlias(src_reg, cmov_abi_size),
7312 cc,
7313 ),
7314 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
7315 registerAlias(tmp_reg, cmov_abi_size),
7316 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
7317 .memory => |addr| .{
7318 .base = .{ .reg = .ds },
7319 .disp = @intCast(@as(i64, @bitCast(addr))),
7320 },
7321 .indirect => |reg_off| .{
7322 .base = .{ .reg = reg_off.reg },
7323 .disp = reg_off.off,
7324 },
7325 .load_frame => |frame_addr| .{
7326 .base = .{ .frame = frame_addr.index },
7327 .disp = frame_addr.off,
7328 },
7284 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
7285 if (src_mcv.isMemory()) {
7286 try self.asmRegisterMemory(.{ ._, .cmp }, dst_regs[0], src_mcv.mem(.qword));
7287 try self.asmRegisterMemory(
7288 .{ ._, .sbb },
7289 tmp_reg,
7290 src_mcv.address().offset(8).deref().mem(.qword),
7291 );
7292 try self.asmCmovccRegisterMemory(cc, dst_regs[0], src_mcv.mem(.qword));
7293 try self.asmCmovccRegisterMemory(
7294 cc,
7295 dst_regs[1],
7296 src_mcv.address().offset(8).deref().mem(.qword),
7297 );
7298 } else {
7299 try self.asmRegisterRegister(
7300 .{ ._, .cmp },
7301 dst_regs[0],
7302 src_mcv.register_pair[0],
7303 );
7304 try self.asmRegisterRegister(
7305 .{ ._, .sbb },
7306 tmp_reg,
7307 src_mcv.register_pair[1],
7308 );
7309 try self.asmCmovccRegisterRegister(cc, dst_regs[0], src_mcv.register_pair[0]);
7310 try self.asmCmovccRegisterRegister(cc, dst_regs[1], src_mcv.register_pair[1]);
7311 }
7312 try self.genCopy(lhs_ty, dst_mcv, .{ .register_pair = dst_regs });
7313 } else {
7314 const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) {
7315 .immediate,
7316 .eflags,
7317 .register_offset,
7318 .load_direct,
7319 .lea_direct,
7320 .load_got,
7321 .lea_got,
7322 .load_tlv,
7323 .lea_tlv,
7324 .lea_frame,
7325 => true,
7326 .memory => |addr| math.cast(i32, @as(i64, @bitCast(addr))) == null,
7327 else => false,
7328 .register_pair,
7329 .register_overflow,
7330 => unreachable,
7331 })
7332 .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) }
7333 else
7334 resolved_src_mcv;
7335 const mat_mcv_lock = switch (mat_src_mcv) {
7336 .register => |reg| self.register_manager.lockReg(reg),
7337 else => null,
7338 };
7339 defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock);
7340
7341 try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
7342
7343 const int_info = lhs_ty.intInfo(mod);
7344 const cc: Condition = switch (int_info.signedness) {
7345 .unsigned => switch (air_tag) {
7346 .min => .a,
7347 .max => .b,
73297348 else => unreachable,
7330 }),
7331 cc,
7332 ),
7349 },
7350 .signed => switch (air_tag) {
7351 .min => .g,
7352 .max => .l,
7353 else => unreachable,
7354 },
7355 };
7356
7357 const cmov_abi_size = @max(@as(u32, @intCast(lhs_ty.abiSize(mod))), 2);
7358 const tmp_reg = switch (dst_mcv) {
7359 .register => |reg| reg,
7360 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),
7361 };
7362 const tmp_lock = self.register_manager.lockReg(tmp_reg);
7363 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
7364 switch (mat_src_mcv) {
7365 .none,
7366 .unreach,
7367 .dead,
7368 .undef,
7369 .immediate,
7370 .eflags,
7371 .register_pair,
7372 .register_offset,
7373 .register_overflow,
7374 .load_direct,
7375 .lea_direct,
7376 .load_got,
7377 .lea_got,
7378 .load_tlv,
7379 .lea_tlv,
7380 .lea_frame,
7381 .reserved_frame,
7382 .air_ref,
7383 => unreachable,
7384 .register => |src_reg| try self.asmCmovccRegisterRegister(
7385 cc,
7386 registerAlias(tmp_reg, cmov_abi_size),
7387 registerAlias(src_reg, cmov_abi_size),
7388 ),
7389 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
7390 cc,
7391 registerAlias(tmp_reg, cmov_abi_size),
7392 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
7393 .memory => |addr| .{
7394 .base = .{ .reg = .ds },
7395 .disp = @intCast(@as(i64, @bitCast(addr))),
7396 },
7397 .indirect => |reg_off| .{
7398 .base = .{ .reg = reg_off.reg },
7399 .disp = reg_off.off,
7400 },
7401 .load_frame => |frame_addr| .{
7402 .base = .{ .frame = frame_addr.index },
7403 .disp = frame_addr.off,
7404 },
7405 else => unreachable,
7406 }),
7407 ),
7408 }
7409 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
73337410 }
7334 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
73357411 },
73367412
73377413 else => return self.fail("TODO implement genBinOp for {s} {}", .{
......@@ -12053,21 +12129,21 @@ fn atomicOp(
1205312129 const cmov_abi_size = @max(val_abi_size, 2);
1205412130 switch (val_mcv) {
1205512131 .register => |val_reg| try self.asmCmovccRegisterRegister(
12132 cc,
1205612133 registerAlias(tmp_reg, cmov_abi_size),
1205712134 registerAlias(val_reg, cmov_abi_size),
12058 cc,
1205912135 ),
1206012136 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
12137 cc,
1206112138 registerAlias(tmp_reg, cmov_abi_size),
1206212139 val_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
12063 cc,
1206412140 ),
1206512141 else => {
1206612142 const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
1206712143 try self.asmCmovccRegisterRegister(
12144 cc,
1206812145 registerAlias(tmp_reg, cmov_abi_size),
1206912146 registerAlias(val_reg, cmov_abi_size),
12070 cc,
1207112147 );
1207212148 },
1207312149 }