| ... | ... | @@ -1120,7 +1120,7 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 { |
| 1120 | 1120 | } |
| 1121 | 1121 | |
| 1122 | 1122 | /// A `cc` of `.z_and_np` clobbers `reg2`! |
| 1123 | | fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void { |
| 1123 | fn asmCmovccRegisterRegister(self: *Self, cc: bits.Condition, reg1: Register, reg2: Register) !void { |
| 1124 | 1124 | _ = try self.addInst(.{ |
| 1125 | 1125 | .tag = switch (cc) { |
| 1126 | 1126 | else => .cmov, |
| ... | ... | @@ -1143,7 +1143,7 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi |
| 1143 | 1143 | } |
| 1144 | 1144 | |
| 1145 | 1145 | /// A `cc` of `.z_and_np` is not supported by this encoding! |
| 1146 | | fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void { |
| 1146 | fn asmCmovccRegisterMemory(self: *Self, cc: bits.Condition, reg: Register, m: Memory) !void { |
| 1147 | 1147 | _ = try self.addInst(.{ |
| 1148 | 1148 | .tag = switch (cc) { |
| 1149 | 1149 | else => .cmov, |
| ... | ... | @@ -3232,9 +3232,9 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { |
| 3232 | 3232 | |
| 3233 | 3233 | const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2); |
| 3234 | 3234 | try self.asmCmovccRegisterRegister( |
| 3235 | cc, |
| 3235 | 3236 | registerAlias(dst_reg, cmov_abi_size), |
| 3236 | 3237 | registerAlias(limit_reg, cmov_abi_size), |
| 3237 | | cc, |
| 3238 | 3238 | ); |
| 3239 | 3239 | |
| 3240 | 3240 | if (reg_extra_bits > 0 and ty.isSignedInt(mod)) { |
| ... | ... | @@ -3308,9 +3308,9 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void { |
| 3308 | 3308 | |
| 3309 | 3309 | const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2); |
| 3310 | 3310 | try self.asmCmovccRegisterRegister( |
| 3311 | cc, |
| 3311 | 3312 | registerAlias(dst_reg, cmov_abi_size), |
| 3312 | 3313 | registerAlias(limit_reg, cmov_abi_size), |
| 3313 | | cc, |
| 3314 | 3314 | ); |
| 3315 | 3315 | |
| 3316 | 3316 | if (reg_extra_bits > 0 and ty.isSignedInt(mod)) { |
| ... | ... | @@ -3371,9 +3371,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { |
| 3371 | 3371 | const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv); |
| 3372 | 3372 | const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2); |
| 3373 | 3373 | try self.asmCmovccRegisterRegister( |
| 3374 | cc, |
| 3374 | 3375 | registerAlias(dst_mcv.register, cmov_abi_size), |
| 3375 | 3376 | registerAlias(limit_reg, cmov_abi_size), |
| 3376 | | cc, |
| 3377 | 3377 | ); |
| 3378 | 3378 | |
| 3379 | 3379 | return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); |
| ... | ... | @@ -3586,7 +3586,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 3586 | 3586 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 3587 | 3587 | const dst_ty = self.typeOf(bin_op.lhs); |
| 3588 | 3588 | const result: MCValue = switch (dst_ty.zigTypeTag(mod)) { |
| 3589 | | .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}), |
| 3589 | .Vector => return self.fail("TODO implement airMulWithOverflow for {}", .{dst_ty.fmt(mod)}), |
| 3590 | 3590 | .Int => result: { |
| 3591 | 3591 | try self.spillEflagsIfOccupied(); |
| 3592 | 3592 | try self.spillRegisters(&.{ .rax, .rdx }); |
| ... | ... | @@ -3743,9 +3743,9 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa |
| 3743 | 3743 | registerAlias(.rdx, abi_size), |
| 3744 | 3744 | ); |
| 3745 | 3745 | try self.asmCmovccRegisterRegister( |
| 3746 | .z, |
| 3746 | 3747 | registerAlias(divisor, abi_size), |
| 3747 | 3748 | registerAlias(.rdx, abi_size), |
| 3748 | | .z, |
| 3749 | 3749 | ); |
| 3750 | 3750 | try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax }); |
| 3751 | 3751 | return MCValue{ .register = divisor }; |
| ... | ... | @@ -4626,7 +4626,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { |
| 4626 | 4626 | tmp_mcv, |
| 4627 | 4627 | mat_src_mcv.address().offset(8).deref(), |
| 4628 | 4628 | ); |
| 4629 | | try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc); |
| 4629 | try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32()); |
| 4630 | 4630 | |
| 4631 | 4631 | if (src_bits < 128) { |
| 4632 | 4632 | try self.genBinOpMir( |
| ... | ... | @@ -4660,9 +4660,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { |
| 4660 | 4660 | |
| 4661 | 4661 | const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2); |
| 4662 | 4662 | try self.asmCmovccRegisterRegister( |
| 4663 | .z, |
| 4663 | 4664 | registerAlias(dst_reg, cmov_abi_size), |
| 4664 | 4665 | registerAlias(imm_reg, cmov_abi_size), |
| 4665 | | .z, |
| 4666 | 4666 | ); |
| 4667 | 4667 | |
| 4668 | 4668 | try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 }); |
| ... | ... | @@ -4687,9 +4687,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { |
| 4687 | 4687 | |
| 4688 | 4688 | const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2); |
| 4689 | 4689 | try self.asmCmovccRegisterRegister( |
| 4690 | .nz, |
| 4690 | 4691 | registerAlias(imm_reg, cmov_abi_size), |
| 4691 | 4692 | registerAlias(dst_reg, cmov_abi_size), |
| 4692 | | .nz, |
| 4693 | 4693 | ); |
| 4694 | 4694 | |
| 4695 | 4695 | try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 }); |
| ... | ... | @@ -4764,7 +4764,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { |
| 4764 | 4764 | try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv); |
| 4765 | 4765 | try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 }); |
| 4766 | 4766 | try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv); |
| 4767 | | try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc); |
| 4767 | try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32()); |
| 4768 | 4768 | } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)}); |
| 4769 | 4769 | break :result dst_mcv; |
| 4770 | 4770 | } |
| ... | ... | @@ -4786,9 +4786,9 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { |
| 4786 | 4786 | |
| 4787 | 4787 | const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2); |
| 4788 | 4788 | try self.asmCmovccRegisterRegister( |
| 4789 | .z, |
| 4789 | 4790 | registerAlias(dst_reg, cmov_abi_size), |
| 4790 | 4791 | registerAlias(width_reg, cmov_abi_size), |
| 4791 | | .z, |
| 4792 | 4792 | ); |
| 4793 | 4793 | break :result dst_mcv; |
| 4794 | 4794 | }; |
| ... | ... | @@ -5389,37 +5389,73 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void { |
| 5389 | 5389 | const result: MCValue = result: { |
| 5390 | 5390 | const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) { |
| 5391 | 5391 | else => null, |
| 5392 | | .Int => { |
| 5393 | | if (ty.abiSize(mod) > 8) { |
| 5394 | | return self.fail("TODO implement abs for integer abi sizes larger than 8", .{}); |
| 5395 | | } |
| 5396 | | const src_mcv = try self.resolveInst(ty_op.operand); |
| 5397 | | const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); |
| 5392 | .Int => switch (ty.abiSize(mod)) { |
| 5393 | 1...8 => { |
| 5394 | try self.spillEflagsIfOccupied(); |
| 5395 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 5396 | const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); |
| 5398 | 5397 | |
| 5399 | | try self.genUnOpMir(.{ ._, .neg }, ty, dst_mcv); |
| 5398 | try self.genUnOpMir(.{ ._, .neg }, ty, dst_mcv); |
| 5400 | 5399 | |
| 5401 | | const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2); |
| 5402 | | switch (src_mcv) { |
| 5403 | | .register => |val_reg| try self.asmCmovccRegisterRegister( |
| 5404 | | registerAlias(dst_mcv.register, cmov_abi_size), |
| 5405 | | registerAlias(val_reg, cmov_abi_size), |
| 5406 | | .l, |
| 5407 | | ), |
| 5408 | | .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( |
| 5409 | | registerAlias(dst_mcv.register, cmov_abi_size), |
| 5410 | | src_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)), |
| 5411 | | .l, |
| 5412 | | ), |
| 5413 | | else => { |
| 5414 | | const val_reg = try self.copyToTmpRegister(ty, src_mcv); |
| 5415 | | try self.asmCmovccRegisterRegister( |
| 5400 | const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(mod))), 2); |
| 5401 | switch (src_mcv) { |
| 5402 | .register => |val_reg| try self.asmCmovccRegisterRegister( |
| 5403 | .l, |
| 5416 | 5404 | registerAlias(dst_mcv.register, cmov_abi_size), |
| 5417 | 5405 | registerAlias(val_reg, cmov_abi_size), |
| 5406 | ), |
| 5407 | .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( |
| 5418 | 5408 | .l, |
| 5409 | registerAlias(dst_mcv.register, cmov_abi_size), |
| 5410 | src_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)), |
| 5411 | ), |
| 5412 | else => { |
| 5413 | const val_reg = try self.copyToTmpRegister(ty, src_mcv); |
| 5414 | try self.asmCmovccRegisterRegister( |
| 5415 | .l, |
| 5416 | registerAlias(dst_mcv.register, cmov_abi_size), |
| 5417 | registerAlias(val_reg, cmov_abi_size), |
| 5418 | ); |
| 5419 | }, |
| 5420 | } |
| 5421 | break :result dst_mcv; |
| 5422 | }, |
| 5423 | 9...16 => { |
| 5424 | try self.spillEflagsIfOccupied(); |
| 5425 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 5426 | const dst_mcv = if (src_mcv == .register_pair and |
| 5427 | self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { |
| 5428 | const dst_regs = try self.register_manager.allocRegs( |
| 5429 | 2, |
| 5430 | .{ inst, inst }, |
| 5431 | abi.RegisterClass.gp, |
| 5419 | 5432 | ); |
| 5420 | | }, |
| 5421 | | } |
| 5422 | | break :result dst_mcv; |
| 5433 | const dst_mcv: MCValue = .{ .register_pair = dst_regs }; |
| 5434 | const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); |
| 5435 | defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); |
| 5436 | |
| 5437 | try self.genCopy(ty, dst_mcv, src_mcv); |
| 5438 | break :dst dst_mcv; |
| 5439 | }; |
| 5440 | const dst_regs = dst_mcv.register_pair; |
| 5441 | const dst_locks = self.register_manager.lockRegs(2, dst_regs); |
| 5442 | defer for (dst_locks) |dst_lock| if (dst_lock) |lock| |
| 5443 | self.register_manager.unlockReg(lock); |
| 5444 | |
| 5445 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 5446 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 5447 | defer self.register_manager.unlockReg(tmp_lock); |
| 5448 | |
| 5449 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); |
| 5450 | try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63)); |
| 5451 | try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg); |
| 5452 | try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg); |
| 5453 | try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg); |
| 5454 | try self.asmRegisterRegister(.{ ._, .sbb }, dst_regs[1], tmp_reg); |
| 5455 | |
| 5456 | break :result dst_mcv; |
| 5457 | }, |
| 5458 | else => return self.fail("TODO implement abs for {}", .{ty.fmt(mod)}), |
| 5423 | 5459 | }, |
| 5424 | 5460 | .Float => return self.floatSign(inst, ty_op.operand, ty), |
| 5425 | 5461 | .Vector => switch (ty.childType(mod).zigTypeTag(mod)) { |
| ... | ... | @@ -6063,6 +6099,126 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 6063 | 6099 | }; |
| 6064 | 6100 | |
| 6065 | 6101 | switch (src_mcv) { |
| 6102 | .register => |src_reg| { |
| 6103 | const src_reg_lock = self.register_manager.lockRegAssumeUnused(src_reg); |
| 6104 | defer self.register_manager.unlockReg(src_reg_lock); |
| 6105 | |
| 6106 | const dst_reg = if (field_rc.supersetOf(container_rc) and |
| 6107 | self.reuseOperand(inst, operand, 0, src_mcv)) |
| 6108 | src_reg |
| 6109 | else |
| 6110 | try self.copyToTmpRegister(Type.usize, .{ .register = src_reg }); |
| 6111 | const dst_mcv = MCValue{ .register = dst_reg }; |
| 6112 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 6113 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 6114 | |
| 6115 | if (field_off > 0) try self.genShiftBinOpMir( |
| 6116 | .{ ._r, .sh }, |
| 6117 | Type.usize, |
| 6118 | dst_mcv, |
| 6119 | .{ .immediate = field_off }, |
| 6120 | ); |
| 6121 | if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg); |
| 6122 | |
| 6123 | break :result if (field_rc.supersetOf(abi.RegisterClass.gp)) |
| 6124 | dst_mcv |
| 6125 | else |
| 6126 | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); |
| 6127 | }, |
| 6128 | .register_pair => |src_regs| { |
| 6129 | const src_regs_lock = self.register_manager.lockRegsAssumeUnused(2, src_regs); |
| 6130 | defer for (src_regs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6131 | |
| 6132 | const field_bit_size: u32 = @intCast(field_ty.bitSize(mod)); |
| 6133 | const src_reg = if (field_off + field_bit_size <= 64) |
| 6134 | src_regs[0] |
| 6135 | else if (field_off >= 64) |
| 6136 | src_regs[1] |
| 6137 | else { |
| 6138 | const dst_regs: [2]Register = if (field_rc.supersetOf(container_rc) and |
| 6139 | self.reuseOperand(inst, operand, 0, src_mcv)) src_regs else dst: { |
| 6140 | const dst_regs = |
| 6141 | try self.register_manager.allocRegs(2, .{ null, null }, field_rc); |
| 6142 | const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); |
| 6143 | defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); |
| 6144 | |
| 6145 | try self.genCopy(container_ty, .{ .register_pair = dst_regs }, src_mcv); |
| 6146 | break :dst dst_regs; |
| 6147 | }; |
| 6148 | const dst_mcv = MCValue{ .register_pair = dst_regs }; |
| 6149 | const dst_locks = self.register_manager.lockRegs(2, dst_regs); |
| 6150 | defer for (dst_locks) |dst_lock| if (dst_lock) |lock| |
| 6151 | self.register_manager.unlockReg(lock); |
| 6152 | |
| 6153 | if (field_off > 0) try self.genShiftBinOpMir( |
| 6154 | .{ ._r, .sh }, |
| 6155 | Type.u128, |
| 6156 | dst_mcv, |
| 6157 | .{ .immediate = field_off }, |
| 6158 | ); |
| 6159 | |
| 6160 | if (field_bit_size <= 64) { |
| 6161 | if (self.regExtraBits(field_ty) > 0) |
| 6162 | try self.truncateRegister(field_ty, dst_regs[0]); |
| 6163 | break :result if (field_rc.supersetOf(abi.RegisterClass.gp)) |
| 6164 | .{ .register = dst_regs[0] } |
| 6165 | else |
| 6166 | try self.copyToRegisterWithInstTracking(inst, field_ty, .{ |
| 6167 | .register = dst_regs[0], |
| 6168 | }); |
| 6169 | } |
| 6170 | |
| 6171 | if (field_bit_size < 128) try self.truncateRegister( |
| 6172 | try mod.intType(.unsigned, @intCast(field_bit_size - 64)), |
| 6173 | dst_regs[1], |
| 6174 | ); |
| 6175 | break :result if (field_rc.supersetOf(abi.RegisterClass.gp)) |
| 6176 | dst_mcv |
| 6177 | else |
| 6178 | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); |
| 6179 | }; |
| 6180 | |
| 6181 | const dst_reg = try self.copyToTmpRegister(Type.usize, .{ .register = src_reg }); |
| 6182 | const dst_mcv = MCValue{ .register = dst_reg }; |
| 6183 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 6184 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 6185 | |
| 6186 | if (field_off % 64 > 0) try self.genShiftBinOpMir( |
| 6187 | .{ ._r, .sh }, |
| 6188 | Type.usize, |
| 6189 | dst_mcv, |
| 6190 | .{ .immediate = field_off % 64 }, |
| 6191 | ); |
| 6192 | if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg); |
| 6193 | |
| 6194 | break :result if (field_rc.supersetOf(abi.RegisterClass.gp)) |
| 6195 | dst_mcv |
| 6196 | else |
| 6197 | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); |
| 6198 | }, |
| 6199 | .register_overflow => |ro| { |
| 6200 | switch (index) { |
| 6201 | // Get wrapped value for overflow operation. |
| 6202 | 0 => break :result if (self.liveness.operandDies(inst, 0)) |
| 6203 | .{ .register = ro.reg } |
| 6204 | else |
| 6205 | try self.copyToRegisterWithInstTracking( |
| 6206 | inst, |
| 6207 | Type.usize, |
| 6208 | .{ .register = ro.reg }, |
| 6209 | ), |
| 6210 | // Get overflow bit. |
| 6211 | 1 => if (self.liveness.operandDies(inst, 0)) { |
| 6212 | self.eflags_inst = inst; |
| 6213 | break :result .{ .eflags = ro.eflags }; |
| 6214 | } else { |
| 6215 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); |
| 6216 | try self.asmSetccRegister(dst_reg.to8(), ro.eflags); |
| 6217 | break :result .{ .register = dst_reg.to8() }; |
| 6218 | }, |
| 6219 | else => unreachable, |
| 6220 | } |
| 6221 | }, |
| 6066 | 6222 | .load_frame => |frame_addr| { |
| 6067 | 6223 | const field_abi_size: u32 = @intCast(field_ty.abiSize(mod)); |
| 6068 | 6224 | if (field_off % 8 == 0) { |
| ... | ... | @@ -6166,56 +6322,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 6166 | 6322 | else |
| 6167 | 6323 | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); |
| 6168 | 6324 | }, |
| 6169 | | .register => |reg| { |
| 6170 | | const reg_lock = self.register_manager.lockRegAssumeUnused(reg); |
| 6171 | | defer self.register_manager.unlockReg(reg_lock); |
| 6172 | | |
| 6173 | | const dst_reg = if (src_mcv.isRegister() and field_rc.supersetOf(container_rc) and |
| 6174 | | self.reuseOperand(inst, operand, 0, src_mcv)) |
| 6175 | | src_mcv.getReg().? |
| 6176 | | else |
| 6177 | | try self.copyToTmpRegister(Type.usize, .{ .register = reg.to64() }); |
| 6178 | | const dst_mcv = MCValue{ .register = dst_reg }; |
| 6179 | | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 6180 | | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 6181 | | |
| 6182 | | try self.genShiftBinOpMir( |
| 6183 | | .{ ._r, .sh }, |
| 6184 | | Type.usize, |
| 6185 | | dst_mcv, |
| 6186 | | .{ .immediate = field_off }, |
| 6187 | | ); |
| 6188 | | if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg); |
| 6189 | | |
| 6190 | | break :result if (field_rc.supersetOf(abi.RegisterClass.gp)) |
| 6191 | | dst_mcv |
| 6192 | | else |
| 6193 | | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); |
| 6194 | | }, |
| 6195 | | .register_overflow => |ro| { |
| 6196 | | switch (index) { |
| 6197 | | // Get wrapped value for overflow operation. |
| 6198 | | 0 => break :result if (self.liveness.operandDies(inst, 0)) |
| 6199 | | .{ .register = ro.reg } |
| 6200 | | else |
| 6201 | | try self.copyToRegisterWithInstTracking( |
| 6202 | | inst, |
| 6203 | | Type.usize, |
| 6204 | | .{ .register = ro.reg }, |
| 6205 | | ), |
| 6206 | | // Get overflow bit. |
| 6207 | | 1 => if (self.liveness.operandDies(inst, 0)) { |
| 6208 | | self.eflags_inst = inst; |
| 6209 | | break :result .{ .eflags = ro.eflags }; |
| 6210 | | } else { |
| 6211 | | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); |
| 6212 | | try self.asmSetccRegister(dst_reg.to8(), ro.eflags); |
| 6213 | | break :result .{ .register = dst_reg.to8() }; |
| 6214 | | }, |
| 6215 | | else => unreachable, |
| 6216 | | } |
| 6217 | | }, |
| 6218 | | else => return self.fail("TODO implement codegen struct_field_val for {}", .{src_mcv}), |
| 6325 | else => return self.fail("TODO implement airStructFieldVal for {}", .{src_mcv}), |
| 6219 | 6326 | } |
| 6220 | 6327 | }; |
| 6221 | 6328 | return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); |
| ... | ... | @@ -6427,139 +6534,233 @@ fn genShiftBinOpMir( |
| 6427 | 6534 | }), |
| 6428 | 6535 | } |
| 6429 | 6536 | } else if (abi_size <= 16) { |
| 6430 | | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6431 | | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 6432 | | defer self.register_manager.unlockReg(tmp_lock); |
| 6433 | | |
| 6434 | | const info: struct { offsets: [2]i32, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) { |
| 6435 | | ._l => .{ .offsets = .{ 0, 8 }, .double_tag = .{ ._ld, .sh } }, |
| 6436 | | ._r => .{ .offsets = .{ 8, 0 }, .double_tag = .{ ._rd, .sh } }, |
| 6537 | const info: struct { indices: [2]u31, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) { |
| 6538 | ._l => .{ .indices = .{ 0, 1 }, .double_tag = .{ ._ld, .sh } }, |
| 6539 | ._r => .{ .indices = .{ 1, 0 }, .double_tag = .{ ._rd, .sh } }, |
| 6437 | 6540 | else => unreachable, |
| 6438 | 6541 | }; |
| 6439 | 6542 | switch (lhs_mcv) { |
| 6440 | | .load_frame => |dst_frame_addr| switch (rhs_mcv) { |
| 6441 | | .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) { |
| 6442 | | try self.asmRegisterMemory( |
| 6443 | | .{ ._, .mov }, |
| 6444 | | tmp_reg, |
| 6445 | | Memory.sib(.qword, .{ |
| 6446 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6447 | | .disp = dst_frame_addr.off + info.offsets[0], |
| 6448 | | }), |
| 6449 | | ); |
| 6450 | | try self.asmMemoryRegisterImmediate( |
| 6543 | .register_pair => |lhs_regs| switch (rhs_mcv) { |
| 6544 | .immediate => |rhs_imm| if (rhs_imm > 0 and rhs_imm < 64) { |
| 6545 | try self.asmRegisterRegisterImmediate( |
| 6451 | 6546 | info.double_tag, |
| 6452 | | Memory.sib(.qword, .{ |
| 6453 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6454 | | .disp = dst_frame_addr.off + info.offsets[1], |
| 6455 | | }), |
| 6456 | | tmp_reg, |
| 6547 | lhs_regs[info.indices[1]], |
| 6548 | lhs_regs[info.indices[0]], |
| 6457 | 6549 | Immediate.u(rhs_imm), |
| 6458 | 6550 | ); |
| 6459 | | try self.asmMemoryImmediate( |
| 6551 | try self.asmRegisterImmediate( |
| 6460 | 6552 | tag, |
| 6461 | | Memory.sib(.qword, .{ |
| 6462 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6463 | | .disp = dst_frame_addr.off + info.offsets[0], |
| 6464 | | }), |
| 6553 | lhs_regs[info.indices[0]], |
| 6465 | 6554 | Immediate.u(rhs_imm), |
| 6466 | 6555 | ); |
| 6467 | 6556 | } else { |
| 6468 | 6557 | assert(rhs_imm < 128); |
| 6469 | | try self.asmRegisterMemory( |
| 6470 | | .{ ._, .mov }, |
| 6471 | | tmp_reg, |
| 6472 | | Memory.sib(.qword, .{ |
| 6473 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6474 | | .disp = dst_frame_addr.off + info.offsets[0], |
| 6475 | | }), |
| 6476 | | ); |
| 6477 | | if (rhs_imm > 64) { |
| 6478 | | try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64)); |
| 6479 | | } |
| 6480 | | try self.asmMemoryRegister( |
| 6558 | try self.asmRegisterRegister( |
| 6481 | 6559 | .{ ._, .mov }, |
| 6482 | | Memory.sib(.qword, .{ |
| 6483 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6484 | | .disp = dst_frame_addr.off + info.offsets[1], |
| 6485 | | }), |
| 6486 | | tmp_reg, |
| 6560 | lhs_regs[info.indices[1]], |
| 6561 | lhs_regs[info.indices[0]], |
| 6487 | 6562 | ); |
| 6488 | | if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate( |
| 6563 | if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate( |
| 6489 | 6564 | tag, |
| 6490 | | Memory.sib(.qword, .{ |
| 6491 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6492 | | .disp = dst_frame_addr.off + info.offsets[0], |
| 6493 | | }), |
| 6565 | lhs_regs[info.indices[0]], |
| 6494 | 6566 | Immediate.u(63), |
| 6495 | | ) else { |
| 6496 | | try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32()); |
| 6497 | | try self.asmMemoryRegister( |
| 6498 | | .{ ._, .mov }, |
| 6499 | | Memory.sib(.qword, .{ |
| 6500 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6501 | | .disp = dst_frame_addr.off + info.offsets[0], |
| 6502 | | }), |
| 6503 | | tmp_reg, |
| 6504 | | ); |
| 6505 | | } |
| 6506 | | }, |
| 6507 | | else => { |
| 6508 | | const first_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6509 | | const first_lock = self.register_manager.lockRegAssumeUnused(first_reg); |
| 6510 | | defer self.register_manager.unlockReg(first_lock); |
| 6511 | | |
| 6512 | | const second_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6513 | | const second_lock = self.register_manager.lockRegAssumeUnused(second_reg); |
| 6514 | | defer self.register_manager.unlockReg(second_lock); |
| 6515 | | |
| 6516 | | try self.genSetReg(.cl, Type.u8, rhs_mcv); |
| 6517 | | try self.asmRegisterMemory( |
| 6518 | | .{ ._, .mov }, |
| 6519 | | first_reg, |
| 6520 | | Memory.sib(.qword, .{ |
| 6521 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6522 | | .disp = dst_frame_addr.off + info.offsets[0], |
| 6523 | | }), |
| 6567 | ) else try self.asmRegisterRegister( |
| 6568 | .{ ._, .xor }, |
| 6569 | lhs_regs[info.indices[0]], |
| 6570 | lhs_regs[info.indices[0]], |
| 6524 | 6571 | ); |
| 6525 | | try self.asmRegisterMemory( |
| 6526 | | .{ ._, .mov }, |
| 6527 | | second_reg, |
| 6528 | | Memory.sib(.qword, .{ |
| 6529 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6530 | | .disp = dst_frame_addr.off + info.offsets[1], |
| 6531 | | }), |
| 6572 | if (rhs_imm > 64) try self.asmRegisterImmediate( |
| 6573 | tag, |
| 6574 | lhs_regs[info.indices[1]], |
| 6575 | Immediate.u(rhs_imm - 64), |
| 6532 | 6576 | ); |
| 6577 | }, |
| 6578 | .register => |rhs_reg| { |
| 6579 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6580 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 6581 | defer self.register_manager.unlockReg(tmp_lock); |
| 6582 | |
| 6533 | 6583 | if (tag[0] == ._r and tag[1] == .sa) { |
| 6534 | | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg); |
| 6584 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]); |
| 6535 | 6585 | try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63)); |
| 6536 | 6586 | } else try self.asmRegisterRegister( |
| 6537 | 6587 | .{ ._, .xor }, |
| 6538 | 6588 | tmp_reg.to32(), |
| 6539 | 6589 | tmp_reg.to32(), |
| 6540 | 6590 | ); |
| 6541 | | try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl); |
| 6542 | | try self.asmRegisterRegister(tag, first_reg, .cl); |
| 6543 | | try self.asmRegisterImmediate(.{ ._, .cmp }, .cl, Immediate.u(64)); |
| 6544 | | try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae); |
| 6545 | | try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae); |
| 6546 | | try self.asmMemoryRegister( |
| 6547 | | .{ ._, .mov }, |
| 6548 | | Memory.sib(.qword, .{ |
| 6549 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6550 | | .disp = dst_frame_addr.off + info.offsets[1], |
| 6551 | | }), |
| 6552 | | second_reg, |
| 6591 | try self.asmRegisterRegisterRegister( |
| 6592 | info.double_tag, |
| 6593 | lhs_regs[info.indices[1]], |
| 6594 | lhs_regs[info.indices[0]], |
| 6595 | registerAlias(rhs_reg, 1), |
| 6553 | 6596 | ); |
| 6554 | | try self.asmMemoryRegister( |
| 6555 | | .{ ._, .mov }, |
| 6556 | | Memory.sib(.qword, .{ |
| 6557 | | .base = .{ .frame = dst_frame_addr.index }, |
| 6558 | | .disp = dst_frame_addr.off + info.offsets[0], |
| 6559 | | }), |
| 6560 | | first_reg, |
| 6597 | try self.asmRegisterRegister( |
| 6598 | tag, |
| 6599 | lhs_regs[info.indices[0]], |
| 6600 | registerAlias(rhs_reg, 1), |
| 6561 | 6601 | ); |
| 6602 | try self.asmRegisterImmediate( |
| 6603 | .{ ._, .cmp }, |
| 6604 | registerAlias(rhs_reg, 1), |
| 6605 | Immediate.u(64), |
| 6606 | ); |
| 6607 | try self.asmCmovccRegisterRegister( |
| 6608 | .ae, |
| 6609 | lhs_regs[info.indices[1]], |
| 6610 | lhs_regs[info.indices[0]], |
| 6611 | ); |
| 6612 | try self.asmCmovccRegisterRegister(.ae, lhs_regs[info.indices[0]], tmp_reg); |
| 6562 | 6613 | }, |
| 6614 | else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ |
| 6615 | @tagName(lhs_mcv), |
| 6616 | @tagName(rhs_mcv), |
| 6617 | }), |
| 6618 | }, |
| 6619 | .load_frame => |dst_frame_addr| { |
| 6620 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6621 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 6622 | defer self.register_manager.unlockReg(tmp_lock); |
| 6623 | |
| 6624 | switch (rhs_mcv) { |
| 6625 | .immediate => |rhs_imm| if (rhs_imm > 0 and rhs_imm < 64) { |
| 6626 | try self.asmRegisterMemory( |
| 6627 | .{ ._, .mov }, |
| 6628 | tmp_reg, |
| 6629 | Memory.sib(.qword, .{ |
| 6630 | .base = .{ .frame = dst_frame_addr.index }, |
| 6631 | .disp = dst_frame_addr.off + info.indices[0] * 8, |
| 6632 | }), |
| 6633 | ); |
| 6634 | try self.asmMemoryRegisterImmediate( |
| 6635 | info.double_tag, |
| 6636 | Memory.sib(.qword, .{ |
| 6637 | .base = .{ .frame = dst_frame_addr.index }, |
| 6638 | .disp = dst_frame_addr.off + info.indices[1] * 8, |
| 6639 | }), |
| 6640 | tmp_reg, |
| 6641 | Immediate.u(rhs_imm), |
| 6642 | ); |
| 6643 | try self.asmMemoryImmediate( |
| 6644 | tag, |
| 6645 | Memory.sib(.qword, .{ |
| 6646 | .base = .{ .frame = dst_frame_addr.index }, |
| 6647 | .disp = dst_frame_addr.off + info.indices[0] * 8, |
| 6648 | }), |
| 6649 | Immediate.u(rhs_imm), |
| 6650 | ); |
| 6651 | } else { |
| 6652 | assert(rhs_imm < 128); |
| 6653 | try self.asmRegisterMemory( |
| 6654 | .{ ._, .mov }, |
| 6655 | tmp_reg, |
| 6656 | Memory.sib(.qword, .{ |
| 6657 | .base = .{ .frame = dst_frame_addr.index }, |
| 6658 | .disp = dst_frame_addr.off + info.indices[0] * 8, |
| 6659 | }), |
| 6660 | ); |
| 6661 | if (rhs_imm > 64) try self.asmRegisterImmediate( |
| 6662 | tag, |
| 6663 | tmp_reg, |
| 6664 | Immediate.u(rhs_imm - 64), |
| 6665 | ); |
| 6666 | try self.asmMemoryRegister( |
| 6667 | .{ ._, .mov }, |
| 6668 | Memory.sib(.qword, .{ |
| 6669 | .base = .{ .frame = dst_frame_addr.index }, |
| 6670 | .disp = dst_frame_addr.off + info.indices[1] * 8, |
| 6671 | }), |
| 6672 | tmp_reg, |
| 6673 | ); |
| 6674 | if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate( |
| 6675 | tag, |
| 6676 | Memory.sib(.qword, .{ |
| 6677 | .base = .{ .frame = dst_frame_addr.index }, |
| 6678 | .disp = dst_frame_addr.off + info.indices[0] * 8, |
| 6679 | }), |
| 6680 | Immediate.u(63), |
| 6681 | ) else { |
| 6682 | try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32()); |
| 6683 | try self.asmMemoryRegister( |
| 6684 | .{ ._, .mov }, |
| 6685 | Memory.sib(.qword, .{ |
| 6686 | .base = .{ .frame = dst_frame_addr.index }, |
| 6687 | .disp = dst_frame_addr.off + info.indices[0] * 8, |
| 6688 | }), |
| 6689 | tmp_reg, |
| 6690 | ); |
| 6691 | } |
| 6692 | }, |
| 6693 | .register => |rhs_reg| { |
| 6694 | const first_reg = |
| 6695 | try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6696 | const first_lock = self.register_manager.lockRegAssumeUnused(first_reg); |
| 6697 | defer self.register_manager.unlockReg(first_lock); |
| 6698 | |
| 6699 | const second_reg = |
| 6700 | try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6701 | const second_lock = self.register_manager.lockRegAssumeUnused(second_reg); |
| 6702 | defer self.register_manager.unlockReg(second_lock); |
| 6703 | |
| 6704 | try self.asmRegisterMemory( |
| 6705 | .{ ._, .mov }, |
| 6706 | first_reg, |
| 6707 | Memory.sib(.qword, .{ |
| 6708 | .base = .{ .frame = dst_frame_addr.index }, |
| 6709 | .disp = dst_frame_addr.off + info.indices[0] * 8, |
| 6710 | }), |
| 6711 | ); |
| 6712 | try self.asmRegisterMemory( |
| 6713 | .{ ._, .mov }, |
| 6714 | second_reg, |
| 6715 | Memory.sib(.qword, .{ |
| 6716 | .base = .{ .frame = dst_frame_addr.index }, |
| 6717 | .disp = dst_frame_addr.off + info.indices[1] * 8, |
| 6718 | }), |
| 6719 | ); |
| 6720 | if (tag[0] == ._r and tag[1] == .sa) { |
| 6721 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg); |
| 6722 | try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63)); |
| 6723 | } else try self.asmRegisterRegister( |
| 6724 | .{ ._, .xor }, |
| 6725 | tmp_reg.to32(), |
| 6726 | tmp_reg.to32(), |
| 6727 | ); |
| 6728 | try self.asmRegisterRegisterRegister( |
| 6729 | info.double_tag, |
| 6730 | second_reg, |
| 6731 | first_reg, |
| 6732 | registerAlias(rhs_reg, 1), |
| 6733 | ); |
| 6734 | try self.asmRegisterRegister(tag, first_reg, registerAlias(rhs_reg, 1)); |
| 6735 | try self.asmRegisterImmediate( |
| 6736 | .{ ._, .cmp }, |
| 6737 | registerAlias(rhs_reg, 1), |
| 6738 | Immediate.u(64), |
| 6739 | ); |
| 6740 | try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg); |
| 6741 | try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg); |
| 6742 | try self.asmMemoryRegister( |
| 6743 | .{ ._, .mov }, |
| 6744 | Memory.sib(.qword, .{ |
| 6745 | .base = .{ .frame = dst_frame_addr.index }, |
| 6746 | .disp = dst_frame_addr.off + info.indices[1] * 8, |
| 6747 | }), |
| 6748 | second_reg, |
| 6749 | ); |
| 6750 | try self.asmMemoryRegister( |
| 6751 | .{ ._, .mov }, |
| 6752 | Memory.sib(.qword, .{ |
| 6753 | .base = .{ .frame = dst_frame_addr.index }, |
| 6754 | .disp = dst_frame_addr.off + info.indices[0] * 8, |
| 6755 | }), |
| 6756 | first_reg, |
| 6757 | ); |
| 6758 | }, |
| 6759 | else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ |
| 6760 | @tagName(lhs_mcv), |
| 6761 | @tagName(rhs_mcv), |
| 6762 | }), |
| 6763 | } |
| 6563 | 6764 | }, |
| 6564 | 6765 | else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ |
| 6565 | 6766 | @tagName(lhs_mcv), |
| ... | ... | @@ -6646,29 +6847,99 @@ fn genMulDivBinOp( |
| 6646 | 6847 | maybe_inst: ?Air.Inst.Index, |
| 6647 | 6848 | dst_ty: Type, |
| 6648 | 6849 | src_ty: Type, |
| 6649 | | lhs: MCValue, |
| 6650 | | rhs: MCValue, |
| 6850 | lhs_mcv: MCValue, |
| 6851 | rhs_mcv: MCValue, |
| 6651 | 6852 | ) !MCValue { |
| 6652 | 6853 | const mod = self.bin_file.options.module.?; |
| 6653 | 6854 | if (dst_ty.zigTypeTag(mod) == .Vector or dst_ty.zigTypeTag(mod) == .Float) return self.fail( |
| 6654 | | "TODO implement genMulDivBinOp for {}", |
| 6655 | | .{dst_ty.fmt(mod)}, |
| 6855 | "TODO implement genMulDivBinOp for {s} from {} to {}", |
| 6856 | .{ @tagName(tag), src_ty.fmt(mod), dst_ty.fmt(mod) }, |
| 6656 | 6857 | ); |
| 6657 | 6858 | const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod)); |
| 6658 | 6859 | const src_abi_size: u32 = @intCast(src_ty.abiSize(mod)); |
| 6860 | |
| 6861 | assert(self.register_manager.isRegFree(.rax)); |
| 6862 | assert(self.register_manager.isRegFree(.rdx)); |
| 6863 | assert(self.eflags_inst == null); |
| 6864 | |
| 6865 | if (dst_abi_size == 16 and src_abi_size == 16) { |
| 6866 | switch (tag) { |
| 6867 | else => unreachable, |
| 6868 | .mul, .mul_wrap => {}, |
| 6869 | .div_trunc, .div_floor, .div_exact, .rem, .mod => { |
| 6870 | const signed = dst_ty.isSignedInt(mod); |
| 6871 | if (signed) switch (tag) { |
| 6872 | .div_floor, .mod => return self.fail( |
| 6873 | "TODO implement genMulDivBinOp for {s} from {} to {}", |
| 6874 | .{ @tagName(tag), src_ty.fmt(mod), dst_ty.fmt(mod) }, |
| 6875 | ), |
| 6876 | else => {}, |
| 6877 | }; |
| 6878 | var callee: ["__udiv?i3".len]u8 = undefined; |
| 6879 | return try self.genCall(.{ .lib = .{ |
| 6880 | .return_type = dst_ty.toIntern(), |
| 6881 | .param_types = &.{ src_ty.toIntern(), src_ty.toIntern() }, |
| 6882 | .callee = std.fmt.bufPrint(&callee, "__{s}{s}{c}i3", .{ |
| 6883 | if (signed) "" else "u", |
| 6884 | switch (tag) { |
| 6885 | .div_trunc, .div_exact => "div", |
| 6886 | .div_floor => if (signed) unreachable else "div", |
| 6887 | .rem => "mod", |
| 6888 | .mod => if (signed) unreachable else "mod", |
| 6889 | else => unreachable, |
| 6890 | }, |
| 6891 | intCompilerRtAbiName(@intCast(dst_ty.bitSize(mod))), |
| 6892 | }) catch unreachable, |
| 6893 | } }, &.{ src_ty, src_ty }, &.{ lhs_mcv, rhs_mcv }); |
| 6894 | }, |
| 6895 | } |
| 6896 | |
| 6897 | const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); |
| 6898 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); |
| 6899 | |
| 6900 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 6901 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 6902 | defer self.register_manager.unlockReg(tmp_lock); |
| 6903 | |
| 6904 | if (lhs_mcv.isMemory()) |
| 6905 | try self.asmRegisterMemory(.{ ._, .mov }, .rax, lhs_mcv.mem(.qword)) |
| 6906 | else |
| 6907 | try self.asmRegisterRegister(.{ ._, .mov }, .rax, lhs_mcv.register_pair[0]); |
| 6908 | if (rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 6909 | .{ ._, .mov }, |
| 6910 | tmp_reg, |
| 6911 | rhs_mcv.address().offset(8).deref().mem(.qword), |
| 6912 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, rhs_mcv.register_pair[1]); |
| 6913 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax); |
| 6914 | if (rhs_mcv.isMemory()) |
| 6915 | try self.asmMemory(.{ ._, .mul }, rhs_mcv.mem(.qword)) |
| 6916 | else |
| 6917 | try self.asmRegister(.{ ._, .mul }, rhs_mcv.register_pair[0]); |
| 6918 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg); |
| 6919 | if (lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 6920 | .{ ._, .mov }, |
| 6921 | tmp_reg, |
| 6922 | lhs_mcv.address().offset(8).deref().mem(.qword), |
| 6923 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_mcv.register_pair[1]); |
| 6924 | if (rhs_mcv.isMemory()) |
| 6925 | try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, rhs_mcv.mem(.qword)) |
| 6926 | else |
| 6927 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, rhs_mcv.register_pair[0]); |
| 6928 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg); |
| 6929 | return .{ .register_pair = .{ .rax, .rdx } }; |
| 6930 | } |
| 6931 | |
| 6659 | 6932 | if (switch (tag) { |
| 6660 | 6933 | else => unreachable, |
| 6661 | 6934 | .mul, .mul_wrap => dst_abi_size != src_abi_size and dst_abi_size != src_abi_size * 2, |
| 6662 | 6935 | .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_abi_size != src_abi_size, |
| 6663 | | } or src_abi_size > 8) return self.fail("TODO implement genMulDivBinOp from {} to {}", .{ |
| 6664 | | src_ty.fmt(mod), dst_ty.fmt(mod), |
| 6665 | | }); |
| 6936 | } or src_abi_size > 8) return self.fail( |
| 6937 | "TODO implement genMulDivBinOp for {s} from {} to {}", |
| 6938 | .{ @tagName(tag), src_ty.fmt(mod), dst_ty.fmt(mod) }, |
| 6939 | ); |
| 6666 | 6940 | const ty = if (dst_abi_size <= 8) dst_ty else src_ty; |
| 6667 | 6941 | const abi_size = if (dst_abi_size <= 8) dst_abi_size else src_abi_size; |
| 6668 | 6942 | |
| 6669 | | assert(self.register_manager.isRegFree(.rax)); |
| 6670 | | assert(self.register_manager.isRegFree(.rdx)); |
| 6671 | | |
| 6672 | 6943 | const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx }); |
| 6673 | 6944 | defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); |
| 6674 | 6945 | |
| ... | ... | @@ -6703,7 +6974,7 @@ fn genMulDivBinOp( |
| 6703 | 6974 | .div_trunc, .div_exact, .rem => .{ ._, .div }, |
| 6704 | 6975 | else => unreachable, |
| 6705 | 6976 | }, |
| 6706 | | }, ty, lhs, rhs); |
| 6977 | }, ty, lhs_mcv, rhs_mcv); |
| 6707 | 6978 | |
| 6708 | 6979 | if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) { |
| 6709 | 6980 | .mul, .mul_wrap, .div_trunc, .div_exact => .rax, |
| ... | ... | @@ -6737,37 +7008,37 @@ fn genMulDivBinOp( |
| 6737 | 7008 | |
| 6738 | 7009 | switch (signedness) { |
| 6739 | 7010 | .signed => { |
| 6740 | | const lhs_lock = switch (lhs) { |
| 7011 | const lhs_lock = switch (lhs_mcv) { |
| 6741 | 7012 | .register => |reg| self.register_manager.lockReg(reg), |
| 6742 | 7013 | else => null, |
| 6743 | 7014 | }; |
| 6744 | 7015 | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6745 | | const rhs_lock = switch (rhs) { |
| 7016 | const rhs_lock = switch (rhs_mcv) { |
| 6746 | 7017 | .register => |reg| self.register_manager.lockReg(reg), |
| 6747 | 7018 | else => null, |
| 6748 | 7019 | }; |
| 6749 | 7020 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6750 | 7021 | |
| 6751 | 7022 | // hack around hazard between rhs and div_floor by copying rhs to another register |
| 6752 | | const rhs_copy = try self.copyToTmpRegister(ty, rhs); |
| 7023 | const rhs_copy = try self.copyToTmpRegister(ty, rhs_mcv); |
| 6753 | 7024 | const rhs_copy_lock = self.register_manager.lockRegAssumeUnused(rhs_copy); |
| 6754 | 7025 | defer self.register_manager.unlockReg(rhs_copy_lock); |
| 6755 | 7026 | |
| 6756 | | const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs); |
| 7027 | const div_floor = try self.genInlineIntDivFloor(ty, lhs_mcv, rhs_mcv); |
| 6757 | 7028 | try self.genIntMulComplexOpMir(ty, div_floor, .{ .register = rhs_copy }); |
| 6758 | 7029 | const div_floor_lock = self.register_manager.lockReg(div_floor.register); |
| 6759 | 7030 | defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock); |
| 6760 | 7031 | |
| 6761 | 7032 | const result: MCValue = if (maybe_inst) |inst| |
| 6762 | | try self.copyToRegisterWithInstTracking(inst, ty, lhs) |
| 7033 | try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv) |
| 6763 | 7034 | else |
| 6764 | | .{ .register = try self.copyToTmpRegister(ty, lhs) }; |
| 7035 | .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) }; |
| 6765 | 7036 | try self.genBinOpMir(.{ ._, .sub }, ty, result, div_floor); |
| 6766 | 7037 | |
| 6767 | 7038 | return result; |
| 6768 | 7039 | }, |
| 6769 | 7040 | .unsigned => { |
| 6770 | | try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, rhs); |
| 7041 | try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs_mcv, rhs_mcv); |
| 6771 | 7042 | return .{ .register = registerAlias(.rdx, abi_size) }; |
| 6772 | 7043 | }, |
| 6773 | 7044 | } |
| ... | ... | @@ -6777,39 +7048,39 @@ fn genMulDivBinOp( |
| 6777 | 7048 | try self.register_manager.getReg(.rax, if (signedness == .unsigned) maybe_inst else null); |
| 6778 | 7049 | try self.register_manager.getReg(.rdx, null); |
| 6779 | 7050 | |
| 6780 | | const lhs_lock: ?RegisterLock = switch (lhs) { |
| 7051 | const lhs_lock: ?RegisterLock = switch (lhs_mcv) { |
| 6781 | 7052 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 6782 | 7053 | else => null, |
| 6783 | 7054 | }; |
| 6784 | 7055 | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6785 | 7056 | |
| 6786 | | const actual_rhs: MCValue = blk: { |
| 7057 | const actual_rhs_mcv: MCValue = blk: { |
| 6787 | 7058 | switch (signedness) { |
| 6788 | 7059 | .signed => { |
| 6789 | | const rhs_lock: ?RegisterLock = switch (rhs) { |
| 7060 | const rhs_lock: ?RegisterLock = switch (rhs_mcv) { |
| 6790 | 7061 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 6791 | 7062 | else => null, |
| 6792 | 7063 | }; |
| 6793 | 7064 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6794 | 7065 | |
| 6795 | 7066 | if (maybe_inst) |inst| { |
| 6796 | | break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs); |
| 7067 | break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv); |
| 6797 | 7068 | } |
| 6798 | | break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; |
| 7069 | break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs_mcv) }; |
| 6799 | 7070 | }, |
| 6800 | | .unsigned => break :blk rhs, |
| 7071 | .unsigned => break :blk rhs_mcv, |
| 6801 | 7072 | } |
| 6802 | 7073 | }; |
| 6803 | | const rhs_lock: ?RegisterLock = switch (actual_rhs) { |
| 7074 | const rhs_lock: ?RegisterLock = switch (actual_rhs_mcv) { |
| 6804 | 7075 | .register => |reg| self.register_manager.lockReg(reg), |
| 6805 | 7076 | else => null, |
| 6806 | 7077 | }; |
| 6807 | 7078 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6808 | 7079 | |
| 6809 | 7080 | switch (signedness) { |
| 6810 | | .signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs), |
| 7081 | .signed => return try self.genInlineIntDivFloor(ty, lhs_mcv, actual_rhs_mcv), |
| 6811 | 7082 | .unsigned => { |
| 6812 | | try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, actual_rhs); |
| 7083 | try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs_mcv, actual_rhs_mcv); |
| 6813 | 7084 | return .{ .register = registerAlias(.rax, abi_size) }; |
| 6814 | 7085 | }, |
| 6815 | 7086 | } |
| ... | ... | @@ -6837,7 +7108,7 @@ fn genBinOp( |
| 6837 | 7108 | 80, 128 => true, |
| 6838 | 7109 | else => unreachable, |
| 6839 | 7110 | })) { |
| 6840 | | var callee: ["__add?f3".len]u8 = undefined; |
| 7111 | var callee: ["__mod?f3".len]u8 = undefined; |
| 6841 | 7112 | const result = try self.genCall(.{ .lib = .{ |
| 6842 | 7113 | .return_type = lhs_ty.toIntern(), |
| 6843 | 7114 | .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() }, |
| ... | ... | @@ -6937,17 +7208,22 @@ fn genBinOp( |
| 6937 | 7208 | .Float, .Vector => true, |
| 6938 | 7209 | }; |
| 6939 | 7210 | |
| 6940 | | const lhs_lock: ?RegisterLock = switch (lhs_mcv) { |
| 6941 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 6942 | | else => null, |
| 7211 | const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) { |
| 7212 | .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null }, |
| 7213 | .register_pair => |lhs_regs| locks: { |
| 7214 | const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs); |
| 7215 | break :locks .{ locks[0], locks[1] }; |
| 7216 | }, |
| 7217 | else => .{ null, null }, |
| 6943 | 7218 | }; |
| 6944 | | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 7219 | defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6945 | 7220 | |
| 6946 | | const rhs_lock: ?RegisterLock = switch (rhs_mcv) { |
| 6947 | | .register => |reg| self.register_manager.lockReg(reg), |
| 6948 | | else => null, |
| 7221 | const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) { |
| 7222 | .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null }, |
| 7223 | .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs), |
| 7224 | else => .{ null, null }, |
| 6949 | 7225 | }; |
| 6950 | | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 7226 | defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 6951 | 7227 | |
| 6952 | 7228 | var flipped = false; |
| 6953 | 7229 | var copied_to_dst = true; |
| ... | ... | @@ -6970,11 +7246,12 @@ fn genBinOp( |
| 6970 | 7246 | try self.genCopy(lhs_ty, dst_mcv, lhs_mcv); |
| 6971 | 7247 | break :dst dst_mcv; |
| 6972 | 7248 | }; |
| 6973 | | const dst_lock: ?RegisterLock = switch (dst_mcv) { |
| 6974 | | .register => |reg| self.register_manager.lockReg(reg), |
| 6975 | | else => null, |
| 7249 | const dst_locks: [2]?RegisterLock = switch (dst_mcv) { |
| 7250 | .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null }, |
| 7251 | .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs), |
| 7252 | else => .{ null, null }, |
| 6976 | 7253 | }; |
| 6977 | | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 7254 | defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 6978 | 7255 | |
| 6979 | 7256 | const unmat_src_mcv = if (flipped) lhs_mcv else rhs_mcv; |
| 6980 | 7257 | const src_mcv: MCValue = if (maybe_mask_reg) |mask_reg| |
| ... | ... | @@ -6985,6 +7262,12 @@ fn genBinOp( |
| 6985 | 7262 | } |
| 6986 | 7263 | else |
| 6987 | 7264 | unmat_src_mcv; |
| 7265 | const src_locks: [2]?RegisterLock = switch (src_mcv) { |
| 7266 | .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null }, |
| 7267 | .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs), |
| 7268 | else => .{ null, null }, |
| 7269 | }; |
| 7270 | defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock); |
| 6988 | 7271 | |
| 6989 | 7272 | if (!vec_op) { |
| 6990 | 7273 | switch (air_tag) { |
| ... | ... | @@ -7035,102 +7318,166 @@ fn genBinOp( |
| 7035 | 7318 | else => src_mcv, |
| 7036 | 7319 | .air_ref => |src_ref| try self.resolveInst(src_ref), |
| 7037 | 7320 | }; |
| 7038 | | const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) { |
| 7039 | | .immediate, |
| 7040 | | .eflags, |
| 7041 | | .register_offset, |
| 7042 | | .load_direct, |
| 7043 | | .lea_direct, |
| 7044 | | .load_got, |
| 7045 | | .lea_got, |
| 7046 | | .load_tlv, |
| 7047 | | .lea_tlv, |
| 7048 | | .lea_frame, |
| 7049 | | => true, |
| 7050 | | .memory => |addr| math.cast(i32, @as(i64, @bitCast(addr))) == null, |
| 7051 | | else => false, |
| 7052 | | .register_pair, |
| 7053 | | .register_overflow, |
| 7054 | | => unreachable, |
| 7055 | | }) |
| 7056 | | .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) } |
| 7057 | | else |
| 7058 | | resolved_src_mcv; |
| 7059 | | const mat_mcv_lock = switch (mat_src_mcv) { |
| 7060 | | .register => |reg| self.register_manager.lockReg(reg), |
| 7061 | | else => null, |
| 7062 | | }; |
| 7063 | | defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock); |
| 7064 | 7321 | |
| 7065 | | try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv); |
| 7322 | if (abi_size > 8) { |
| 7323 | const dst_regs = switch (dst_mcv) { |
| 7324 | .register_pair => |dst_regs| dst_regs, |
| 7325 | else => dst: { |
| 7326 | const dst_regs = try self.register_manager.allocRegs( |
| 7327 | 2, |
| 7328 | .{ null, null }, |
| 7329 | abi.RegisterClass.gp, |
| 7330 | ); |
| 7331 | const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs); |
| 7332 | defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock| |
| 7333 | self.register_manager.unlockReg(lock); |
| 7066 | 7334 | |
| 7067 | | const int_info = lhs_ty.intInfo(mod); |
| 7068 | | const cc: Condition = switch (int_info.signedness) { |
| 7069 | | .unsigned => switch (air_tag) { |
| 7070 | | .min => .a, |
| 7071 | | .max => .b, |
| 7072 | | else => unreachable, |
| 7073 | | }, |
| 7074 | | .signed => switch (air_tag) { |
| 7075 | | .min => .g, |
| 7076 | | .max => .l, |
| 7335 | try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv); |
| 7336 | break :dst dst_regs; |
| 7337 | }, |
| 7338 | }; |
| 7339 | const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs); |
| 7340 | defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock| |
| 7341 | self.register_manager.unlockReg(lock); |
| 7342 | |
| 7343 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 7344 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7345 | defer self.register_manager.unlockReg(tmp_lock); |
| 7346 | |
| 7347 | const signed = lhs_ty.isSignedInt(mod); |
| 7348 | const cc: Condition = switch (air_tag) { |
| 7349 | .min => if (signed) .nl else .nb, |
| 7350 | .max => if (signed) .nge else .nae, |
| 7077 | 7351 | else => unreachable, |
| 7078 | | }, |
| 7079 | | }; |
| 7352 | }; |
| 7080 | 7353 | |
| 7081 | | const cmov_abi_size = @max(@as(u32, @intCast(lhs_ty.abiSize(mod))), 2); |
| 7082 | | const tmp_reg = switch (dst_mcv) { |
| 7083 | | .register => |reg| reg, |
| 7084 | | else => try self.copyToTmpRegister(lhs_ty, dst_mcv), |
| 7085 | | }; |
| 7086 | | const tmp_lock = self.register_manager.lockReg(tmp_reg); |
| 7087 | | defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock); |
| 7088 | | switch (mat_src_mcv) { |
| 7089 | | .none, |
| 7090 | | .unreach, |
| 7091 | | .dead, |
| 7092 | | .undef, |
| 7093 | | .immediate, |
| 7094 | | .eflags, |
| 7095 | | .register_pair, |
| 7096 | | .register_offset, |
| 7097 | | .register_overflow, |
| 7098 | | .load_direct, |
| 7099 | | .lea_direct, |
| 7100 | | .load_got, |
| 7101 | | .lea_got, |
| 7102 | | .load_tlv, |
| 7103 | | .lea_tlv, |
| 7104 | | .lea_frame, |
| 7105 | | .reserved_frame, |
| 7106 | | .air_ref, |
| 7107 | | => unreachable, |
| 7108 | | .register => |src_reg| try self.asmCmovccRegisterRegister( |
| 7109 | | registerAlias(tmp_reg, cmov_abi_size), |
| 7110 | | registerAlias(src_reg, cmov_abi_size), |
| 7111 | | cc, |
| 7112 | | ), |
| 7113 | | .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( |
| 7114 | | registerAlias(tmp_reg, cmov_abi_size), |
| 7115 | | Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) { |
| 7116 | | .memory => |addr| .{ |
| 7117 | | .base = .{ .reg = .ds }, |
| 7118 | | .disp = @intCast(@as(i64, @bitCast(addr))), |
| 7119 | | }, |
| 7120 | | .indirect => |reg_off| .{ |
| 7121 | | .base = .{ .reg = reg_off.reg }, |
| 7122 | | .disp = reg_off.off, |
| 7123 | | }, |
| 7124 | | .load_frame => |frame_addr| .{ |
| 7125 | | .base = .{ .frame = frame_addr.index }, |
| 7126 | | .disp = frame_addr.off, |
| 7127 | | }, |
| 7354 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); |
| 7355 | if (src_mcv.isMemory()) { |
| 7356 | try self.asmRegisterMemory(.{ ._, .cmp }, dst_regs[0], src_mcv.mem(.qword)); |
| 7357 | try self.asmRegisterMemory( |
| 7358 | .{ ._, .sbb }, |
| 7359 | tmp_reg, |
| 7360 | src_mcv.address().offset(8).deref().mem(.qword), |
| 7361 | ); |
| 7362 | try self.asmCmovccRegisterMemory(cc, dst_regs[0], src_mcv.mem(.qword)); |
| 7363 | try self.asmCmovccRegisterMemory( |
| 7364 | cc, |
| 7365 | dst_regs[1], |
| 7366 | src_mcv.address().offset(8).deref().mem(.qword), |
| 7367 | ); |
| 7368 | } else { |
| 7369 | try self.asmRegisterRegister( |
| 7370 | .{ ._, .cmp }, |
| 7371 | dst_regs[0], |
| 7372 | src_mcv.register_pair[0], |
| 7373 | ); |
| 7374 | try self.asmRegisterRegister( |
| 7375 | .{ ._, .sbb }, |
| 7376 | tmp_reg, |
| 7377 | src_mcv.register_pair[1], |
| 7378 | ); |
| 7379 | try self.asmCmovccRegisterRegister(cc, dst_regs[0], src_mcv.register_pair[0]); |
| 7380 | try self.asmCmovccRegisterRegister(cc, dst_regs[1], src_mcv.register_pair[1]); |
| 7381 | } |
| 7382 | try self.genCopy(lhs_ty, dst_mcv, .{ .register_pair = dst_regs }); |
| 7383 | } else { |
| 7384 | const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) { |
| 7385 | .immediate, |
| 7386 | .eflags, |
| 7387 | .register_offset, |
| 7388 | .load_direct, |
| 7389 | .lea_direct, |
| 7390 | .load_got, |
| 7391 | .lea_got, |
| 7392 | .load_tlv, |
| 7393 | .lea_tlv, |
| 7394 | .lea_frame, |
| 7395 | => true, |
| 7396 | .memory => |addr| math.cast(i32, @as(i64, @bitCast(addr))) == null, |
| 7397 | else => false, |
| 7398 | .register_pair, |
| 7399 | .register_overflow, |
| 7400 | => unreachable, |
| 7401 | }) |
| 7402 | .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) } |
| 7403 | else |
| 7404 | resolved_src_mcv; |
| 7405 | const mat_mcv_lock = switch (mat_src_mcv) { |
| 7406 | .register => |reg| self.register_manager.lockReg(reg), |
| 7407 | else => null, |
| 7408 | }; |
| 7409 | defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock); |
| 7410 | |
| 7411 | try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv); |
| 7412 | |
| 7413 | const int_info = lhs_ty.intInfo(mod); |
| 7414 | const cc: Condition = switch (int_info.signedness) { |
| 7415 | .unsigned => switch (air_tag) { |
| 7416 | .min => .a, |
| 7417 | .max => .b, |
| 7128 | 7418 | else => unreachable, |
| 7129 | | }), |
| 7130 | | cc, |
| 7131 | | ), |
| 7419 | }, |
| 7420 | .signed => switch (air_tag) { |
| 7421 | .min => .g, |
| 7422 | .max => .l, |
| 7423 | else => unreachable, |
| 7424 | }, |
| 7425 | }; |
| 7426 | |
| 7427 | const cmov_abi_size = @max(@as(u32, @intCast(lhs_ty.abiSize(mod))), 2); |
| 7428 | const tmp_reg = switch (dst_mcv) { |
| 7429 | .register => |reg| reg, |
| 7430 | else => try self.copyToTmpRegister(lhs_ty, dst_mcv), |
| 7431 | }; |
| 7432 | const tmp_lock = self.register_manager.lockReg(tmp_reg); |
| 7433 | defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock); |
| 7434 | switch (mat_src_mcv) { |
| 7435 | .none, |
| 7436 | .unreach, |
| 7437 | .dead, |
| 7438 | .undef, |
| 7439 | .immediate, |
| 7440 | .eflags, |
| 7441 | .register_pair, |
| 7442 | .register_offset, |
| 7443 | .register_overflow, |
| 7444 | .load_direct, |
| 7445 | .lea_direct, |
| 7446 | .load_got, |
| 7447 | .lea_got, |
| 7448 | .load_tlv, |
| 7449 | .lea_tlv, |
| 7450 | .lea_frame, |
| 7451 | .reserved_frame, |
| 7452 | .air_ref, |
| 7453 | => unreachable, |
| 7454 | .register => |src_reg| try self.asmCmovccRegisterRegister( |
| 7455 | cc, |
| 7456 | registerAlias(tmp_reg, cmov_abi_size), |
| 7457 | registerAlias(src_reg, cmov_abi_size), |
| 7458 | ), |
| 7459 | .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( |
| 7460 | cc, |
| 7461 | registerAlias(tmp_reg, cmov_abi_size), |
| 7462 | Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) { |
| 7463 | .memory => |addr| .{ |
| 7464 | .base = .{ .reg = .ds }, |
| 7465 | .disp = @intCast(@as(i64, @bitCast(addr))), |
| 7466 | }, |
| 7467 | .indirect => |reg_off| .{ |
| 7468 | .base = .{ .reg = reg_off.reg }, |
| 7469 | .disp = reg_off.off, |
| 7470 | }, |
| 7471 | .load_frame => |frame_addr| .{ |
| 7472 | .base = .{ .frame = frame_addr.index }, |
| 7473 | .disp = frame_addr.off, |
| 7474 | }, |
| 7475 | else => unreachable, |
| 7476 | }), |
| 7477 | ), |
| 7478 | } |
| 7479 | try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg }); |
| 7132 | 7480 | } |
| 7133 | | try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg }); |
| 7134 | 7481 | }, |
| 7135 | 7482 | |
| 7136 | 7483 | else => return self.fail("TODO implement genBinOp for {s} {}", .{ |
| ... | ... | @@ -11852,21 +12199,21 @@ fn atomicOp( |
| 11852 | 12199 | const cmov_abi_size = @max(val_abi_size, 2); |
| 11853 | 12200 | switch (val_mcv) { |
| 11854 | 12201 | .register => |val_reg| try self.asmCmovccRegisterRegister( |
| 12202 | cc, |
| 11855 | 12203 | registerAlias(tmp_reg, cmov_abi_size), |
| 11856 | 12204 | registerAlias(val_reg, cmov_abi_size), |
| 11857 | | cc, |
| 11858 | 12205 | ), |
| 11859 | 12206 | .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( |
| 12207 | cc, |
| 11860 | 12208 | registerAlias(tmp_reg, cmov_abi_size), |
| 11861 | 12209 | val_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)), |
| 11862 | | cc, |
| 11863 | 12210 | ), |
| 11864 | 12211 | else => { |
| 11865 | 12212 | const val_reg = try self.copyToTmpRegister(val_ty, val_mcv); |
| 11866 | 12213 | try self.asmCmovccRegisterRegister( |
| 12214 | cc, |
| 11867 | 12215 | registerAlias(tmp_reg, cmov_abi_size), |
| 11868 | 12216 | registerAlias(val_reg, cmov_abi_size), |
| 11869 | | cc, |
| 11870 | 12217 | ); |
| 11871 | 12218 | }, |
| 11872 | 12219 | } |
Jacob Young <jacobly@ziglang.org> 2023-10-09 16:18:50-04:00
GitHub <noreply@github.com> 2023-10-09 16:18:50-04:00
Signed by PGP key 4AEE18F83AFDEB23
src