| ... | @@ -4799,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -4799,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void { |
| 4799 | switch (lhs_ty.zigTypeTag(mod)) { | 4799 | switch (lhs_ty.zigTypeTag(mod)) { |
| 4800 | .Int => { | 4800 | .Int => { |
| 4801 | try self.spillRegisters(&.{.rcx}); | 4801 | try self.spillRegisters(&.{.rcx}); |
| 4802 | try self.register_manager.getReg(.rcx, null); | 4802 | try self.register_manager.getKnownReg(.rcx, null); |
| 4803 | const lhs_mcv = try self.resolveInst(bin_op.lhs); | 4803 | const lhs_mcv = try self.resolveInst(bin_op.lhs); |
| 4804 | const rhs_mcv = try self.resolveInst(bin_op.rhs); | 4804 | const rhs_mcv = try self.resolveInst(bin_op.rhs); |
| 4805 | | 4805 | |
| ... | @@ -6825,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) | ... | @@ -6825,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) |
| 6825 | | 6825 | |
| 6826 | const dst_mcv: MCValue = .{ .register = .st0 }; | 6826 | const dst_mcv: MCValue = .{ .register = .st0 }; |
| 6827 | if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv)) | 6827 | if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv)) |
| 6828 | try self.register_manager.getReg(.st0, inst); | 6828 | try self.register_manager.getKnownReg(.st0, inst); |
| 6829 | | 6829 | |
| 6830 | try self.genCopy(ty, dst_mcv, src_mcv, .{}); | 6830 | try self.genCopy(ty, dst_mcv, src_mcv, .{}); |
| 6831 | switch (tag) { | 6831 | switch (tag) { |
| ... | @@ -9059,7 +9059,7 @@ fn genShiftBinOp( | ... | @@ -9059,7 +9059,7 @@ fn genShiftBinOp( |
| 9059 | lhs_ty.fmt(mod), | 9059 | lhs_ty.fmt(mod), |
| 9060 | }); | 9060 | }); |
| 9061 | | 9061 | |
| 9062 | try self.register_manager.getReg(.rcx, null); | 9062 | try self.register_manager.getKnownReg(.rcx, null); |
| 9063 | const rcx_lock = self.register_manager.lockReg(.rcx); | 9063 | const rcx_lock = self.register_manager.lockReg(.rcx); |
| 9064 | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); | 9064 | defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); |
| 9065 | | 9065 | |
| ... | @@ -9382,8 +9382,8 @@ fn genMulDivBinOp( | ... | @@ -9382,8 +9382,8 @@ fn genMulDivBinOp( |
| 9382 | .rem => maybe_inst, | 9382 | .rem => maybe_inst, |
| 9383 | else => null, | 9383 | else => null, |
| 9384 | }; | 9384 | }; |
| 9385 | try self.register_manager.getReg(.rax, track_inst_rax); | 9385 | try self.register_manager.getKnownReg(.rax, track_inst_rax); |
| 9386 | try self.register_manager.getReg(.rdx, track_inst_rdx); | 9386 | try self.register_manager.getKnownReg(.rdx, track_inst_rdx); |
| 9387 | | 9387 | |
| 9388 | try self.genIntMulDivOpMir(switch (signedness) { | 9388 | try self.genIntMulDivOpMir(switch (signedness) { |
| 9389 | .signed => switch (tag) { | 9389 | .signed => switch (tag) { |
| ... | @@ -9423,8 +9423,11 @@ fn genMulDivBinOp( | ... | @@ -9423,8 +9423,11 @@ fn genMulDivBinOp( |
| 9423 | }, | 9423 | }, |
| 9424 | | 9424 | |
| 9425 | .mod => { | 9425 | .mod => { |
| 9426 | try self.register_manager.getReg(.rax, null); | 9426 | try self.register_manager.getKnownReg(.rax, null); |
| 9427 | try self.register_manager.getReg(.rdx, if (signedness == .unsigned) maybe_inst else null); | 9427 | try self.register_manager.getKnownReg( |
| | 9428 | .rdx, |
| | 9429 | if (signedness == .unsigned) maybe_inst else null, |
| | 9430 | ); |
| 9428 | | 9431 | |
| 9429 | switch (signedness) { | 9432 | switch (signedness) { |
| 9430 | .signed => { | 9433 | .signed => { |
| ... | @@ -9465,8 +9468,11 @@ fn genMulDivBinOp( | ... | @@ -9465,8 +9468,11 @@ fn genMulDivBinOp( |
| 9465 | }, | 9468 | }, |
| 9466 | | 9469 | |
| 9467 | .div_floor => { | 9470 | .div_floor => { |
| 9468 | try self.register_manager.getReg(.rax, if (signedness == .unsigned) maybe_inst else null); | 9471 | try self.register_manager.getKnownReg( |
| 9469 | try self.register_manager.getReg(.rdx, null); | 9472 | .rax, |
| | 9473 | if (signedness == .unsigned) maybe_inst else null, |
| | 9474 | ); |
| | 9475 | try self.register_manager.getKnownReg(.rdx, null); |
| 9470 | | 9476 | |
| 9471 | const lhs_lock: ?RegisterLock = switch (lhs_mcv) { | 9477 | const lhs_lock: ?RegisterLock = switch (lhs_mcv) { |
| 9472 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | 9478 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| ... | @@ -9710,7 +9716,7 @@ fn genBinOp( | ... | @@ -9710,7 +9716,7 @@ fn genBinOp( |
| 9710 | .rem, .mod => unreachable, | 9716 | .rem, .mod => unreachable, |
| 9711 | .max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias( | 9717 | .max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias( |
| 9712 | if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: { | 9718 | if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: { |
| 9713 | try self.register_manager.getReg(.xmm0, null); | 9719 | try self.register_manager.getKnownReg(.xmm0, null); |
| 9714 | break :mask .xmm0; | 9720 | break :mask .xmm0; |
| 9715 | } else try self.register_manager.allocReg(null, abi.RegisterClass.sse), | 9721 | } else try self.register_manager.allocReg(null, abi.RegisterClass.sse), |
| 9716 | abi_size, | 9722 | abi_size, |
| ... | @@ -16769,7 +16775,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -16769,7 +16775,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { |
| 16769 | .register => |pred_reg| switch (pred_reg.class()) { | 16775 | .register => |pred_reg| switch (pred_reg.class()) { |
| 16770 | .general_purpose => {}, | 16776 | .general_purpose => {}, |
| 16771 | .sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) { | 16777 | .sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) { |
| 16772 | try self.register_manager.getReg(.xmm0, null); | 16778 | try self.register_manager.getKnownReg(.xmm0, null); |
| 16773 | try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{}); | 16779 | try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{}); |
| 16774 | break :mask .xmm0; | 16780 | break :mask .xmm0; |
| 16775 | } else break :mask if (has_blend) | 16781 | } else break :mask if (has_blend) |
| ... | @@ -16781,7 +16787,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -16781,7 +16787,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { |
| 16781 | else => {}, | 16787 | else => {}, |
| 16782 | } | 16788 | } |
| 16783 | const mask_reg: Register = if (need_xmm0) mask_reg: { | 16789 | const mask_reg: Register = if (need_xmm0) mask_reg: { |
| 16784 | try self.register_manager.getReg(.xmm0, null); | 16790 | try self.register_manager.getKnownReg(.xmm0, null); |
| 16785 | break :mask_reg .xmm0; | 16791 | break :mask_reg .xmm0; |
| 16786 | } else try self.register_manager.allocReg(null, abi.RegisterClass.sse); | 16792 | } else try self.register_manager.allocReg(null, abi.RegisterClass.sse); |
| 16787 | const mask_alias = registerAlias(mask_reg, abi_size); | 16793 | const mask_alias = registerAlias(mask_reg, abi_size); |
| ... | @@ -17093,7 +17099,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17093,7 +17099,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17093 | | 17099 | |
| 17094 | const dst_ty = self.typeOfIndex(inst); | 17100 | const dst_ty = self.typeOfIndex(inst); |
| 17095 | const elem_ty = dst_ty.childType(mod); | 17101 | const elem_ty = dst_ty.childType(mod); |
| 17096 | const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod)); | 17102 | const elem_abi_size: u16 = @intCast(elem_ty.abiSize(mod)); |
| 17097 | const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod)); | 17103 | const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod)); |
| 17098 | const lhs_ty = self.typeOf(extra.a); | 17104 | const lhs_ty = self.typeOf(extra.a); |
| 17099 | const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod)); | 17105 | const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod)); |
| ... | @@ -17109,8 +17115,8 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17109,8 +17115,8 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17109 | const mask_elems = try allocator.alloc(?i32, extra.mask_len); | 17115 | const mask_elems = try allocator.alloc(?i32, extra.mask_len); |
| 17110 | defer allocator.free(mask_elems); | 17116 | defer allocator.free(mask_elems); |
| 17111 | for (mask_elems, 0..) |*mask_elem, elem_index| { | 17117 | for (mask_elems, 0..) |*mask_elem, elem_index| { |
| 17112 | const mask_elem_val = Value | 17118 | const mask_elem_val = |
| 17113 | .fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable; | 17119 | Value.fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable; |
| 17114 | mask_elem.* = if (mask_elem_val.isUndef(mod)) | 17120 | mask_elem.* = if (mask_elem_val.isUndef(mod)) |
| 17115 | null | 17121 | null |
| 17116 | else | 17122 | else |
| ... | @@ -17124,7 +17130,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17124,7 +17130,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17124 | } else break :result try self.allocRegOrMem(inst, true); | 17130 | } else break :result try self.allocRegOrMem(inst, true); |
| 17125 | | 17131 | |
| 17126 | for (mask_elems, 0..) |mask_elem, elem_index| { | 17132 | for (mask_elems, 0..) |mask_elem, elem_index| { |
| 17127 | if (mask_elem orelse continue != @as(i32, @intCast(elem_index))) break; | 17133 | if (mask_elem orelse continue != elem_index) break; |
| 17128 | } else { | 17134 | } else { |
| 17129 | const lhs_mcv = try self.resolveInst(extra.a); | 17135 | const lhs_mcv = try self.resolveInst(extra.a); |
| 17130 | if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv; | 17136 | if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv; |
| ... | @@ -17134,7 +17140,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17134,7 +17140,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17134 | } | 17140 | } |
| 17135 | | 17141 | |
| 17136 | for (mask_elems, 0..) |mask_elem, elem_index| { | 17142 | for (mask_elems, 0..) |mask_elem, elem_index| { |
| 17137 | if (mask_elem orelse continue != ~@as(i32, @intCast(elem_index))) break; | 17143 | if (~(mask_elem orelse continue) != elem_index) break; |
| 17138 | } else { | 17144 | } else { |
| 17139 | const rhs_mcv = try self.resolveInst(extra.b); | 17145 | const rhs_mcv = try self.resolveInst(extra.b); |
| 17140 | if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv; | 17146 | if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv; |
| ... | @@ -17143,6 +17149,219 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17143,6 +17149,219 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17143 | break :result dst_mcv; | 17149 | break :result dst_mcv; |
| 17144 | } | 17150 | } |
| 17145 | | 17151 | |
| | 17152 | for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { |
| | 17153 | if (elem_abi_size > 8) break :unpck; |
| | 17154 | if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4) |
| | 17155 | has_avx |
| | 17156 | else |
| | 17157 | self.hasFeature(.avx2)) 32 else 16)) break :unpck; |
| | 17158 | |
| | 17159 | var sources = [1]?u1{null} ** 2; |
| | 17160 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { |
| | 17161 | const mask_elem = maybe_mask_elem orelse continue; |
| | 17162 | const mask_elem_index = |
| | 17163 | math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck; |
| | 17164 | const elem_byte = (elem_index >> 1) * elem_abi_size; |
| | 17165 | if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) { |
| | 17166 | .unpckl => 0b0000, |
| | 17167 | .unpckh => 0b1000, |
| | 17168 | else => unreachable, |
| | 17169 | }) | (elem_byte << 1 & 0b10000)) break :unpck; |
| | 17170 | |
| | 17171 | const source = @intFromBool(mask_elem < 0); |
| | 17172 | if (sources[elem_index & 0b00001]) |prev_source| { |
| | 17173 | if (source != prev_source) break :unpck; |
| | 17174 | } else sources[elem_index & 0b00001] = source; |
| | 17175 | } |
| | 17176 | if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck; |
| | 17177 | |
| | 17178 | const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; |
| | 17179 | const operand_tys = [2]Type{ lhs_ty, rhs_ty }; |
| | 17180 | const lhs_mcv = try self.resolveInst(operands[sources[0].?]); |
| | 17181 | const rhs_mcv = try self.resolveInst(operands[sources[1].?]); |
| | 17182 | |
| | 17183 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and |
| | 17184 | self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) |
| | 17185 | lhs_mcv |
| | 17186 | else if (has_avx and lhs_mcv.isRegister()) |
| | 17187 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } |
| | 17188 | else |
| | 17189 | try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); |
| | 17190 | const dst_reg = dst_mcv.getReg().?; |
| | 17191 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| | 17192 | |
| | 17193 | const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or |
| | 17194 | (dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) { |
| | 17195 | 4 => if (has_avx) .v_ps else ._ps, |
| | 17196 | 8 => if (has_avx) .v_pd else ._pd, |
| | 17197 | else => unreachable, |
| | 17198 | }, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) { |
| | 17199 | .unpckl => switch (elem_abi_size) { |
| | 17200 | 1 => .unpcklbw, |
| | 17201 | 2 => .unpcklwd, |
| | 17202 | 4 => .unpckldq, |
| | 17203 | 8 => .unpcklqdq, |
| | 17204 | else => unreachable, |
| | 17205 | }, |
| | 17206 | .unpckh => switch (elem_abi_size) { |
| | 17207 | 1 => .unpckhbw, |
| | 17208 | 2 => .unpckhwd, |
| | 17209 | 4 => .unpckhdq, |
| | 17210 | 8 => .unpckhqdq, |
| | 17211 | else => unreachable, |
| | 17212 | }, |
| | 17213 | else => unreachable, |
| | 17214 | } }; |
| | 17215 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| | 17216 | mir_tag, |
| | 17217 | dst_alias, |
| | 17218 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| | 17219 | try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), |
| | 17220 | ) else try self.asmRegisterRegisterRegister( |
| | 17221 | mir_tag, |
| | 17222 | dst_alias, |
| | 17223 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| | 17224 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17225 | rhs_mcv.getReg().? |
| | 17226 | else |
| | 17227 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| | 17228 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| | 17229 | mir_tag, |
| | 17230 | dst_alias, |
| | 17231 | try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), |
| | 17232 | ) else try self.asmRegisterRegister( |
| | 17233 | mir_tag, |
| | 17234 | dst_alias, |
| | 17235 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17236 | rhs_mcv.getReg().? |
| | 17237 | else |
| | 17238 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| | 17239 | ); |
| | 17240 | break :result dst_mcv; |
| | 17241 | } |
| | 17242 | |
| | 17243 | pshufd: { |
| | 17244 | if (elem_abi_size != 4) break :pshufd; |
| | 17245 | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd; |
| | 17246 | |
| | 17247 | var control: u8 = 0b00_00_00_00; |
| | 17248 | var sources = [1]?u1{null} ** 1; |
| | 17249 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { |
| | 17250 | const mask_elem = maybe_mask_elem orelse continue; |
| | 17251 | const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); |
| | 17252 | if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd; |
| | 17253 | |
| | 17254 | const source = @intFromBool(mask_elem < 0); |
| | 17255 | if (sources[0]) |prev_source| { |
| | 17256 | if (source != prev_source) break :pshufd; |
| | 17257 | } else sources[(elem_index & 0b010) >> 1] = source; |
| | 17258 | |
| | 17259 | const select_bit: u3 = @intCast((elem_index & 0b011) << 1); |
| | 17260 | const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; |
| | 17261 | if (elem_index & 0b100 == 0) |
| | 17262 | control |= select |
| | 17263 | else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd; |
| | 17264 | } |
| | 17265 | |
| | 17266 | const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; |
| | 17267 | const operand_tys = [2]Type{ lhs_ty, rhs_ty }; |
| | 17268 | const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]); |
| | 17269 | |
| | 17270 | const dst_reg = if (src_mcv.isRegister() and |
| | 17271 | self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv)) |
| | 17272 | src_mcv.getReg().? |
| | 17273 | else |
| | 17274 | try self.register_manager.allocReg(inst, abi.RegisterClass.sse); |
| | 17275 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| | 17276 | |
| | 17277 | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| | 17278 | .{ if (has_avx) .vp_d else .p_d, .shuf }, |
| | 17279 | dst_alias, |
| | 17280 | try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), |
| | 17281 | Immediate.u(control), |
| | 17282 | ) else try self.asmRegisterRegisterImmediate( |
| | 17283 | .{ if (has_avx) .vp_d else .p_d, .shuf }, |
| | 17284 | dst_alias, |
| | 17285 | registerAlias(if (src_mcv.isRegister()) |
| | 17286 | src_mcv.getReg().? |
| | 17287 | else |
| | 17288 | try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size), |
| | 17289 | Immediate.u(control), |
| | 17290 | ); |
| | 17291 | break :result .{ .register = dst_reg }; |
| | 17292 | } |
| | 17293 | |
| | 17294 | shufps: { |
| | 17295 | if (elem_abi_size != 4) break :shufps; |
| | 17296 | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps; |
| | 17297 | |
| | 17298 | var control: u8 = 0b00_00_00_00; |
| | 17299 | var sources = [1]?u1{null} ** 2; |
| | 17300 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { |
| | 17301 | const mask_elem = maybe_mask_elem orelse continue; |
| | 17302 | const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); |
| | 17303 | if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps; |
| | 17304 | |
| | 17305 | const source = @intFromBool(mask_elem < 0); |
| | 17306 | if (sources[(elem_index & 0b010) >> 1]) |prev_source| { |
| | 17307 | if (source != prev_source) break :shufps; |
| | 17308 | } else sources[(elem_index & 0b010) >> 1] = source; |
| | 17309 | |
| | 17310 | const select_bit: u3 = @intCast((elem_index & 0b011) << 1); |
| | 17311 | const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; |
| | 17312 | if (elem_index & 0b100 == 0) |
| | 17313 | control |= select |
| | 17314 | else if (control & @as(u8, 0b11) << select_bit != select) break :shufps; |
| | 17315 | } |
| | 17316 | if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps; |
| | 17317 | |
| | 17318 | const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; |
| | 17319 | const operand_tys = [2]Type{ lhs_ty, rhs_ty }; |
| | 17320 | const lhs_mcv = try self.resolveInst(operands[sources[0].?]); |
| | 17321 | const rhs_mcv = try self.resolveInst(operands[sources[1].?]); |
| | 17322 | |
| | 17323 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and |
| | 17324 | self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) |
| | 17325 | lhs_mcv |
| | 17326 | else if (has_avx and lhs_mcv.isRegister()) |
| | 17327 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } |
| | 17328 | else |
| | 17329 | try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); |
| | 17330 | const dst_reg = dst_mcv.getReg().?; |
| | 17331 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| | 17332 | |
| | 17333 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| | 17334 | .{ .v_ps, .shuf }, |
| | 17335 | dst_alias, |
| | 17336 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| | 17337 | try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), |
| | 17338 | Immediate.u(control), |
| | 17339 | ) else try self.asmRegisterRegisterRegisterImmediate( |
| | 17340 | .{ .v_ps, .shuf }, |
| | 17341 | dst_alias, |
| | 17342 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| | 17343 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17344 | rhs_mcv.getReg().? |
| | 17345 | else |
| | 17346 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| | 17347 | Immediate.u(control), |
| | 17348 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| | 17349 | .{ ._ps, .shuf }, |
| | 17350 | dst_alias, |
| | 17351 | try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), |
| | 17352 | Immediate.u(control), |
| | 17353 | ) else try self.asmRegisterRegisterImmediate( |
| | 17354 | .{ ._ps, .shuf }, |
| | 17355 | dst_alias, |
| | 17356 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17357 | rhs_mcv.getReg().? |
| | 17358 | else |
| | 17359 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| | 17360 | Immediate.u(control), |
| | 17361 | ); |
| | 17362 | break :result dst_mcv; |
| | 17363 | } |
| | 17364 | |
| 17146 | shufpd: { | 17365 | shufpd: { |
| 17147 | if (elem_abi_size != 8) break :shufpd; | 17366 | if (elem_abi_size != 8) break :shufpd; |
| 17148 | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd; | 17367 | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd; |
| ... | @@ -17163,8 +17382,8 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17163,8 +17382,8 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17163 | } | 17382 | } |
| 17164 | if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd; | 17383 | if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd; |
| 17165 | | 17384 | |
| 17166 | const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; | 17385 | const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b }; |
| 17167 | const operand_tys = [2]Type{ lhs_ty, rhs_ty }; | 17386 | const operand_tys: [2]Type = .{ lhs_ty, rhs_ty }; |
| 17168 | const lhs_mcv = try self.resolveInst(operands[sources[0].?]); | 17387 | const lhs_mcv = try self.resolveInst(operands[sources[0].?]); |
| 17169 | const rhs_mcv = try self.resolveInst(operands[sources[1].?]); | 17388 | const rhs_mcv = try self.resolveInst(operands[sources[1].?]); |
| 17170 | | 17389 | |
| ... | @@ -17210,6 +17429,355 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17210,6 +17429,355 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17210 | break :result dst_mcv; | 17429 | break :result dst_mcv; |
| 17211 | } | 17430 | } |
| 17212 | | 17431 | |
| | 17432 | blend: { |
| | 17433 | if (elem_abi_size < 2) break :blend; |
| | 17434 | if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend; |
| | 17435 | if (!self.hasFeature(.sse4_1)) break :blend; |
| | 17436 | |
| | 17437 | var control: u8 = 0b0_0_0_0_0_0_0_0; |
| | 17438 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { |
| | 17439 | const mask_elem = maybe_mask_elem orelse continue; |
| | 17440 | const mask_elem_index = |
| | 17441 | math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend; |
| | 17442 | if (mask_elem_index != elem_index) break :blend; |
| | 17443 | |
| | 17444 | const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index); |
| | 17445 | if (elem_index & 0b1000 == 0) |
| | 17446 | control |= select |
| | 17447 | else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend; |
| | 17448 | } |
| | 17449 | |
| | 17450 | if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: { |
| | 17451 | const expanded_control = switch (elem_abi_size) { |
| | 17452 | 4 => control, |
| | 17453 | 8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | |
| | 17454 | @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | |
| | 17455 | @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | |
| | 17456 | @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00), |
| | 17457 | else => break :vpblendd, |
| | 17458 | }; |
| | 17459 | |
| | 17460 | const lhs_mcv = try self.resolveInst(extra.a); |
| | 17461 | const lhs_reg = if (lhs_mcv.isRegister()) |
| | 17462 | lhs_mcv.getReg().? |
| | 17463 | else |
| | 17464 | try self.copyToTmpRegister(dst_ty, lhs_mcv); |
| | 17465 | const lhs_lock = self.register_manager.lockReg(lhs_reg); |
| | 17466 | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| | 17467 | |
| | 17468 | const rhs_mcv = try self.resolveInst(extra.b); |
| | 17469 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); |
| | 17470 | if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| | 17471 | .{ .vp_d, .blend }, |
| | 17472 | registerAlias(dst_reg, dst_abi_size), |
| | 17473 | registerAlias(lhs_reg, dst_abi_size), |
| | 17474 | try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17475 | Immediate.u(expanded_control), |
| | 17476 | ) else try self.asmRegisterRegisterRegisterImmediate( |
| | 17477 | .{ .vp_d, .blend }, |
| | 17478 | registerAlias(dst_reg, dst_abi_size), |
| | 17479 | registerAlias(lhs_reg, dst_abi_size), |
| | 17480 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17481 | rhs_mcv.getReg().? |
| | 17482 | else |
| | 17483 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| | 17484 | Immediate.u(expanded_control), |
| | 17485 | ); |
| | 17486 | break :result .{ .register = dst_reg }; |
| | 17487 | } |
| | 17488 | |
| | 17489 | if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: { |
| | 17490 | const expanded_control = switch (elem_abi_size) { |
| | 17491 | 2 => control, |
| | 17492 | 4 => if (dst_abi_size <= 16 or |
| | 17493 | @as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0))) |
| | 17494 | @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | |
| | 17495 | @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | |
| | 17496 | @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | |
| | 17497 | @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00) |
| | 17498 | else |
| | 17499 | break :pblendw, |
| | 17500 | 8 => if (dst_abi_size <= 16 or |
| | 17501 | @as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0))) |
| | 17502 | @as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) | |
| | 17503 | @as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000) |
| | 17504 | else |
| | 17505 | break :pblendw, |
| | 17506 | 16 => break :pblendw, |
| | 17507 | else => unreachable, |
| | 17508 | }; |
| | 17509 | |
| | 17510 | const lhs_mcv = try self.resolveInst(extra.a); |
| | 17511 | const rhs_mcv = try self.resolveInst(extra.b); |
| | 17512 | |
| | 17513 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and |
| | 17514 | self.reuseOperand(inst, extra.a, 0, lhs_mcv)) |
| | 17515 | lhs_mcv |
| | 17516 | else if (has_avx and lhs_mcv.isRegister()) |
| | 17517 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } |
| | 17518 | else |
| | 17519 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); |
| | 17520 | const dst_reg = dst_mcv.getReg().?; |
| | 17521 | |
| | 17522 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| | 17523 | .{ .vp_w, .blend }, |
| | 17524 | registerAlias(dst_reg, dst_abi_size), |
| | 17525 | registerAlias(if (lhs_mcv.isRegister()) |
| | 17526 | lhs_mcv.getReg().? |
| | 17527 | else |
| | 17528 | dst_reg, dst_abi_size), |
| | 17529 | try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17530 | Immediate.u(expanded_control), |
| | 17531 | ) else try self.asmRegisterRegisterRegisterImmediate( |
| | 17532 | .{ .vp_w, .blend }, |
| | 17533 | registerAlias(dst_reg, dst_abi_size), |
| | 17534 | registerAlias(if (lhs_mcv.isRegister()) |
| | 17535 | lhs_mcv.getReg().? |
| | 17536 | else |
| | 17537 | dst_reg, dst_abi_size), |
| | 17538 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17539 | rhs_mcv.getReg().? |
| | 17540 | else |
| | 17541 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| | 17542 | Immediate.u(expanded_control), |
| | 17543 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| | 17544 | .{ .p_w, .blend }, |
| | 17545 | registerAlias(dst_reg, dst_abi_size), |
| | 17546 | try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17547 | Immediate.u(expanded_control), |
| | 17548 | ) else try self.asmRegisterRegisterImmediate( |
| | 17549 | .{ .p_w, .blend }, |
| | 17550 | registerAlias(dst_reg, dst_abi_size), |
| | 17551 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17552 | rhs_mcv.getReg().? |
| | 17553 | else |
| | 17554 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| | 17555 | Immediate.u(expanded_control), |
| | 17556 | ); |
| | 17557 | break :result .{ .register = dst_reg }; |
| | 17558 | } |
| | 17559 | |
| | 17560 | const expanded_control = switch (elem_abi_size) { |
| | 17561 | 4, 8 => control, |
| | 17562 | 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) | |
| | 17563 | @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00), |
| | 17564 | else => unreachable, |
| | 17565 | }; |
| | 17566 | |
| | 17567 | const lhs_mcv = try self.resolveInst(extra.a); |
| | 17568 | const rhs_mcv = try self.resolveInst(extra.b); |
| | 17569 | |
| | 17570 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and |
| | 17571 | self.reuseOperand(inst, extra.a, 0, lhs_mcv)) |
| | 17572 | lhs_mcv |
| | 17573 | else if (has_avx and lhs_mcv.isRegister()) |
| | 17574 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } |
| | 17575 | else |
| | 17576 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); |
| | 17577 | const dst_reg = dst_mcv.getReg().?; |
| | 17578 | |
| | 17579 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| | 17580 | switch (elem_abi_size) { |
| | 17581 | 4 => .{ .v_ps, .blend }, |
| | 17582 | 8, 16 => .{ .v_pd, .blend }, |
| | 17583 | else => unreachable, |
| | 17584 | }, |
| | 17585 | registerAlias(dst_reg, dst_abi_size), |
| | 17586 | registerAlias(if (lhs_mcv.isRegister()) |
| | 17587 | lhs_mcv.getReg().? |
| | 17588 | else |
| | 17589 | dst_reg, dst_abi_size), |
| | 17590 | try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17591 | Immediate.u(expanded_control), |
| | 17592 | ) else try self.asmRegisterRegisterRegisterImmediate( |
| | 17593 | switch (elem_abi_size) { |
| | 17594 | 4 => .{ .v_ps, .blend }, |
| | 17595 | 8, 16 => .{ .v_pd, .blend }, |
| | 17596 | else => unreachable, |
| | 17597 | }, |
| | 17598 | registerAlias(dst_reg, dst_abi_size), |
| | 17599 | registerAlias(if (lhs_mcv.isRegister()) |
| | 17600 | lhs_mcv.getReg().? |
| | 17601 | else |
| | 17602 | dst_reg, dst_abi_size), |
| | 17603 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17604 | rhs_mcv.getReg().? |
| | 17605 | else |
| | 17606 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| | 17607 | Immediate.u(expanded_control), |
| | 17608 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| | 17609 | switch (elem_abi_size) { |
| | 17610 | 4 => .{ ._ps, .blend }, |
| | 17611 | 8, 16 => .{ ._pd, .blend }, |
| | 17612 | else => unreachable, |
| | 17613 | }, |
| | 17614 | registerAlias(dst_reg, dst_abi_size), |
| | 17615 | try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17616 | Immediate.u(expanded_control), |
| | 17617 | ) else try self.asmRegisterRegisterImmediate( |
| | 17618 | switch (elem_abi_size) { |
| | 17619 | 4 => .{ ._ps, .blend }, |
| | 17620 | 8, 16 => .{ ._pd, .blend }, |
| | 17621 | else => unreachable, |
| | 17622 | }, |
| | 17623 | registerAlias(dst_reg, dst_abi_size), |
| | 17624 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17625 | rhs_mcv.getReg().? |
| | 17626 | else |
| | 17627 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| | 17628 | Immediate.u(expanded_control), |
| | 17629 | ); |
| | 17630 | break :result .{ .register = dst_reg }; |
| | 17631 | } |
| | 17632 | |
| | 17633 | blendv: { |
| | 17634 | if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4) |
| | 17635 | has_avx |
| | 17636 | else |
| | 17637 | self.hasFeature(.avx2)) 32 else 16)) break :blendv; |
| | 17638 | |
| | 17639 | const select_mask_elem_ty = try mod.intType(.unsigned, elem_abi_size * 8); |
| | 17640 | const select_mask_ty = try mod.vectorType(.{ |
| | 17641 | .len = @intCast(mask_elems.len), |
| | 17642 | .child = select_mask_elem_ty.toIntern(), |
| | 17643 | }); |
| | 17644 | var select_mask_elems: [32]InternPool.Index = undefined; |
| | 17645 | for ( |
| | 17646 | select_mask_elems[0..mask_elems.len], |
| | 17647 | mask_elems, |
| | 17648 | 0.., |
| | 17649 | ) |*select_mask_elem, maybe_mask_elem, elem_index| { |
| | 17650 | const mask_elem = maybe_mask_elem orelse continue; |
| | 17651 | const mask_elem_index = |
| | 17652 | math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv; |
| | 17653 | if (mask_elem_index != elem_index) break :blendv; |
| | 17654 | |
| | 17655 | select_mask_elem.* = (if (mask_elem < 0) |
| | 17656 | try select_mask_elem_ty.maxIntScalar(mod, select_mask_elem_ty) |
| | 17657 | else |
| | 17658 | try select_mask_elem_ty.minIntScalar(mod, select_mask_elem_ty)).toIntern(); |
| | 17659 | } |
| | 17660 | const select_mask_mcv = try self.genTypedValue(.{ |
| | 17661 | .ty = select_mask_ty, |
| | 17662 | .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{ |
| | 17663 | .ty = select_mask_ty.toIntern(), |
| | 17664 | .storage = .{ .elems = select_mask_elems[0..mask_elems.len] }, |
| | 17665 | } })), |
| | 17666 | }); |
| | 17667 | |
| | 17668 | if (self.hasFeature(.sse4_1)) { |
| | 17669 | const mir_tag: Mir.Inst.FixedTag = .{ |
| | 17670 | if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or |
| | 17671 | (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) { |
| | 17672 | 4 => if (has_avx) .v_ps else ._ps, |
| | 17673 | 8 => if (has_avx) .v_pd else ._pd, |
| | 17674 | else => unreachable, |
| | 17675 | } else if (has_avx) .vp_b else .p_b, |
| | 17676 | .blendv, |
| | 17677 | }; |
| | 17678 | |
| | 17679 | const select_mask_reg = if (!has_avx) reg: { |
| | 17680 | try self.register_manager.getKnownReg(.xmm0, null); |
| | 17681 | try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{}); |
| | 17682 | break :reg .xmm0; |
| | 17683 | } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); |
| | 17684 | const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size); |
| | 17685 | const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg); |
| | 17686 | defer self.register_manager.unlockReg(select_mask_lock); |
| | 17687 | |
| | 17688 | const lhs_mcv = try self.resolveInst(extra.a); |
| | 17689 | const rhs_mcv = try self.resolveInst(extra.b); |
| | 17690 | |
| | 17691 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and |
| | 17692 | self.reuseOperand(inst, extra.a, 0, lhs_mcv)) |
| | 17693 | lhs_mcv |
| | 17694 | else if (has_avx and lhs_mcv.isRegister()) |
| | 17695 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } |
| | 17696 | else |
| | 17697 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); |
| | 17698 | const dst_reg = dst_mcv.getReg().?; |
| | 17699 | const dst_alias = registerAlias(dst_reg, dst_abi_size); |
| | 17700 | |
| | 17701 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister( |
| | 17702 | mir_tag, |
| | 17703 | dst_alias, |
| | 17704 | if (lhs_mcv.isRegister()) |
| | 17705 | registerAlias(lhs_mcv.getReg().?, dst_abi_size) |
| | 17706 | else |
| | 17707 | dst_alias, |
| | 17708 | try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17709 | select_mask_alias, |
| | 17710 | ) else try self.asmRegisterRegisterRegisterRegister( |
| | 17711 | mir_tag, |
| | 17712 | dst_alias, |
| | 17713 | if (lhs_mcv.isRegister()) |
| | 17714 | registerAlias(lhs_mcv.getReg().?, dst_abi_size) |
| | 17715 | else |
| | 17716 | dst_alias, |
| | 17717 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17718 | rhs_mcv.getReg().? |
| | 17719 | else |
| | 17720 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| | 17721 | select_mask_alias, |
| | 17722 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister( |
| | 17723 | mir_tag, |
| | 17724 | dst_alias, |
| | 17725 | try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17726 | select_mask_alias, |
| | 17727 | ) else try self.asmRegisterRegisterRegister( |
| | 17728 | mir_tag, |
| | 17729 | dst_alias, |
| | 17730 | registerAlias(if (rhs_mcv.isRegister()) |
| | 17731 | rhs_mcv.getReg().? |
| | 17732 | else |
| | 17733 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| | 17734 | select_mask_alias, |
| | 17735 | ); |
| | 17736 | break :result dst_mcv; |
| | 17737 | } |
| | 17738 | |
| | 17739 | const lhs_mcv = try self.resolveInst(extra.a); |
| | 17740 | const rhs_mcv = try self.resolveInst(extra.b); |
| | 17741 | |
| | 17742 | const dst_mcv: MCValue = if (rhs_mcv.isRegister() and |
| | 17743 | self.reuseOperand(inst, extra.b, 1, rhs_mcv)) |
| | 17744 | rhs_mcv |
| | 17745 | else |
| | 17746 | try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv); |
| | 17747 | const dst_reg = dst_mcv.getReg().?; |
| | 17748 | const dst_alias = registerAlias(dst_reg, dst_abi_size); |
| | 17749 | |
| | 17750 | const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); |
| | 17751 | const mask_alias = registerAlias(mask_reg, dst_abi_size); |
| | 17752 | const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); |
| | 17753 | defer self.register_manager.unlockReg(mask_lock); |
| | 17754 | |
| | 17755 | const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat()) |
| | 17756 | switch (elem_ty.floatBits(self.target.*)) { |
| | 17757 | 16, 80, 128 => .p_, |
| | 17758 | 32 => ._ps, |
| | 17759 | 64 => ._pd, |
| | 17760 | else => unreachable, |
| | 17761 | } |
| | 17762 | else |
| | 17763 | .p_; |
| | 17764 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); |
| | 17765 | if (lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| | 17766 | .{ mir_fixes, .andn }, |
| | 17767 | mask_alias, |
| | 17768 | try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), |
| | 17769 | ) else try self.asmRegisterRegister( |
| | 17770 | .{ mir_fixes, .andn }, |
| | 17771 | mask_alias, |
| | 17772 | if (lhs_mcv.isRegister()) |
| | 17773 | lhs_mcv.getReg().? |
| | 17774 | else |
| | 17775 | try self.copyToTmpRegister(dst_ty, lhs_mcv), |
| | 17776 | ); |
| | 17777 | try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias); |
| | 17778 | break :result dst_mcv; |
| | 17779 | } |
| | 17780 | |
| 17213 | pshufb: { | 17781 | pshufb: { |
| 17214 | if (max_abi_size > 16) break :pshufb; | 17782 | if (max_abi_size > 16) break :pshufb; |
| 17215 | if (!self.hasFeature(.ssse3)) break :pshufb; | 17783 | if (!self.hasFeature(.ssse3)) break :pshufb; |
| ... | @@ -17330,38 +17898,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -17330,38 +17898,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 17330 | break :result .{ .register = temp_regs[0] }; | 17898 | break :result .{ .register = temp_regs[0] }; |
| 17331 | } | 17899 | } |
| 17332 | | 17900 | |
| 17333 | if (max_abi_size <= 16) { | | |
| 17334 | const lhs_mcv = try self.resolveInst(extra.a); | | |
| 17335 | const lhs_reg = if (lhs_mcv.isRegister()) | | |
| 17336 | lhs_mcv.getReg().? | | |
| 17337 | else | | |
| 17338 | try self.copyToTmpRegister(lhs_ty, lhs_mcv); | | |
| 17339 | const lhs_lock = self.register_manager.lockRegAssumeUnused(lhs_reg); | | |
| 17340 | defer self.register_manager.unlockReg(lhs_lock); | | |
| 17341 | | | |
| 17342 | const rhs_mcv = try self.resolveInst(extra.b); | | |
| 17343 | const rhs_reg = if (rhs_mcv.isRegister()) | | |
| 17344 | rhs_mcv.getReg().? | | |
| 17345 | else | | |
| 17346 | try self.copyToTmpRegister(rhs_ty, rhs_mcv); | | |
| 17347 | const rhs_lock = self.register_manager.lockReg(rhs_reg); | | |
| 17348 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 17349 | | | |
| 17350 | //const dst_mcv = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); | | |
| 17351 | switch (elem_ty.zigTypeTag(mod)) { | | |
| 17352 | .Float => switch (elem_ty.floatBits(self.target.*)) { | | |
| 17353 | 16, 32 => {}, | | |
| 17354 | 64 => unreachable, // fully handled by shufpd | | |
| 17355 | 80, 128 => unreachable, // all possible masks already handled | | |
| 17356 | else => unreachable, | | |
| 17357 | }, | | |
| 17358 | else => {}, | | |
| 17359 | } | | |
| 17360 | } | | |
| 17361 | | | |
| 17362 | break :result null; | 17901 | break :result null; |
| 17363 | }) orelse return self.fail("TODO implement airShuffle from {} and {} to {}", .{ | 17902 | }) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{ |
| 17364 | lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod), | 17903 | lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod), |
| | 17904 | Value.fromInterned(extra.mask).fmtValue( |
| | 17905 | Type.fromInterned(mod.intern_pool.typeOf(extra.mask)), |
| | 17906 | mod, |
| | 17907 | ), |
| 17365 | }); | 17908 | }); |
| 17366 | return self.finishAir(inst, result, .{ extra.a, extra.b, .none }); | 17909 | return self.finishAir(inst, result, .{ extra.a, extra.b, .none }); |
| 17367 | } | 17910 | } |
Jacob Young <jacobly@ziglang.org> 2024-02-17 09:24:08+01:00
lib