| ... | ... | @@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory( |
| 1547 | 1547 | }); |
| 1548 | 1548 | } |
| 1549 | 1549 | |
| 1550 | fn asmRegisterRegisterMemoryRegister( |
| 1551 | self: *Self, |
| 1552 | tag: Mir.Inst.FixedTag, |
| 1553 | reg1: Register, |
| 1554 | reg2: Register, |
| 1555 | m: Memory, |
| 1556 | reg3: Register, |
| 1557 | ) !void { |
| 1558 | _ = try self.addInst(.{ |
| 1559 | .tag = tag[1], |
| 1560 | .ops = .rrmr, |
| 1561 | .data = .{ .rrrx = .{ |
| 1562 | .fixes = tag[0], |
| 1563 | .r1 = reg1, |
| 1564 | .r2 = reg2, |
| 1565 | .r3 = reg3, |
| 1566 | .payload = try self.addExtra(Mir.Memory.encode(m)), |
| 1567 | } }, |
| 1568 | }); |
| 1569 | } |
| 1570 | |
| 1550 | 1571 | fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void { |
| 1551 | 1572 | _ = try self.addInst(.{ |
| 1552 | 1573 | .tag = tag[1], |
| ... | ... | @@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo |
| 1570 | 1591 | }); |
| 1571 | 1592 | } |
| 1572 | 1593 | |
| 1594 | fn asmRegisterMemoryRegister( |
| 1595 | self: *Self, |
| 1596 | tag: Mir.Inst.FixedTag, |
| 1597 | reg1: Register, |
| 1598 | m: Memory, |
| 1599 | reg2: Register, |
| 1600 | ) !void { |
| 1601 | _ = try self.addInst(.{ |
| 1602 | .tag = tag[1], |
| 1603 | .ops = .rmr, |
| 1604 | .data = .{ .rrx = .{ |
| 1605 | .fixes = tag[0], |
| 1606 | .r1 = reg1, |
| 1607 | .r2 = reg2, |
| 1608 | .payload = try self.addExtra(Mir.Memory.encode(m)), |
| 1609 | } }, |
| 1610 | }); |
| 1611 | } |
| 1612 | |
| 1573 | 1613 | fn asmRegisterMemoryImmediate( |
| 1574 | 1614 | self: *Self, |
| 1575 | 1615 | tag: Mir.Inst.FixedTag, |
| ... | ... | @@ -10820,96 +10860,35 @@ fn genBinOp( |
| 10820 | 10860 | lhs_copy_reg.?, |
| 10821 | 10861 | mask_reg, |
| 10822 | 10862 | ) else { |
| 10823 | | try self.asmRegisterRegister( |
| 10824 | | @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) { |
| 10825 | | .Float => switch (lhs_ty.floatBits(self.target.*)) { |
| 10826 | | 32 => .{ ._ps, .@"and" }, |
| 10827 | | 64 => .{ ._pd, .@"and" }, |
| 10828 | | 16, 80, 128 => null, |
| 10829 | | else => unreachable, |
| 10830 | | }, |
| 10831 | | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { |
| 10832 | | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { |
| 10833 | | 32 => switch (lhs_ty.vectorLen(mod)) { |
| 10834 | | 1...4 => .{ ._ps, .@"and" }, |
| 10835 | | else => null, |
| 10836 | | }, |
| 10837 | | 64 => switch (lhs_ty.vectorLen(mod)) { |
| 10838 | | 1...2 => .{ ._pd, .@"and" }, |
| 10839 | | else => null, |
| 10840 | | }, |
| 10841 | | 16, 80, 128 => null, |
| 10842 | | else => unreachable, |
| 10843 | | }, |
| 10844 | | else => unreachable, |
| 10845 | | }, |
| 10863 | const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(mod)) { |
| 10864 | .Float => switch (lhs_ty.floatBits(self.target.*)) { |
| 10865 | 32 => ._ps, |
| 10866 | 64 => ._pd, |
| 10867 | 16, 80, 128 => null, |
| 10846 | 10868 | else => unreachable, |
| 10847 | | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ |
| 10848 | | @tagName(air_tag), lhs_ty.fmt(mod), |
| 10849 | | }), |
| 10850 | | dst_reg, |
| 10851 | | mask_reg, |
| 10852 | | ); |
| 10853 | | try self.asmRegisterRegister( |
| 10854 | | @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) { |
| 10855 | | .Float => switch (lhs_ty.floatBits(self.target.*)) { |
| 10856 | | 32 => .{ ._ps, .andn }, |
| 10857 | | 64 => .{ ._pd, .andn }, |
| 10858 | | 16, 80, 128 => null, |
| 10859 | | else => unreachable, |
| 10860 | | }, |
| 10861 | | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { |
| 10862 | | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { |
| 10863 | | 32 => switch (lhs_ty.vectorLen(mod)) { |
| 10864 | | 1...4 => .{ ._ps, .andn }, |
| 10865 | | else => null, |
| 10866 | | }, |
| 10867 | | 64 => switch (lhs_ty.vectorLen(mod)) { |
| 10868 | | 1...2 => .{ ._pd, .andn }, |
| 10869 | | else => null, |
| 10870 | | }, |
| 10871 | | 16, 80, 128 => null, |
| 10872 | | else => unreachable, |
| 10869 | }, |
| 10870 | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { |
| 10871 | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { |
| 10872 | 32 => switch (lhs_ty.vectorLen(mod)) { |
| 10873 | 1...4 => ._ps, |
| 10874 | else => null, |
| 10873 | 10875 | }, |
| 10874 | | else => unreachable, |
| 10875 | | }, |
| 10876 | | else => unreachable, |
| 10877 | | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ |
| 10878 | | @tagName(air_tag), lhs_ty.fmt(mod), |
| 10879 | | }), |
| 10880 | | mask_reg, |
| 10881 | | lhs_copy_reg.?, |
| 10882 | | ); |
| 10883 | | try self.asmRegisterRegister( |
| 10884 | | @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) { |
| 10885 | | .Float => switch (lhs_ty.floatBits(self.target.*)) { |
| 10886 | | 32 => .{ ._ps, .@"or" }, |
| 10887 | | 64 => .{ ._pd, .@"or" }, |
| 10888 | | 16, 80, 128 => null, |
| 10889 | | else => unreachable, |
| 10890 | | }, |
| 10891 | | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { |
| 10892 | | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { |
| 10893 | | 32 => switch (lhs_ty.vectorLen(mod)) { |
| 10894 | | 1...4 => .{ ._ps, .@"or" }, |
| 10895 | | else => null, |
| 10896 | | }, |
| 10897 | | 64 => switch (lhs_ty.vectorLen(mod)) { |
| 10898 | | 1...2 => .{ ._pd, .@"or" }, |
| 10899 | | else => null, |
| 10900 | | }, |
| 10901 | | 16, 80, 128 => null, |
| 10902 | | else => unreachable, |
| 10876 | 64 => switch (lhs_ty.vectorLen(mod)) { |
| 10877 | 1...2 => ._pd, |
| 10878 | else => null, |
| 10903 | 10879 | }, |
| 10880 | 16, 80, 128 => null, |
| 10904 | 10881 | else => unreachable, |
| 10905 | 10882 | }, |
| 10906 | 10883 | else => unreachable, |
| 10907 | | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ |
| 10908 | | @tagName(air_tag), lhs_ty.fmt(mod), |
| 10909 | | }), |
| 10910 | | dst_reg, |
| 10911 | | mask_reg, |
| 10912 | | ); |
| 10884 | }, |
| 10885 | else => unreachable, |
| 10886 | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ |
| 10887 | @tagName(air_tag), lhs_ty.fmt(mod), |
| 10888 | }); |
| 10889 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg); |
| 10890 | try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?); |
| 10891 | try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg); |
| 10913 | 10892 | } |
| 10914 | 10893 | }, |
| 10915 | 10894 | .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => { |
| ... | ... | @@ -16353,7 +16332,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { |
| 16353 | 16332 | .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl }, |
| 16354 | 16333 | dst_alias, |
| 16355 | 16334 | dst_alias, |
| 16356 | | Immediate.u(0), |
| 16335 | Immediate.u(0b00_00_00_00), |
| 16357 | 16336 | ); |
| 16358 | 16337 | if (switch (scalar_bits) { |
| 16359 | 16338 | 1...8 => vector_len > 4, |
| ... | ... | @@ -16564,11 +16543,341 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { |
| 16564 | 16543 | } |
| 16565 | 16544 | |
| 16566 | 16545 | fn airSelect(self: *Self, inst: Air.Inst.Index) !void { |
| 16546 | const mod = self.bin_file.comp.module.?; |
| 16567 | 16547 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 16568 | 16548 | const extra = self.air.extraData(Air.Bin, pl_op.payload).data; |
| 16569 | | _ = extra; |
| 16570 | | return self.fail("TODO implement airSelect for x86_64", .{}); |
| 16571 | | //return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); |
| 16549 | const ty = self.typeOfIndex(inst); |
| 16550 | const vec_len = ty.vectorLen(mod); |
| 16551 | const elem_ty = ty.childType(mod); |
| 16552 | const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod)); |
| 16553 | const abi_size = elem_abi_size * vec_len; |
| 16554 | const pred_ty = self.typeOf(pl_op.operand); |
| 16555 | |
| 16556 | const result = result: { |
| 16557 | const has_blend = self.hasFeature(.sse4_1); |
| 16558 | const has_avx = self.hasFeature(.avx); |
| 16559 | const need_xmm0 = has_blend and !has_avx; |
| 16560 | const pred_mcv = try self.resolveInst(pl_op.operand); |
| 16561 | const mask_reg = mask: { |
| 16562 | switch (pred_mcv) { |
| 16563 | .register => |pred_reg| switch (pred_reg.class()) { |
| 16564 | .general_purpose => {}, |
| 16565 | .sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) { |
| 16566 | try self.register_manager.getReg(.xmm0, null); |
| 16567 | try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{}); |
| 16568 | break :mask .xmm0; |
| 16569 | } else break :mask if (has_blend) |
| 16570 | pred_reg |
| 16571 | else |
| 16572 | try self.copyToTmpRegister(pred_ty, pred_mcv), |
| 16573 | else => unreachable, |
| 16574 | }, |
| 16575 | else => {}, |
| 16576 | } |
| 16577 | const mask_reg: Register = if (need_xmm0) mask_reg: { |
| 16578 | try self.register_manager.getReg(.xmm0, null); |
| 16579 | break :mask_reg .xmm0; |
| 16580 | } else try self.register_manager.allocReg(null, abi.RegisterClass.sse); |
| 16581 | const mask_alias = registerAlias(mask_reg, abi_size); |
| 16582 | const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); |
| 16583 | defer self.register_manager.unlockReg(mask_lock); |
| 16584 | |
| 16585 | const pred_fits_in_elem = vec_len <= elem_abi_size; |
| 16586 | if (self.hasFeature(.avx2) and abi_size <= 32) { |
| 16587 | if (pred_mcv.isRegister()) broadcast: { |
| 16588 | try self.asmRegisterRegister( |
| 16589 | .{ .v_d, .mov }, |
| 16590 | mask_reg.to128(), |
| 16591 | pred_mcv.getReg().?.to32(), |
| 16592 | ); |
| 16593 | if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister( |
| 16594 | .{ switch (elem_abi_size) { |
| 16595 | 1 => .vp_b, |
| 16596 | 2 => .vp_w, |
| 16597 | 3...4 => .vp_d, |
| 16598 | 5...8 => .vp_q, |
| 16599 | 9...16 => { |
| 16600 | try self.asmRegisterRegisterRegisterImmediate( |
| 16601 | .{ .v_f128, .insert }, |
| 16602 | mask_alias, |
| 16603 | mask_alias, |
| 16604 | mask_reg.to128(), |
| 16605 | Immediate.u(1), |
| 16606 | ); |
| 16607 | break :broadcast; |
| 16608 | }, |
| 16609 | 17...32 => break :broadcast, |
| 16610 | else => unreachable, |
| 16611 | }, .broadcast }, |
| 16612 | mask_alias, |
| 16613 | mask_reg.to128(), |
| 16614 | ); |
| 16615 | } else try self.asmRegisterMemory( |
| 16616 | .{ switch (vec_len) { |
| 16617 | 1...8 => .vp_b, |
| 16618 | 9...16 => .vp_w, |
| 16619 | 17...32 => .vp_d, |
| 16620 | else => unreachable, |
| 16621 | }, .broadcast }, |
| 16622 | mask_alias, |
| 16623 | if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{ |
| 16624 | .base = .{ .reg = (try self.copyToTmpRegister( |
| 16625 | Type.usize, |
| 16626 | pred_mcv.address(), |
| 16627 | )).to64() }, |
| 16628 | .mod = .{ .rm = .{ .size = .byte } }, |
| 16629 | }, |
| 16630 | ); |
| 16631 | } else if (abi_size <= 16) broadcast: { |
| 16632 | try self.asmRegisterRegister( |
| 16633 | .{ if (has_avx) .v_d else ._d, .mov }, |
| 16634 | mask_alias, |
| 16635 | (if (pred_mcv.isRegister()) |
| 16636 | pred_mcv.getReg().? |
| 16637 | else |
| 16638 | try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(), |
| 16639 | ); |
| 16640 | if (!pred_fits_in_elem or vec_len == 1) break :broadcast; |
| 16641 | if (elem_abi_size <= 1) { |
| 16642 | if (has_avx) try self.asmRegisterRegisterRegister( |
| 16643 | .{ .vp_, .unpcklbw }, |
| 16644 | mask_alias, |
| 16645 | mask_alias, |
| 16646 | mask_alias, |
| 16647 | ) else try self.asmRegisterRegister( |
| 16648 | .{ .p_, .unpcklbw }, |
| 16649 | mask_alias, |
| 16650 | mask_alias, |
| 16651 | ); |
| 16652 | if (abi_size <= 2) break :broadcast; |
| 16653 | } |
| 16654 | if (elem_abi_size <= 2) { |
| 16655 | try self.asmRegisterRegisterImmediate( |
| 16656 | .{ if (has_avx) .vp_w else .p_w, .shufl }, |
| 16657 | mask_alias, |
| 16658 | mask_alias, |
| 16659 | Immediate.u(0b00_00_00_00), |
| 16660 | ); |
| 16661 | if (abi_size <= 8) break :broadcast; |
| 16662 | } |
| 16663 | try self.asmRegisterRegisterImmediate( |
| 16664 | .{ if (has_avx) .vp_d else .p_d, .shuf }, |
| 16665 | mask_alias, |
| 16666 | mask_alias, |
| 16667 | Immediate.u(switch (elem_abi_size) { |
| 16668 | 1...2, 5...8 => 0b01_00_01_00, |
| 16669 | 3...4 => 0b00_00_00_00, |
| 16670 | else => unreachable, |
| 16671 | }), |
| 16672 | ); |
| 16673 | } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| 16674 | const elem_bits: u16 = @intCast(elem_abi_size * 8); |
| 16675 | const mask_elem_ty = try mod.intType(.unsigned, elem_bits); |
| 16676 | const mask_ty = try mod.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() }); |
| 16677 | if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) { |
| 16678 | var mask_elems: [32]InternPool.Index = undefined; |
| 16679 | for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{ |
| 16680 | .ty = mask_elem_ty.toIntern(), |
| 16681 | .storage = .{ .u64 = bit / elem_bits }, |
| 16682 | } }); |
| 16683 | const mask_mcv = try self.genTypedValue(.{ |
| 16684 | .ty = mask_ty, |
| 16685 | .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{ |
| 16686 | .ty = mask_ty.toIntern(), |
| 16687 | .storage = .{ .elems = mask_elems[0..vec_len] }, |
| 16688 | } })), |
| 16689 | }); |
| 16690 | const mask_mem: Memory = .{ |
| 16691 | .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) }, |
| 16692 | .mod = .{ .rm = .{ .size = self.memSize(ty) } }, |
| 16693 | }; |
| 16694 | if (has_avx) try self.asmRegisterRegisterMemory( |
| 16695 | .{ .vp_b, .shuf }, |
| 16696 | mask_alias, |
| 16697 | mask_alias, |
| 16698 | mask_mem, |
| 16699 | ) else try self.asmRegisterMemory( |
| 16700 | .{ .p_b, .shuf }, |
| 16701 | mask_alias, |
| 16702 | mask_mem, |
| 16703 | ); |
| 16704 | } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| 16705 | { |
| 16706 | var mask_elems: [32]InternPool.Index = undefined; |
| 16707 | for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{ |
| 16708 | .ty = mask_elem_ty.toIntern(), |
| 16709 | .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) }, |
| 16710 | } }); |
| 16711 | const mask_mcv = try self.genTypedValue(.{ |
| 16712 | .ty = mask_ty, |
| 16713 | .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{ |
| 16714 | .ty = mask_ty.toIntern(), |
| 16715 | .storage = .{ .elems = mask_elems[0..vec_len] }, |
| 16716 | } })), |
| 16717 | }); |
| 16718 | const mask_mem: Memory = .{ |
| 16719 | .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) }, |
| 16720 | .mod = .{ .rm = .{ .size = self.memSize(ty) } }, |
| 16721 | }; |
| 16722 | if (has_avx) { |
| 16723 | try self.asmRegisterRegisterMemory( |
| 16724 | .{ .vp_, .@"and" }, |
| 16725 | mask_alias, |
| 16726 | mask_alias, |
| 16727 | mask_mem, |
| 16728 | ); |
| 16729 | try self.asmRegisterRegisterMemory( |
| 16730 | .{ .vp_d, .cmpeq }, |
| 16731 | mask_alias, |
| 16732 | mask_alias, |
| 16733 | mask_mem, |
| 16734 | ); |
| 16735 | } else { |
| 16736 | try self.asmRegisterMemory( |
| 16737 | .{ .p_, .@"and" }, |
| 16738 | mask_alias, |
| 16739 | mask_mem, |
| 16740 | ); |
| 16741 | try self.asmRegisterMemory( |
| 16742 | .{ .p_d, .cmpeq }, |
| 16743 | mask_alias, |
| 16744 | mask_mem, |
| 16745 | ); |
| 16746 | } |
| 16747 | } |
| 16748 | break :mask mask_reg; |
| 16749 | }; |
| 16750 | const mask_alias = registerAlias(mask_reg, abi_size); |
| 16751 | const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); |
| 16752 | defer self.register_manager.unlockReg(mask_lock); |
| 16753 | |
| 16754 | const lhs_mcv = try self.resolveInst(extra.lhs); |
| 16755 | const lhs_lock = switch (lhs_mcv) { |
| 16756 | .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg), |
| 16757 | else => null, |
| 16758 | }; |
| 16759 | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 16760 | |
| 16761 | const rhs_mcv = try self.resolveInst(extra.rhs); |
| 16762 | const rhs_lock = switch (rhs_mcv) { |
| 16763 | .register => |rhs_reg| self.register_manager.lockReg(rhs_reg), |
| 16764 | else => null, |
| 16765 | }; |
| 16766 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 16767 | |
| 16768 | const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv; |
| 16769 | const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand( |
| 16770 | inst, |
| 16771 | if (has_blend) extra.rhs else extra.lhs, |
| 16772 | @intFromBool(has_blend), |
| 16773 | reuse_mcv, |
| 16774 | )) reuse_mcv else if (has_avx) |
| 16775 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } |
| 16776 | else |
| 16777 | try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv); |
| 16778 | const dst_reg = dst_mcv.getReg().?; |
| 16779 | const dst_alias = registerAlias(dst_reg, abi_size); |
| 16780 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 16781 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 16782 | |
| 16783 | const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.childType(mod).zigTypeTag(mod)) { |
| 16784 | else => null, |
| 16785 | .Int => switch (abi_size) { |
| 16786 | 0 => unreachable, |
| 16787 | 1...16 => if (has_avx) |
| 16788 | .{ .vp_b, .blendv } |
| 16789 | else if (has_blend) |
| 16790 | .{ .p_b, .blendv } |
| 16791 | else |
| 16792 | .{ .p_, undefined }, |
| 16793 | 17...32 => if (self.hasFeature(.avx2)) |
| 16794 | .{ .vp_b, .blendv } |
| 16795 | else |
| 16796 | null, |
| 16797 | else => null, |
| 16798 | }, |
| 16799 | .Float => switch (ty.childType(mod).floatBits(self.target.*)) { |
| 16800 | else => unreachable, |
| 16801 | 16, 80, 128 => null, |
| 16802 | 32 => switch (vec_len) { |
| 16803 | 0 => unreachable, |
| 16804 | 1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv }, |
| 16805 | 5...8 => if (has_avx) .{ .v_ps, .blendv } else null, |
| 16806 | else => null, |
| 16807 | }, |
| 16808 | 64 => switch (vec_len) { |
| 16809 | 0 => unreachable, |
| 16810 | 1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv }, |
| 16811 | 3...4 => if (has_avx) .{ .v_pd, .blendv } else null, |
| 16812 | else => null, |
| 16813 | }, |
| 16814 | }, |
| 16815 | }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| 16816 | if (has_avx) { |
| 16817 | const rhs_alias = if (rhs_mcv.isRegister()) |
| 16818 | registerAlias(rhs_mcv.getReg().?, abi_size) |
| 16819 | else rhs: { |
| 16820 | try self.genSetReg(dst_reg, ty, rhs_mcv, .{}); |
| 16821 | break :rhs dst_alias; |
| 16822 | }; |
| 16823 | if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister( |
| 16824 | mir_tag, |
| 16825 | dst_alias, |
| 16826 | rhs_alias, |
| 16827 | try lhs_mcv.mem(self, self.memSize(ty)), |
| 16828 | mask_alias, |
| 16829 | ) else try self.asmRegisterRegisterRegisterRegister( |
| 16830 | mir_tag, |
| 16831 | dst_alias, |
| 16832 | rhs_alias, |
| 16833 | registerAlias(if (lhs_mcv.isRegister()) |
| 16834 | lhs_mcv.getReg().? |
| 16835 | else |
| 16836 | try self.copyToTmpRegister(ty, lhs_mcv), abi_size), |
| 16837 | mask_alias, |
| 16838 | ); |
| 16839 | } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister( |
| 16840 | mir_tag, |
| 16841 | dst_alias, |
| 16842 | try lhs_mcv.mem(self, self.memSize(ty)), |
| 16843 | mask_alias, |
| 16844 | ) else try self.asmRegisterRegisterRegister( |
| 16845 | mir_tag, |
| 16846 | dst_alias, |
| 16847 | registerAlias(if (lhs_mcv.isRegister()) |
| 16848 | lhs_mcv.getReg().? |
| 16849 | else |
| 16850 | try self.copyToTmpRegister(ty, lhs_mcv), abi_size), |
| 16851 | mask_alias, |
| 16852 | ) else { |
| 16853 | const mir_fixes = @as(?Mir.Inst.Fixes, switch (elem_ty.zigTypeTag(mod)) { |
| 16854 | else => null, |
| 16855 | .Int => .p_, |
| 16856 | .Float => switch (elem_ty.floatBits(self.target.*)) { |
| 16857 | 32 => ._ps, |
| 16858 | 64 => ._pd, |
| 16859 | 16, 80, 128 => null, |
| 16860 | else => unreachable, |
| 16861 | }, |
| 16862 | }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| 16863 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); |
| 16864 | if (rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 16865 | .{ mir_fixes, .andn }, |
| 16866 | mask_alias, |
| 16867 | try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)), |
| 16868 | ) else try self.asmRegisterRegister( |
| 16869 | .{ mir_fixes, .andn }, |
| 16870 | mask_alias, |
| 16871 | if (rhs_mcv.isRegister()) |
| 16872 | rhs_mcv.getReg().? |
| 16873 | else |
| 16874 | try self.copyToTmpRegister(ty, rhs_mcv), |
| 16875 | ); |
| 16876 | try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias); |
| 16877 | } |
| 16878 | break :result dst_mcv; |
| 16879 | }; |
| 16880 | return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); |
| 16572 | 16881 | } |
| 16573 | 16882 | |
| 16574 | 16883 | fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
Jacob Young <jacobly@ziglang.org> 2024-02-13 08:12:56+01:00
src