| ... | @@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory( | ... | @@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory( |
| 1547 | }); | 1547 | }); |
| 1548 | } | 1548 | } |
| 1549 | | 1549 | |
| | 1550 | fn asmRegisterRegisterMemoryRegister( |
| | 1551 | self: *Self, |
| | 1552 | tag: Mir.Inst.FixedTag, |
| | 1553 | reg1: Register, |
| | 1554 | reg2: Register, |
| | 1555 | m: Memory, |
| | 1556 | reg3: Register, |
| | 1557 | ) !void { |
| | 1558 | _ = try self.addInst(.{ |
| | 1559 | .tag = tag[1], |
| | 1560 | .ops = .rrmr, |
| | 1561 | .data = .{ .rrrx = .{ |
| | 1562 | .fixes = tag[0], |
| | 1563 | .r1 = reg1, |
| | 1564 | .r2 = reg2, |
| | 1565 | .r3 = reg3, |
| | 1566 | .payload = try self.addExtra(Mir.Memory.encode(m)), |
| | 1567 | } }, |
| | 1568 | }); |
| | 1569 | } |
| | 1570 | |
| 1550 | fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void { | 1571 | fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void { |
| 1551 | _ = try self.addInst(.{ | 1572 | _ = try self.addInst(.{ |
| 1552 | .tag = tag[1], | 1573 | .tag = tag[1], |
| ... | @@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo | ... | @@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo |
| 1570 | }); | 1591 | }); |
| 1571 | } | 1592 | } |
| 1572 | | 1593 | |
| | 1594 | fn asmRegisterMemoryRegister( |
| | 1595 | self: *Self, |
| | 1596 | tag: Mir.Inst.FixedTag, |
| | 1597 | reg1: Register, |
| | 1598 | m: Memory, |
| | 1599 | reg2: Register, |
| | 1600 | ) !void { |
| | 1601 | _ = try self.addInst(.{ |
| | 1602 | .tag = tag[1], |
| | 1603 | .ops = .rmr, |
| | 1604 | .data = .{ .rrx = .{ |
| | 1605 | .fixes = tag[0], |
| | 1606 | .r1 = reg1, |
| | 1607 | .r2 = reg2, |
| | 1608 | .payload = try self.addExtra(Mir.Memory.encode(m)), |
| | 1609 | } }, |
| | 1610 | }); |
| | 1611 | } |
| | 1612 | |
| 1573 | fn asmRegisterMemoryImmediate( | 1613 | fn asmRegisterMemoryImmediate( |
| 1574 | self: *Self, | 1614 | self: *Self, |
| 1575 | tag: Mir.Inst.FixedTag, | 1615 | tag: Mir.Inst.FixedTag, |
| ... | @@ -10820,96 +10860,35 @@ fn genBinOp( | ... | @@ -10820,96 +10860,35 @@ fn genBinOp( |
| 10820 | lhs_copy_reg.?, | 10860 | lhs_copy_reg.?, |
| 10821 | mask_reg, | 10861 | mask_reg, |
| 10822 | ) else { | 10862 | ) else { |
| 10823 | try self.asmRegisterRegister( | 10863 | const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(mod)) { |
| 10824 | @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) { | 10864 | .Float => switch (lhs_ty.floatBits(self.target.*)) { |
| 10825 | .Float => switch (lhs_ty.floatBits(self.target.*)) { | 10865 | 32 => ._ps, |
| 10826 | 32 => .{ ._ps, .@"and" }, | 10866 | 64 => ._pd, |
| 10827 | 64 => .{ ._pd, .@"and" }, | 10867 | 16, 80, 128 => null, |
| 10828 | 16, 80, 128 => null, | | |
| 10829 | else => unreachable, | | |
| 10830 | }, | | |
| 10831 | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { | | |
| 10832 | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { | | |
| 10833 | 32 => switch (lhs_ty.vectorLen(mod)) { | | |
| 10834 | 1...4 => .{ ._ps, .@"and" }, | | |
| 10835 | else => null, | | |
| 10836 | }, | | |
| 10837 | 64 => switch (lhs_ty.vectorLen(mod)) { | | |
| 10838 | 1...2 => .{ ._pd, .@"and" }, | | |
| 10839 | else => null, | | |
| 10840 | }, | | |
| 10841 | 16, 80, 128 => null, | | |
| 10842 | else => unreachable, | | |
| 10843 | }, | | |
| 10844 | else => unreachable, | | |
| 10845 | }, | | |
| 10846 | else => unreachable, | 10868 | else => unreachable, |
| 10847 | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ | 10869 | }, |
| 10848 | @tagName(air_tag), lhs_ty.fmt(mod), | 10870 | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { |
| 10849 | }), | 10871 | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { |
| 10850 | dst_reg, | 10872 | 32 => switch (lhs_ty.vectorLen(mod)) { |
| 10851 | mask_reg, | 10873 | 1...4 => ._ps, |
| 10852 | ); | 10874 | else => null, |
| 10853 | try self.asmRegisterRegister( | | |
| 10854 | @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) { | | |
| 10855 | .Float => switch (lhs_ty.floatBits(self.target.*)) { | | |
| 10856 | 32 => .{ ._ps, .andn }, | | |
| 10857 | 64 => .{ ._pd, .andn }, | | |
| 10858 | 16, 80, 128 => null, | | |
| 10859 | else => unreachable, | | |
| 10860 | }, | | |
| 10861 | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { | | |
| 10862 | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { | | |
| 10863 | 32 => switch (lhs_ty.vectorLen(mod)) { | | |
| 10864 | 1...4 => .{ ._ps, .andn }, | | |
| 10865 | else => null, | | |
| 10866 | }, | | |
| 10867 | 64 => switch (lhs_ty.vectorLen(mod)) { | | |
| 10868 | 1...2 => .{ ._pd, .andn }, | | |
| 10869 | else => null, | | |
| 10870 | }, | | |
| 10871 | 16, 80, 128 => null, | | |
| 10872 | else => unreachable, | | |
| 10873 | }, | 10875 | }, |
| 10874 | else => unreachable, | 10876 | 64 => switch (lhs_ty.vectorLen(mod)) { |
| 10875 | }, | 10877 | 1...2 => ._pd, |
| 10876 | else => unreachable, | 10878 | else => null, |
| 10877 | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ | | |
| 10878 | @tagName(air_tag), lhs_ty.fmt(mod), | | |
| 10879 | }), | | |
| 10880 | mask_reg, | | |
| 10881 | lhs_copy_reg.?, | | |
| 10882 | ); | | |
| 10883 | try self.asmRegisterRegister( | | |
| 10884 | @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) { | | |
| 10885 | .Float => switch (lhs_ty.floatBits(self.target.*)) { | | |
| 10886 | 32 => .{ ._ps, .@"or" }, | | |
| 10887 | 64 => .{ ._pd, .@"or" }, | | |
| 10888 | 16, 80, 128 => null, | | |
| 10889 | else => unreachable, | | |
| 10890 | }, | | |
| 10891 | .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) { | | |
| 10892 | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { | | |
| 10893 | 32 => switch (lhs_ty.vectorLen(mod)) { | | |
| 10894 | 1...4 => .{ ._ps, .@"or" }, | | |
| 10895 | else => null, | | |
| 10896 | }, | | |
| 10897 | 64 => switch (lhs_ty.vectorLen(mod)) { | | |
| 10898 | 1...2 => .{ ._pd, .@"or" }, | | |
| 10899 | else => null, | | |
| 10900 | }, | | |
| 10901 | 16, 80, 128 => null, | | |
| 10902 | else => unreachable, | | |
| 10903 | }, | 10879 | }, |
| | 10880 | 16, 80, 128 => null, |
| 10904 | else => unreachable, | 10881 | else => unreachable, |
| 10905 | }, | 10882 | }, |
| 10906 | else => unreachable, | 10883 | else => unreachable, |
| 10907 | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ | 10884 | }, |
| 10908 | @tagName(air_tag), lhs_ty.fmt(mod), | 10885 | else => unreachable, |
| 10909 | }), | 10886 | }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{ |
| 10910 | dst_reg, | 10887 | @tagName(air_tag), lhs_ty.fmt(mod), |
| 10911 | mask_reg, | 10888 | }); |
| 10912 | ); | 10889 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg); |
| | 10890 | try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?); |
| | 10891 | try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg); |
| 10913 | } | 10892 | } |
| 10914 | }, | 10893 | }, |
| 10915 | .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => { | 10894 | .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => { |
| ... | @@ -16353,7 +16332,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -16353,7 +16332,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { |
| 16353 | .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl }, | 16332 | .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl }, |
| 16354 | dst_alias, | 16333 | dst_alias, |
| 16355 | dst_alias, | 16334 | dst_alias, |
| 16356 | Immediate.u(0), | 16335 | Immediate.u(0b00_00_00_00), |
| 16357 | ); | 16336 | ); |
| 16358 | if (switch (scalar_bits) { | 16337 | if (switch (scalar_bits) { |
| 16359 | 1...8 => vector_len > 4, | 16338 | 1...8 => vector_len > 4, |
| ... | @@ -16564,11 +16543,341 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -16564,11 +16543,341 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { |
| 16564 | } | 16543 | } |
| 16565 | | 16544 | |
| 16566 | fn airSelect(self: *Self, inst: Air.Inst.Index) !void { | 16545 | fn airSelect(self: *Self, inst: Air.Inst.Index) !void { |
| | 16546 | const mod = self.bin_file.comp.module.?; |
| 16567 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | 16547 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 16568 | const extra = self.air.extraData(Air.Bin, pl_op.payload).data; | 16548 | const extra = self.air.extraData(Air.Bin, pl_op.payload).data; |
| 16569 | _ = extra; | 16549 | const ty = self.typeOfIndex(inst); |
| 16570 | return self.fail("TODO implement airSelect for x86_64", .{}); | 16550 | const vec_len = ty.vectorLen(mod); |
| 16571 | //return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); | 16551 | const elem_ty = ty.childType(mod); |
| | 16552 | const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod)); |
| | 16553 | const abi_size = elem_abi_size * vec_len; |
| | 16554 | const pred_ty = self.typeOf(pl_op.operand); |
| | 16555 | |
| | 16556 | const result = result: { |
| | 16557 | const has_blend = self.hasFeature(.sse4_1); |
| | 16558 | const has_avx = self.hasFeature(.avx); |
| | 16559 | const need_xmm0 = has_blend and !has_avx; |
| | 16560 | const pred_mcv = try self.resolveInst(pl_op.operand); |
| | 16561 | const mask_reg = mask: { |
| | 16562 | switch (pred_mcv) { |
| | 16563 | .register => |pred_reg| switch (pred_reg.class()) { |
| | 16564 | .general_purpose => {}, |
| | 16565 | .sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) { |
| | 16566 | try self.register_manager.getReg(.xmm0, null); |
| | 16567 | try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{}); |
| | 16568 | break :mask .xmm0; |
| | 16569 | } else break :mask if (has_blend) |
| | 16570 | pred_reg |
| | 16571 | else |
| | 16572 | try self.copyToTmpRegister(pred_ty, pred_mcv), |
| | 16573 | else => unreachable, |
| | 16574 | }, |
| | 16575 | else => {}, |
| | 16576 | } |
| | 16577 | const mask_reg: Register = if (need_xmm0) mask_reg: { |
| | 16578 | try self.register_manager.getReg(.xmm0, null); |
| | 16579 | break :mask_reg .xmm0; |
| | 16580 | } else try self.register_manager.allocReg(null, abi.RegisterClass.sse); |
| | 16581 | const mask_alias = registerAlias(mask_reg, abi_size); |
| | 16582 | const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); |
| | 16583 | defer self.register_manager.unlockReg(mask_lock); |
| | 16584 | |
| | 16585 | const pred_fits_in_elem = vec_len <= elem_abi_size; |
| | 16586 | if (self.hasFeature(.avx2) and abi_size <= 32) { |
| | 16587 | if (pred_mcv.isRegister()) broadcast: { |
| | 16588 | try self.asmRegisterRegister( |
| | 16589 | .{ .v_d, .mov }, |
| | 16590 | mask_reg.to128(), |
| | 16591 | pred_mcv.getReg().?.to32(), |
| | 16592 | ); |
| | 16593 | if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister( |
| | 16594 | .{ switch (elem_abi_size) { |
| | 16595 | 1 => .vp_b, |
| | 16596 | 2 => .vp_w, |
| | 16597 | 3...4 => .vp_d, |
| | 16598 | 5...8 => .vp_q, |
| | 16599 | 9...16 => { |
| | 16600 | try self.asmRegisterRegisterRegisterImmediate( |
| | 16601 | .{ .v_f128, .insert }, |
| | 16602 | mask_alias, |
| | 16603 | mask_alias, |
| | 16604 | mask_reg.to128(), |
| | 16605 | Immediate.u(1), |
| | 16606 | ); |
| | 16607 | break :broadcast; |
| | 16608 | }, |
| | 16609 | 17...32 => break :broadcast, |
| | 16610 | else => unreachable, |
| | 16611 | }, .broadcast }, |
| | 16612 | mask_alias, |
| | 16613 | mask_reg.to128(), |
| | 16614 | ); |
| | 16615 | } else try self.asmRegisterMemory( |
| | 16616 | .{ switch (vec_len) { |
| | 16617 | 1...8 => .vp_b, |
| | 16618 | 9...16 => .vp_w, |
| | 16619 | 17...32 => .vp_d, |
| | 16620 | else => unreachable, |
| | 16621 | }, .broadcast }, |
| | 16622 | mask_alias, |
| | 16623 | if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{ |
| | 16624 | .base = .{ .reg = (try self.copyToTmpRegister( |
| | 16625 | Type.usize, |
| | 16626 | pred_mcv.address(), |
| | 16627 | )).to64() }, |
| | 16628 | .mod = .{ .rm = .{ .size = .byte } }, |
| | 16629 | }, |
| | 16630 | ); |
| | 16631 | } else if (abi_size <= 16) broadcast: { |
| | 16632 | try self.asmRegisterRegister( |
| | 16633 | .{ if (has_avx) .v_d else ._d, .mov }, |
| | 16634 | mask_alias, |
| | 16635 | (if (pred_mcv.isRegister()) |
| | 16636 | pred_mcv.getReg().? |
| | 16637 | else |
| | 16638 | try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(), |
| | 16639 | ); |
| | 16640 | if (!pred_fits_in_elem or vec_len == 1) break :broadcast; |
| | 16641 | if (elem_abi_size <= 1) { |
| | 16642 | if (has_avx) try self.asmRegisterRegisterRegister( |
| | 16643 | .{ .vp_, .unpcklbw }, |
| | 16644 | mask_alias, |
| | 16645 | mask_alias, |
| | 16646 | mask_alias, |
| | 16647 | ) else try self.asmRegisterRegister( |
| | 16648 | .{ .p_, .unpcklbw }, |
| | 16649 | mask_alias, |
| | 16650 | mask_alias, |
| | 16651 | ); |
| | 16652 | if (abi_size <= 2) break :broadcast; |
| | 16653 | } |
| | 16654 | if (elem_abi_size <= 2) { |
| | 16655 | try self.asmRegisterRegisterImmediate( |
| | 16656 | .{ if (has_avx) .vp_w else .p_w, .shufl }, |
| | 16657 | mask_alias, |
| | 16658 | mask_alias, |
| | 16659 | Immediate.u(0b00_00_00_00), |
| | 16660 | ); |
| | 16661 | if (abi_size <= 8) break :broadcast; |
| | 16662 | } |
| | 16663 | try self.asmRegisterRegisterImmediate( |
| | 16664 | .{ if (has_avx) .vp_d else .p_d, .shuf }, |
| | 16665 | mask_alias, |
| | 16666 | mask_alias, |
| | 16667 | Immediate.u(switch (elem_abi_size) { |
| | 16668 | 1...2, 5...8 => 0b01_00_01_00, |
| | 16669 | 3...4 => 0b00_00_00_00, |
| | 16670 | else => unreachable, |
| | 16671 | }), |
| | 16672 | ); |
| | 16673 | } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| | 16674 | const elem_bits: u16 = @intCast(elem_abi_size * 8); |
| | 16675 | const mask_elem_ty = try mod.intType(.unsigned, elem_bits); |
| | 16676 | const mask_ty = try mod.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() }); |
| | 16677 | if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) { |
| | 16678 | var mask_elems: [32]InternPool.Index = undefined; |
| | 16679 | for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{ |
| | 16680 | .ty = mask_elem_ty.toIntern(), |
| | 16681 | .storage = .{ .u64 = bit / elem_bits }, |
| | 16682 | } }); |
| | 16683 | const mask_mcv = try self.genTypedValue(.{ |
| | 16684 | .ty = mask_ty, |
| | 16685 | .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{ |
| | 16686 | .ty = mask_ty.toIntern(), |
| | 16687 | .storage = .{ .elems = mask_elems[0..vec_len] }, |
| | 16688 | } })), |
| | 16689 | }); |
| | 16690 | const mask_mem: Memory = .{ |
| | 16691 | .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) }, |
| | 16692 | .mod = .{ .rm = .{ .size = self.memSize(ty) } }, |
| | 16693 | }; |
| | 16694 | if (has_avx) try self.asmRegisterRegisterMemory( |
| | 16695 | .{ .vp_b, .shuf }, |
| | 16696 | mask_alias, |
| | 16697 | mask_alias, |
| | 16698 | mask_mem, |
| | 16699 | ) else try self.asmRegisterMemory( |
| | 16700 | .{ .p_b, .shuf }, |
| | 16701 | mask_alias, |
| | 16702 | mask_mem, |
| | 16703 | ); |
| | 16704 | } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| | 16705 | { |
| | 16706 | var mask_elems: [32]InternPool.Index = undefined; |
| | 16707 | for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{ |
| | 16708 | .ty = mask_elem_ty.toIntern(), |
| | 16709 | .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) }, |
| | 16710 | } }); |
| | 16711 | const mask_mcv = try self.genTypedValue(.{ |
| | 16712 | .ty = mask_ty, |
| | 16713 | .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{ |
| | 16714 | .ty = mask_ty.toIntern(), |
| | 16715 | .storage = .{ .elems = mask_elems[0..vec_len] }, |
| | 16716 | } })), |
| | 16717 | }); |
| | 16718 | const mask_mem: Memory = .{ |
| | 16719 | .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) }, |
| | 16720 | .mod = .{ .rm = .{ .size = self.memSize(ty) } }, |
| | 16721 | }; |
| | 16722 | if (has_avx) { |
| | 16723 | try self.asmRegisterRegisterMemory( |
| | 16724 | .{ .vp_, .@"and" }, |
| | 16725 | mask_alias, |
| | 16726 | mask_alias, |
| | 16727 | mask_mem, |
| | 16728 | ); |
| | 16729 | try self.asmRegisterRegisterMemory( |
| | 16730 | .{ .vp_d, .cmpeq }, |
| | 16731 | mask_alias, |
| | 16732 | mask_alias, |
| | 16733 | mask_mem, |
| | 16734 | ); |
| | 16735 | } else { |
| | 16736 | try self.asmRegisterMemory( |
| | 16737 | .{ .p_, .@"and" }, |
| | 16738 | mask_alias, |
| | 16739 | mask_mem, |
| | 16740 | ); |
| | 16741 | try self.asmRegisterMemory( |
| | 16742 | .{ .p_d, .cmpeq }, |
| | 16743 | mask_alias, |
| | 16744 | mask_mem, |
| | 16745 | ); |
| | 16746 | } |
| | 16747 | } |
| | 16748 | break :mask mask_reg; |
| | 16749 | }; |
| | 16750 | const mask_alias = registerAlias(mask_reg, abi_size); |
| | 16751 | const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); |
| | 16752 | defer self.register_manager.unlockReg(mask_lock); |
| | 16753 | |
| | 16754 | const lhs_mcv = try self.resolveInst(extra.lhs); |
| | 16755 | const lhs_lock = switch (lhs_mcv) { |
| | 16756 | .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg), |
| | 16757 | else => null, |
| | 16758 | }; |
| | 16759 | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| | 16760 | |
| | 16761 | const rhs_mcv = try self.resolveInst(extra.rhs); |
| | 16762 | const rhs_lock = switch (rhs_mcv) { |
| | 16763 | .register => |rhs_reg| self.register_manager.lockReg(rhs_reg), |
| | 16764 | else => null, |
| | 16765 | }; |
| | 16766 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| | 16767 | |
| | 16768 | const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv; |
| | 16769 | const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand( |
| | 16770 | inst, |
| | 16771 | if (has_blend) extra.rhs else extra.lhs, |
| | 16772 | @intFromBool(has_blend), |
| | 16773 | reuse_mcv, |
| | 16774 | )) reuse_mcv else if (has_avx) |
| | 16775 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } |
| | 16776 | else |
| | 16777 | try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv); |
| | 16778 | const dst_reg = dst_mcv.getReg().?; |
| | 16779 | const dst_alias = registerAlias(dst_reg, abi_size); |
| | 16780 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| | 16781 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| | 16782 | |
| | 16783 | const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.childType(mod).zigTypeTag(mod)) { |
| | 16784 | else => null, |
| | 16785 | .Int => switch (abi_size) { |
| | 16786 | 0 => unreachable, |
| | 16787 | 1...16 => if (has_avx) |
| | 16788 | .{ .vp_b, .blendv } |
| | 16789 | else if (has_blend) |
| | 16790 | .{ .p_b, .blendv } |
| | 16791 | else |
| | 16792 | .{ .p_, undefined }, |
| | 16793 | 17...32 => if (self.hasFeature(.avx2)) |
| | 16794 | .{ .vp_b, .blendv } |
| | 16795 | else |
| | 16796 | null, |
| | 16797 | else => null, |
| | 16798 | }, |
| | 16799 | .Float => switch (ty.childType(mod).floatBits(self.target.*)) { |
| | 16800 | else => unreachable, |
| | 16801 | 16, 80, 128 => null, |
| | 16802 | 32 => switch (vec_len) { |
| | 16803 | 0 => unreachable, |
| | 16804 | 1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv }, |
| | 16805 | 5...8 => if (has_avx) .{ .v_ps, .blendv } else null, |
| | 16806 | else => null, |
| | 16807 | }, |
| | 16808 | 64 => switch (vec_len) { |
| | 16809 | 0 => unreachable, |
| | 16810 | 1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv }, |
| | 16811 | 3...4 => if (has_avx) .{ .v_pd, .blendv } else null, |
| | 16812 | else => null, |
| | 16813 | }, |
| | 16814 | }, |
| | 16815 | }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| | 16816 | if (has_avx) { |
| | 16817 | const rhs_alias = if (rhs_mcv.isRegister()) |
| | 16818 | registerAlias(rhs_mcv.getReg().?, abi_size) |
| | 16819 | else rhs: { |
| | 16820 | try self.genSetReg(dst_reg, ty, rhs_mcv, .{}); |
| | 16821 | break :rhs dst_alias; |
| | 16822 | }; |
| | 16823 | if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister( |
| | 16824 | mir_tag, |
| | 16825 | dst_alias, |
| | 16826 | rhs_alias, |
| | 16827 | try lhs_mcv.mem(self, self.memSize(ty)), |
| | 16828 | mask_alias, |
| | 16829 | ) else try self.asmRegisterRegisterRegisterRegister( |
| | 16830 | mir_tag, |
| | 16831 | dst_alias, |
| | 16832 | rhs_alias, |
| | 16833 | registerAlias(if (lhs_mcv.isRegister()) |
| | 16834 | lhs_mcv.getReg().? |
| | 16835 | else |
| | 16836 | try self.copyToTmpRegister(ty, lhs_mcv), abi_size), |
| | 16837 | mask_alias, |
| | 16838 | ); |
| | 16839 | } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister( |
| | 16840 | mir_tag, |
| | 16841 | dst_alias, |
| | 16842 | try lhs_mcv.mem(self, self.memSize(ty)), |
| | 16843 | mask_alias, |
| | 16844 | ) else try self.asmRegisterRegisterRegister( |
| | 16845 | mir_tag, |
| | 16846 | dst_alias, |
| | 16847 | registerAlias(if (lhs_mcv.isRegister()) |
| | 16848 | lhs_mcv.getReg().? |
| | 16849 | else |
| | 16850 | try self.copyToTmpRegister(ty, lhs_mcv), abi_size), |
| | 16851 | mask_alias, |
| | 16852 | ) else { |
| | 16853 | const mir_fixes = @as(?Mir.Inst.Fixes, switch (elem_ty.zigTypeTag(mod)) { |
| | 16854 | else => null, |
| | 16855 | .Int => .p_, |
| | 16856 | .Float => switch (elem_ty.floatBits(self.target.*)) { |
| | 16857 | 32 => ._ps, |
| | 16858 | 64 => ._pd, |
| | 16859 | 16, 80, 128 => null, |
| | 16860 | else => unreachable, |
| | 16861 | }, |
| | 16862 | }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)}); |
| | 16863 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); |
| | 16864 | if (rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| | 16865 | .{ mir_fixes, .andn }, |
| | 16866 | mask_alias, |
| | 16867 | try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)), |
| | 16868 | ) else try self.asmRegisterRegister( |
| | 16869 | .{ mir_fixes, .andn }, |
| | 16870 | mask_alias, |
| | 16871 | if (rhs_mcv.isRegister()) |
| | 16872 | rhs_mcv.getReg().? |
| | 16873 | else |
| | 16874 | try self.copyToTmpRegister(ty, rhs_mcv), |
| | 16875 | ); |
| | 16876 | try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias); |
| | 16877 | } |
| | 16878 | break :result dst_mcv; |
| | 16879 | }; |
| | 16880 | return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); |
| 16572 | } | 16881 | } |
| 16573 | | 16882 | |
| 16574 | fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { | 16883 | fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
Jacob Young <jacobly@ziglang.org> 2024-02-13 08:12:56+01:00
src