| author |  Jacob Young <jacobly@ziglang.org> 2023-05-17 00:23:11-04:00 |
| committer | Jacob Young <jacobly@ziglang.org> 2023-05-18 20:42:38-04:00 |
| log | 35da95fe8765874a1ccffb0d7bfd523b14f44a4a |
| tree | 7a4bbc7644c11e42d3b742d04df133835fae32e3 |
| parent | 28c445addde840eec49ed0fd19cc19384a040085 |
x86_64: implement integer vector `@truncate`
5 files changed, 136 insertions(+), 21 deletions(-)
src- test
- behavior
src/arch/x86_64/CodeGen.zig+104-20
| ... | ... | @@ -2709,28 +2709,112 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { |
| 2709 | 2709 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 2710 | 2710 | |
| 2711 | 2711 | const dst_ty = self.air.typeOfIndex(inst); |
| 2712 | const dst_abi_size = dst_ty.abiSize(self.target.*); | |
| 2713 | if (dst_abi_size > 8) { | |
| 2714 | return self.fail("TODO implement trunc for abi sizes larger than 8", .{}); | |
| 2715 | } | |
| 2712 | const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); | |
| 2713 | const src_ty = self.air.typeOf(ty_op.operand); | |
| 2714 | const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); | |
| 2716 | 2715 | |
| 2717 | const src_mcv = try self.resolveInst(ty_op.operand); | |
| 2718 | const src_lock = switch (src_mcv) { | |
| 2719 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | |
| 2720 | else => null, | |
| 2721 | }; | |
| 2722 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); | |
| 2716 | const result = result: { | |
| 2717 | const src_mcv = try self.resolveInst(ty_op.operand); | |
| 2718 | const src_lock = | |
| 2719 | if (src_mcv.getReg()) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; | |
| 2720 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); | |
| 2723 | 2721 | |
| 2724 | const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) | |
| 2725 | src_mcv | |
| 2726 | else | |
| 2727 | try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); | |
| 2722 | const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) | |
| 2723 | src_mcv | |
| 2724 | else | |
| 2725 | try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); | |
| 2726 | ||
| 2727 | if (dst_ty.zigTypeTag() == .Vector) { | |
| 2728 | assert(src_ty.zigTypeTag() == .Vector and dst_ty.vectorLen() == src_ty.vectorLen()); | |
| 2729 | const dst_info = dst_ty.childType().intInfo(self.target.*); | |
| 2730 | const src_info = src_ty.childType().intInfo(self.target.*); | |
| 2731 | const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (dst_info.bits) { | |
| 2732 | 8 => switch (src_info.bits) { | |
| 2733 | 16 => switch (dst_ty.vectorLen()) { | |
| 2734 | 1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw }, | |
| 2735 | 9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null, | |
| 2736 | else => null, | |
| 2737 | }, | |
| 2738 | else => null, | |
| 2739 | }, | |
| 2740 | 16 => switch (src_info.bits) { | |
| 2741 | 32 => switch (dst_ty.vectorLen()) { | |
| 2742 | 1...4 => if (self.hasFeature(.avx)) | |
| 2743 | .{ .vp_w, .ackusd } | |
| 2744 | else if (self.hasFeature(.sse4_1)) | |
| 2745 | .{ .p_w, .ackusd } | |
| 2746 | else | |
| 2747 | null, | |
| 2748 | 5...8 => if (self.hasFeature(.avx2)) .{ .vp_w, .ackusd } else null, | |
| 2749 | else => null, | |
| 2750 | }, | |
| 2751 | else => null, | |
| 2752 | }, | |
| 2753 | else => null, | |
| 2754 | })) |tag| tag else return self.fail("TODO implement airTrunc for {}", .{ | |
| 2755 | dst_ty.fmt(self.bin_file.options.module.?), | |
| 2756 | }); | |
| 2728 | 2757 | |
| 2729 | // when truncating a `u16` to `u5`, for example, those top 3 bits in the result | |
| 2730 | // have to be removed. this only happens if the dst if not a power-of-two size. | |
| 2731 | if (self.regExtraBits(dst_ty) > 0) try self.truncateRegister(dst_ty, dst_mcv.register.to64()); | |
| 2758 | var mask_pl = Value.Payload.U64{ | |
| 2759 | .base = .{ .tag = .int_u64 }, | |
| 2760 | .data = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - dst_info.bits), | |
| 2761 | }; | |
| 2762 | const mask_val = Value.initPayload(&mask_pl.base); | |
| 2732 | 2763 | |
| 2733 | return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); | |
| 2764 | var splat_pl = Value.Payload.SubValue{ | |
| 2765 | .base = .{ .tag = .repeated }, | |
| 2766 | .data = mask_val, | |
| 2767 | }; | |
| 2768 | const splat_val = Value.initPayload(&splat_pl.base); | |
| 2769 | ||
| 2770 | var full_pl = Type.Payload.Array{ | |
| 2771 | .base = .{ .tag = .vector }, | |
| 2772 | .data = .{ | |
| 2773 | .len = @divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits), | |
| 2774 | .elem_type = src_ty.childType(), | |
| 2775 | }, | |
| 2776 | }; | |
| 2777 | const full_ty = Type.initPayload(&full_pl.base); | |
| 2778 | const full_abi_size = @intCast(u32, full_ty.abiSize(self.target.*)); | |
| 2779 | ||
| 2780 | const splat_mcv = try self.genTypedValue(.{ .ty = full_ty, .val = splat_val }); | |
| 2781 | const splat_addr_mcv: MCValue = switch (splat_mcv) { | |
| 2782 | .memory, .indirect, .load_frame => splat_mcv.address(), | |
| 2783 | else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) }, | |
| 2784 | }; | |
| 2785 | ||
| 2786 | const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size); | |
| 2787 | if (self.hasFeature(.avx)) { | |
| 2788 | try self.asmRegisterRegisterMemory( | |
| 2789 | .{ .vp_, .@"and" }, | |
| 2790 | dst_reg, | |
| 2791 | dst_reg, | |
| 2792 | splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(full_abi_size)), | |
| 2793 | ); | |
| 2794 | try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg); | |
| 2795 | } else { | |
| 2796 | try self.asmRegisterMemory( | |
| 2797 | .{ .p_, .@"and" }, | |
| 2798 | dst_reg, | |
| 2799 | splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(full_abi_size)), | |
| 2800 | ); | |
| 2801 | try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg); | |
| 2802 | } | |
| 2803 | break :result dst_mcv; | |
| 2804 | } | |
| 2805 | ||
| 2806 | if (dst_abi_size > 8) { | |
| 2807 | return self.fail("TODO implement trunc for abi sizes larger than 8", .{}); | |
| 2808 | } | |
| 2809 | ||
| 2810 | // when truncating a `u16` to `u5`, for example, those top 3 bits in the result | |
| 2811 | // have to be removed. this only happens if the dst if not a power-of-two size. | |
| 2812 | if (self.regExtraBits(dst_ty) > 0) | |
| 2813 | try self.truncateRegister(dst_ty, dst_mcv.register.to64()); | |
| 2814 | ||
| 2815 | break :result dst_mcv; | |
| 2816 | }; | |
| 2817 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); | |
| 2734 | 2818 | } |
| 2735 | 2819 | |
| 2736 | 2820 | fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void { |
| ... | ... | @@ -11081,8 +11165,8 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { |
| 11081 | 11165 | } |
| 11082 | 11166 | |
| 11083 | 11167 | fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { |
| 11084 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; | |
| 11085 | _ = ty_op; | |
| 11168 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; | |
| 11169 | _ = ty_pl; | |
| 11086 | 11170 | return self.fail("TODO implement airShuffle for x86_64", .{}); |
| 11087 | 11171 | //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 11088 | 11172 | } |
src/arch/x86_64/Encoding.zig+3
| ... | ... | @@ -263,6 +263,7 @@ pub const Mnemonic = enum { |
| 263 | 263 | fisttp, fld, |
| 264 | 264 | // MMX |
| 265 | 265 | movd, movq, |
| 266 | packssdw, packsswb, packuswb, | |
| 266 | 267 | paddb, paddd, paddq, paddsb, paddsw, paddusb, paddusw, paddw, |
| 267 | 268 | pand, pandn, por, pxor, |
| 268 | 269 | pmulhw, pmullw, |
| ... | ... | @@ -319,6 +320,7 @@ pub const Mnemonic = enum { |
| 319 | 320 | blendpd, blendps, blendvpd, blendvps, |
| 320 | 321 | extractps, |
| 321 | 322 | insertps, |
| 323 | packusdw, | |
| 322 | 324 | pextrb, pextrd, pextrq, |
| 323 | 325 | pinsrb, pinsrd, pinsrq, |
| 324 | 326 | pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw, |
| ... | ... | @@ -351,6 +353,7 @@ pub const Mnemonic = enum { |
| 351 | 353 | vmovupd, vmovups, |
| 352 | 354 | vmulpd, vmulps, vmulsd, vmulss, |
| 353 | 355 | vorpd, vorps, |
| 356 | vpackssdw, vpacksswb, vpackusdw, vpackuswb, | |
| 354 | 357 | vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw, |
| 355 | 358 | vpand, vpandn, |
| 356 | 359 | vpextrb, vpextrd, vpextrq, vpextrw, |
src/arch/x86_64/Mir.zig+8
| ... | ... | @@ -446,6 +446,12 @@ pub const Inst = struct { |
| 446 | 446 | /// Bitwise logical xor of packed double-precision floating-point values |
| 447 | 447 | xor, |
| 448 | 448 | |
| 449 | /// Pack with signed saturation | |
| 450 | ackssw, | |
| 451 | /// Pack with signed saturation | |
| 452 | ackssd, | |
| 453 | /// Pack with unsigned saturation | |
| 454 | ackusw, | |
| 449 | 455 | /// Add packed signed integers with signed saturation |
| 450 | 456 | adds, |
| 451 | 457 | /// Add packed unsigned integers with unsigned saturation |
| ... | ... | @@ -596,6 +602,8 @@ pub const Inst = struct { |
| 596 | 602 | /// Replicate single floating-point values |
| 597 | 603 | movsldup, |
| 598 | 604 | |
| 605 | /// Pack with unsigned saturation | |
| 606 | ackusd, | |
| 599 | 607 | /// Blend packed single-precision floating-point values |
| 600 | 608 | /// Blend scalar single-precision floating-point values |
| 601 | 609 | /// Blend packed double-precision floating-point values |
src/arch/x86_64/encodings.zig+21
| ... | ... | @@ -996,6 +996,11 @@ pub const table = [_]Entry{ |
| 996 | 996 | |
| 997 | 997 | .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 }, |
| 998 | 998 | |
| 999 | .{ .packsswb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x63 }, 0, .none, .sse2 }, | |
| 1000 | .{ .packssdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6b }, 0, .none, .sse2 }, | |
| 1001 | ||
| 1002 | .{ .packuswb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x67 }, 0, .none, .sse2 }, | |
| 1003 | ||
| 999 | 1004 | .{ .paddb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .none, .sse2 }, |
| 1000 | 1005 | .{ .paddw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .none, .sse2 }, |
| 1001 | 1006 | .{ .paddd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .none, .sse2 }, |
| ... | ... | @@ -1101,6 +1106,8 @@ pub const table = [_]Entry{ |
| 1101 | 1106 | |
| 1102 | 1107 | .{ .insertps, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .none, .sse4_1 }, |
| 1103 | 1108 | |
| 1109 | .{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 }, | |
| 1110 | ||
| 1104 | 1111 | .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 }, |
| 1105 | 1112 | .{ .pextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 }, |
| 1106 | 1113 | .{ .pextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 }, |
| ... | ... | @@ -1346,6 +1353,13 @@ pub const table = [_]Entry{ |
| 1346 | 1353 | .{ .vorps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .vex_128_wig, .avx }, |
| 1347 | 1354 | .{ .vorps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x56 }, 0, .vex_256_wig, .avx }, |
| 1348 | 1355 | |
| 1356 | .{ .vpacksswb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x63 }, 0, .vex_128_wig, .avx }, | |
| 1357 | .{ .vpackssdw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6b }, 0, .vex_128_wig, .avx }, | |
| 1358 | ||
| 1359 | .{ .vpackusdw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_128_wig, .avx }, | |
| 1360 | ||
| 1361 | .{ .vpackuswb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x67 }, 0, .vex_128_wig, .avx }, | |
| 1362 | ||
| 1349 | 1363 | .{ .vpaddb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_128_wig, .avx }, |
| 1350 | 1364 | .{ .vpaddw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_128_wig, .avx }, |
| 1351 | 1365 | .{ .vpaddd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_128_wig, .avx }, |
| ... | ... | @@ -1508,6 +1522,13 @@ pub const table = [_]Entry{ |
| 1508 | 1522 | .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 }, |
| 1509 | 1523 | .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 }, |
| 1510 | 1524 | |
| 1525 | .{ .vpacksswb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x63 }, 0, .vex_256_wig, .avx2 }, | |
| 1526 | .{ .vpackssdw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6b }, 0, .vex_256_wig, .avx2 }, | |
| 1527 | ||
| 1528 | .{ .vpackusdw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_256_wig, .avx2 }, | |
| 1529 | ||
| 1530 | .{ .vpackuswb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x67 }, 0, .vex_256_wig, .avx2 }, | |
| 1531 | ||
| 1511 | 1532 | .{ .vpaddb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_256_wig, .avx2 }, |
| 1512 | 1533 | .{ .vpaddw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_256_wig, .avx2 }, |
| 1513 | 1534 | .{ .vpaddd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_256_wig, .avx2 }, |
test/behavior/truncate.zig-1
| ... | ... | @@ -61,7 +61,6 @@ test "truncate on comptime integer" { |
| 61 | 61 | |
| 62 | 62 | test "truncate on vectors" { |
| 63 | 63 | if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; |
| 64 | if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; | |
| 65 | 64 | if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; |
| 66 | 65 | if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; |
| 67 | 66 | if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO |