authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-16 07:59:16+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-25 11:22:10+01:00
log513c4c145ee163bb178c615a216534ac1f7e9e91
tree6cd1ae03642e2f260675f44c9669173a48aebb36
parent2fcb2f597549edd0b1241cebf98c11efe2f25884

x86_64: fix avx2 `@truncacte`

4 files changed, 39 insertions(+), 14 deletions(-)

src/arch/x86_64/CodeGen.zig+28-10
......@@ -3274,8 +3274,8 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
32743274 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
32753275 break :dst dst_mcv;
32763276 } else dst: {
3277 const dst_mcv = try self.allocRegOrMem(inst, true);
3278 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
3277 const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true);
3278 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
32793279 break :dst dst_mcv;
32803280 };
32813281
......@@ -3333,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
33333333 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
33343334 };
33353335
3336 const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size);
3336 const dst_reg = dst_mcv.getReg().?;
3337 const dst_alias = registerAlias(dst_reg, src_abi_size);
33373338 if (self.hasFeature(.avx)) {
33383339 try self.asmRegisterRegisterMemory(
33393340 .{ .vp_, .@"and" },
3340 dst_reg,
3341 dst_reg,
3341 dst_alias,
3342 dst_alias,
33423343 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
33433344 );
3344 try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
3345 if (src_abi_size > 16) {
3346 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
3347 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
3348 defer self.register_manager.unlockReg(temp_lock);
3349
3350 try self.asmRegisterRegisterImmediate(
3351 .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
3352 registerAlias(temp_reg, dst_abi_size),
3353 dst_alias,
3354 Immediate.u(1),
3355 );
3356 try self.asmRegisterRegisterRegister(
3357 mir_tag,
3358 registerAlias(dst_reg, dst_abi_size),
3359 registerAlias(dst_reg, dst_abi_size),
3360 registerAlias(temp_reg, dst_abi_size),
3361 );
3362 } else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias);
33453363 } else {
33463364 try self.asmRegisterMemory(
33473365 .{ .p_, .@"and" },
3348 dst_reg,
3366 dst_alias,
33493367 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
33503368 );
3351 try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
3369 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
33523370 }
33533371 break :result dst_mcv;
33543372 }
......@@ -16404,7 +16422,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1640416422 },
1640516423 65...128 => switch (vector_len) {
1640616424 else => null,
16407 1...2 => .{ .vp_i128, .broadcast },
16425 1...2 => .{ .v_i128, .broadcast },
1640816426 },
1640916427 }) orelse break :avx2;
1641016428
......@@ -16418,7 +16436,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1641816436 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
1641916437 try src_mcv.mem(self, self.memSize(scalar_ty)),
1642016438 ) else {
16421 if (mir_tag[0] == .vp_i128) break :avx2;
16439 if (mir_tag[0] == .v_i128) break :avx2;
1642216440 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
1642316441 try self.asmRegisterRegister(
1642416442 mir_tag,
src/arch/x86_64/Encoding.zig+2-1
......@@ -415,7 +415,8 @@ pub const Mnemonic = enum {
415415 vfmadd132sd, vfmadd213sd, vfmadd231sd,
416416 vfmadd132ss, vfmadd213ss, vfmadd231ss,
417417 // AVX2
418 vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
418 vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw,
419 vextracti128, vinserti128,
419420 // zig fmt: on
420421};
421422
src/arch/x86_64/Mir.zig+4-2
......@@ -230,6 +230,8 @@ pub const Inst = struct {
230230 v_d,
231231 /// VEX-Encoded ___ QuadWord
232232 v_q,
233 /// VEX-Encoded ___ Integer Data
234 v_i128,
233235 /// VEX-Encoded Packed ___
234236 vp_,
235237 /// VEX-Encoded Packed ___ Byte
......@@ -242,8 +244,6 @@ pub const Inst = struct {
242244 vp_q,
243245 /// VEX-Encoded Packed ___ Double Quadword
244246 vp_dq,
245 /// VEX-Encoded Packed ___ Integer Data
246 vp_i128,
247247 /// VEX-Encoded ___ Scalar Single-Precision Values
248248 v_ss,
249249 /// VEX-Encoded ___ Packed Single-Precision Values
......@@ -654,6 +654,7 @@ pub const Inst = struct {
654654 /// Variable blend scalar double-precision floating-point values
655655 blendv,
656656 /// Extract packed floating-point values
657 /// Extract packed integer values
657658 extract,
658659 /// Insert scalar single-precision floating-point value
659660 /// Insert packed floating-point values
......@@ -696,6 +697,7 @@ pub const Inst = struct {
696697 sha256rnds2,
697698
698699 /// Load with broadcast floating-point data
700 /// Load integer and broadcast
699701 broadcast,
700702
701703 /// Convert 16-bit floating-point values to single-precision floating-point values
src/arch/x86_64/encodings.zig+5-1
......@@ -1769,6 +1769,10 @@ pub const table = [_]Entry{
17691769 .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
17701770 .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
17711771
1772 .{ .vextracti128, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 },
1773
1774 .{ .vinserti128, .rvmi, &.{ .ymm, .ymm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 },
1775
17721776 .{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },
17731777 .{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },
17741778 .{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },
......@@ -1809,7 +1813,7 @@ pub const table = [_]Entry{
18091813 .{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
18101814 .{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
18111815 .{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
1812 .{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
1816 .{ .vbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
18131817
18141818 .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
18151819 .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },