authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:35:31-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:36:20-04:00
log1f5aa7747f5710e281cd2190508ce562a4bfd35f
treeff832710e67a7ac1256ed0ced050db411fd23551
parentecb5feaf94bf49dc4c180f09c170223d6c1898b3

x86_64: finish optimizing mir tag usage

Final tag count is 95.

2 files changed, 266 insertions(+), 466 deletions(-)

src/arch/x86_64/CodeGen.zig+184-184
......@@ -2443,7 +2443,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
24432443 else
24442444 try self.copyToTmpRegister(src_ty, src_mcv);
24452445 try self.asmRegisterRegisterImmediate(
2446 .{ ._, .vcvtps2ph },
2446 .{ .v_, .cvtps2ph },
24472447 dst_reg,
24482448 mat_src_reg.to128(),
24492449 Immediate.u(0b1_00),
......@@ -2455,12 +2455,12 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
24552455 }
24562456 } else if (src_bits == 64 and dst_bits == 32) {
24572457 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2458 .{ ._, .vcvtsd2ss },
2458 .{ .v_, .cvtsd2ss },
24592459 dst_reg,
24602460 dst_reg,
24612461 src_mcv.mem(.qword),
24622462 ) else try self.asmRegisterRegisterRegister(
2463 .{ ._, .vcvtsd2ss },
2463 .{ .v_, .cvtsd2ss },
24642464 dst_reg,
24652465 dst_reg,
24662466 (if (src_mcv.isRegister())
......@@ -2506,22 +2506,22 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
25062506 src_mcv.getReg().?
25072507 else
25082508 try self.copyToTmpRegister(src_ty, src_mcv);
2509 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
2509 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
25102510 switch (dst_bits) {
25112511 32 => {},
2512 64 => try self.asmRegisterRegisterRegister(.{ ._, .vcvtss2sd }, dst_reg, dst_reg, dst_reg),
2512 64 => try self.asmRegisterRegisterRegister(.{ .v_, .cvtss2sd }, dst_reg, dst_reg, dst_reg),
25132513 else => return self.fail("TODO implement airFpext from {} to {}", .{
25142514 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
25152515 }),
25162516 }
25172517 } else if (src_bits == 32 and dst_bits == 64) {
25182518 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2519 .{ ._, .vcvtss2sd },
2519 .{ .v_, .cvtss2sd },
25202520 dst_reg,
25212521 dst_reg,
25222522 src_mcv.mem(.dword),
25232523 ) else try self.asmRegisterRegisterRegister(
2524 .{ ._, .vcvtss2sd },
2524 .{ .v_, .cvtss2sd },
25252525 dst_reg,
25262526 dst_reg,
25272527 (if (src_mcv.isRegister())
......@@ -4678,8 +4678,8 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
46784678 try self.genBinOpMir(switch (ty_bits) {
46794679 // No point using an extra prefix byte for *pd which performs the same operation.
46804680 16, 32, 64, 128 => switch (tag) {
4681 .neg => .{ ._, .xorps },
4682 .fabs => .{ ._, .andnps },
4681 .neg => .{ ._ps, .xor },
4682 .fabs => .{ ._ps, .andn },
46834683 else => unreachable,
46844684 },
46854685 80 => return self.fail("TODO implement airFloatSign for {}", .{
......@@ -4712,23 +4712,23 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
47124712
47134713 const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
47144714 .Float => switch (ty.floatBits(self.target.*)) {
4715 32 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
4716 64 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
4715 32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
4716 64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
47174717 16, 80, 128 => null,
47184718 else => unreachable,
47194719 },
47204720 .Vector => switch (ty.childType().zigTypeTag()) {
47214721 .Float => switch (ty.childType().floatBits(self.target.*)) {
47224722 32 => switch (ty.vectorLen()) {
4723 1 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
4724 2...4 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else .{ ._, .roundps },
4725 5...8 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else null,
4723 1 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
4724 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else .{ ._ps, .round },
4725 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else null,
47264726 else => null,
47274727 },
47284728 64 => switch (ty.vectorLen()) {
4729 1 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
4730 2 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else .{ ._, .roundpd },
4731 3...4 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else null,
4729 1 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
4730 2 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else .{ ._pd, .round },
4731 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else null,
47324732 else => null,
47334733 },
47344734 16, 80, 128 => null,
......@@ -4743,8 +4743,8 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
47434743
47444744 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
47454745 const dst_alias = registerAlias(dst_reg, abi_size);
4746 switch (mir_tag[1]) {
4747 .vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
4746 switch (mir_tag[0]) {
4747 .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
47484748 mir_tag,
47494749 dst_alias,
47504750 dst_alias,
......@@ -4799,18 +4799,18 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
47994799 src_mcv.getReg().?
48004800 else
48014801 try self.copyToTmpRegister(ty, src_mcv);
4802 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
4803 try self.asmRegisterRegisterRegister(.{ ._, .vsqrtss }, dst_reg, dst_reg, dst_reg);
4802 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
4803 try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg);
48044804 try self.asmRegisterRegisterImmediate(
4805 .{ ._, .vcvtps2ph },
4805 .{ .v_, .cvtps2ph },
48064806 dst_reg,
48074807 dst_reg,
48084808 Immediate.u(0b1_00),
48094809 );
48104810 break :result dst_mcv;
48114811 } else null,
4812 32 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
4813 64 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
4812 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
4813 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
48144814 80, 128 => null,
48154815 else => unreachable,
48164816 },
......@@ -4819,7 +4819,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
48194819 16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
48204820 1 => {
48214821 try self.asmRegisterRegister(
4822 .{ ._, .vcvtph2ps },
4822 .{ .v_, .cvtph2ps },
48234823 dst_reg,
48244824 (if (src_mcv.isRegister())
48254825 src_mcv.getReg().?
......@@ -4827,13 +4827,13 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
48274827 try self.copyToTmpRegister(ty, src_mcv)).to128(),
48284828 );
48294829 try self.asmRegisterRegisterRegister(
4830 .{ ._, .vsqrtss },
4830 .{ .v_ss, .sqrt },
48314831 dst_reg,
48324832 dst_reg,
48334833 dst_reg,
48344834 );
48354835 try self.asmRegisterRegisterImmediate(
4836 .{ ._, .vcvtps2ph },
4836 .{ .v_, .cvtps2ph },
48374837 dst_reg,
48384838 dst_reg,
48394839 Immediate.u(0b1_00),
......@@ -4843,22 +4843,22 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
48434843 2...8 => {
48444844 const wide_reg = registerAlias(dst_reg, abi_size * 2);
48454845 if (src_mcv.isMemory()) try self.asmRegisterMemory(
4846 .{ ._, .vcvtph2ps },
4846 .{ .v_, .cvtph2ps },
48474847 wide_reg,
48484848 src_mcv.mem(Memory.PtrSize.fromSize(
48494849 @intCast(u32, @divExact(wide_reg.bitSize(), 16)),
48504850 )),
48514851 ) else try self.asmRegisterRegister(
4852 .{ ._, .vcvtph2ps },
4852 .{ .v_, .cvtph2ps },
48534853 wide_reg,
48544854 (if (src_mcv.isRegister())
48554855 src_mcv.getReg().?
48564856 else
48574857 try self.copyToTmpRegister(ty, src_mcv)).to128(),
48584858 );
4859 try self.asmRegisterRegister(.{ ._, .vsqrtps }, wide_reg, wide_reg);
4859 try self.asmRegisterRegister(.{ .v_ps, .sqrt }, wide_reg, wide_reg);
48604860 try self.asmRegisterRegisterImmediate(
4861 .{ ._, .vcvtps2ph },
4861 .{ .v_, .cvtps2ph },
48624862 dst_reg,
48634863 wide_reg,
48644864 Immediate.u(0b1_00),
......@@ -4868,15 +4868,15 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
48684868 else => null,
48694869 } else null,
48704870 32 => switch (ty.vectorLen()) {
4871 1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
4872 2...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else .{ ._, .sqrtps },
4873 5...8 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else null,
4871 1 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
4872 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else .{ ._ps, .sqrt },
4873 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else null,
48744874 else => null,
48754875 },
48764876 64 => switch (ty.vectorLen()) {
4877 1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
4878 2 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else .{ ._, .sqrtpd },
4879 3...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else null,
4877 1 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
4878 2 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else .{ ._pd, .sqrt },
4879 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else null,
48804880 else => null,
48814881 },
48824882 80, 128 => null,
......@@ -4888,8 +4888,8 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
48884888 })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
48894889 ty.fmt(self.bin_file.options.module.?),
48904890 });
4891 switch (mir_tag[1]) {
4892 .vsqrtss, .vsqrtsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
4891 switch (mir_tag[0]) {
4892 .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
48934893 mir_tag,
48944894 dst_reg,
48954895 dst_reg,
......@@ -6325,13 +6325,13 @@ fn genBinOp(
63256325 defer self.register_manager.unlockReg(tmp_lock);
63266326
63276327 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6328 .{ ._, .vpinsrw },
6328 .{ .vp_w, .insr },
63296329 dst_reg,
63306330 dst_reg,
63316331 src_mcv.mem(.word),
63326332 Immediate.u(1),
63336333 ) else try self.asmRegisterRegisterRegister(
6334 .{ ._, .vpunpcklwd },
6334 .{ .vp_, .unpcklwd },
63356335 dst_reg,
63366336 dst_reg,
63376337 (if (src_mcv.isRegister())
......@@ -6339,15 +6339,15 @@ fn genBinOp(
63396339 else
63406340 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
63416341 );
6342 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6343 try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
6342 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
6343 try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
63446344 try self.asmRegisterRegisterRegister(
63456345 switch (air_tag) {
6346 .add => .{ ._, .vaddss },
6347 .sub => .{ ._, .vsubss },
6348 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
6349 .max => .{ ._, .vmaxss },
6350 .min => .{ ._, .vmaxss },
6346 .add => .{ .v_ss, .add },
6347 .sub => .{ .v_ss, .sub },
6348 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
6349 .max => .{ .v_ss, .max },
6350 .min => .{ .v_ss, .max },
63516351 else => unreachable,
63526352 },
63536353 dst_reg,
......@@ -6355,7 +6355,7 @@ fn genBinOp(
63556355 tmp_reg,
63566356 );
63576357 try self.asmRegisterRegisterImmediate(
6358 .{ ._, .vcvtps2ph },
6358 .{ .v_, .cvtps2ph },
63596359 dst_reg,
63606360 dst_reg,
63616361 Immediate.u(0b1_00),
......@@ -6363,29 +6363,29 @@ fn genBinOp(
63636363 return dst_mcv;
63646364 } else null,
63656365 32 => switch (air_tag) {
6366 .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
6367 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
6368 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
6366 .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
6367 .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
6368 .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
63696369 .div_float,
63706370 .div_trunc,
63716371 .div_floor,
63726372 .div_exact,
6373 => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
6374 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
6375 .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
6373 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
6374 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
6375 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
63766376 else => unreachable,
63776377 },
63786378 64 => switch (air_tag) {
6379 .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
6380 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
6381 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
6379 .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
6380 .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
6381 .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
63826382 .div_float,
63836383 .div_trunc,
63846384 .div_floor,
63856385 .div_exact,
6386 => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
6387 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
6388 .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
6386 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
6387 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
6388 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
63896389 else => unreachable,
63906390 },
63916391 80, 128 => null,
......@@ -6401,13 +6401,13 @@ fn genBinOp(
64016401 defer self.register_manager.unlockReg(tmp_lock);
64026402
64036403 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6404 .{ ._, .vpinsrw },
6404 .{ .vp_w, .insr },
64056405 dst_reg,
64066406 dst_reg,
64076407 src_mcv.mem(.word),
64086408 Immediate.u(1),
64096409 ) else try self.asmRegisterRegisterRegister(
6410 .{ ._, .vpunpcklwd },
6410 .{ .vp_, .unpcklwd },
64116411 dst_reg,
64126412 dst_reg,
64136413 (if (src_mcv.isRegister())
......@@ -6415,15 +6415,15 @@ fn genBinOp(
64156415 else
64166416 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
64176417 );
6418 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6419 try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
6418 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
6419 try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
64206420 try self.asmRegisterRegisterRegister(
64216421 switch (air_tag) {
6422 .add => .{ ._, .vaddss },
6423 .sub => .{ ._, .vsubss },
6424 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
6425 .max => .{ ._, .vmaxss },
6426 .min => .{ ._, .vmaxss },
6422 .add => .{ .v_ss, .add },
6423 .sub => .{ .v_ss, .sub },
6424 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
6425 .max => .{ .v_ss, .max },
6426 .min => .{ .v_ss, .max },
64276427 else => unreachable,
64286428 },
64296429 dst_reg,
......@@ -6431,7 +6431,7 @@ fn genBinOp(
64316431 tmp_reg,
64326432 );
64336433 try self.asmRegisterRegisterImmediate(
6434 .{ ._, .vcvtps2ph },
6434 .{ .v_, .cvtps2ph },
64356435 dst_reg,
64366436 dst_reg,
64376437 Immediate.u(0b1_00),
......@@ -6444,12 +6444,12 @@ fn genBinOp(
64446444 defer self.register_manager.unlockReg(tmp_lock);
64456445
64466446 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
6447 .{ ._, .vpinsrd },
6447 .{ .vp_d, .insr },
64486448 dst_reg,
64496449 src_mcv.mem(.dword),
64506450 Immediate.u(1),
64516451 ) else try self.asmRegisterRegisterRegister(
6452 .{ ._, .vunpcklps },
6452 .{ .v_ps, .unpckl },
64536453 dst_reg,
64546454 dst_reg,
64556455 (if (src_mcv.isRegister())
......@@ -6457,20 +6457,20 @@ fn genBinOp(
64576457 else
64586458 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
64596459 );
6460 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6460 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
64616461 try self.asmRegisterRegisterRegister(
6462 .{ ._, .vmovhlps },
6462 .{ .v_ps, .movhl },
64636463 tmp_reg,
64646464 dst_reg,
64656465 dst_reg,
64666466 );
64676467 try self.asmRegisterRegisterRegister(
64686468 switch (air_tag) {
6469 .add => .{ ._, .vaddps },
6470 .sub => .{ ._, .vsubps },
6471 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6472 .max => .{ ._, .vmaxps },
6473 .min => .{ ._, .vmaxps },
6469 .add => .{ .v_ps, .add },
6470 .sub => .{ .v_ps, .sub },
6471 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
6472 .max => .{ .v_ps, .max },
6473 .min => .{ .v_ps, .max },
64746474 else => unreachable,
64756475 },
64766476 dst_reg,
......@@ -6478,7 +6478,7 @@ fn genBinOp(
64786478 tmp_reg,
64796479 );
64806480 try self.asmRegisterRegisterImmediate(
6481 .{ ._, .vcvtps2ph },
6481 .{ .v_, .cvtps2ph },
64826482 dst_reg,
64836483 dst_reg,
64846484 Immediate.u(0b1_00),
......@@ -6490,13 +6490,13 @@ fn genBinOp(
64906490 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
64916491 defer self.register_manager.unlockReg(tmp_lock);
64926492
6493 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6493 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
64946494 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6495 .{ ._, .vcvtph2ps },
6495 .{ .v_, .cvtph2ps },
64966496 tmp_reg,
64976497 src_mcv.mem(.qword),
64986498 ) else try self.asmRegisterRegister(
6499 .{ ._, .vcvtph2ps },
6499 .{ .v_, .cvtph2ps },
65006500 tmp_reg,
65016501 (if (src_mcv.isRegister())
65026502 src_mcv.getReg().?
......@@ -6505,11 +6505,11 @@ fn genBinOp(
65056505 );
65066506 try self.asmRegisterRegisterRegister(
65076507 switch (air_tag) {
6508 .add => .{ ._, .vaddps },
6509 .sub => .{ ._, .vsubps },
6510 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6511 .max => .{ ._, .vmaxps },
6512 .min => .{ ._, .vmaxps },
6508 .add => .{ .v_ps, .add },
6509 .sub => .{ .v_ps, .sub },
6510 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
6511 .max => .{ .v_ps, .max },
6512 .min => .{ .v_ps, .max },
65136513 else => unreachable,
65146514 },
65156515 dst_reg,
......@@ -6517,7 +6517,7 @@ fn genBinOp(
65176517 tmp_reg,
65186518 );
65196519 try self.asmRegisterRegisterImmediate(
6520 .{ ._, .vcvtps2ph },
6520 .{ .v_, .cvtps2ph },
65216521 dst_reg,
65226522 dst_reg,
65236523 Immediate.u(0b1_00),
......@@ -6529,13 +6529,13 @@ fn genBinOp(
65296529 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
65306530 defer self.register_manager.unlockReg(tmp_lock);
65316531
6532 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg.to256(), dst_reg);
6532 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg.to256(), dst_reg);
65336533 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6534 .{ ._, .vcvtph2ps },
6534 .{ .v_, .cvtph2ps },
65356535 tmp_reg,
65366536 src_mcv.mem(.xword),
65376537 ) else try self.asmRegisterRegister(
6538 .{ ._, .vcvtph2ps },
6538 .{ .v_, .cvtph2ps },
65396539 tmp_reg,
65406540 (if (src_mcv.isRegister())
65416541 src_mcv.getReg().?
......@@ -6544,11 +6544,11 @@ fn genBinOp(
65446544 );
65456545 try self.asmRegisterRegisterRegister(
65466546 switch (air_tag) {
6547 .add => .{ ._, .vaddps },
6548 .sub => .{ ._, .vsubps },
6549 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6550 .max => .{ ._, .vmaxps },
6551 .min => .{ ._, .vmaxps },
6547 .add => .{ .v_ps, .add },
6548 .sub => .{ .v_ps, .sub },
6549 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
6550 .max => .{ .v_ps, .max },
6551 .min => .{ .v_ps, .max },
65526552 else => unreachable,
65536553 },
65546554 dst_reg.to256(),
......@@ -6556,7 +6556,7 @@ fn genBinOp(
65566556 tmp_reg,
65576557 );
65586558 try self.asmRegisterRegisterImmediate(
6559 .{ ._, .vcvtps2ph },
6559 .{ .v_, .cvtps2ph },
65606560 dst_reg,
65616561 dst_reg.to256(),
65626562 Immediate.u(0b1_00),
......@@ -6567,76 +6567,76 @@ fn genBinOp(
65676567 } else null,
65686568 32 => switch (lhs_ty.vectorLen()) {
65696569 1 => switch (air_tag) {
6570 .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
6571 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
6572 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
6570 .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
6571 .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
6572 .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
65736573 .div_float,
65746574 .div_trunc,
65756575 .div_floor,
65766576 .div_exact,
6577 => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
6578 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
6579 .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
6577 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
6578 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
6579 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
65806580 else => unreachable,
65816581 },
65826582 2...4 => switch (air_tag) {
6583 .add => if (self.hasFeature(.avx)) .{ ._, .vaddps } else .{ ._, .addps },
6584 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubps } else .{ ._, .subps },
6585 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulps } else .{ ._, .mulps },
6583 .add => if (self.hasFeature(.avx)) .{ .v_ps, .add } else .{ ._ps, .add },
6584 .sub => if (self.hasFeature(.avx)) .{ .v_ps, .sub } else .{ ._ps, .sub },
6585 .mul => if (self.hasFeature(.avx)) .{ .v_ps, .mul } else .{ ._ps, .mul },
65866586 .div_float,
65876587 .div_trunc,
65886588 .div_floor,
65896589 .div_exact,
6590 => if (self.hasFeature(.avx)) .{ ._, .vdivps } else .{ ._, .divps },
6591 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxps } else .{ ._, .maxps },
6592 .min => if (self.hasFeature(.avx)) .{ ._, .vminps } else .{ ._, .minps },
6590 => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
6591 .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
6592 .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
65936593 else => unreachable,
65946594 },
65956595 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
6596 .add => .{ ._, .vaddps },
6597 .sub => .{ ._, .vsubps },
6598 .mul => .{ ._, .vmulps },
6599 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
6600 .max => .{ ._, .vmaxps },
6601 .min => .{ ._, .vminps },
6596 .add => .{ .v_ps, .add },
6597 .sub => .{ .v_ps, .sub },
6598 .mul => .{ .v_ps, .mul },
6599 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
6600 .max => .{ .v_ps, .max },
6601 .min => .{ .v_ps, .min },
66026602 else => unreachable,
66036603 } else null,
66046604 else => null,
66056605 },
66066606 64 => switch (lhs_ty.vectorLen()) {
66076607 1 => switch (air_tag) {
6608 .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
6609 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
6610 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
6608 .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
6609 .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
6610 .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
66116611 .div_float,
66126612 .div_trunc,
66136613 .div_floor,
66146614 .div_exact,
6615 => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
6616 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
6617 .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
6615 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
6616 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
6617 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
66186618 else => unreachable,
66196619 },
66206620 2 => switch (air_tag) {
6621 .add => if (self.hasFeature(.avx)) .{ ._, .vaddpd } else .{ ._, .addpd },
6622 .sub => if (self.hasFeature(.avx)) .{ ._, .vsubpd } else .{ ._, .subpd },
6623 .mul => if (self.hasFeature(.avx)) .{ ._, .vmulpd } else .{ ._, .mulpd },
6621 .add => if (self.hasFeature(.avx)) .{ .v_pd, .add } else .{ ._pd, .add },
6622 .sub => if (self.hasFeature(.avx)) .{ .v_pd, .sub } else .{ ._pd, .sub },
6623 .mul => if (self.hasFeature(.avx)) .{ .v_pd, .mul } else .{ ._pd, .mul },
66246624 .div_float,
66256625 .div_trunc,
66266626 .div_floor,
66276627 .div_exact,
6628 => if (self.hasFeature(.avx)) .{ ._, .vdivpd } else .{ ._, .divpd },
6629 .max => if (self.hasFeature(.avx)) .{ ._, .vmaxpd } else .{ ._, .maxpd },
6630 .min => if (self.hasFeature(.avx)) .{ ._, .vminpd } else .{ ._, .minpd },
6628 => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
6629 .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
6630 .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
66316631 else => unreachable,
66326632 },
66336633 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
6634 .add => .{ ._, .vaddpd },
6635 .sub => .{ ._, .vsubpd },
6636 .mul => .{ ._, .vmulpd },
6637 .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivpd },
6638 .max => .{ ._, .vmaxpd },
6639 .min => .{ ._, .vminpd },
6634 .add => .{ .v_pd, .add },
6635 .sub => .{ .v_pd, .sub },
6636 .mul => .{ .v_pd, .mul },
6637 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
6638 .max => .{ .v_pd, .max },
6639 .min => .{ .v_pd, .min },
66406640 else => unreachable,
66416641 } else null,
66426642 else => null,
......@@ -7563,13 +7563,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
75637563 defer self.register_manager.unlockReg(tmp2_lock);
75647564
75657565 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
7566 .{ ._, .vpinsrw },
7566 .{ .vp_w, .insr },
75677567 tmp1_reg,
75687568 dst_reg.to128(),
75697569 src_mcv.mem(.word),
75707570 Immediate.u(1),
75717571 ) else try self.asmRegisterRegisterRegister(
7572 .{ ._, .vpunpcklwd },
7572 .{ .vp_, .unpcklwd },
75737573 tmp1_reg,
75747574 dst_reg.to128(),
75757575 (if (src_mcv.isRegister())
......@@ -7577,20 +7577,20 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
75777577 else
75787578 try self.copyToTmpRegister(ty, src_mcv)).to128(),
75797579 );
7580 try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, tmp1_reg, tmp1_reg);
7581 try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp2_reg, tmp1_reg);
7582 try self.genBinOpMir(.{ ._, .ucomiss }, ty, tmp1_mcv, tmp2_mcv);
7580 try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, tmp1_reg, tmp1_reg);
7581 try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg);
7582 try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv);
75837583 } else return self.fail("TODO implement airCmp for {}", .{
75847584 ty.fmt(self.bin_file.options.module.?),
75857585 }),
75867586 32 => try self.genBinOpMir(
7587 .{ ._, .ucomiss },
7587 .{ ._ss, .ucomi },
75887588 ty,
75897589 .{ .register = dst_reg },
75907590 src_mcv,
75917591 ),
75927592 64 => try self.genBinOpMir(
7593 .{ ._, .ucomisd },
7593 .{ ._sd, .ucomi },
75947594 ty,
75957595 .{ .register = dst_reg },
75967596 src_mcv,
......@@ -8573,42 +8573,42 @@ fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
85738573 else => return .{ ._, .mov },
85748574 .Float => switch (ty.floatBits(self.target.*)) {
85758575 16 => unreachable, // needs special handling
8576 32 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
8577 64 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8576 32 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
8577 64 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
85788578 128 => return if (self.hasFeature(.avx))
8579 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8580 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8579 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8580 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
85818581 else => {},
85828582 },
85838583 .Vector => switch (ty.childType().zigTypeTag()) {
85848584 .Float => switch (ty.childType().floatBits(self.target.*)) {
85858585 16 => switch (ty.vectorLen()) {
85868586 1 => unreachable, // needs special handling
8587 2 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
8588 3...4 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8587 2 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
8588 3...4 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
85898589 5...8 => return if (self.hasFeature(.avx))
8590 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8591 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8590 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8591 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
85928592 9...16 => if (self.hasFeature(.avx))
8593 return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8593 return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
85948594 else => {},
85958595 },
85968596 32 => switch (ty.vectorLen()) {
8597 1 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
8597 1 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
85988598 2...4 => return if (self.hasFeature(.avx))
8599 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8600 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8599 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8600 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
86018601 5...8 => if (self.hasFeature(.avx))
8602 return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8602 return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
86038603 else => {},
86048604 },
86058605 64 => switch (ty.vectorLen()) {
8606 1 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8606 1 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
86078607 2 => return if (self.hasFeature(.avx))
8608 if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
8609 else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8608 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8609 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
86108610 3...4 => if (self.hasFeature(.avx))
8611 return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8611 return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
86128612 else => {},
86138613 },
86148614 else => {},
......@@ -8724,11 +8724,11 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
87248724 if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
87258725 switch (ty.zigTypeTag()) {
87268726 else => .{ ._, .mov },
8727 .Float, .Vector => .{ ._, .movaps },
8727 .Float, .Vector => .{ ._ps, .mova },
87288728 }
87298729 else switch (abi_size) {
87308730 2 => return try self.asmRegisterRegisterImmediate(
8731 if (dst_reg.class() == .floating_point) .{ ._, .pinsrw } else .{ ._, .pextrw },
8731 if (dst_reg.class() == .floating_point) .{ .p_w, .insr } else .{ .p_w, .extr },
87328732 registerAlias(dst_reg, 4),
87338733 registerAlias(src_reg, 4),
87348734 Immediate.u(0),
......@@ -8761,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
87618761 });
87628762 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
87638763 try self.asmRegisterMemoryImmediate(
8764 .{ ._, .pinsrw },
8764 .{ .p_w, .insr },
87658765 registerAlias(dst_reg, abi_size),
87668766 src_mem,
87678767 Immediate.u(0),
......@@ -8794,7 +8794,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
87948794 });
87958795 return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
87968796 self.asmRegisterMemoryImmediate(
8797 .{ ._, .pinsrw },
8797 .{ .p_w, .insr },
87988798 registerAlias(dst_reg, abi_size),
87998799 src_mem,
88008800 Immediate.u(0),
......@@ -8838,7 +8838,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
88388838 });
88398839 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
88408840 try self.asmRegisterMemoryImmediate(
8841 .{ ._, .pinsrw },
8841 .{ .p_w, .insr },
88428842 registerAlias(dst_reg, abi_size),
88438843 src_mem,
88448844 Immediate.u(0),
......@@ -8952,7 +8952,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
89528952 );
89538953 if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
89548954 try self.asmMemoryRegisterImmediate(
8955 .{ ._, .pextrw },
8955 .{ .p_w, .extr },
89568956 dst_mem,
89578957 src_reg.to128(),
89588958 Immediate.u(0),
......@@ -9069,7 +9069,7 @@ fn genInlineMemcpyRegisterRegister(
90699069 try self.asmMemoryRegister(
90709070 switch (src_reg.class()) {
90719071 .general_purpose, .segment => .{ ._, .mov },
9072 .floating_point => .{ ._, .movss },
9072 .floating_point => .{ ._ss, .mov },
90739073 },
90749074 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
90759075 registerAlias(src_reg, abi_size),
......@@ -10197,21 +10197,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
1019710197 if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
1019810198 switch (ty.zigTypeTag()) {
1019910199 .Float => switch (ty.floatBits(self.target.*)) {
10200 32 => .{ ._, .vfmadd132ss },
10201 64 => .{ ._, .vfmadd132sd },
10200 32 => .{ .v_ss, .fmadd132 },
10201 64 => .{ .v_sd, .fmadd132 },
1020210202 16, 80, 128 => null,
1020310203 else => unreachable,
1020410204 },
1020510205 .Vector => switch (ty.childType().zigTypeTag()) {
1020610206 .Float => switch (ty.childType().floatBits(self.target.*)) {
1020710207 32 => switch (ty.vectorLen()) {
10208 1 => .{ ._, .vfmadd132ss },
10209 2...8 => .{ ._, .vfmadd132ps },
10208 1 => .{ .v_ss, .fmadd132 },
10209 2...8 => .{ .v_ps, .fmadd132 },
1021010210 else => null,
1021110211 },
1021210212 64 => switch (ty.vectorLen()) {
10213 1 => .{ ._, .vfmadd132sd },
10214 2...4 => .{ ._, .vfmadd132pd },
10213 1 => .{ .v_sd, .fmadd132 },
10214 2...4 => .{ .v_pd, .fmadd132 },
1021510215 else => null,
1021610216 },
1021710217 16, 80, 128 => null,
......@@ -10224,21 +10224,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
1022410224 else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
1022510225 switch (ty.zigTypeTag()) {
1022610226 .Float => switch (ty.floatBits(self.target.*)) {
10227 32 => .{ ._, .vfmadd213ss },
10228 64 => .{ ._, .vfmadd213sd },
10227 32 => .{ .v_ss, .fmadd213 },
10228 64 => .{ .v_sd, .fmadd213 },
1022910229 16, 80, 128 => null,
1023010230 else => unreachable,
1023110231 },
1023210232 .Vector => switch (ty.childType().zigTypeTag()) {
1023310233 .Float => switch (ty.childType().floatBits(self.target.*)) {
1023410234 32 => switch (ty.vectorLen()) {
10235 1 => .{ ._, .vfmadd213ss },
10236 2...8 => .{ ._, .vfmadd213ps },
10235 1 => .{ .v_ss, .fmadd213 },
10236 2...8 => .{ .v_ps, .fmadd213 },
1023710237 else => null,
1023810238 },
1023910239 64 => switch (ty.vectorLen()) {
10240 1 => .{ ._, .vfmadd213sd },
10241 2...4 => .{ ._, .vfmadd213pd },
10240 1 => .{ .v_sd, .fmadd213 },
10241 2...4 => .{ .v_pd, .fmadd213 },
1024210242 else => null,
1024310243 },
1024410244 16, 80, 128 => null,
......@@ -10251,21 +10251,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
1025110251 else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
1025210252 switch (ty.zigTypeTag()) {
1025310253 .Float => switch (ty.floatBits(self.target.*)) {
10254 32 => .{ ._, .vfmadd231ss },
10255 64 => .{ ._, .vfmadd231sd },
10254 32 => .{ .v_ss, .fmadd231 },
10255 64 => .{ .v_sd, .fmadd231 },
1025610256 16, 80, 128 => null,
1025710257 else => unreachable,
1025810258 },
1025910259 .Vector => switch (ty.childType().zigTypeTag()) {
1026010260 .Float => switch (ty.childType().floatBits(self.target.*)) {
1026110261 32 => switch (ty.vectorLen()) {
10262 1 => .{ ._, .vfmadd231ss },
10263 2...8 => .{ ._, .vfmadd231ps },
10262 1 => .{ .v_ss, .fmadd231 },
10263 2...8 => .{ .v_ps, .fmadd231 },
1026410264 else => null,
1026510265 },
1026610266 64 => switch (ty.vectorLen()) {
10267 1 => .{ ._, .vfmadd231sd },
10268 2...4 => .{ ._, .vfmadd231pd },
10267 1 => .{ .v_sd, .fmadd231 },
10268 2...4 => .{ .v_pd, .fmadd231 },
1026910269 else => null,
1027010270 },
1027110271 16, 80, 128 => null,
src/arch/x86_64/Mir.zig+82-282
......@@ -278,8 +278,14 @@ pub const Inst = struct {
278278 /// Add with carry
279279 adc,
280280 /// Add
281 /// Add packed single-precision floating-point values
282 /// Add scalar single-precision floating-point values
283 /// Add packed double-precision floating-point values
284 /// Add scalar double-precision floating-point values
281285 add,
282286 /// Logical and
287 /// Bitwise logical and of packed single-precision floating-point values
288 /// Bitwise logical and of packed double-precision floating-point values
283289 @"and",
284290 /// Bit scan forward
285291 bsf,
......@@ -304,6 +310,8 @@ pub const Inst = struct {
304310 cmov,
305311 /// Logical compare
306312 /// Compare string
313 /// Compare scalar single-precision floating-point values
314 /// Compare scalar double-precision floating-point values
307315 cmp,
308316 /// Compare and exchange
309317 /// Compare and exchange bytes
......@@ -316,6 +324,10 @@ pub const Inst = struct {
316324 cwde,
317325 /// Unsigned division
318326 /// Signed division
327 /// Divide packed single-precision floating-point values
328 /// Divide scalar single-precision floating-point values
329 /// Divide packed double-precision floating-point values
330 /// Divide scalar double-precision floating-point values
319331 div,
320332 ///
321333 int3,
......@@ -339,6 +351,8 @@ pub const Inst = struct {
339351 mfence,
340352 /// Move
341353 /// Move data from string to string
354 /// Move scalar single-precision floating-point value
355 /// Move scalar double-precision floating-point value
342356 /// Move doubleword
343357 /// Move quadword
344358 mov,
......@@ -350,6 +364,10 @@ pub const Inst = struct {
350364 movzx,
351365 /// Multiply
352366 /// Signed multiplication
367 /// Multiply packed single-precision floating-point values
368 /// Multiply scalar single-precision floating-point values
369 /// Multiply packed double-precision floating-point values
370 /// Multiply scalar double-precision floating-point values
353371 mul,
354372 /// Two's complement negation
355373 neg,
......@@ -358,6 +376,8 @@ pub const Inst = struct {
358376 /// One's complement negation
359377 not,
360378 /// Logical or
379 /// Bitwise logical or of packed single-precision floating-point values
380 /// Bitwise logical or of packed double-precision floating-point values
361381 @"or",
362382 /// Pop
363383 pop,
......@@ -390,6 +410,10 @@ pub const Inst = struct {
390410 /// Double precision shift right
391411 sh,
392412 /// Subtract
413 /// Subtract packed single-precision floating-point values
414 /// Subtract scalar single-precision floating-point values
415 /// Subtract packed double-precision floating-point values
416 /// Subtract scalar double-precision floating-point values
393417 sub,
394418 /// Store string
395419 sto,
......@@ -406,145 +430,88 @@ pub const Inst = struct {
406430 /// Exchange register/memory with register
407431 xchg,
408432 /// Logical exclusive-or
433 /// Bitwise logical xor of packed single-precision floating-point values
434 /// Bitwise logical xor of packed double-precision floating-point values
409435 xor,
410436
411 /// Add packed single-precision floating-point values
412 addps,
413 /// Add scalar single-precision floating-point values
414 addss,
415 /// Bitwise logical and of packed single precision floating-point values
416 andps,
417 /// Bitwise logical and not of packed single precision floating-point values
418 andnps,
419 /// Compare scalar single-precision floating-point values
420 cmpss,
437 /// Bitwise logical and not of packed single-precision floating-point values
438 /// Bitwise logical and not of packed double-precision floating-point values
439 andn,
421440 /// Convert doubleword integer to scalar single-precision floating-point value
422441 cvtsi2ss,
423 /// Divide packed single-precision floating-point values
424 divps,
425 /// Divide scalar single-precision floating-point values
426 divss,
427442 /// Maximum of packed single-precision floating-point values
428 maxps,
429443 /// Maximum of scalar single-precision floating-point values
430 maxss,
444 /// Maximum of packed double-precision floating-point values
445 /// Maximum of scalar double-precision floating-point values
446 max,
431447 /// Minimum of packed single-precision floating-point values
432 minps,
433448 /// Minimum of scalar single-precision floating-point values
434 minss,
449 /// Minimum of packed double-precision floating-point values
450 /// Minimum of scalar double-precision floating-point values
451 min,
435452 /// Move aligned packed single-precision floating-point values
436 movaps,
453 /// Move aligned packed double-precision floating-point values
454 mova,
437455 /// Move packed single-precision floating-point values high to low
438 movhlps,
439 /// Move scalar single-precision floating-point value
440 movss,
456 movhl,
441457 /// Move unaligned packed single-precision floating-point values
442 movups,
443 /// Multiply packed single-precision floating-point values
444 mulps,
445 /// Multiply scalar single-precision floating-point values
446 mulss,
447 /// Bitwise logical or of packed single precision floating-point values
448 orps,
458 /// Move unaligned packed double-precision floating-point values
459 movu,
460 /// Extract byte
449461 /// Extract word
450 pextrw,
462 /// Extract doubleword
463 /// Extract quadword
464 extr,
465 /// Insert byte
451466 /// Insert word
452 pinsrw,
467 /// Insert doubleword
468 /// Insert quadword
469 insr,
453470 /// Square root of packed single-precision floating-point values
454 sqrtps,
455471 /// Square root of scalar single-precision floating-point value
456 sqrtss,
457 /// Subtract packed single-precision floating-point values
458 subps,
459 /// Subtract scalar single-precision floating-point values
460 subss,
472 /// Square root of packed double-precision floating-point values
473 /// Square root of scalar double-precision floating-point value
474 sqrt,
461475 /// Unordered compare scalar single-precision floating-point values
462 ucomiss,
476 /// Unordered compare scalar double-precision floating-point values
477 ucomi,
463478 /// Unpack and interleave high packed single-precision floating-point values
464 unpckhps,
479 /// Unpack and interleave high packed double-precision floating-point values
480 unpckh,
465481 /// Unpack and interleave low packed single-precision floating-point values
466 unpcklps,
467 /// Bitwise logical xor of packed single precision floating-point values
468 xorps,
482 /// Unpack and interleave low packed double-precision floating-point values
483 unpckl,
469484
470 /// Add packed double-precision floating-point values
471 addpd,
472 /// Add scalar double-precision floating-point values
473 addsd,
474 /// Bitwise logical and not of packed double precision floating-point values
475 andnpd,
476 /// Bitwise logical and of packed double precision floating-point values
477 andpd,
478 /// Compare scalar double-precision floating-point values
479 cmpsd,
480485 /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
481486 cvtsd2ss,
482487 /// Convert doubleword integer to scalar double-precision floating-point value
483488 cvtsi2sd,
484489 /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
485490 cvtss2sd,
486 /// Divide packed double-precision floating-point values
487 divpd,
488 /// Divide scalar double-precision floating-point values
489 divsd,
490 /// Maximum of packed double-precision floating-point values
491 maxpd,
492 /// Maximum of scalar double-precision floating-point values
493 maxsd,
494 /// Minimum of packed double-precision floating-point values
495 minpd,
496 /// Minimum of scalar double-precision floating-point values
497 minsd,
498 /// Move scalar double-precision floating-point value
499 movsd,
500 /// Multiply packed double-precision floating-point values
501 mulpd,
502 /// Multiply scalar double-precision floating-point values
503 mulsd,
504 /// Bitwise logical or of packed double precision floating-point values
505 orpd,
506491 /// Shuffle packed high words
507 pshufhw,
492 shufh,
508493 /// Shuffle packed low words
509 pshuflw,
494 shufl,
510495 /// Shift packed data right logical
511 psrld,
512496 /// Shift packed data right logical
513 psrlq,
514497 /// Shift packed data right logical
515 psrlw,
498 srl,
516499 /// Unpack high data
517 punpckhbw,
500 unpckhbw,
518501 /// Unpack high data
519 punpckhdq,
502 unpckhdq,
520503 /// Unpack high data
521 punpckhqdq,
504 unpckhqdq,
522505 /// Unpack high data
523 punpckhwd,
506 unpckhwd,
524507 /// Unpack low data
525 punpcklbw,
508 unpcklbw,
526509 /// Unpack low data
527 punpckldq,
510 unpckldq,
528511 /// Unpack low data
529 punpcklqdq,
512 unpcklqdq,
530513 /// Unpack low data
531 punpcklwd,
532 /// Square root of double precision floating-point values
533 sqrtpd,
534 /// Square root of scalar double precision floating-point value
535 sqrtsd,
536 /// Subtract packed double-precision floating-point values
537 subpd,
538 /// Subtract scalar double-precision floating-point values
539 subsd,
540 /// Unordered compare scalar double-precision floating-point values
541 ucomisd,
542 /// Unpack and interleave high packed double-precision floating-point values
543 unpckhpd,
544 /// Unpack and interleave low packed double-precision floating-point values
545 unpcklpd,
546 /// Bitwise logical xor of packed double precision floating-point values
547 xorpd,
514 unpcklwd,
548515
549516 /// Replicate double floating-point values
550517 movddup,
......@@ -553,199 +520,32 @@ pub const Inst = struct {
553520 /// Replicate single floating-point values
554521 movsldup,
555522
556 /// Extract Byte
557 pextrb,
558 /// Extract Doubleword
559 pextrd,
560 /// Extract Quadword
561 pextrq,
562 /// Insert Byte
563 pinsrb,
564 /// Insert Doubleword
565 pinsrd,
566 /// Insert Quadword
567 pinsrq,
568 /// Round packed double-precision floating-point values
569 roundpd,
570523 /// Round packed single-precision floating-point values
571 roundps,
572 /// Round scalar double-precision floating-point value
573 roundsd,
574524 /// Round scalar single-precision floating-point value
575 roundss,
576
577 /// Add packed double-precision floating-point values
578 vaddpd,
579 /// Add packed single-precision floating-point values
580 vaddps,
581 /// Add scalar double-precision floating-point values
582 vaddsd,
583 /// Add scalar single-precision floating-point values
584 vaddss,
585 /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
586 vcvtsd2ss,
587 /// Convert doubleword integer to scalar double-precision floating-point value
588 vcvtsi2sd,
589 /// Convert doubleword integer to scalar single-precision floating-point value
590 vcvtsi2ss,
591 /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
592 vcvtss2sd,
593 /// Divide packed double-precision floating-point values
594 vdivpd,
595 /// Divide packed single-precision floating-point values
596 vdivps,
597 /// Divide scalar double-precision floating-point values
598 vdivsd,
599 /// Divide scalar single-precision floating-point values
600 vdivss,
601 /// Maximum of packed double-precision floating-point values
602 vmaxpd,
603 /// Maximum of packed single-precision floating-point values
604 vmaxps,
605 /// Maximum of scalar double-precision floating-point values
606 vmaxsd,
607 /// Maximum of scalar single-precision floating-point values
608 vmaxss,
609 /// Minimum of packed double-precision floating-point values
610 vminpd,
611 /// Minimum of packed single-precision floating-point values
612 vminps,
613 /// Minimum of scalar double-precision floating-point values
614 vminsd,
615 /// Minimum of scalar single-precision floating-point values
616 vminss,
617 /// Move aligned packed double-precision floating-point values
618 vmovapd,
619 /// Move aligned packed single-precision floating-point values
620 vmovaps,
621 /// Move packed single-precision floating-point values high to low
622 vmovhlps,
623 /// Replicate double floating-point values
624 vmovddup,
625 /// Move or merge scalar double-precision floating-point value
626 vmovsd,
627 /// Replicate single floating-point values
628 vmovshdup,
629 /// Replicate single floating-point values
630 vmovsldup,
631 /// Move or merge scalar single-precision floating-point value
632 vmovss,
633 /// Move unaligned packed double-precision floating-point values
634 vmovupd,
635 /// Move unaligned packed single-precision floating-point values
636 vmovups,
637 /// Multiply packed double-precision floating-point values
638 vmulpd,
639 /// Multiply packed single-precision floating-point values
640 vmulps,
641 /// Multiply scalar double-precision floating-point values
642 vmulsd,
643 /// Multiply scalar single-precision floating-point values
644 vmulss,
645 /// Extract Byte
646 vpextrb,
647 /// Extract Doubleword
648 vpextrd,
649 /// Extract Quadword
650 vpextrq,
651 /// Extract word
652 vpextrw,
653 /// Insert Byte
654 vpinsrb,
655 /// Insert Doubleword
656 vpinsrd,
657 /// Insert Quadword
658 vpinsrq,
659 /// Insert word
660 vpinsrw,
661 /// Shuffle packed high words
662 vpshufhw,
663 /// Shuffle packed low words
664 vpshuflw,
665 /// Shift packed data right logical
666 vpsrld,
667 /// Shift packed data right logical
668 vpsrlq,
669 /// Shift packed data right logical
670 vpsrlw,
671 /// Unpack high data
672 vpunpckhbw,
673 /// Unpack high data
674 vpunpckhdq,
675 /// Unpack high data
676 vpunpckhqdq,
677 /// Unpack high data
678 vpunpckhwd,
679 /// Unpack low data
680 vpunpcklbw,
681 /// Unpack low data
682 vpunpckldq,
683 /// Unpack low data
684 vpunpcklqdq,
685 /// Unpack low data
686 vpunpcklwd,
687525 /// Round packed double-precision floating-point values
688 vroundpd,
689 /// Round packed single-precision floating-point values
690 vroundps,
691526 /// Round scalar double-precision floating-point value
692 vroundsd,
693 /// Round scalar single-precision floating-point value
694 vroundss,
695 /// Square root of packed double-precision floating-point value
696 vsqrtpd,
697 /// Square root of packed single-precision floating-point value
698 vsqrtps,
699 /// Square root of scalar double-precision floating-point value
700 vsqrtsd,
701 /// Square root of scalar single-precision floating-point value
702 vsqrtss,
703 /// Subtract packed double-precision floating-point values
704 vsubpd,
705 /// Subtract packed single-precision floating-point values
706 vsubps,
707 /// Subtract scalar double-precision floating-point values
708 vsubsd,
709 /// Subtract scalar single-precision floating-point values
710 vsubss,
711 /// Unpack and interleave high packed double-precision floating-point values
712 vunpckhpd,
713 /// Unpack and interleave high packed single-precision floating-point values
714 vunpckhps,
715 /// Unpack and interleave low packed double-precision floating-point values
716 vunpcklpd,
717 /// Unpack and interleave low packed single-precision floating-point values
718 vunpcklps,
527 round,
719528
720529 /// Convert 16-bit floating-point values to single-precision floating-point values
721 vcvtph2ps,
530 cvtph2ps,
722531 /// Convert single-precision floating-point values to 16-bit floating-point values
723 vcvtps2ph,
532 cvtps2ph,
724533
725 /// Fused multiply-add of packed double-precision floating-point values
726 vfmadd132pd,
727 /// Fused multiply-add of packed double-precision floating-point values
728 vfmadd213pd,
729 /// Fused multiply-add of packed double-precision floating-point values
730 vfmadd231pd,
731 /// Fused multiply-add of packed single-precision floating-point values
732 vfmadd132ps,
733534 /// Fused multiply-add of packed single-precision floating-point values
734 vfmadd213ps,
735 /// Fused multiply-add of packed single-precision floating-point values
736 vfmadd231ps,
737 /// Fused multiply-add of scalar double-precision floating-point values
738 vfmadd132sd,
739 /// Fused multiply-add of scalar double-precision floating-point values
740 vfmadd213sd,
741 /// Fused multiply-add of scalar double-precision floating-point values
742 vfmadd231sd,
743535 /// Fused multiply-add of scalar single-precision floating-point values
744 vfmadd132ss,
536 /// Fused multiply-add of packed double-precision floating-point values
537 /// Fused multiply-add of scalar double-precision floating-point values
538 fmadd132,
539 /// Fused multiply-add of packed single-precision floating-point values
745540 /// Fused multiply-add of scalar single-precision floating-point values
746 vfmadd213ss,
541 /// Fused multiply-add of packed double-precision floating-point values
542 /// Fused multiply-add of scalar double-precision floating-point values
543 fmadd213,
544 /// Fused multiply-add of packed single-precision floating-point values
747545 /// Fused multiply-add of scalar single-precision floating-point values
748 vfmadd231ss,
546 /// Fused multiply-add of packed double-precision floating-point values
547 /// Fused multiply-add of scalar double-precision floating-point values
548 fmadd231,
749549
750550 /// A pseudo instruction that requires special lowering.
751551 /// This should be the only tag in this enum that doesn't