authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-24 11:37:13-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-28 15:10:22-04:00
loga4a1ebdeed16494270a0111aeee6fec8e04ef7ee
tree6a3cac6bbd6efde299d15991fa04b261e4459890
parent612f5784cf0a02b4d3be5aaf9e3822d72e220c96

x86_64: implement optimized float `@reduce(.Mul)`

14 files changed, 2082 insertions(+), 349 deletions(-)

lib/std/zig/Zir.zig+5-1
......@@ -2142,7 +2142,7 @@ pub const Inst = struct {
21422142 ref_start_index = static_len,
21432143 _,
21442144
2145 pub const static_len = 105;
2145 pub const static_len = 109;
21462146
21472147 pub fn toRef(i: Index) Inst.Ref {
21482148 return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i));
......@@ -2255,11 +2255,15 @@ pub const Inst = struct {
22552255 vector_1_u256_type,
22562256 vector_4_f16_type,
22572257 vector_8_f16_type,
2258 vector_16_f16_type,
2259 vector_32_f16_type,
22582260 vector_2_f32_type,
22592261 vector_4_f32_type,
22602262 vector_8_f32_type,
2263 vector_16_f32_type,
22612264 vector_2_f64_type,
22622265 vector_4_f64_type,
2266 vector_8_f64_type,
22632267 optional_noreturn_type,
22642268 anyerror_void_error_union_type,
22652269 adhoc_inferred_error_set_type,
src/Air.zig+4
......@@ -1038,11 +1038,15 @@ pub const Inst = struct {
10381038 vector_1_u256_type = @intFromEnum(InternPool.Index.vector_1_u256_type),
10391039 vector_4_f16_type = @intFromEnum(InternPool.Index.vector_4_f16_type),
10401040 vector_8_f16_type = @intFromEnum(InternPool.Index.vector_8_f16_type),
1041 vector_16_f16_type = @intFromEnum(InternPool.Index.vector_16_f16_type),
1042 vector_32_f16_type = @intFromEnum(InternPool.Index.vector_32_f16_type),
10411043 vector_2_f32_type = @intFromEnum(InternPool.Index.vector_2_f32_type),
10421044 vector_4_f32_type = @intFromEnum(InternPool.Index.vector_4_f32_type),
10431045 vector_8_f32_type = @intFromEnum(InternPool.Index.vector_8_f32_type),
1046 vector_16_f32_type = @intFromEnum(InternPool.Index.vector_16_f32_type),
10441047 vector_2_f64_type = @intFromEnum(InternPool.Index.vector_2_f64_type),
10451048 vector_4_f64_type = @intFromEnum(InternPool.Index.vector_4_f64_type),
1049 vector_8_f64_type = @intFromEnum(InternPool.Index.vector_8_f64_type),
10461050 optional_noreturn_type = @intFromEnum(InternPool.Index.optional_noreturn_type),
10471051 anyerror_void_error_union_type = @intFromEnum(InternPool.Index.anyerror_void_error_union_type),
10481052 adhoc_inferred_error_set_type = @intFromEnum(InternPool.Index.adhoc_inferred_error_set_type),
src/InternPool.zig+20
......@@ -4615,11 +4615,15 @@ pub const Index = enum(u32) {
46154615 vector_1_u256_type,
46164616 vector_4_f16_type,
46174617 vector_8_f16_type,
4618 vector_16_f16_type,
4619 vector_32_f16_type,
46184620 vector_2_f32_type,
46194621 vector_4_f32_type,
46204622 vector_8_f32_type,
4623 vector_16_f32_type,
46214624 vector_2_f64_type,
46224625 vector_4_f64_type,
4626 vector_8_f64_type,
46234627
46244628 optional_noreturn_type,
46254629 anyerror_void_error_union_type,
......@@ -5174,16 +5178,24 @@ pub const static_keys = [_]Key{
51745178 .{ .vector_type = .{ .len = 4, .child = .f16_type } },
51755179 // @Vector(8, f16)
51765180 .{ .vector_type = .{ .len = 8, .child = .f16_type } },
5181 // @Vector(16, f16)
5182 .{ .vector_type = .{ .len = 16, .child = .f16_type } },
5183 // @Vector(32, f16)
5184 .{ .vector_type = .{ .len = 32, .child = .f16_type } },
51775185 // @Vector(2, f32)
51785186 .{ .vector_type = .{ .len = 2, .child = .f32_type } },
51795187 // @Vector(4, f32)
51805188 .{ .vector_type = .{ .len = 4, .child = .f32_type } },
51815189 // @Vector(8, f32)
51825190 .{ .vector_type = .{ .len = 8, .child = .f32_type } },
5191 // @Vector(16, f32)
5192 .{ .vector_type = .{ .len = 16, .child = .f32_type } },
51835193 // @Vector(2, f64)
51845194 .{ .vector_type = .{ .len = 2, .child = .f64_type } },
51855195 // @Vector(4, f64)
51865196 .{ .vector_type = .{ .len = 4, .child = .f64_type } },
5197 // @Vector(8, f64)
5198 .{ .vector_type = .{ .len = 8, .child = .f64_type } },
51875199
51885200 // ?noreturn
51895201 .{ .opt_type = .noreturn_type },
......@@ -11847,11 +11859,15 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {
1184711859 .vector_1_u256_type,
1184811860 .vector_4_f16_type,
1184911861 .vector_8_f16_type,
11862 .vector_16_f16_type,
11863 .vector_32_f16_type,
1185011864 .vector_2_f32_type,
1185111865 .vector_4_f32_type,
1185211866 .vector_8_f32_type,
11867 .vector_16_f32_type,
1185311868 .vector_2_f64_type,
1185411869 .vector_4_f64_type,
11870 .vector_8_f64_type,
1185511871 .optional_noreturn_type,
1185611872 .anyerror_void_error_union_type,
1185711873 .adhoc_inferred_error_set_type,
......@@ -12175,11 +12191,15 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {
1217512191 .vector_1_u256_type,
1217612192 .vector_4_f16_type,
1217712193 .vector_8_f16_type,
12194 .vector_16_f16_type,
12195 .vector_32_f16_type,
1217812196 .vector_2_f32_type,
1217912197 .vector_4_f32_type,
1218012198 .vector_8_f32_type,
12199 .vector_16_f32_type,
1218112200 .vector_2_f64_type,
1218212201 .vector_4_f64_type,
12202 .vector_8_f64_type,
1218312203 => .vector,
1218412204
1218512205 .optional_noreturn_type => .optional,
src/Sema.zig+4
......@@ -36571,11 +36571,15 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {
3657136571 .vector_1_u256_type,
3657236572 .vector_4_f16_type,
3657336573 .vector_8_f16_type,
36574 .vector_16_f16_type,
36575 .vector_32_f16_type,
3657436576 .vector_2_f32_type,
3657536577 .vector_4_f32_type,
3657636578 .vector_8_f32_type,
36579 .vector_16_f32_type,
3657736580 .vector_2_f64_type,
3657836581 .vector_4_f64_type,
36582 .vector_8_f64_type,
3657936583 .anyerror_void_error_union_type,
3658036584 => null,
3658136585 .void_type => Value.void,
src/Type.zig+4
......@@ -4136,11 +4136,15 @@ pub const vector_2_u128: Type = .{ .ip_index = .vector_2_u128_type };
41364136pub const vector_1_u256: Type = .{ .ip_index = .vector_1_u256_type };
41374137pub const vector_4_f16: Type = .{ .ip_index = .vector_4_f16_type };
41384138pub const vector_8_f16: Type = .{ .ip_index = .vector_8_f16_type };
4139pub const vector_16_f16: Type = .{ .ip_index = .vector_16_f16_type };
4140pub const vector_32_f16: Type = .{ .ip_index = .vector_32_f16_type };
41394141pub const vector_2_f32: Type = .{ .ip_index = .vector_2_f32_type };
41404142pub const vector_4_f32: Type = .{ .ip_index = .vector_4_f32_type };
41414143pub const vector_8_f32: Type = .{ .ip_index = .vector_8_f32_type };
4144pub const vector_16_f32: Type = .{ .ip_index = .vector_16_f32_type };
41424145pub const vector_2_f64: Type = .{ .ip_index = .vector_2_f64_type };
41434146pub const vector_4_f64: Type = .{ .ip_index = .vector_4_f64_type };
4147pub const vector_8_f64: Type = .{ .ip_index = .vector_8_f64_type };
41444148
41454149pub const empty_tuple: Type = .{ .ip_index = .empty_tuple_type };
41464150
src/arch/x86_64/CodeGen.zig+1950-334
......@@ -2389,7 +2389,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
23892389}
23902390
23912391fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2392 @setEvalBranchQuota(23_600);
2392 @setEvalBranchQuota(23_800);
23932393 const pt = cg.pt;
23942394 const zcu = pt.zcu;
23952395 const ip = &zcu.intern_pool;
......@@ -68441,7 +68441,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6844168441 .{ .src = .{ .to_sse, .none, .none } },
6844268442 },
6844368443 .extra_temps = .{
68444 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
68444 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6844568445 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6844668446 .unused,
6844768447 .unused,
......@@ -68465,7 +68465,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6846568465 .{ .src = .{ .to_mut_sse, .none, .none } },
6846668466 },
6846768467 .extra_temps = .{
68468 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
68468 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6846968469 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6847068470 .unused,
6847168471 .unused,
......@@ -68489,7 +68489,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6848968489 .{ .src = .{ .to_sse, .none, .none } },
6849068490 },
6849168491 .extra_temps = .{
68492 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68492 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6849368493 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6849468494 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6849568495 .unused,
......@@ -68517,7 +68517,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6851768517 .{ .src = .{ .to_sse, .none, .none } },
6851868518 },
6851968519 .extra_temps = .{
68520 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68520 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6852168521 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6852268522 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6852368523 .unused,
......@@ -68546,7 +68546,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6854668546 .{ .src = .{ .to_mut_sse, .none, .none } },
6854768547 },
6854868548 .extra_temps = .{
68549 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68549 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6855068550 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6855168551 .unused,
6855268552 .unused,
......@@ -68575,7 +68575,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6857568575 .{ .src = .{ .to_mut_sse, .none, .none } },
6857668576 },
6857768577 .extra_temps = .{
68578 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68578 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6857968579 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6858068580 .unused,
6858168581 .unused,
......@@ -68642,7 +68642,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6864268642 .{ .src = .{ .to_sse, .none, .none } },
6864368643 },
6864468644 .extra_temps = .{
68645 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
68645 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6864668646 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6864768647 .unused,
6864868648 .unused,
......@@ -68668,7 +68668,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6866868668 .{ .src = .{ .to_mut_sse, .none, .none } },
6866968669 },
6867068670 .extra_temps = .{
68671 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
68671 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6867268672 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6867368673 .unused,
6867468674 .unused,
......@@ -68694,7 +68694,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6869468694 .{ .src = .{ .to_sse, .none, .none } },
6869568695 },
6869668696 .extra_temps = .{
68697 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68697 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6869868698 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6869968699 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6870068700 .unused,
......@@ -68724,7 +68724,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6872468724 .{ .src = .{ .to_sse, .none, .none } },
6872568725 },
6872668726 .extra_temps = .{
68727 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68727 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6872868728 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6872968729 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6873068730 .unused,
......@@ -68755,7 +68755,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6875568755 .{ .src = .{ .to_mut_sse, .none, .none } },
6875668756 },
6875768757 .extra_temps = .{
68758 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68758 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6875968759 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6876068760 .unused,
6876168761 .unused,
......@@ -68786,7 +68786,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6878668786 .{ .src = .{ .to_mut_sse, .none, .none } },
6878768787 },
6878868788 .extra_temps = .{
68789 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68789 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6879068790 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6879168791 .unused,
6879268792 .unused,
......@@ -68856,7 +68856,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6885668856 .{ .src = .{ .to_sse, .none, .none } },
6885768857 },
6885868858 .extra_temps = .{
68859 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
68859 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6886068860 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6886168861 .unused,
6886268862 .unused,
......@@ -68882,7 +68882,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6888268882 .{ .src = .{ .to_mut_sse, .none, .none } },
6888368883 },
6888468884 .extra_temps = .{
68885 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
68885 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6888668886 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6888768887 .unused,
6888868888 .unused,
......@@ -68908,7 +68908,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6890868908 .{ .src = .{ .to_sse, .none, .none } },
6890968909 },
6891068910 .extra_temps = .{
68911 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68911 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6891268912 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6891368913 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6891468914 .unused,
......@@ -68938,7 +68938,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6893868938 .{ .src = .{ .to_sse, .none, .none } },
6893968939 },
6894068940 .extra_temps = .{
68941 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68941 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6894268942 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6894368943 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6894468944 .unused,
......@@ -68969,7 +68969,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6896968969 .{ .src = .{ .to_mut_sse, .none, .none } },
6897068970 },
6897168971 .extra_temps = .{
68972 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
68972 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6897368973 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6897468974 .unused,
6897568975 .unused,
......@@ -69000,7 +69000,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6900069000 .{ .src = .{ .to_mut_sse, .none, .none } },
6900169001 },
6900269002 .extra_temps = .{
69003 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
69003 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6900469004 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6900569005 .unused,
6900669006 .unused,
......@@ -69070,7 +69070,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6907069070 .{ .src = .{ .to_sse, .none, .none } },
6907169071 },
6907269072 .extra_temps = .{
69073 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69073 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6907469074 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
6907569075 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6907669076 .unused,
......@@ -69096,7 +69096,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6909669096 .{ .src = .{ .to_mut_sse, .none, .none } },
6909769097 },
6909869098 .extra_temps = .{
69099 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69099 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6910069100 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
6910169101 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6910269102 .unused,
......@@ -69122,7 +69122,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6912269122 .{ .src = .{ .to_mut_sse, .none, .none } },
6912369123 },
6912469124 .extra_temps = .{
69125 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69125 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6912669126 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6912769127 .unused,
6912869128 .unused,
......@@ -69149,7 +69149,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6914969149 },
6915069150 .extra_temps = .{
6915169151 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
69152 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
69152 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6915369153 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6915469154 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6915569155 .unused,
......@@ -69178,7 +69178,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6917869178 },
6917969179 .extra_temps = .{
6918069180 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
69181 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
69181 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6918269182 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6918369183 .unused,
6918469184 .unused,
......@@ -69247,7 +69247,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6924769247 .{ .src = .{ .to_sse, .none, .none } },
6924869248 },
6924969249 .extra_temps = .{
69250 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69250 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6925169251 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
6925269252 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6925369253 .unused,
......@@ -69274,7 +69274,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6927469274 .{ .src = .{ .to_mut_sse, .none, .none } },
6927569275 },
6927669276 .extra_temps = .{
69277 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69277 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6927869278 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
6927969279 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6928069280 .unused,
......@@ -69301,7 +69301,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6930169301 .{ .src = .{ .to_mut_sse, .none, .none } },
6930269302 },
6930369303 .extra_temps = .{
69304 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69304 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6930569305 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6930669306 .unused,
6930769307 .unused,
......@@ -69329,7 +69329,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6932969329 },
6933069330 .extra_temps = .{
6933169331 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
69332 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
69332 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6933369333 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6933469334 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6933569335 .unused,
......@@ -69360,7 +69360,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6936069360 },
6936169361 .extra_temps = .{
6936269362 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
69363 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
69363 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6936469364 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6936569365 .unused,
6936669366 .unused,
......@@ -69432,7 +69432,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6943269432 .{ .src = .{ .to_sse, .none, .none } },
6943369433 },
6943469434 .extra_temps = .{
69435 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69435 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6943669436 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
6943769437 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6943869438 .unused,
......@@ -69459,7 +69459,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6945969459 .{ .src = .{ .to_mut_sse, .none, .none } },
6946069460 },
6946169461 .extra_temps = .{
69462 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69462 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6946369463 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
6946469464 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6946569465 .unused,
......@@ -69486,7 +69486,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6948669486 .{ .src = .{ .to_mut_sse, .none, .none } },
6948769487 },
6948869488 .extra_temps = .{
69489 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69489 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6949069490 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6949169491 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6949269492 .unused,
......@@ -69515,7 +69515,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6951569515 },
6951669516 .extra_temps = .{
6951769517 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
69518 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
69518 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6951969519 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6952069520 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6952169521 .unused,
......@@ -69546,7 +69546,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6954669546 },
6954769547 .extra_temps = .{
6954869548 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
69549 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
69549 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
6955069550 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6955169551 .unused,
6955269552 .unused,
......@@ -69618,7 +69618,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6961869618 .{ .src = .{ .to_sse, .none, .none } },
6961969619 },
6962069620 .extra_temps = .{
69621 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69621 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6962269622 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
6962369623 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6962469624 .unused,
......@@ -69644,7 +69644,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6964469644 .{ .src = .{ .to_mut_sse, .none, .none } },
6964569645 },
6964669646 .extra_temps = .{
69647 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69647 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6964869648 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
6964969649 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6965069650 .unused,
......@@ -69670,7 +69670,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6967069670 .{ .src = .{ .to_mut_sse, .none, .none } },
6967169671 },
6967269672 .extra_temps = .{
69673 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69673 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6967469674 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6967569675 .unused,
6967669676 .unused,
......@@ -69698,7 +69698,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6969869698 },
6969969699 .extra_temps = .{
6970069700 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
69701 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
69701 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },
6970269702 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6970369703 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
6970469704 .unused,
......@@ -69768,7 +69768,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6976869768 .{ .src = .{ .to_sse, .none, .none } },
6976969769 },
6977069770 .extra_temps = .{
69771 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69771 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6977269772 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
6977369773 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6977469774 .unused,
......@@ -69795,7 +69795,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6979569795 .{ .src = .{ .to_mut_sse, .none, .none } },
6979669796 },
6979769797 .extra_temps = .{
69798 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69798 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6979969799 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
6980069800 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6980169801 .unused,
......@@ -69822,7 +69822,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6982269822 .{ .src = .{ .to_mut_sse, .none, .none } },
6982369823 },
6982469824 .extra_temps = .{
69825 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69825 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6982669826 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6982769827 .unused,
6982869828 .unused,
......@@ -69851,7 +69851,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6985169851 },
6985269852 .extra_temps = .{
6985369853 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
69854 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
69854 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },
6985569855 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
6985669856 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
6985769857 .unused,
......@@ -69924,7 +69924,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6992469924 .{ .src = .{ .to_sse, .none, .none } },
6992569925 },
6992669926 .extra_temps = .{
69927 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69927 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6992869928 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
6992969929 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6993069930 .unused,
......@@ -69951,7 +69951,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6995169951 .{ .src = .{ .to_mut_sse, .none, .none } },
6995269952 },
6995369953 .extra_temps = .{
69954 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69954 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6995569955 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
6995669956 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6995769957 .unused,
......@@ -69978,7 +69978,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6997869978 .{ .src = .{ .to_mut_sse, .none, .none } },
6997969979 },
6998069980 .extra_temps = .{
69981 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
69981 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
6998269982 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6998369983 .unused,
6998469984 .unused,
......@@ -70007,7 +70007,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7000770007 },
7000870008 .extra_temps = .{
7000970009 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
70010 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
70010 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },
7001170011 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7001270012 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7001370013 .unused,
......@@ -70080,7 +70080,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7008070080 .{ .src = .{ .to_sse, .none, .none } },
7008170081 },
7008270082 .extra_temps = .{
70083 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70083 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7008470084 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
7008570085 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7008670086 .unused,
......@@ -70106,7 +70106,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7010670106 .{ .src = .{ .to_mut_sse, .none, .none } },
7010770107 },
7010870108 .extra_temps = .{
70109 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70109 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7011070110 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
7011170111 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7011270112 .unused,
......@@ -70132,7 +70132,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7013270132 .{ .src = .{ .to_mut_sse, .none, .none } },
7013370133 },
7013470134 .extra_temps = .{
70135 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70135 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7013670136 .unused,
7013770137 .unused,
7013870138 .unused,
......@@ -70203,7 +70203,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7020370203 .{ .src = .{ .to_sse, .none, .none } },
7020470204 },
7020570205 .extra_temps = .{
70206 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70206 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7020770207 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7020870208 .unused,
7020970209 .unused,
......@@ -70231,7 +70231,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7023170231 .{ .src = .{ .to_mut_sse, .none, .none } },
7023270232 },
7023370233 .extra_temps = .{
70234 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70234 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7023570235 .unused,
7023670236 .unused,
7023770237 .unused,
......@@ -70305,7 +70305,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7030570305 .{ .src = .{ .to_sse, .none, .none } },
7030670306 },
7030770307 .extra_temps = .{
70308 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70308 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7030970309 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
7031070310 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7031170311 .unused,
......@@ -70332,7 +70332,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7033270332 .{ .src = .{ .to_mut_sse, .none, .none } },
7033370333 },
7033470334 .extra_temps = .{
70335 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70335 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7033670336 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
7033770337 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7033870338 .unused,
......@@ -70359,7 +70359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7035970359 .{ .src = .{ .to_mut_sse, .none, .none } },
7036070360 },
7036170361 .extra_temps = .{
70362 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70362 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7036370363 .unused,
7036470364 .unused,
7036570365 .unused,
......@@ -70433,7 +70433,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7043370433 .{ .src = .{ .to_sse, .none, .none } },
7043470434 },
7043570435 .extra_temps = .{
70436 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70436 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7043770437 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7043870438 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7043970439 .unused,
......@@ -70459,7 +70459,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7045970459 .{ .src = .{ .to_mut_sse, .none, .none } },
7046070460 },
7046170461 .extra_temps = .{
70462 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70462 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7046370463 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7046470464 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7046570465 .unused,
......@@ -70485,7 +70485,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7048570485 .{ .src = .{ .to_sse, .none, .none } },
7048670486 },
7048770487 .extra_temps = .{
70488 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70488 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7048970489 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7049070490 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7049170491 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
......@@ -70518,7 +70518,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7051870518 .{ .src = .{ .to_mut_sse, .none, .none } },
7051970519 },
7052070520 .extra_temps = .{
70521 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70521 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7052270522 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7052370523 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7052470524 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
......@@ -70551,7 +70551,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7055170551 .{ .src = .{ .to_sse, .none, .none } },
7055270552 },
7055370553 .extra_temps = .{
70554 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70554 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7055570555 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7055670556 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7055770557 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
......@@ -70584,7 +70584,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7058470584 .{ .src = .{ .to_mut_sse, .none, .none } },
7058570585 },
7058670586 .extra_temps = .{
70587 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70587 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7058870588 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7058970589 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7059070590 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
......@@ -70617,7 +70617,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7061770617 .{ .src = .{ .to_sse, .none, .none } },
7061870618 },
7061970619 .extra_temps = .{
70620 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
70620 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },
7062170621 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
7062270622 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7062370623 .unused,
......@@ -70644,7 +70644,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7064470644 .{ .src = .{ .to_sse, .none, .none } },
7064570645 },
7064670646 .extra_temps = .{
70647 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
70647 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },
7064870648 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7064970649 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7065070650 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -70676,7 +70676,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7067670676 .extra_temps = .{
7067770677 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7067870678 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
70679 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
70679 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },
7068070680 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
7068170681 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7068270682 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
......@@ -70712,7 +70712,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7071270712 .extra_temps = .{
7071370713 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7071470714 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
70715 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
70715 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
7071670716 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
7071770717 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7071870718 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -129477,8 +129477,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
129477129477 switch (reduce.operation) {
129478129478 .And, .Or, .Xor => unreachable,
129479129479 .Min, .Max => break :fallback try cg.airReduce(inst),
129480 .Add => {},
129481 .Mul => break :fallback try cg.airReduce(inst),
129480 .Add, .Mul => {},
129482129481 }
129483129482 var ops = try cg.tempsFromOperands(inst, .{reduce.operand});
129484129483 var res: [1]Temp = undefined;
......@@ -129894,7 +129893,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
129894129893 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
129895129894 } },
129896129895 }, .{
129897 .required_features = .{ .avx512f, .f16c, null, null },
129896 .required_features = .{ .avx512f, null, null, null },
129898129897 .dst_constraints = .{ .{ .float = .word }, .any },
129899129898 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any },
129900129899 .patterns = &.{
......@@ -129938,7 +129937,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
129938129937 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
129939129938 } },
129940129939 }, .{
129941 .required_features = .{ .avx512f, .f16c, null, null },
129940 .required_features = .{ .avx512f, null, null, null },
129942129941 .dst_constraints = .{ .{ .float = .word }, .any },
129943129942 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any },
129944129943 .patterns = &.{
......@@ -130051,7 +130050,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
130051130050 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
130052130051 } },
130053130052 }, .{
130054 .required_features = .{ .avx512f, .f16c, null, null },
130053 .required_features = .{ .avx512f, null, null, null },
130055130054 .dst_constraints = .{ .{ .float = .word }, .any },
130056130055 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any },
130057130056 .patterns = &.{
......@@ -130685,10 +130684,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
130685130684 .dst_temps = .{ .{ .rc = .sse }, .unused },
130686130685 .each = .{ .once = &.{
130687130686 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
130688 .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ },
130689 .{ ._, .v_ps, .mova, .tmp2y, .memd(.src0y, 32), ._, ._ },
130690 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ },
130691 .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ },
130687 .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
130688 .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
130689 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ },
130690 .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ },
130692130691 .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ },
130693130692 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
130694130693 .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ },
......@@ -131117,33 +131116,1720 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131117131116 .each = .{ .once = &.{
131118131117 .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ },
131119131118 .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ },
131120 .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ },
131121 } },
131122 }, .{
131123 .required_features = .{ .avx, .fast_hops, null, null },
131124 .dst_constraints = .{ .{ .float = .qword }, .any },
131125 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131126 .patterns = &.{
131127 .{ .src = .{ .to_sse, .none, .none } },
131128 },
131129 .extra_temps = .{
131130 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131131 .unused,
131132 .unused,
131133 .unused,
131134 .unused,
131135 .unused,
131136 .unused,
131137 .unused,
131138 .unused,
131139 .unused,
131140 .unused,
131141 },
131142 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131143 .each = .{ .once = &.{
131144 .{ ._, .vh_pd, .add, .tmp0x, .src0x, .src0x, ._ },
131145 .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ },
131146 .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ },
131119 .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ },
131120 } },
131121 }, .{
131122 .required_features = .{ .avx, .fast_hops, null, null },
131123 .dst_constraints = .{ .{ .float = .qword }, .any },
131124 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131125 .patterns = &.{
131126 .{ .src = .{ .to_sse, .none, .none } },
131127 },
131128 .extra_temps = .{
131129 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131130 .unused,
131131 .unused,
131132 .unused,
131133 .unused,
131134 .unused,
131135 .unused,
131136 .unused,
131137 .unused,
131138 .unused,
131139 .unused,
131140 },
131141 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131142 .each = .{ .once = &.{
131143 .{ ._, .vh_pd, .add, .tmp0x, .src0x, .src0x, ._ },
131144 .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ },
131145 .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ },
131146 } },
131147 }, .{
131148 .required_features = .{ .avx, null, null, null },
131149 .dst_constraints = .{ .{ .float = .qword }, .any },
131150 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131151 .patterns = &.{
131152 .{ .src = .{ .to_sse, .none, .none } },
131153 },
131154 .extra_temps = .{
131155 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131156 .unused,
131157 .unused,
131158 .unused,
131159 .unused,
131160 .unused,
131161 .unused,
131162 .unused,
131163 .unused,
131164 .unused,
131165 .unused,
131166 },
131167 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131168 .each = .{ .once = &.{
131169 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
131170 .{ ._, .v_pd, .add, .tmp0x, .src0x, .tmp0x, ._ },
131171 .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ },
131172 .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ },
131173 } },
131174 }, .{
131175 .required_features = .{ .avx, .fast_hops, null, null },
131176 .dst_constraints = .{ .{ .float = .qword }, .any },
131177 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131178 .patterns = &.{
131179 .{ .src = .{ .to_sse, .none, .none } },
131180 },
131181 .extra_temps = .{
131182 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131183 .unused,
131184 .unused,
131185 .unused,
131186 .unused,
131187 .unused,
131188 .unused,
131189 .unused,
131190 .unused,
131191 .unused,
131192 .unused,
131193 },
131194 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131195 .each = .{ .once = &.{
131196 .{ ._, .vh_pd, .add, .dst0y, .src0y, .src0y, ._ },
131197 .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
131198 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ },
131199 } },
131200 }, .{
131201 .required_features = .{ .avx, null, null, null },
131202 .dst_constraints = .{ .{ .float = .qword }, .any },
131203 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131204 .patterns = &.{
131205 .{ .src = .{ .to_sse, .none, .none } },
131206 },
131207 .extra_temps = .{
131208 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131209 .unused,
131210 .unused,
131211 .unused,
131212 .unused,
131213 .unused,
131214 .unused,
131215 .unused,
131216 .unused,
131217 .unused,
131218 .unused,
131219 },
131220 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131221 .each = .{ .once = &.{
131222 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
131223 .{ ._, .v_pd, .add, .dst0x, .src0x, .tmp0x, ._ },
131224 .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ },
131225 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ },
131226 } },
131227 }, .{
131228 .required_features = .{ .avx512f, null, null, null },
131229 .dst_constraints = .{ .{ .float = .qword }, .any },
131230 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any },
131231 .patterns = &.{
131232 .{ .src = .{ .to_mem, .none, .none } },
131233 },
131234 .extra_temps = .{
131235 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
131236 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131237 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131238 .unused,
131239 .unused,
131240 .unused,
131241 .unused,
131242 .unused,
131243 .unused,
131244 .unused,
131245 .unused,
131246 },
131247 .dst_temps = .{ .{ .rc = .sse }, .unused },
131248 .each = .{ .once = &.{
131249 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131250 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131251 .{ ._, .v_pd, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
131252 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ },
131253 .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ },
131254 .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ },
131255 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131256 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131257 .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ },
131258 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131259 } },
131260 }, .{
131261 .required_features = .{ .avx512f, null, null, null },
131262 .dst_constraints = .{ .{ .float = .qword }, .any },
131263 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any },
131264 .patterns = &.{
131265 .{ .src = .{ .to_mem, .none, .none } },
131266 },
131267 .extra_temps = .{
131268 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
131269 .unused,
131270 .unused,
131271 .unused,
131272 .unused,
131273 .unused,
131274 .unused,
131275 .unused,
131276 .unused,
131277 .unused,
131278 .unused,
131279 },
131280 .dst_temps = .{ .{ .rc = .sse }, .unused },
131281 .each = .{ .once = &.{
131282 .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ },
131283 .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ },
131284 .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp0y, ._ },
131285 .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
131286 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp0x, ._ },
131287 .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ },
131288 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ },
131289 } },
131290 }, .{
131291 .required_features = .{ .avx, .fast_hops, null, null },
131292 .dst_constraints = .{ .{ .float = .qword }, .any },
131293 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131294 .patterns = &.{
131295 .{ .src = .{ .to_mem, .none, .none } },
131296 },
131297 .extra_temps = .{
131298 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131299 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131300 .unused,
131301 .unused,
131302 .unused,
131303 .unused,
131304 .unused,
131305 .unused,
131306 .unused,
131307 .unused,
131308 .unused,
131309 },
131310 .dst_temps = .{ .{ .rc = .sse }, .unused },
131311 .clobbers = .{ .eflags = true },
131312 .each = .{ .once = &.{
131313 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ },
131314 .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
131315 .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
131316 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
131317 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131318 .{ ._, .vh_pd, .add, .dst0y, .dst0y, .dst0y, ._ },
131319 .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
131320 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp1x, ._ },
131321 } },
131322 }, .{
131323 .required_features = .{ .avx, null, null, null },
131324 .dst_constraints = .{ .{ .float = .qword }, .any },
131325 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131326 .patterns = &.{
131327 .{ .src = .{ .to_mem, .none, .none } },
131328 },
131329 .extra_temps = .{
131330 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131331 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
131332 .unused,
131333 .unused,
131334 .unused,
131335 .unused,
131336 .unused,
131337 .unused,
131338 .unused,
131339 .unused,
131340 .unused,
131341 },
131342 .dst_temps = .{ .{ .rc = .sse }, .unused },
131343 .clobbers = .{ .eflags = true },
131344 .each = .{ .once = &.{
131345 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ },
131346 .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
131347 .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
131348 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
131349 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131350 .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
131351 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp1x, ._ },
131352 .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ },
131353 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp1x, ._ },
131354 } },
131355 }, .{
131356 .required_features = .{ .sse3, .fast_hops, null, null },
131357 .dst_constraints = .{ .{ .float = .qword }, .any },
131358 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
131359 .patterns = &.{
131360 .{ .src = .{ .to_mem, .none, .none } },
131361 },
131362 .extra_temps = .{
131363 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131364 .unused,
131365 .unused,
131366 .unused,
131367 .unused,
131368 .unused,
131369 .unused,
131370 .unused,
131371 .unused,
131372 .unused,
131373 .unused,
131374 },
131375 .dst_temps = .{ .{ .rc = .sse }, .unused },
131376 .clobbers = .{ .eflags = true },
131377 .each = .{ .once = &.{
131378 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131379 .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
131380 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
131381 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131382 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131383 .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ },
131384 } },
131385 }, .{
131386 .required_features = .{ .sse2, null, null, null },
131387 .dst_constraints = .{ .{ .float = .qword }, .any },
131388 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
131389 .patterns = &.{
131390 .{ .src = .{ .to_mem, .none, .none } },
131391 },
131392 .extra_temps = .{
131393 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131394 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131395 .unused,
131396 .unused,
131397 .unused,
131398 .unused,
131399 .unused,
131400 .unused,
131401 .unused,
131402 .unused,
131403 .unused,
131404 },
131405 .dst_temps = .{ .{ .rc = .sse }, .unused },
131406 .clobbers = .{ .eflags = true },
131407 .each = .{ .once = &.{
131408 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131409 .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
131410 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
131411 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131412 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131413 .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ },
131414 .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ },
131415 .{ ._, ._sd, .add, .dst0x, .tmp1x, ._, ._ },
131416 } },
131417 }, .{
131418 .required_features = .{ .avx512f, null, null, null },
131419 .dst_constraints = .{ .{ .float = .qword }, .any },
131420 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any },
131421 .patterns = &.{
131422 .{ .src = .{ .to_mem, .none, .none } },
131423 },
131424 .extra_temps = .{
131425 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131426 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131427 .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } },
131428 .unused,
131429 .unused,
131430 .unused,
131431 .unused,
131432 .unused,
131433 .unused,
131434 .unused,
131435 .unused,
131436 },
131437 .dst_temps = .{ .{ .rc = .sse }, .unused },
131438 .clobbers = .{ .eflags = true },
131439 .each = .{ .once = &.{
131440 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131441 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131442 .{ ._, .v_pd, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
131443 .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ },
131444 .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
131445 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ },
131446 .{ .@"0:", .v_pd, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ },
131447 .{ ._, .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
131448 .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ },
131449 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131450 .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ },
131451 .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131452 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131453 .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ },
131454 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131455 } },
131456 }, .{
131457 .required_features = .{ .avx, .fast_hops, null, null },
131458 .dst_constraints = .{ .{ .float = .qword }, .any },
131459 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131460 .patterns = &.{
131461 .{ .src = .{ .to_mem, .none, .none } },
131462 },
131463 .extra_temps = .{
131464 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131465 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131466 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131467 .unused,
131468 .unused,
131469 .unused,
131470 .unused,
131471 .unused,
131472 .unused,
131473 .unused,
131474 .unused,
131475 },
131476 .dst_temps = .{ .{ .rc = .sse }, .unused },
131477 .clobbers = .{ .eflags = true },
131478 .each = .{ .once = &.{
131479 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131480 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131481 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
131482 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ },
131483 .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131484 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131485 .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
131486 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131487 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131488 .{ ._, .vh_pd, .add, .dst0x, .dst0x, .dst0x, ._ },
131489 } },
131490 }, .{
131491 .required_features = .{ .avx, null, null, null },
131492 .dst_constraints = .{ .{ .float = .qword }, .any },
131493 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131494 .patterns = &.{
131495 .{ .src = .{ .to_mem, .none, .none } },
131496 },
131497 .extra_temps = .{
131498 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131499 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131500 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131501 .unused,
131502 .unused,
131503 .unused,
131504 .unused,
131505 .unused,
131506 .unused,
131507 .unused,
131508 .unused,
131509 },
131510 .dst_temps = .{ .{ .rc = .sse }, .unused },
131511 .clobbers = .{ .eflags = true },
131512 .each = .{ .once = &.{
131513 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131514 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131515 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
131516 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ },
131517 .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131518 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131519 .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
131520 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131521 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131522 .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ },
131523 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131524 } },
131525 }, .{
131526 .required_features = .{ .sse3, .fast_hops, null, null },
131527 .dst_constraints = .{ .{ .float = .qword }, .any },
131528 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
131529 .patterns = &.{
131530 .{ .src = .{ .to_mem, .none, .none } },
131531 },
131532 .extra_temps = .{
131533 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131534 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131535 .unused,
131536 .unused,
131537 .unused,
131538 .unused,
131539 .unused,
131540 .unused,
131541 .unused,
131542 .unused,
131543 .unused,
131544 },
131545 .dst_temps = .{ .{ .rc = .sse }, .unused },
131546 .clobbers = .{ .eflags = true },
131547 .each = .{ .once = &.{
131548 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131549 .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ },
131550 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
131551 .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
131552 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
131553 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131554 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131555 .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ },
131556 } },
131557 }, .{
131558 .required_features = .{ .sse2, null, null, null },
131559 .dst_constraints = .{ .{ .float = .qword }, .any },
131560 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
131561 .patterns = &.{
131562 .{ .src = .{ .to_mem, .none, .none } },
131563 },
131564 .extra_temps = .{
131565 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131566 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131567 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131568 .unused,
131569 .unused,
131570 .unused,
131571 .unused,
131572 .unused,
131573 .unused,
131574 .unused,
131575 .unused,
131576 },
131577 .dst_temps = .{ .{ .rc = .sse }, .unused },
131578 .clobbers = .{ .eflags = true },
131579 .each = .{ .once = &.{
131580 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131581 .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ },
131582 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
131583 .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
131584 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
131585 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131586 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131587 .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ },
131588 .{ ._, ._ps, .movhl, .tmp2x, .dst0x, ._, ._ },
131589 .{ ._, ._pd, .add, .dst0x, .tmp2x, ._, ._ },
131590 } },
131591 }, .{
131592 .required_features = .{ .x87, null, null, null },
131593 .dst_constraints = .{ .{ .float = .qword }, .any },
131594 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any },
131595 .patterns = &.{
131596 .{ .src = .{ .to_mem, .none, .none } },
131597 },
131598 .extra_temps = .{
131599 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
131600 .{ .type = .f64, .kind = .{ .reg = .st7 } },
131601 .unused,
131602 .unused,
131603 .unused,
131604 .unused,
131605 .unused,
131606 .unused,
131607 .unused,
131608 .unused,
131609 .unused,
131610 },
131611 .dst_temps = .{ .mem, .unused },
131612 .clobbers = .{ .eflags = true },
131613 .each = .{ .once = &.{
131614 .{ ._, ._, .mov, .tmp0p, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
131615 .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ },
131616 .{ .@"0:", .f_, .add, .memi(.src0q, .tmp0), ._, ._, ._ },
131617 .{ ._, ._, .sub, .tmp0p, .si(8), ._, ._ },
131618 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131619 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },
131620 } },
131621 }, .{
131622 .required_features = .{ .x87, null, null, null },
131623 .dst_constraints = .{ .{ .float = .tbyte }, .any },
131624 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .tbyte } }, .any, .any },
131625 .patterns = &.{
131626 .{ .src = .{ .to_mem, .none, .none } },
131627 },
131628 .extra_temps = .{
131629 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
131630 .{ .type = .f80, .kind = .{ .reg = .st6 } },
131631 .{ .type = .f80, .kind = .{ .reg = .st7 } },
131632 .unused,
131633 .unused,
131634 .unused,
131635 .unused,
131636 .unused,
131637 .unused,
131638 .unused,
131639 .unused,
131640 },
131641 .dst_temps = .{ .mem, .unused },
131642 .clobbers = .{ .eflags = true },
131643 .each = .{ .once = &.{
131644 .{ ._, ._, .mov, .tmp0p, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131645 .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ },
131646 .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ },
131647 .{ ._, .f_p, .add, ._, ._, ._, ._ },
131648 .{ ._, ._, .sub, .tmp0p, .si(16), ._, ._ },
131649 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131650 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
131651 } },
131652 }, .{
131653 .required_features = .{ .avx, null, null, null },
131654 .dst_constraints = .{ .{ .float = .xword }, .any },
131655 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
131656 .patterns = &.{
131657 .{ .src = .{ .to_mem, .none, .none } },
131658 },
131659 .call_frame = .{ .alignment = .@"16" },
131660 .extra_temps = .{
131661 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131662 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
131663 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__addtf3" } } },
131664 .unused,
131665 .unused,
131666 .unused,
131667 .unused,
131668 .unused,
131669 .unused,
131670 .unused,
131671 .unused,
131672 },
131673 .dst_temps = .{ .{ .reg = .xmm0 }, .unused },
131674 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
131675 .each = .{ .once = &.{
131676 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131677 .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
131678 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
131679 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
131680 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131681 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131682 } },
131683 }, .{
131684 .required_features = .{ .sse2, null, null, null },
131685 .dst_constraints = .{ .{ .float = .xword }, .any },
131686 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
131687 .patterns = &.{
131688 .{ .src = .{ .to_mem, .none, .none } },
131689 },
131690 .call_frame = .{ .alignment = .@"16" },
131691 .extra_temps = .{
131692 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131693 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
131694 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__addtf3" } } },
131695 .unused,
131696 .unused,
131697 .unused,
131698 .unused,
131699 .unused,
131700 .unused,
131701 .unused,
131702 .unused,
131703 },
131704 .dst_temps = .{ .{ .reg = .xmm0 }, .unused },
131705 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
131706 .each = .{ .once = &.{
131707 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131708 .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
131709 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
131710 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
131711 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131712 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131713 } },
131714 }, .{
131715 .required_features = .{ .sse, null, null, null },
131716 .dst_constraints = .{ .{ .float = .xword }, .any },
131717 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
131718 .patterns = &.{
131719 .{ .src = .{ .to_mem, .none, .none } },
131720 },
131721 .call_frame = .{ .alignment = .@"16" },
131722 .extra_temps = .{
131723 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131724 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
131725 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__addtf3" } } },
131726 .unused,
131727 .unused,
131728 .unused,
131729 .unused,
131730 .unused,
131731 .unused,
131732 .unused,
131733 .unused,
131734 },
131735 .dst_temps = .{ .{ .reg = .xmm0 }, .unused },
131736 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
131737 .each = .{ .once = &.{
131738 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131739 .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
131740 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
131741 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
131742 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131743 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131744 } },
131745 } },
131746 .Mul => comptime &.{ .{
131747 .required_features = .{ .f16c, null, null, null },
131748 .dst_constraints = .{ .{ .float = .word }, .any },
131749 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any },
131750 .patterns = &.{
131751 .{ .src = .{ .mem, .none, .none } },
131752 .{ .src = .{ .to_sse, .none, .none } },
131753 },
131754 .extra_temps = .{
131755 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
131756 .unused,
131757 .unused,
131758 .unused,
131759 .unused,
131760 .unused,
131761 .unused,
131762 .unused,
131763 .unused,
131764 .unused,
131765 .unused,
131766 },
131767 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131768 .each = .{ .once = &.{
131769 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
131770 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
131771 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131772 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
131773 } },
131774 }, .{
131775 .required_features = .{ .f16c, null, null, null },
131776 .dst_constraints = .{ .{ .float = .word }, .any },
131777 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any },
131778 .patterns = &.{
131779 .{ .src = .{ .mem, .none, .none } },
131780 .{ .src = .{ .to_sse, .none, .none } },
131781 },
131782 .extra_temps = .{
131783 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
131784 .unused,
131785 .unused,
131786 .unused,
131787 .unused,
131788 .unused,
131789 .unused,
131790 .unused,
131791 .unused,
131792 .unused,
131793 .unused,
131794 },
131795 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131796 .each = .{ .once = &.{
131797 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
131798 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
131799 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131800 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
131801 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131802 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
131803 } },
131804 }, .{
131805 .required_features = .{ .f16c, null, null, null },
131806 .dst_constraints = .{ .{ .float = .word }, .any },
131807 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any },
131808 .patterns = &.{
131809 .{ .src = .{ .mem, .none, .none } },
131810 .{ .src = .{ .to_sse, .none, .none } },
131811 },
131812 .extra_temps = .{
131813 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
131814 .unused,
131815 .unused,
131816 .unused,
131817 .unused,
131818 .unused,
131819 .unused,
131820 .unused,
131821 .unused,
131822 .unused,
131823 .unused,
131824 },
131825 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131826 .each = .{ .once = &.{
131827 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
131828 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
131829 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131830 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
131831 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131832 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
131833 } },
131834 }, .{
131835 .required_features = .{ .f16c, null, null, null },
131836 .dst_constraints = .{ .{ .float = .word }, .any },
131837 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any },
131838 .patterns = &.{
131839 .{ .src = .{ .to_sse, .none, .none } },
131840 },
131841 .extra_temps = .{
131842 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
131843 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131844 .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
131845 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131846 .unused,
131847 .unused,
131848 .unused,
131849 .unused,
131850 .unused,
131851 .unused,
131852 .unused,
131853 },
131854 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131855 .each = .{ .once = &.{
131856 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131857 .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ },
131858 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
131859 .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ },
131860 .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ },
131861 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
131862 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131863 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
131864 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131865 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
131866 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131867 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
131868 } },
131869 }, .{
131870 .required_features = .{ .f16c, null, null, null },
131871 .dst_constraints = .{ .{ .float = .word }, .any },
131872 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any },
131873 .patterns = &.{
131874 .{ .src = .{ .mem, .none, .none } },
131875 .{ .src = .{ .to_sse, .none, .none } },
131876 },
131877 .extra_temps = .{
131878 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131879 .unused,
131880 .unused,
131881 .unused,
131882 .unused,
131883 .unused,
131884 .unused,
131885 .unused,
131886 .unused,
131887 .unused,
131888 .unused,
131889 },
131890 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131891 .each = .{ .once = &.{
131892 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
131893 .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
131894 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131895 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
131896 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131897 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
131898 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131899 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
131900 } },
131901 }, .{
131902 .required_features = .{ .f16c, null, null, null },
131903 .dst_constraints = .{ .{ .float = .word }, .any },
131904 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any },
131905 .patterns = &.{
131906 .{ .src = .{ .to_sse, .none, .none } },
131907 },
131908 .extra_temps = .{
131909 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
131910 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131911 .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
131912 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131913 .unused,
131914 .unused,
131915 .unused,
131916 .unused,
131917 .unused,
131918 .unused,
131919 .unused,
131920 },
131921 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131922 .each = .{ .once = &.{
131923 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131924 .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ },
131925 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
131926 .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
131927 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
131928 .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ },
131929 .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ },
131930 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
131931 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
131932 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131933 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
131934 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131935 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
131936 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131937 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
131938 } },
131939 }, .{
131940 .required_features = .{ .f16c, null, null, null },
131941 .dst_constraints = .{ .{ .float = .word }, .any },
131942 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any },
131943 .patterns = &.{
131944 .{ .src = .{ .to_sse, .none, .none } },
131945 },
131946 .extra_temps = .{
131947 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131948 .unused,
131949 .unused,
131950 .unused,
131951 .unused,
131952 .unused,
131953 .unused,
131954 .unused,
131955 .unused,
131956 .unused,
131957 .unused,
131958 },
131959 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131960 .each = .{ .once = &.{
131961 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
131962 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
131963 .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ },
131964 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ },
131965 .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
131966 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131967 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
131968 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131969 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
131970 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131971 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
131972 } },
131973 }, .{
131974 .required_features = .{ .avx512f, null, null, null },
131975 .dst_constraints = .{ .{ .float = .word }, .any },
131976 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any },
131977 .patterns = &.{
131978 .{ .src = .{ .to_mem, .none, .none } },
131979 },
131980 .extra_temps = .{
131981 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
131982 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131983 .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
131984 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131985 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131986 .unused,
131987 .unused,
131988 .unused,
131989 .unused,
131990 .unused,
131991 .unused,
131992 },
131993 .dst_temps = .{ .{ .rc = .sse }, .unused },
131994 .each = .{ .once = &.{
131995 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131996 .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131997 .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ },
131998 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
131999 .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ },
132000 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ },
132001 .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
132002 .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ },
132003 .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ },
132004 .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ },
132005 .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ },
132006 .{ ._, .v_ps, .mul, .tmp3y, .tmp3y, .tmp4y, ._ },
132007 .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ },
132008 .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ },
132009 .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ },
132010 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp4y, ._ },
132011 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132012 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132013 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132014 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
132015 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132016 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132017 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132018 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
132019 } },
132020 }, .{
132021 .required_features = .{ .avx512f, null, null, null },
132022 .dst_constraints = .{ .{ .float = .word }, .any },
132023 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any },
132024 .patterns = &.{
132025 .{ .src = .{ .to_mem, .none, .none } },
132026 },
132027 .extra_temps = .{
132028 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
132029 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132030 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132031 .unused,
132032 .unused,
132033 .unused,
132034 .unused,
132035 .unused,
132036 .unused,
132037 .unused,
132038 .unused,
132039 },
132040 .dst_temps = .{ .{ .rc = .sse }, .unused },
132041 .each = .{ .once = &.{
132042 .{ ._, .v_ps, .mova, .tmp1y, .memd(.src0y, 32), ._, ._ },
132043 .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ },
132044 .{ ._, .v_f128, .extract, .tmp2x, .tmp1y, .ui(1), ._ },
132045 .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ },
132046 .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ },
132047 .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ },
132048 .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
132049 .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ },
132050 .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ },
132051 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp2y, ._ },
132052 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp1y, ._ },
132053 .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
132054 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132055 .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
132056 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132057 .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132058 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132059 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
132060 } },
132061 }, .{
132062 .required_features = .{ .f16c, null, null, null },
132063 .dst_constraints = .{ .{ .float = .word }, .any },
132064 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any },
132065 .patterns = &.{
132066 .{ .src = .{ .to_mem, .none, .none } },
132067 },
132068 .extra_temps = .{
132069 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132070 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132071 .unused,
132072 .unused,
132073 .unused,
132074 .unused,
132075 .unused,
132076 .unused,
132077 .unused,
132078 .unused,
132079 .unused,
132080 },
132081 .dst_temps = .{ .{ .rc = .sse }, .unused },
132082 .clobbers = .{ .eflags = true },
132083 .each = .{ .once = &.{
132084 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
132085 .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
132086 .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ },
132087 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp1y, ._ },
132088 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
132089 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132090 .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
132091 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132092 .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
132093 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132094 .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132095 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132096 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
132097 } },
132098 }, .{
132099 .required_features = .{ .avx512f, null, null, null },
132100 .dst_constraints = .{ .{ .float = .word }, .any },
132101 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any },
132102 .patterns = &.{
132103 .{ .src = .{ .to_mem, .none, .none } },
132104 },
132105 .extra_temps = .{
132106 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132107 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132108 .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
132109 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132110 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132111 .unused,
132112 .unused,
132113 .unused,
132114 .unused,
132115 .unused,
132116 .unused,
132117 },
132118 .dst_temps = .{ .{ .rc = .sse }, .unused },
132119 .clobbers = .{ .eflags = true },
132120 .each = .{ .once = &.{
132121 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
132122 .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
132123 .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ },
132124 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132125 .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ },
132126 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ },
132127 .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
132128 .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ },
132129 .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ },
132130 .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ },
132131 .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ },
132132 .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ },
132133 .{ ._, .v_ps, .mul, .tmp3y, .tmp3y, .tmp4y, ._ },
132134 .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ },
132135 .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ },
132136 .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ },
132137 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp4y, ._ },
132138 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132139 .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ },
132140 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132141 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
132142 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132143 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132144 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132145 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
132146 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132147 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132148 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132149 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
132150 } },
132151 }, .{
132152 .required_features = .{ .f16c, null, null, null },
132153 .dst_constraints = .{ .{ .float = .word }, .any },
132154 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any },
132155 .patterns = &.{
132156 .{ .src = .{ .to_mem, .none, .none } },
132157 },
132158 .extra_temps = .{
132159 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132160 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132161 .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
132162 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132163 .unused,
132164 .unused,
132165 .unused,
132166 .unused,
132167 .unused,
132168 .unused,
132169 .unused,
132170 },
132171 .dst_temps = .{ .{ .rc = .sse }, .unused },
132172 .clobbers = .{ .eflags = true },
132173 .each = .{ .once = &.{
132174 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
132175 .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
132176 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132177 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ },
132178 .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
132179 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
132180 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132181 .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ },
132182 .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ },
132183 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132184 .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ },
132185 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132186 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
132187 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132188 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132189 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132190 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10_11_10) },
132191 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132192 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132193 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132194 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
132195 } },
132196 }, .{
132197 .required_features = .{ .avx, null, null, null },
132198 .dst_constraints = .{ .{ .float = .word }, .any },
132199 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any },
132200 .patterns = &.{
132201 .{ .src = .{ .to_mem, .none, .none } },
132202 },
132203 .call_frame = .{ .alignment = .@"16" },
132204 .extra_temps = .{
132205 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132206 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
132207 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
132208 .unused,
132209 .unused,
132210 .unused,
132211 .unused,
132212 .unused,
132213 .unused,
132214 .unused,
132215 .unused,
132216 },
132217 .dst_temps = .{ .{ .reg = .xmm0 }, .unused },
132218 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
132219 .each = .{ .once = &.{
132220 .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ },
132221 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ },
132222 .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) },
132223 .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
132224 .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) },
132225 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
132226 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
132227 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132228 } },
132229 }, .{
132230 .required_features = .{ .sse2, null, null, null },
132231 .dst_constraints = .{ .{ .float = .word }, .any },
132232 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any },
132233 .patterns = &.{
132234 .{ .src = .{ .to_mem, .none, .none } },
132235 },
132236 .call_frame = .{ .alignment = .@"16" },
132237 .extra_temps = .{
132238 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132239 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
132240 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
132241 .unused,
132242 .unused,
132243 .unused,
132244 .unused,
132245 .unused,
132246 .unused,
132247 .unused,
132248 .unused,
132249 },
132250 .dst_temps = .{ .{ .reg = .xmm0 }, .unused },
132251 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
132252 .each = .{ .once = &.{
132253 .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ },
132254 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ },
132255 .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ },
132256 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
132257 .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ },
132258 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
132259 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
132260 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132261 } },
132262 }, .{
132263 .required_features = .{ .sse, null, null, null },
132264 .dst_constraints = .{ .{ .float = .word }, .any },
132265 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any },
132266 .patterns = &.{
132267 .{ .src = .{ .to_mem, .none, .none } },
132268 },
132269 .call_frame = .{ .alignment = .@"16" },
132270 .extra_temps = .{
132271 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132272 .{ .type = .f16, .kind = .{ .reg = .ax } },
132273 .{ .type = .f32, .kind = .mem },
132274 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
132275 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
132276 .unused,
132277 .unused,
132278 .unused,
132279 .unused,
132280 .unused,
132281 .unused,
132282 },
132283 .dst_temps = .{ .{ .reg = .xmm0 }, .unused },
132284 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
132285 .each = .{ .once = &.{
132286 .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ },
132287 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ },
132288 .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ },
132289 .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ },
132290 .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ },
132291 .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ },
132292 .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ },
132293 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
132294 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
132295 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132296 } },
132297 }, .{
132298 .required_features = .{ .avx, null, null, null },
132299 .dst_constraints = .{ .{ .float = .dword }, .any },
132300 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any },
132301 .patterns = &.{
132302 .{ .src = .{ .to_sse, .none, .none } },
132303 },
132304 .extra_temps = .{
132305 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132306 .unused,
132307 .unused,
132308 .unused,
132309 .unused,
132310 .unused,
132311 .unused,
132312 .unused,
132313 .unused,
132314 .unused,
132315 .unused,
132316 },
132317 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
132318 .each = .{ .once = &.{
132319 .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01_01_01_01) },
132320 .{ ._, .v_ss, .mul, .dst0x, .src0x, .tmp0x, ._ },
132321 } },
132322 }, .{
132323 .required_features = .{ .sse, null, null, null },
132324 .dst_constraints = .{ .{ .float = .dword }, .any },
132325 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any },
132326 .patterns = &.{
132327 .{ .src = .{ .to_mut_sse, .none, .none } },
132328 },
132329 .extra_temps = .{
132330 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132331 .unused,
132332 .unused,
132333 .unused,
132334 .unused,
132335 .unused,
132336 .unused,
132337 .unused,
132338 .unused,
132339 .unused,
132340 .unused,
132341 },
132342 .dst_temps = .{ .{ .ref = .src0 }, .unused },
132343 .each = .{ .once = &.{
132344 .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ },
132345 .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01_01_01_01), ._ },
132346 .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ },
132347 } },
132348 }, .{
132349 .required_features = .{ .avx, null, null, null },
132350 .dst_constraints = .{ .{ .float = .dword }, .any },
132351 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any },
132352 .patterns = &.{
132353 .{ .src = .{ .to_sse, .none, .none } },
132354 },
132355 .extra_temps = .{
132356 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132357 .unused,
132358 .unused,
132359 .unused,
132360 .unused,
132361 .unused,
132362 .unused,
132363 .unused,
132364 .unused,
132365 .unused,
132366 .unused,
132367 },
132368 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
132369 .each = .{ .once = &.{
132370 .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ },
132371 .{ ._, .v_ps, .mul, .tmp0x, .src0x, .tmp0x, ._ },
132372 .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01_01_01_01) },
132373 .{ ._, .v_ss, .mul, .dst0x, .tmp0x, .dst0x, ._ },
132374 } },
132375 }, .{
132376 .required_features = .{ .sse, null, null, null },
132377 .dst_constraints = .{ .{ .float = .dword }, .any },
132378 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any },
132379 .patterns = &.{
132380 .{ .src = .{ .to_mut_sse, .none, .none } },
132381 },
132382 .extra_temps = .{
132383 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132384 .unused,
132385 .unused,
132386 .unused,
132387 .unused,
132388 .unused,
132389 .unused,
132390 .unused,
132391 .unused,
132392 .unused,
132393 .unused,
132394 },
132395 .dst_temps = .{ .{ .ref = .src0 }, .unused },
132396 .each = .{ .once = &.{
132397 .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ },
132398 .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ },
132399 .{ ._, ._ss, .mul, .tmp0x, .src0x, ._, ._ },
132400 .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01_01_01_01), ._ },
132401 .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ },
132402 } },
132403 }, .{
132404 .required_features = .{ .avx, null, null, null },
132405 .dst_constraints = .{ .{ .float = .dword }, .any },
132406 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any },
132407 .patterns = &.{
132408 .{ .src = .{ .to_sse, .none, .none } },
132409 },
132410 .extra_temps = .{
132411 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132412 .unused,
132413 .unused,
132414 .unused,
132415 .unused,
132416 .unused,
132417 .unused,
132418 .unused,
132419 .unused,
132420 .unused,
132421 .unused,
132422 },
132423 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
132424 .each = .{ .once = &.{
132425 .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ },
132426 .{ ._, .v_ps, .mul, .dst0x, .src0x, .tmp0x, ._ },
132427 .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01_01_01_01) },
132428 .{ ._, .v_ss, .mul, .dst0x, .src0x, .tmp0x, ._ },
132429 } },
132430 }, .{
132431 .required_features = .{ .sse, null, null, null },
132432 .dst_constraints = .{ .{ .float = .dword }, .any },
132433 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any },
132434 .patterns = &.{
132435 .{ .src = .{ .to_mut_sse, .none, .none } },
132436 },
132437 .extra_temps = .{
132438 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132439 .unused,
132440 .unused,
132441 .unused,
132442 .unused,
132443 .unused,
132444 .unused,
132445 .unused,
132446 .unused,
132447 .unused,
132448 .unused,
132449 },
132450 .dst_temps = .{ .{ .ref = .src0 }, .unused },
132451 .each = .{ .once = &.{
132452 .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ },
132453 .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ },
132454 .{ ._, ._ps, .mul, .dst0x, .tmp0x, ._, ._ },
132455 .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ },
132456 .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01_01_01_01), ._ },
132457 .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ },
132458 } },
132459 }, .{
132460 .required_features = .{ .avx, null, null, null },
132461 .dst_constraints = .{ .{ .float = .dword }, .any },
132462 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any },
132463 .patterns = &.{
132464 .{ .src = .{ .to_sse, .none, .none } },
132465 },
132466 .extra_temps = .{
132467 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
132468 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132469 .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
132470 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132471 .unused,
132472 .unused,
132473 .unused,
132474 .unused,
132475 .unused,
132476 .unused,
132477 .unused,
132478 },
132479 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
132480 .each = .{ .once = &.{
132481 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
132482 .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ },
132483 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132484 .{ ._, .v_ps, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ },
132485 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132486 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132487 .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ },
132488 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132489 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132490 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132491 } },
132492 }, .{
132493 .required_features = .{ .avx, null, null, null },
132494 .dst_constraints = .{ .{ .float = .dword }, .any },
132495 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any },
132496 .patterns = &.{
132497 .{ .src = .{ .to_sse, .none, .none } },
132498 },
132499 .extra_temps = .{
132500 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
132501 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132502 .unused,
132503 .unused,
132504 .unused,
132505 .unused,
132506 .unused,
132507 .unused,
132508 .unused,
132509 .unused,
132510 .unused,
132511 },
132512 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
132513 .each = .{ .once = &.{
132514 .{ ._, .v_f128, .extract, .tmp1x, .src0y, .ui(1), ._ },
132515 .{ ._, .v_ps, .mul, .dst0x, .src0x, .tmp1x, ._ },
132516 .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ },
132517 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132518 .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132519 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132520 } },
132521 }, .{
132522 .required_features = .{ .avx512f, null, null, null },
132523 .dst_constraints = .{ .{ .float = .dword }, .any },
132524 .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any },
132525 .patterns = &.{
132526 .{ .src = .{ .to_mem, .none, .none } },
132527 },
132528 .extra_temps = .{
132529 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
132530 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132531 .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
132532 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132533 .unused,
132534 .unused,
132535 .unused,
132536 .unused,
132537 .unused,
132538 .unused,
132539 .unused,
132540 },
132541 .dst_temps = .{ .{ .rc = .sse }, .unused },
132542 .each = .{ .once = &.{
132543 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
132544 .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
132545 .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ },
132546 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132547 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ },
132548 .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ },
132549 .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
132550 .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ },
132551 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132552 .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132553 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132554 .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ },
132555 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132556 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132557 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132558 } },
132559 }, .{
132560 .required_features = .{ .avx512f, null, null, null },
132561 .dst_constraints = .{ .{ .float = .dword }, .any },
132562 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any },
132563 .patterns = &.{
132564 .{ .src = .{ .to_mem, .none, .none } },
132565 },
132566 .extra_temps = .{
132567 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132568 .unused,
132569 .unused,
132570 .unused,
132571 .unused,
132572 .unused,
132573 .unused,
132574 .unused,
132575 .unused,
132576 .unused,
132577 .unused,
132578 },
132579 .dst_temps = .{ .{ .rc = .sse }, .unused },
132580 .each = .{ .once = &.{
132581 .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ },
132582 .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ },
132583 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ },
132584 .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
132585 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
132586 .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ },
132587 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
132588 .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132589 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ },
132590 } },
132591 }, .{
132592 .required_features = .{ .avx, null, null, null },
132593 .dst_constraints = .{ .{ .float = .dword }, .any },
132594 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any },
132595 .patterns = &.{
132596 .{ .src = .{ .to_mem, .none, .none } },
132597 },
132598 .extra_temps = .{
132599 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132600 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132601 .unused,
132602 .unused,
132603 .unused,
132604 .unused,
132605 .unused,
132606 .unused,
132607 .unused,
132608 .unused,
132609 .unused,
132610 },
132611 .dst_temps = .{ .{ .rc = .sse }, .unused },
132612 .clobbers = .{ .eflags = true },
132613 .each = .{ .once = &.{
132614 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ },
132615 .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
132616 .{ .@"0:", .v_ps, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
132617 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
132618 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132619 .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
132620 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132621 .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ },
132622 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132623 .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132624 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ },
132625 } },
132626 }, .{
132627 .required_features = .{ .sse, null, null, null },
132628 .dst_constraints = .{ .{ .float = .dword }, .any },
132629 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any },
132630 .patterns = &.{
132631 .{ .src = .{ .to_mem, .none, .none } },
132632 },
132633 .extra_temps = .{
132634 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132635 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132636 .unused,
132637 .unused,
132638 .unused,
132639 .unused,
132640 .unused,
132641 .unused,
132642 .unused,
132643 .unused,
132644 .unused,
132645 },
132646 .dst_temps = .{ .{ .rc = .sse }, .unused },
132647 .clobbers = .{ .eflags = true },
132648 .each = .{ .once = &.{
132649 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
132650 .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
132651 .{ .@"0:", ._ps, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
132652 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
132653 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132654 .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ },
132655 .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ },
132656 .{ ._, ._ps, .mul, .dst0x, .tmp1x, ._, ._ },
132657 .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ },
132658 .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01_01_01_01), ._ },
132659 .{ ._, ._ss, .mul, .dst0x, .tmp1x, ._, ._ },
132660 } },
132661 }, .{
132662 .required_features = .{ .avx512f, null, null, null },
132663 .dst_constraints = .{ .{ .float = .dword }, .any },
132664 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any },
132665 .patterns = &.{
132666 .{ .src = .{ .to_mem, .none, .none } },
132667 },
132668 .extra_temps = .{
132669 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132670 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132671 .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
132672 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
132673 .unused,
132674 .unused,
132675 .unused,
132676 .unused,
132677 .unused,
132678 .unused,
132679 .unused,
132680 },
132681 .dst_temps = .{ .{ .rc = .sse }, .unused },
132682 .clobbers = .{ .eflags = true },
132683 .each = .{ .once = &.{
132684 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
132685 .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
132686 .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ },
132687 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132688 .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ },
132689 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ },
132690 .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
132691 .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ },
132692 .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ },
132693 .{ .@"0:", .v_ps, .mul, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ },
132694 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
132695 .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ },
132696 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132697 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132698 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132699 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132700 .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ },
132701 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132702 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132703 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132704 } },
132705 }, .{
132706 .required_features = .{ .avx, null, null, null },
132707 .dst_constraints = .{ .{ .float = .dword }, .any },
132708 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any },
132709 .patterns = &.{
132710 .{ .src = .{ .to_mem, .none, .none } },
132711 },
132712 .extra_temps = .{
132713 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132714 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132715 .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
132716 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132717 .unused,
132718 .unused,
132719 .unused,
132720 .unused,
132721 .unused,
132722 .unused,
132723 .unused,
132724 },
132725 .dst_temps = .{ .{ .rc = .sse }, .unused },
132726 .clobbers = .{ .eflags = true },
132727 .each = .{ .once = &.{
132728 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
132729 .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
132730 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132731 .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ },
132732 .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
132733 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
132734 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132735 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132736 .{ .@"0:", .v_ps, .mul, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
132737 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
132738 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132739 .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ },
132740 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132741 .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01_01_01_01) },
132742 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132743 } },
132744 }, .{
132745 .required_features = .{ .sse, null, null, null },
132746 .dst_constraints = .{ .{ .float = .dword }, .any },
132747 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any },
132748 .patterns = &.{
132749 .{ .src = .{ .to_mem, .none, .none } },
132750 },
132751 .extra_temps = .{
132752 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
132753 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132754 .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
132755 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
132756 .unused,
132757 .unused,
132758 .unused,
132759 .unused,
132760 .unused,
132761 .unused,
132762 .unused,
132763 },
132764 .dst_temps = .{ .{ .rc = .sse }, .unused },
132765 .clobbers = .{ .eflags = true },
132766 .each = .{ .once = &.{
132767 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
132768 .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ },
132769 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132770 .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
132771 .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ },
132772 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
132773 .{ .@"0:", ._ps, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
132774 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
132775 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
132776 .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ },
132777 .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ },
132778 .{ ._, ._ps, .mul, .dst0x, .tmp3x, ._, ._ },
132779 .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ },
132780 .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01_01_01_01), ._ },
132781 .{ ._, ._ss, .mul, .dst0x, .tmp3x, ._, ._ },
132782 } },
132783 }, .{
132784 .required_features = .{ .avx, null, null, null },
132785 .dst_constraints = .{ .{ .float = .qword }, .any },
132786 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
132787 .patterns = &.{
132788 .{ .src = .{ .to_sse, .none, .none } },
132789 },
132790 .extra_temps = .{
132791 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
132792 .unused,
132793 .unused,
132794 .unused,
132795 .unused,
132796 .unused,
132797 .unused,
132798 .unused,
132799 .unused,
132800 .unused,
132801 .unused,
132802 },
132803 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
132804 .each = .{ .once = &.{
132805 .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ },
132806 .{ ._, .v_sd, .mul, .dst0x, .src0x, .tmp0x, ._ },
132807 } },
132808 }, .{
132809 .required_features = .{ .sse2, null, null, null },
132810 .dst_constraints = .{ .{ .float = .qword }, .any },
132811 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
132812 .patterns = &.{
132813 .{ .src = .{ .to_mut_sse, .none, .none } },
132814 },
132815 .extra_temps = .{
132816 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
132817 .unused,
132818 .unused,
132819 .unused,
132820 .unused,
132821 .unused,
132822 .unused,
132823 .unused,
132824 .unused,
132825 .unused,
132826 .unused,
132827 },
132828 .dst_temps = .{ .{ .ref = .src0 }, .unused },
132829 .each = .{ .once = &.{
132830 .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ },
132831 .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ },
132832 .{ ._, ._sd, .mul, .dst0x, .tmp0x, ._, ._ },
131147132833 } },
131148132834 }, .{
131149132835 .required_features = .{ .avx, null, null, null },
......@@ -131168,35 +132854,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131168132854 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131169132855 .each = .{ .once = &.{
131170132856 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
131171 .{ ._, .v_pd, .add, .tmp0x, .src0x, .tmp0x, ._ },
132857 .{ ._, .v_pd, .mul, .tmp0x, .src0x, .tmp0x, ._ },
131172132858 .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ },
131173 .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ },
131174 } },
131175 }, .{
131176 .required_features = .{ .avx, .fast_hops, null, null },
131177 .dst_constraints = .{ .{ .float = .qword }, .any },
131178 .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131179 .patterns = &.{
131180 .{ .src = .{ .to_sse, .none, .none } },
131181 },
131182 .extra_temps = .{
131183 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131184 .unused,
131185 .unused,
131186 .unused,
131187 .unused,
131188 .unused,
131189 .unused,
131190 .unused,
131191 .unused,
131192 .unused,
131193 .unused,
131194 },
131195 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131196 .each = .{ .once = &.{
131197 .{ ._, .vh_pd, .add, .dst0y, .src0y, .src0y, ._ },
131198 .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
131199 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ },
132859 .{ ._, .v_sd, .mul, .dst0x, .tmp0x, .dst0x, ._ },
131200132860 } },
131201132861 }, .{
131202132862 .required_features = .{ .avx, null, null, null },
......@@ -131221,9 +132881,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131221132881 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
131222132882 .each = .{ .once = &.{
131223132883 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
131224 .{ ._, .v_pd, .add, .dst0x, .src0x, .tmp0x, ._ },
132884 .{ ._, .v_pd, .mul, .dst0x, .src0x, .tmp0x, ._ },
131225132885 .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ },
131226 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ },
132886 .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131227132887 } },
131228132888 }, .{
131229132889 .required_features = .{ .avx512f, null, null, null },
......@@ -131235,6 +132895,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131235132895 .extra_temps = .{
131236132896 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
131237132897 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
132898 .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
131238132899 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
131239132900 .unused,
131240132901 .unused,
......@@ -131243,20 +132904,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131243132904 .unused,
131244132905 .unused,
131245132906 .unused,
131246 .unused,
131247132907 },
131248132908 .dst_temps = .{ .{ .rc = .sse }, .unused },
131249132909 .each = .{ .once = &.{
131250132910 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131251 .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ },
131252 .{ ._, .v_pd, .mova, .tmp2y, .memd(.src0y, 32), ._, ._ },
131253 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ },
131254 .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ },
131255 .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ },
131256 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131257 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131258 .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ },
131259 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ },
132911 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
132912 .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ },
132913 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
132914 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ },
132915 .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ },
132916 .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
132917 .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ },
132918 .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp3y, ._ },
132919 .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
132920 .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ },
132921 .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ },
132922 .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131260132923 } },
131261132924 }, .{
131262132925 .required_features = .{ .avx512f, null, null, null },
......@@ -131282,43 +132945,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131282132945 .each = .{ .once = &.{
131283132946 .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ },
131284132947 .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ },
131285 .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp0y, ._ },
132948 .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp0y, ._ },
131286132949 .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
131287 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp0x, ._ },
132950 .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131288132951 .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ },
131289 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ },
131290 } },
131291 }, .{
131292 .required_features = .{ .avx, .fast_hops, null, null },
131293 .dst_constraints = .{ .{ .float = .qword }, .any },
131294 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131295 .patterns = &.{
131296 .{ .src = .{ .to_mem, .none, .none } },
131297 },
131298 .extra_temps = .{
131299 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131300 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131301 .unused,
131302 .unused,
131303 .unused,
131304 .unused,
131305 .unused,
131306 .unused,
131307 .unused,
131308 .unused,
131309 .unused,
131310 },
131311 .dst_temps = .{ .{ .rc = .sse }, .unused },
131312 .clobbers = .{ .eflags = true },
131313 .each = .{ .once = &.{
131314 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ },
131315 .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
131316 .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
131317 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
131318 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131319 .{ ._, .vh_pd, .add, .dst0y, .dst0y, .dst0y, ._ },
131320 .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
131321 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp1x, ._ },
132952 .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp0x, ._ },
131322132953 } },
131323132954 }, .{
131324132955 .required_features = .{ .avx, null, null, null },
......@@ -131345,43 +132976,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131345132976 .each = .{ .once = &.{
131346132977 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ },
131347132978 .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
131348 .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
132979 .{ .@"0:", .v_pd, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
131349132980 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
131350132981 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131351132982 .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
131352 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp1x, ._ },
132983 .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp1x, ._ },
131353132984 .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ },
131354 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp1x, ._ },
131355 } },
131356 }, .{
131357 .required_features = .{ .sse3, .fast_hops, null, null },
131358 .dst_constraints = .{ .{ .float = .qword }, .any },
131359 .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
131360 .patterns = &.{
131361 .{ .src = .{ .to_mem, .none, .none } },
131362 },
131363 .extra_temps = .{
131364 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131365 .unused,
131366 .unused,
131367 .unused,
131368 .unused,
131369 .unused,
131370 .unused,
131371 .unused,
131372 .unused,
131373 .unused,
131374 .unused,
131375 },
131376 .dst_temps = .{ .{ .rc = .sse }, .unused },
131377 .clobbers = .{ .eflags = true },
131378 .each = .{ .once = &.{
131379 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131380 .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
131381 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
131382 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131383 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131384 .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ },
132985 .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp1x, ._ },
131385132986 } },
131386132987 }, .{
131387132988 .required_features = .{ .sse2, null, null, null },
......@@ -131408,12 +133009,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131408133009 .each = .{ .once = &.{
131409133010 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131410133011 .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
131411 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
133012 .{ .@"0:", ._pd, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
131412133013 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131413133014 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131414133015 .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ },
131415133016 .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ },
131416 .{ ._, ._sd, .add, .dst0x, .tmp1x, ._, ._ },
133017 .{ ._, ._sd, .mul, .dst0x, .tmp1x, ._, ._ },
131417133018 } },
131418133019 }, .{
131419133020 .required_features = .{ .avx512f, null, null, null },
......@@ -131425,6 +133026,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131425133026 .extra_temps = .{
131426133027 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131427133028 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
133029 .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
131428133030 .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } },
131429133031 .unused,
131430133032 .unused,
......@@ -131433,60 +133035,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131433133035 .unused,
131434133036 .unused,
131435133037 .unused,
131436 .unused,
131437133038 },
131438133039 .dst_temps = .{ .{ .rc = .sse }, .unused },
131439133040 .clobbers = .{ .eflags = true },
131440133041 .each = .{ .once = &.{
131441133042 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131442133043 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131443 .{ ._, .v_pd, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
131444 .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ },
131445 .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
133044 .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ },
133045 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
133046 .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ },
131446133047 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ },
131447 .{ .@"0:", .v_pd, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ },
131448 .{ ._, .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
133048 .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
133049 .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ },
133050 .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ },
133051 .{ .@"0:", .v_pd, .mul, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ },
133052 .{ ._, .v_pd, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
131449133053 .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ },
131450133054 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131451 .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ },
131452 .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131453 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131454 .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ },
131455 .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131456 } },
131457 }, .{
131458 .required_features = .{ .avx, .fast_hops, null, null },
131459 .dst_constraints = .{ .{ .float = .qword }, .any },
131460 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any },
131461 .patterns = &.{
131462 .{ .src = .{ .to_mem, .none, .none } },
131463 },
131464 .extra_temps = .{
131465 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131466 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131467 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131468 .unused,
131469 .unused,
131470 .unused,
131471 .unused,
131472 .unused,
131473 .unused,
131474 .unused,
131475 .unused,
131476 },
131477 .dst_temps = .{ .{ .rc = .sse }, .unused },
131478 .clobbers = .{ .eflags = true },
131479 .each = .{ .once = &.{
131480 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131481 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131482 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
131483 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ },
131484 .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131485 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131486 .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
131487 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131488 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131489 .{ ._, .vh_pd, .add, .dst0x, .dst0x, .dst0x, ._ },
133055 .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp3y, ._ },
133056 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
133057 .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ },
133058 .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ },
133059 .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131490133060 } },
131491133061 }, .{
131492133062 .required_features = .{ .avx, null, null, null },
......@@ -131498,6 +133068,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131498133068 .extra_temps = .{
131499133069 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131500133070 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
133071 .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
131501133072 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131502133073 .unused,
131503133074 .unused,
......@@ -131506,54 +133077,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131506133077 .unused,
131507133078 .unused,
131508133079 .unused,
131509 .unused,
131510133080 },
131511133081 .dst_temps = .{ .{ .rc = .sse }, .unused },
131512133082 .clobbers = .{ .eflags = true },
131513133083 .each = .{ .once = &.{
131514133084 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131515133085 .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ },
131516 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
133086 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
131517133087 .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ },
131518 .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
131519 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131520 .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
131521 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131522 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131523 .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ },
131524 .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ },
131525 } },
131526 }, .{
131527 .required_features = .{ .sse3, .fast_hops, null, null },
131528 .dst_constraints = .{ .{ .float = .qword }, .any },
131529 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any },
131530 .patterns = &.{
131531 .{ .src = .{ .to_mem, .none, .none } },
131532 },
131533 .extra_temps = .{
131534 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131535 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
131536 .unused,
131537 .unused,
131538 .unused,
131539 .unused,
131540 .unused,
131541 .unused,
131542 .unused,
131543 .unused,
131544 .unused,
131545 },
131546 .dst_temps = .{ .{ .rc = .sse }, .unused },
131547 .clobbers = .{ .eflags = true },
131548 .each = .{ .once = &.{
131549 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131550 .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ },
131551 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
131552 .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
131553 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
133088 .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ },
133089 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
133090 .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ },
133091 .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ },
133092 .{ .@"0:", .v_pd, .mul, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
131554133093 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131555133094 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131556 .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ },
133095 .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ },
133096 .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ },
131557133097 } },
131558133098 }, .{
131559133099 .required_features = .{ .sse2, null, null, null },
......@@ -131565,6 +133105,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131565133105 .extra_temps = .{
131566133106 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131567133107 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
133108 .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .val = 1.0, .fill = .outside } } },
131568133109 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
131569133110 .unused,
131570133111 .unused,
......@@ -131573,21 +133114,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131573133114 .unused,
131574133115 .unused,
131575133116 .unused,
131576 .unused,
131577133117 },
131578133118 .dst_temps = .{ .{ .rc = .sse }, .unused },
131579133119 .clobbers = .{ .eflags = true },
131580133120 .each = .{ .once = &.{
131581133121 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
131582133122 .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ },
131583 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
133123 .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ },
131584133124 .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
131585 .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
133125 .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ },
133126 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
133127 .{ .@"0:", ._pd, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
131586133128 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
131587133129 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131588 .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ },
131589 .{ ._, ._ps, .movhl, .tmp2x, .dst0x, ._, ._ },
131590 .{ ._, ._pd, .add, .dst0x, .tmp2x, ._, ._ },
133130 .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ },
133131 .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ },
133132 .{ ._, ._pd, .mul, .dst0x, .tmp3x, ._, ._ },
131591133133 } },
131592133134 }, .{
131593133135 .required_features = .{ .x87, null, null, null },
......@@ -131614,7 +133156,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131614133156 .each = .{ .once = &.{
131615133157 .{ ._, ._, .mov, .tmp0p, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
131616133158 .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ },
131617 .{ .@"0:", .f_, .add, .memi(.src0q, .tmp0), ._, ._, ._ },
133159 .{ .@"0:", .f_, .mul, .memi(.src0q, .tmp0), ._, ._, ._ },
131618133160 .{ ._, ._, .sub, .tmp0p, .si(8), ._, ._ },
131619133161 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131620133162 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },
......@@ -131645,7 +133187,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131645133187 .{ ._, ._, .mov, .tmp0p, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
131646133188 .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ },
131647133189 .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ },
131648 .{ ._, .f_p, .add, ._, ._, ._, ._ },
133190 .{ ._, .f_p, .mul, ._, ._, ._, ._ },
131649133191 .{ ._, ._, .sub, .tmp0p, .si(16), ._, ._ },
131650133192 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131651133193 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
......@@ -131661,7 +133203,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131661133203 .extra_temps = .{
131662133204 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131663133205 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
131664 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__addtf3" } } },
133206 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
131665133207 .unused,
131666133208 .unused,
131667133209 .unused,
......@@ -131692,7 +133234,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131692133234 .extra_temps = .{
131693133235 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131694133236 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
131695 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__addtf3" } } },
133237 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
131696133238 .unused,
131697133239 .unused,
131698133240 .unused,
......@@ -131723,7 +133265,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131723133265 .extra_temps = .{
131724133266 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
131725133267 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
131726 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__addtf3" } } },
133268 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
131727133269 .unused,
131728133270 .unused,
131729133271 .unused,
......@@ -131744,7 +133286,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
131744133286 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
131745133287 } },
131746133288 } },
131747 .Mul => unreachable,
131748133289 }) catch |err| switch (err) {
131749133290 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
131750133291 @tagName(air_tag),
......@@ -149031,7 +150572,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
149031150572 } else if (self.hasFeature(.sse4_1)) {
149032150573 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
149033150574 try self.asmRegisterRegisterImmediate(.{ .p_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1));
149034 } else {
150575 } else if (self.hasFeature(.sse2)) {
149035150576 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
149036150577 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
149037150578 defer self.register_manager.unlockReg(tmp_lock);
......@@ -149039,6 +150580,19 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
149039150580 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
149040150581 try self.asmRegisterRegister(.{ ._ps, .movhl }, tmp_reg.to128(), src_reg.to128());
149041150582 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[1].to64(), tmp_reg.to128());
150583 } else {
150584 const frame_index = try self.allocFrameIndex(.init(.{
150585 .size = 16,
150586 .alignment = .@"16",
150587 }));
150588 try self.asmMemoryRegister(.{ ._ps, .mova }, .{
150589 .base = .{ .frame = frame_index },
150590 .mod = .{ .rm = .{ .size = .xword } },
150591 }, src_reg.to128());
150592 for (dst_regs, 0..) |dst_reg, dst_index| try self.asmRegisterMemory(.{ ._, .mov }, dst_reg.to64(), .{
150593 .base = .{ .frame = frame_index },
150594 .mod = .{ .rm = .{ .size = .qword, .disp = @intCast(8 * dst_index) } },
150595 });
149042150596 }
149043150597 return;
149044150598 } else unreachable,
......@@ -149282,7 +150836,7 @@ fn genSetReg(
149282150836 },
149283150837 ),
149284150838 .x87 => switch (src_reg.class()) {
149285 .general_purpose, .gphi, .segment => unreachable,
150839 .general_purpose, .gphi, .segment, .mmx, .ip, .cr, .dr => unreachable,
149286150840 .x87 => switch (src_reg) {
149287150841 .st0 => try self.asmRegister(.{ .f_, .st }, dst_reg),
149288150842 .st1, .st2, .st3, .st4, .st5, .st6 => switch (dst_reg) {
......@@ -149307,7 +150861,25 @@ fn genSetReg(
149307150861 },
149308150862 else => unreachable,
149309150863 },
149310 .mmx, .sse, .ip, .cr, .dr => unreachable,
150864 .sse => if (abi_size <= 16) {
150865 const frame_index = try self.allocFrameIndex(.init(.{
150866 .size = 16,
150867 .alignment = .@"16",
150868 }));
150869 try self.asmMemoryRegister(if (self.hasFeature(.avx))
150870 .{ .v_dqa, .mov }
150871 else if (self.hasFeature(.sse2))
150872 .{ ._dqa, .mov }
150873 else
150874 .{ ._ps, .mova }, .{
150875 .base = .{ .frame = frame_index },
150876 .mod = .{ .rm = .{ .size = .xword } },
150877 }, src_reg.to128());
150878 try MoveStrategy.read(.load_store_x87, self, dst_reg, .{
150879 .base = .{ .frame = frame_index },
150880 .mod = .{ .rm = .{ .size = .tbyte } },
150881 });
150882 } else unreachable,
149311150883 },
149312150884 .mmx => unreachable,
149313150885 .sse => switch (src_reg.class()) {
......@@ -149349,7 +150921,7 @@ fn genSetReg(
149349150921 .{ .register = try self.copyToTmpRegister(ty, src_mcv) },
149350150922 opts,
149351150923 ),
149352 .x87 => {
150924 .x87 => if (abi_size <= 16) {
149353150925 const frame_index = try self.allocFrameIndex(.init(.{
149354150926 .size = 16,
149355150927 .alignment = .@"16",
......@@ -149367,7 +150939,7 @@ fn genSetReg(
149367150939 .base = .{ .frame = frame_index },
149368150940 .mod = .{ .rm = .{ .size = .xword } },
149369150941 });
149370 },
150942 } else unreachable,
149371150943 .mmx, .ip, .cr, .dr => unreachable,
149372150944 .sse => try self.asmRegisterRegister(
149373150945 @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
......@@ -149431,7 +151003,7 @@ fn genSetReg(
149431151003 } else if (self.hasFeature(.sse4_1)) {
149432151004 try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
149433151005 try self.asmRegisterRegisterImmediate(.{ .p_q, .insr }, dst_reg.to128(), src_regs[1].to64(), .u(1));
149434 } else {
151006 } else if (self.hasFeature(.sse2)) {
149435151007 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
149436151008 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
149437151009 defer self.register_manager.unlockReg(tmp_lock);
......@@ -149439,6 +151011,19 @@ fn genSetReg(
149439151011 try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
149440151012 try self.asmRegisterRegister(.{ ._q, .mov }, tmp_reg.to128(), src_regs[1].to64());
149441151013 try self.asmRegisterRegister(.{ ._ps, .movlh }, dst_reg.to128(), tmp_reg.to128());
151014 } else {
151015 const frame_index = try self.allocFrameIndex(.init(.{
151016 .size = 16,
151017 .alignment = .@"16",
151018 }));
151019 for (src_regs, 0..) |src_reg, src_index| try self.asmMemoryRegister(.{ ._, .mov }, .{
151020 .base = .{ .frame = frame_index },
151021 .mod = .{ .rm = .{ .size = .qword, .disp = @intCast(8 * src_index) } },
151022 }, src_reg.to64());
151023 try self.asmRegisterMemory(.{ ._ps, .mova }, dst_reg.to128(), .{
151024 .base = .{ .frame = frame_index },
151025 .mod = .{ .rm = .{ .size = .xword } },
151026 });
149442151027 }
149443151028 } else unreachable,
149444151029 else => unreachable,
......@@ -149746,11 +151331,18 @@ fn genSetMem(
149746151331 },
149747151332 else => abi_size,
149748151333 };
149749 const src_alias = registerAlias(src_reg, abi_size);
149750 const src_size: u32 = @intCast(switch (src_alias.class()) {
149751 .general_purpose, .gphi, .segment, .x87, .ip, .cr, .dr => @divExact(src_alias.bitSize(), 8),
151334 const src_alias = registerAlias(src_reg, @intCast(self.unalignedSize(ty)));
151335 const src_class = src_alias.class();
151336 const src_size: u32 = switch (src_class) {
151337 .general_purpose, .gphi, .segment, .ip, .cr, .dr => @intCast(@divExact(src_alias.bitSize(), 8)),
149752151338 .mmx, .sse => abi_size,
149753 });
151339 .x87 => switch (abi.classifySystemV(ty, zcu, self.target, .other)[0]) {
151340 else => unreachable,
151341 .float, .float_combine => 4,
151342 .sse => 8,
151343 .x87 => 10,
151344 },
151345 };
149754151346 const src_align: InternPool.Alignment = .fromNonzeroByteUnits(
149755151347 std.math.ceilPowerOfTwoAssert(u32, src_size),
149756151348 );
......@@ -149760,7 +151352,7 @@ fn genSetMem(
149760151352 .alignment = src_align,
149761151353 }));
149762151354 const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } };
149763 try (try self.moveStrategy(ty, src_alias.class(), true)).write(
151355 try (try self.moveStrategy(ty, src_class, true)).write(
149764151356 self,
149765151357 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
149766151358 .size = .fromSize(src_size),
......@@ -149769,7 +151361,7 @@ fn genSetMem(
149769151361 );
149770151362 try self.genSetMem(base, disp, ty, frame_mcv, opts);
149771151363 try self.freeValue(frame_mcv);
149772 } else try (try self.moveStrategy(ty, src_alias.class(), switch (base) {
151364 } else try (try self.moveStrategy(ty, src_class, switch (base) {
149773151365 .none => src_align.check(@as(u32, @bitCast(disp))),
149774151366 .reg => |reg| switch (reg) {
149775151367 .es, .cs, .ss, .ds => src_align.check(@as(u32, @bitCast(disp))),
......@@ -154146,7 +155738,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
154146155738 reg
154147155739 else
154148155740 unreachable,
154149 .x87 => if (size_bytes >= 10 and size_bytes <= 16)
155741 .x87 => if (size_bytes >= 4 and size_bytes <= 16)
154150155742 reg
154151155743 else
154152155744 unreachable,
......@@ -154433,7 +156025,10 @@ fn promoteVarArg(self: *CodeGen, ty: Type) Type {
154433156025fn unalignedSize(cg: *CodeGen, ty: Type) u64 {
154434156026 const zcu = cg.pt.zcu;
154435156027 return switch (zcu.intern_pool.indexToKey(ty.toIntern())) {
154436 .vector_type => |vector_type| Type.fromInterned(vector_type.child).abiSize(zcu) * vector_type.len,
156028 .vector_type => |vector_type| switch (vector_type.child) {
156029 .bool_type => ty.abiSize(zcu),
156030 else => Type.fromInterned(vector_type.child).abiSize(zcu) * vector_type.len,
156031 },
154437156032 else => ty.abiSize(zcu),
154438156033 };
154439156034}
......@@ -155222,7 +156817,7 @@ const Temp = struct {
155222156817 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
155223156818 .register => |val_reg| try src.readReg(opts.disp, val_ty, registerAlias(
155224156819 val_reg,
155225 @intCast(val_ty.abiSize(cg.pt.zcu)),
156820 @intCast(cg.unalignedSize(val_ty)),
155226156821 ), cg),
155227156822 inline .register_pair, .register_triple, .register_quadruple => |val_regs| {
155228156823 var disp = opts.disp;
......@@ -160731,8 +162326,8 @@ const Select = struct {
160731162326 pand_mask_mem: struct { ref: Select.Operand.Ref, invert: bool = false },
160732162327 ptest_mask_mem: Select.Operand.Ref,
160733162328 pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },
160734 forward_bits_mem,
160735 reverse_bits_mem,
162329 bits_mem: enum { forward, reverse },
162330 splat_float_mem: struct { ref: Select.Operand.Ref, val: f16, fill: enum { inside, outside } = .inside },
160736162331 frame: FrameIndex,
160737162332 lazy_symbol: struct { kind: link.File.LazySymbol.Kind, ref: Select.Operand.Ref = .none },
160738162333 symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 },
......@@ -161051,11 +162646,11 @@ const Select = struct {
161051162646 .pand_mask_mem => |mask_spec| {
161052162647 const zcu = pt.zcu;
161053162648 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);
161054 const ty = mask_spec.ref.typeOf(s);
161055 assert(ty.isVector(zcu));
162649 const ref_ty = mask_spec.ref.typeOf(s);
162650 assert(ref_ty.isVector(zcu));
161056162651 var elem_buf: [64]u8 = undefined;
161057162652 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
161058 const mask_len: usize = @intCast(cg.unalignedSize(ty) % elems.len);
162653 const mask_len: usize = @intCast((cg.unalignedSize(ref_ty) - 1) % elems.len + 1);
161059162654 const invert_mask: u8 = switch (mask_spec.invert) {
161060162655 false => std.math.minInt(u8),
161061162656 true => std.math.maxInt(u8),
......@@ -161070,14 +162665,14 @@ const Select = struct {
161070162665 .ptest_mask_mem => |mask_ref| {
161071162666 const zcu = pt.zcu;
161072162667 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);
161073 const ty = mask_ref.typeOf(s);
161074 assert(ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type);
162668 const ref_ty = mask_ref.typeOf(s);
162669 assert(ref_ty.isVector(zcu) and ref_ty.childType(zcu).toIntern() == .bool_type);
161075162670 const mask_info = mask_ref.valueOf(s).register_mask.info;
161076162671 var elem_buf: [64]u8 = @splat(0);
161077162672 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
161078162673 const elem_bytes: u6 = @intCast(@divExact(mask_info.scalar.bitSize(cg.target), 8));
161079162674 var index: u7 = 0;
161080 for (0..@intCast(ty.vectorLen(zcu))) |_| {
162675 for (0..@intCast(ref_ty.vectorLen(zcu))) |_| {
161081162676 switch (mask_info.kind) {
161082162677 .sign => {
161083162678 @memset(elems[index..][0 .. elem_bytes - 1], std.math.minInt(u8));
......@@ -161108,21 +162703,42 @@ const Select = struct {
161108162703 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
161109162704 } }))), true };
161110162705 },
161111 .forward_bits_mem, .reverse_bits_mem => {
162706 .bits_mem => |direction| {
161112162707 const zcu = pt.zcu;
161113162708 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);
161114162709 var bytes: [32]u8 = undefined;
161115162710 const elems = bytes[0..spec.type.vectorLen(zcu)];
161116 for (elems, 0..) |*elem, index| elem.* = switch (spec.kind) {
161117 else => unreachable,
161118 .forward_bits_mem => @as(u8, 1 << 0) << @truncate(index),
161119 .reverse_bits_mem => @as(u8, 1 << 7) >> @truncate(index),
162711 for (elems, 0..) |*elem, index| elem.* = switch (direction) {
162712 .forward => @as(u8, 1 << 0) << @truncate(index),
162713 .reverse => @as(u8, 1 << 7) >> @truncate(index),
161120162714 };
161121162715 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
161122162716 .ty = spec.type.toIntern(),
161123162717 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
161124162718 } }))), true };
161125162719 },
162720 .splat_float_mem => |splat_spec| {
162721 const zcu = pt.zcu;
162722 assert(spec.type.isVector(zcu));
162723 const elem_ty = spec.type.childType(zcu);
162724 const ref_ty = splat_spec.ref.typeOf(s);
162725 assert(ref_ty.isVector(zcu) and ref_ty.childType(zcu).toIntern() == elem_ty.toIntern());
162726 var elem_buf: [@divExact(64, 2)]InternPool.Index = undefined;
162727 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
162728 const inside_len = (ref_ty.vectorLen(zcu) - 1) % elems.len + 1;
162729 @memset(elems[0..inside_len], (try pt.floatValue(elem_ty, switch (splat_spec.fill) {
162730 .inside => splat_spec.val,
162731 .outside => 0.0,
162732 })).toIntern());
162733 @memset(elems[inside_len..], (try pt.floatValue(elem_ty, switch (splat_spec.fill) {
162734 .inside => 0.0,
162735 .outside => splat_spec.val,
162736 })).toIntern());
162737 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
162738 .ty = spec.type.toIntern(),
162739 .storage = .{ .elems = elems },
162740 } }))), true };
162741 },
161126162742 .frame => |frame_index| .{ try cg.tempInit(spec.type, .{ .load_frame = .{ .index = frame_index } }), true },
161127162743 .lazy_symbol => |lazy_symbol_spec| {
161128162744 const ip = &pt.zcu.intern_pool;
src/codegen/c/Type.zig+60
......@@ -1885,6 +1885,36 @@ pub const Pool = struct {
18851885 };
18861886 return pool.fromFields(allocator, .@"struct", &fields, kind);
18871887 },
1888 .vector_16_f16_type => {
1889 const vector_ctype = try pool.getVector(allocator, .{
1890 .elem_ctype = .f16,
1891 .len = 16,
1892 });
1893 if (!kind.isParameter()) return vector_ctype;
1894 var fields = [_]Info.Field{
1895 .{
1896 .name = .{ .index = .array },
1897 .ctype = vector_ctype,
1898 .alignas = AlignAs.fromAbiAlignment(Type.f16.abiAlignment(zcu)),
1899 },
1900 };
1901 return pool.fromFields(allocator, .@"struct", &fields, kind);
1902 },
1903 .vector_32_f16_type => {
1904 const vector_ctype = try pool.getVector(allocator, .{
1905 .elem_ctype = .f16,
1906 .len = 32,
1907 });
1908 if (!kind.isParameter()) return vector_ctype;
1909 var fields = [_]Info.Field{
1910 .{
1911 .name = .{ .index = .array },
1912 .ctype = vector_ctype,
1913 .alignas = AlignAs.fromAbiAlignment(Type.f16.abiAlignment(zcu)),
1914 },
1915 };
1916 return pool.fromFields(allocator, .@"struct", &fields, kind);
1917 },
18881918 .vector_2_f32_type => {
18891919 const vector_ctype = try pool.getVector(allocator, .{
18901920 .elem_ctype = .f32,
......@@ -1930,6 +1960,21 @@ pub const Pool = struct {
19301960 };
19311961 return pool.fromFields(allocator, .@"struct", &fields, kind);
19321962 },
1963 .vector_16_f32_type => {
1964 const vector_ctype = try pool.getVector(allocator, .{
1965 .elem_ctype = .f32,
1966 .len = 16,
1967 });
1968 if (!kind.isParameter()) return vector_ctype;
1969 var fields = [_]Info.Field{
1970 .{
1971 .name = .{ .index = .array },
1972 .ctype = vector_ctype,
1973 .alignas = AlignAs.fromAbiAlignment(Type.f32.abiAlignment(zcu)),
1974 },
1975 };
1976 return pool.fromFields(allocator, .@"struct", &fields, kind);
1977 },
19331978 .vector_2_f64_type => {
19341979 const vector_ctype = try pool.getVector(allocator, .{
19351980 .elem_ctype = .f64,
......@@ -1960,6 +2005,21 @@ pub const Pool = struct {
19602005 };
19612006 return pool.fromFields(allocator, .@"struct", &fields, kind);
19622007 },
2008 .vector_8_f64_type => {
2009 const vector_ctype = try pool.getVector(allocator, .{
2010 .elem_ctype = .f64,
2011 .len = 8,
2012 });
2013 if (!kind.isParameter()) return vector_ctype;
2014 var fields = [_]Info.Field{
2015 .{
2016 .name = .{ .index = .array },
2017 .ctype = vector_ctype,
2018 .alignas = AlignAs.fromAbiAlignment(Type.f64.abiAlignment(zcu)),
2019 },
2020 };
2021 return pool.fromFields(allocator, .@"struct", &fields, kind);
2022 },
19632023
19642024 .undef,
19652025 .zero,
test/behavior/x86_64/math.zig+18-6
......@@ -125,7 +125,7 @@ fn boolOr(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
125125 @compileError("unsupported boolOr type: " ++ @typeName(@TypeOf(lhs)));
126126}
127127
128pub const Compare = enum { strict, relaxed, approx, approx_int };
128pub const Compare = enum { strict, relaxed, approx, approx_int, approx_or_overflow };
129129// noinline for a more helpful stack trace
130130pub noinline fn checkExpected(expected: anytype, actual: @TypeOf(expected), comptime compare: Compare) !void {
131131 const Expected = @TypeOf(expected);
......@@ -137,20 +137,32 @@ pub noinline fn checkExpected(expected: anytype, actual: @TypeOf(expected), comp
137137 break :unexpected switch (compare) {
138138 .strict => boolOr(unequal, sign(expected) != sign(actual)),
139139 .relaxed => unequal,
140 .approx, .approx_int => comptime unreachable,
140 .approx, .approx_int, .approx_or_overflow => comptime unreachable,
141141 };
142142 },
143 .approx, .approx_int => {
143 .approx, .approx_int, .approx_or_overflow => {
144144 const epsilon = math.floatEps(Scalar(Expected));
145 const tolerance = @sqrt(epsilon);
146 break :unexpected @abs(expected - actual) > @max(
145 const tolerance = switch (compare) {
146 .strict, .relaxed => comptime unreachable,
147 .approx, .approx_int => @sqrt(epsilon),
148 .approx_or_overflow => @exp2(@log2(epsilon) * 0.4),
149 };
150 const approx_unequal = @abs(expected - actual) > @max(
147151 @abs(expected) * splat(Expected, tolerance),
148152 splat(Expected, switch (compare) {
149153 .strict, .relaxed => comptime unreachable,
150 .approx => tolerance,
154 .approx, .approx_or_overflow => tolerance,
151155 .approx_int => 1,
152156 }),
153157 );
158 break :unexpected switch (compare) {
159 .strict, .relaxed => comptime unreachable,
160 .approx, .approx_int => approx_unequal,
161 .approx_or_overflow => boolAnd(approx_unequal, boolOr(boolAnd(
162 @abs(expected) != splat(Expected, inf(Expected)),
163 @abs(actual) != splat(Expected, inf(Expected)),
164 ), sign(expected) != sign(actual))),
165 };
154166 },
155167 },
156168 .@"struct" => |@"struct"| inline for (@"struct".fields) |field| {
test/behavior/x86_64/unary.zig+9
......@@ -5119,6 +5119,15 @@ test reduceAddOptimized {
51195119 try test_reduce_add_optimized.testFloatVectors();
51205120}
51215121
5122inline fn reduceMulOptimized(comptime Type: type, rhs: Type) @typeInfo(Type).vector.child {
5123 @setFloatMode(.optimized);
5124 return @reduce(.Mul, rhs);
5125}
5126test reduceMulOptimized {
5127 const test_reduce_mul_optimized = unary(reduceMulOptimized, .{ .compare = .approx_or_overflow });
5128 try test_reduce_mul_optimized.testFloatVectors();
5129}
5130
51225131inline fn splat(comptime Type: type, rhs: Type) Type {
51235132 return @splat(rhs[0]);
51245133}
test/cases/compile_errors/@import_zon_bad_type.zig+3-3
......@@ -117,9 +117,9 @@ export fn testMutablePointer() void {
117117// tmp.zig:37:38: note: imported here
118118// neg_inf.zon:1:1: error: expected type '?u8'
119119// tmp.zig:57:28: note: imported here
120// neg_inf.zon:1:1: error: expected type 'tmp.testNonExhaustiveEnum__enum_505'
120// neg_inf.zon:1:1: error: expected type 'tmp.testNonExhaustiveEnum__enum_509'
121121// tmp.zig:62:39: note: imported here
122// neg_inf.zon:1:1: error: expected type 'tmp.testUntaggedUnion__union_507'
122// neg_inf.zon:1:1: error: expected type 'tmp.testUntaggedUnion__union_511'
123123// tmp.zig:67:44: note: imported here
124// neg_inf.zon:1:1: error: expected type 'tmp.testTaggedUnionVoid__union_510'
124// neg_inf.zon:1:1: error: expected type 'tmp.testTaggedUnionVoid__union_514'
125125// tmp.zig:72:50: note: imported here
test/cases/compile_errors/anytype_param_requires_comptime.zig+1-1
......@@ -15,6 +15,6 @@ pub export fn entry() void {
1515// error
1616//
1717// :7:25: error: unable to resolve comptime value
18// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_479.C' must be comptime-known
18// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_483.C' must be comptime-known
1919// :4:16: note: struct requires comptime because of this field
2020// :4:16: note: types are not available at runtime
test/cases/compile_errors/bogus_method_call_on_slice.zig+1-1
......@@ -16,5 +16,5 @@ pub export fn entry2() void {
1616//
1717// :3:6: error: no field or member function named 'copy' in '[]const u8'
1818// :9:8: error: no field or member function named 'bar' in '@TypeOf(.{})'
19// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_483'
19// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_487'
2020// :12:6: note: struct declared here
test/cases/compile_errors/coerce_anon_struct.zig+1-1
......@@ -6,6 +6,6 @@ export fn foo() void {
66
77// error
88//
9// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_472'
9// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_476'
1010// :3:16: note: struct declared here
1111// :1:11: note: struct declared here
test/cases/compile_errors/redundant_try.zig+2-2
......@@ -44,9 +44,9 @@ comptime {
4444//
4545// :5:23: error: expected error union type, found 'comptime_int'
4646// :10:23: error: expected error union type, found '@TypeOf(.{})'
47// :15:23: error: expected error union type, found 'tmp.test2__struct_509'
47// :15:23: error: expected error union type, found 'tmp.test2__struct_513'
4848// :15:23: note: struct declared here
49// :20:27: error: expected error union type, found 'tmp.test3__struct_511'
49// :20:27: error: expected error union type, found 'tmp.test3__struct_515'
5050// :20:27: note: struct declared here
5151// :25:23: error: expected error union type, found 'struct { comptime *const [5:0]u8 = "hello" }'
5252// :31:13: error: expected error union type, found 'u32'