authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-21 12:27:24-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-28 15:10:22-04:00
log7bfdb7f26d85b8f9df8dfb39de70160d1881208b
tree167d7fecf6588d4938f9e1ff096bb8187e4c9547
parent1f6f8b0ffe33696ce980955cceacd98171d7ea0c

x86_64: implement integer `@reduce(.Mul)`

6 files changed, 3830 insertions(+), 141 deletions(-)

src/arch/x86_64/CodeGen.zig+3766-119
......@@ -2389,7 +2389,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
23892389}
23902390
23912391fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2392 @setEvalBranchQuota(22_500);
2392 @setEvalBranchQuota(23_100);
23932393 const pt = cg.pt;
23942394 const zcu = pt.zcu;
23952395 const ip = &zcu.intern_pool;
......@@ -98880,6 +98880,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
9888098880 .unused,
9888198881 },
9888298882 .dst_temps = .{ .mem, .unused },
98883 .clobbers = .{ .eflags = true },
9888398884 .each = .{ .once = &.{
9888498885 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9888598886 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
......@@ -99134,6 +99135,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
9913499135 .unused,
9913599136 },
9913699137 .dst_temps = .{ .mem, .unused },
99138 .clobbers = .{ .eflags = true },
9913799139 .each = .{ .once = &.{
9913899140 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
9913999141 .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
......@@ -99164,6 +99166,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
9916499166 .unused,
9916599167 },
9916699168 .dst_temps = .{ .mem, .unused },
99169 .clobbers = .{ .eflags = true },
9916799170 .each = .{ .once = &.{
9916899171 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
9916999172 .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
......@@ -99291,6 +99294,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
9929199294 .unused,
9929299295 },
9929399296 .dst_temps = .{ .mem, .unused },
99297 .clobbers = .{ .eflags = true },
9929499298 .each = .{ .once = &.{
9929599299 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
9929699300 .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
......@@ -99320,6 +99324,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
9932099324 .unused,
9932199325 },
9932299326 .dst_temps = .{ .mem, .unused },
99327 .clobbers = .{ .eflags = true },
9932399328 .each = .{ .once = &.{
9932499329 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
9932599330 .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
......@@ -99545,6 +99550,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
9954599550 .unused,
9954699551 },
9954799552 .dst_temps = .{ .mem, .unused },
99553 .clobbers = .{ .eflags = true },
9954899554 .each = .{ .once = &.{
9954999555 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
9955099556 .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
......@@ -100555,6 +100561,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100555100561 .unused,
100556100562 },
100557100563 .dst_temps = .{ .mem, .unused },
100564 .clobbers = .{ .eflags = true },
100558100565 .each = .{ .once = &.{
100559100566 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
100560100567 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
......@@ -100586,6 +100593,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100586100593 .unused,
100587100594 },
100588100595 .dst_temps = .{ .mem, .unused },
100596 .clobbers = .{ .eflags = true },
100589100597 .each = .{ .once = &.{
100590100598 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
100591100599 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
......@@ -100617,6 +100625,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100617100625 .unused,
100618100626 },
100619100627 .dst_temps = .{ .mem, .unused },
100628 .clobbers = .{ .eflags = true },
100620100629 .each = .{ .once = &.{
100621100630 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
100622100631 .{ .@"0:", ._, .cvttss2si, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ },
......@@ -100645,6 +100654,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100645100654 .unused,
100646100655 },
100647100656 .dst_temps = .{ .mem, .unused },
100657 .clobbers = .{ .eflags = true },
100648100658 .each = .{ .once = &.{
100649100659 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
100650100660 .{ .@"0:", ._, .cvttss2si, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ },
......@@ -100849,6 +100859,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100849100859 .unused,
100850100860 },
100851100861 .dst_temps = .{ .mem, .unused },
100862 .clobbers = .{ .eflags = true },
100852100863 .each = .{ .once = &.{
100853100864 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
100854100865 .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memsia(.src0x, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
......@@ -100878,6 +100889,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100878100889 .unused,
100879100890 },
100880100891 .dst_temps = .{ .mem, .unused },
100892 .clobbers = .{ .eflags = true },
100881100893 .each = .{ .once = &.{
100882100894 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
100883100895 .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memsia(.src0x, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
......@@ -100907,6 +100919,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100907100919 .unused,
100908100920 },
100909100921 .dst_temps = .{ .mem, .unused },
100922 .clobbers = .{ .eflags = true },
100910100923 .each = .{ .once = &.{
100911100924 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
100912100925 .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memsia(.src0x, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
......@@ -100936,6 +100949,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
100936100949 .unused,
100937100950 },
100938100951 .dst_temps = .{ .mem, .unused },
100952 .clobbers = .{ .eflags = true },
100939100953 .each = .{ .once = &.{
100940100954 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
100941100955 .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memsia(.src0x, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
......@@ -101051,6 +101065,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101051101065 .unused,
101052101066 },
101053101067 .dst_temps = .{ .mem, .unused },
101068 .clobbers = .{ .eflags = true },
101054101069 .each = .{ .once = &.{
101055101070 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
101056101071 .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
......@@ -101079,6 +101094,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101079101094 .unused,
101080101095 },
101081101096 .dst_temps = .{ .mem, .unused },
101097 .clobbers = .{ .eflags = true },
101082101098 .each = .{ .once = &.{
101083101099 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
101084101100 .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
......@@ -101231,6 +101247,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101231101247 .unused,
101232101248 },
101233101249 .dst_temps = .{ .mem, .unused },
101250 .clobbers = .{ .eflags = true },
101234101251 .each = .{ .once = &.{
101235101252 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
101236101253 .{ .@"0:", .v_, .cvttss2si, .tmp1q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
......@@ -101259,6 +101276,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101259101276 .unused,
101260101277 },
101261101278 .dst_temps = .{ .mem, .unused },
101279 .clobbers = .{ .eflags = true },
101262101280 .each = .{ .once = &.{
101263101281 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
101264101282 .{ .@"0:", ._, .cvttss2si, .tmp1q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
......@@ -101687,6 +101705,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101687101705 },
101688101706 .extra_temps = .{
101689101707 .{ .type = .f64, .kind = .{ .reg = .st7 } },
101708 .{ .type = .u16, .kind = .mem },
101690101709 .{ .type = .i16, .kind = .mem },
101691101710 .unused,
101692101711 .unused,
......@@ -101696,13 +101715,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101696101715 .unused,
101697101716 .unused,
101698101717 .unused,
101699 .unused,
101700101718 },
101701101719 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
101702101720 .each = .{ .once = &.{
101703101721 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
101704 .{ ._, .fi_p, .stt, .tmp1w, ._, ._, ._ },
101705 .{ ._, ._, .movsx, .dst0d, .tmp1b, ._, ._ },
101722 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
101723 .{ ._, ._, .movzx, .dst0d, .tmp1w, ._, ._ },
101724 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
101725 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101726 .{ ._, ._, .mov, .tmp1w, .dst0w, ._, ._ },
101727 .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ },
101728 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101729 .{ ._, ._, .movsx, .dst0d, .tmp2b, ._, ._ },
101706101730 } },
101707101731 }, .{
101708101732 .required_features = .{ .x87, null, null, null },
......@@ -101713,6 +101737,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101713101737 },
101714101738 .extra_temps = .{
101715101739 .{ .type = .f64, .kind = .{ .reg = .st7 } },
101740 .{ .type = .u16, .kind = .mem },
101716101741 .{ .type = .i16, .kind = .mem },
101717101742 .unused,
101718101743 .unused,
......@@ -101722,13 +101747,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101722101747 .unused,
101723101748 .unused,
101724101749 .unused,
101725 .unused,
101726101750 },
101727101751 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
101728101752 .each = .{ .once = &.{
101729101753 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
101730 .{ ._, .fi_p, .stt, .tmp1w, ._, ._, ._ },
101731 .{ ._, ._, .movzx, .dst0d, .tmp1b, ._, ._ },
101754 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
101755 .{ ._, ._, .movzx, .dst0d, .tmp1w, ._, ._ },
101756 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
101757 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101758 .{ ._, ._, .mov, .tmp1w, .dst0w, ._, ._ },
101759 .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ },
101760 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101761 .{ ._, ._, .movzx, .dst0d, .tmp2b, ._, ._ },
101732101762 } },
101733101763 }, .{
101734101764 .required_features = .{ .x87, null, null, null },
......@@ -101739,8 +101769,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101739101769 },
101740101770 .extra_temps = .{
101741101771 .{ .type = .f64, .kind = .{ .reg = .st7 } },
101742 .unused,
101743 .unused,
101772 .{ .type = .u16, .kind = .mem },
101773 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
101744101774 .unused,
101745101775 .unused,
101746101776 .unused,
......@@ -101753,7 +101783,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101753101783 .dst_temps = .{ .mem, .unused },
101754101784 .each = .{ .once = &.{
101755101785 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
101756 .{ ._, .fi_p, .stt, .dst0w, ._, ._, ._ },
101786 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
101787 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
101788 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
101789 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101790 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
101791 .{ ._, .fi_p, .st, .dst0w, ._, ._, ._ },
101792 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101757101793 } },
101758101794 }, .{
101759101795 .required_features = .{ .x87, null, null, null },
......@@ -101764,8 +101800,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101764101800 },
101765101801 .extra_temps = .{
101766101802 .{ .type = .f64, .kind = .{ .reg = .st7 } },
101767 .unused,
101768 .unused,
101803 .{ .type = .u16, .kind = .mem },
101804 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
101769101805 .unused,
101770101806 .unused,
101771101807 .unused,
......@@ -101778,7 +101814,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101778101814 .dst_temps = .{ .mem, .unused },
101779101815 .each = .{ .once = &.{
101780101816 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
101781 .{ ._, .fi_p, .stt, .dst0d, ._, ._, ._ },
101817 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
101818 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
101819 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
101820 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101821 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
101822 .{ ._, .fi_p, .st, .dst0d, ._, ._, ._ },
101823 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101782101824 } },
101783101825 }, .{
101784101826 .required_features = .{ .@"64bit", .avx, null, null },
......@@ -101813,8 +101855,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101813101855 },
101814101856 .extra_temps = .{
101815101857 .{ .type = .f64, .kind = .{ .reg = .st7 } },
101816 .unused,
101817 .unused,
101858 .{ .type = .u16, .kind = .mem },
101859 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
101818101860 .unused,
101819101861 .unused,
101820101862 .unused,
......@@ -101827,7 +101869,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101827101869 .dst_temps = .{ .mem, .unused },
101828101870 .each = .{ .once = &.{
101829101871 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
101830 .{ ._, .fi_p, .stt, .dst0q, ._, ._, ._ },
101872 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
101873 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
101874 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
101875 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101876 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
101877 .{ ._, .fi_p, .st, .dst0q, ._, ._, ._ },
101878 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
101831101879 } },
101832101880 }, .{
101833101881 .required_features = .{ .@"64bit", .avx, null, null },
......@@ -101901,12 +101949,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101901101949 .{ .src = .{ .to_mem, .none, .none } },
101902101950 },
101903101951 .extra_temps = .{
101904 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
101905 .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } },
101906 .{ .type = .f64, .kind = .{ .reg = .st6 } },
101907101952 .{ .type = .f64, .kind = .{ .reg = .st7 } },
101953 .{ .type = .f64, .kind = .{ .reg = .st6 } },
101954 .{ .type = .u16, .kind = .mem },
101955 .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } },
101908101956 .{ .type = .i64, .kind = .mem },
101909 .unused,
101957 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
101910101958 .unused,
101911101959 .unused,
101912101960 .unused,
......@@ -101917,16 +101965,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
101917101965 .clobbers = .{ .eflags = true },
101918101966 .each = .{ .once = &.{
101919101967 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
101920 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
101921 .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ },
101922 .{ ._, .fi_p, .stt, .tmp4q, ._, ._, ._ },
101923 .{ ._, ._, .mov, .dst0q, .tmp4q, ._, ._ },
101924 .{ ._, .f_, .sub, .lea(.tmp0d), ._, ._, ._ },
101925 .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ },
101926 .{ ._, .fi_p, .stt, .tmp4q, ._, ._, ._ },
101968 .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ },
101969 .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ },
101970 .{ ._, ._, .movzx, .dst0d, .tmp2w, ._, ._ },
101971 .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ },
101972 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
101973 .{ ._, ._, .mov, .tmp2w, .dst0w, ._, ._ },
101974 .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ },
101975 .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ },
101976 .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ },
101977 .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ },
101978 .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ },
101979 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
101980 .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ },
101927101981 .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ },
101928101982 .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ },
101929 .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ },
101983 .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ },
101930101984 } },
101931101985 }, .{
101932101986 .required_features = .{ .@"64bit", .sse, null, null },
......@@ -102248,10 +102302,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102248102302 },
102249102303 .extra_temps = .{
102250102304 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
102305 .{ .type = .u16, .kind = .mem },
102306 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
102251102307 .{ .type = .f64, .kind = .{ .reg = .st7 } },
102252102308 .{ .type = .i16, .kind = .mem },
102253 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
102254 .unused,
102255102309 .unused,
102256102310 .unused,
102257102311 .unused,
......@@ -102260,14 +102314,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102260102314 .unused,
102261102315 },
102262102316 .dst_temps = .{ .mem, .unused },
102317 .clobbers = .{ .eflags = true },
102263102318 .each = .{ .once = &.{
102264102319 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
102320 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
102321 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
102322 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
102323 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102324 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
102265102325 .{ .@"0:", .f_, .ld, .memsia(.src0q, .@"8", .tmp0, .add_unaligned_size), ._, ._, ._ },
102266 .{ ._, .fi_p, .stt, .tmp2w, ._, ._, ._ },
102267 .{ ._, ._, .movzx, .tmp3d, .tmp2b, ._, ._ },
102268 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp3b, ._, ._ },
102326 .{ ._, .fi_p, .st, .tmp4w, ._, ._, ._ },
102327 .{ ._, ._, .movzx, .tmp2d, .tmp4b, ._, ._ },
102328 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ },
102269102329 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
102270102330 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
102331 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102271102332 } },
102272102333 }, .{
102273102334 .required_features = .{ .x87, null, null, null },
......@@ -102278,10 +102339,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102278102339 },
102279102340 .extra_temps = .{
102280102341 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
102342 .{ .type = .u16, .kind = .mem },
102343 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
102281102344 .{ .type = .f64, .kind = .{ .reg = .st7 } },
102282102345 .{ .type = .i16, .kind = .mem },
102283 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
102284 .unused,
102285102346 .unused,
102286102347 .unused,
102287102348 .unused,
......@@ -102290,14 +102351,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102290102351 .unused,
102291102352 },
102292102353 .dst_temps = .{ .mem, .unused },
102354 .clobbers = .{ .eflags = true },
102293102355 .each = .{ .once = &.{
102294102356 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
102357 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
102358 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
102359 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
102360 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102361 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
102295102362 .{ .@"0:", .f_, .ld, .memsia(.src0q, .@"8", .tmp0, .add_unaligned_size), ._, ._, ._ },
102296 .{ ._, .fi_p, .stt, .tmp2w, ._, ._, ._ },
102297 .{ ._, ._, .movzx, .tmp3d, .tmp2b, ._, ._ },
102298 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp3b, ._, ._ },
102363 .{ ._, .fi_p, .st, .tmp4w, ._, ._, ._ },
102364 .{ ._, ._, .movzx, .tmp2d, .tmp4b, ._, ._ },
102365 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ },
102299102366 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
102300102367 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
102368 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102301102369 } },
102302102370 }, .{
102303102371 .required_features = .{ .avx, .slow_incdec, null, null },
......@@ -102723,6 +102791,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102723102791 },
102724102792 .extra_temps = .{
102725102793 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
102794 .{ .type = .u16, .kind = .mem },
102726102795 .{ .type = .f64, .kind = .{ .reg = .st7 } },
102727102796 .unused,
102728102797 .unused,
......@@ -102732,15 +102801,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102732102801 .unused,
102733102802 .unused,
102734102803 .unused,
102735 .unused,
102736102804 },
102737102805 .dst_temps = .{ .mem, .unused },
102806 .clobbers = .{ .eflags = true },
102738102807 .each = .{ .once = &.{
102808 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
102809 .{ ._, ._, .movzx, .tmp0d, .tmp1w, ._, ._ },
102810 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
102811 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102812 .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ },
102739102813 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
102740102814 .{ .@"0:", .f_, .ld, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._, ._ },
102741 .{ ._, .fi_p, .stt, .memia(.dst0w, .tmp0, .add_unaligned_size), ._, ._, ._ },
102815 .{ ._, .fi_p, .st, .memia(.dst0w, .tmp0, .add_unaligned_size), ._, ._, ._ },
102742102816 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
102743102817 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
102818 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102744102819 } },
102745102820 }, .{
102746102821 .required_features = .{ .x87, null, null, null },
......@@ -102751,10 +102826,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102751102826 },
102752102827 .extra_temps = .{
102753102828 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
102829 .{ .type = .u16, .kind = .mem },
102830 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
102754102831 .{ .type = .f64, .kind = .{ .reg = .st7 } },
102755102832 .{ .type = .i32, .kind = .mem },
102756 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
102757 .unused,
102758102833 .unused,
102759102834 .unused,
102760102835 .unused,
......@@ -102763,14 +102838,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102763102838 .unused,
102764102839 },
102765102840 .dst_temps = .{ .mem, .unused },
102841 .clobbers = .{ .eflags = true },
102766102842 .each = .{ .once = &.{
102767102843 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
102844 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
102845 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
102846 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
102847 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102848 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
102768102849 .{ .@"0:", .f_, .ld, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._, ._ },
102769 .{ ._, .fi_p, .stt, .tmp2d, ._, ._, ._ },
102770 .{ ._, ._, .mov, .tmp3d, .tmp2d, ._, ._ },
102771 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp3w, ._, ._ },
102850 .{ ._, .fi_p, .st, .tmp4d, ._, ._, ._ },
102851 .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ },
102852 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ },
102772102853 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
102773102854 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
102855 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102774102856 } },
102775102857 }, .{
102776102858 .required_features = .{ .avx, null, null, null },
......@@ -102969,6 +103051,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102969103051 },
102970103052 .extra_temps = .{
102971103053 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
103054 .{ .type = .u16, .kind = .mem },
102972103055 .{ .type = .f64, .kind = .{ .reg = .st7 } },
102973103056 .unused,
102974103057 .unused,
......@@ -102978,15 +103061,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102978103061 .unused,
102979103062 .unused,
102980103063 .unused,
102981 .unused,
102982103064 },
102983103065 .dst_temps = .{ .mem, .unused },
103066 .clobbers = .{ .eflags = true },
102984103067 .each = .{ .once = &.{
103068 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
103069 .{ ._, ._, .movzx, .tmp0d, .tmp1w, ._, ._ },
103070 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
103071 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
103072 .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ },
102985103073 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
102986103074 .{ .@"0:", .f_, .ld, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._, ._ },
102987 .{ ._, .fi_p, .stt, .memia(.dst0d, .tmp0, .add_unaligned_size), ._, ._, ._ },
103075 .{ ._, .fi_p, .st, .memia(.dst0d, .tmp0, .add_unaligned_size), ._, ._, ._ },
102988103076 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
102989103077 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
103078 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
102990103079 } },
102991103080 }, .{
102992103081 .required_features = .{ .x87, null, null, null },
......@@ -102997,10 +103086,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
102997103086 },
102998103087 .extra_temps = .{
102999103088 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
103089 .{ .type = .u16, .kind = .mem },
103090 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
103000103091 .{ .type = .f64, .kind = .{ .reg = .st7 } },
103001103092 .{ .type = .i64, .kind = .mem },
103002 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
103003 .unused,
103004103093 .unused,
103005103094 .unused,
103006103095 .unused,
......@@ -103009,14 +103098,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103009103098 .unused,
103010103099 },
103011103100 .dst_temps = .{ .mem, .unused },
103101 .clobbers = .{ .eflags = true },
103012103102 .each = .{ .once = &.{
103013103103 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
103104 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
103105 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
103106 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
103107 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
103108 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
103014103109 .{ .@"0:", .f_, .ld, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._, ._ },
103015 .{ ._, .fi_p, .stt, .tmp2q, ._, ._, ._ },
103016 .{ ._, ._, .mov, .tmp3d, .tmp2d, ._, ._ },
103017 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp3d, ._, ._ },
103110 .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ },
103111 .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ },
103112 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ },
103018103113 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
103019103114 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
103115 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
103020103116 } },
103021103117 }, .{
103022103118 .required_features = .{ .avx, null, null, null },
......@@ -103273,6 +103369,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103273103369 },
103274103370 .extra_temps = .{
103275103371 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
103372 .{ .type = .u16, .kind = .mem },
103276103373 .{ .type = .f64, .kind = .{ .reg = .st7 } },
103277103374 .unused,
103278103375 .unused,
......@@ -103282,15 +103379,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103282103379 .unused,
103283103380 .unused,
103284103381 .unused,
103285 .unused,
103286103382 },
103287103383 .dst_temps = .{ .mem, .unused },
103384 .clobbers = .{ .eflags = true },
103288103385 .each = .{ .once = &.{
103289 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
103386 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
103387 .{ ._, ._, .movzx, .tmp0d, .tmp1w, ._, ._ },
103388 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
103389 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
103390 .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ },
103391 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
103290103392 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
103291 .{ ._, .fi_p, .stt, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
103393 .{ ._, .fi_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
103292103394 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
103293103395 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
103396 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
103294103397 } },
103295103398 }, .{
103296103399 .required_features = .{ .avx, null, null, null },
......@@ -103881,7 +103984,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103881103984 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
103882103985 } },
103883103986 }, .{
103884 .required_features = .{ .x87, null, null, null },
103987 .required_features = .{ .sse3, .x87, null, null },
103885103988 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
103886103989 .dst_constraints = .{ .{ .signed_int = .byte }, .any },
103887103990 .patterns = &.{
......@@ -103889,7 +103992,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103889103992 .{ .src = .{ .to_x87, .none, .none } },
103890103993 },
103891103994 .extra_temps = .{
103892 .{ .type = .f64, .kind = .{ .reg = .st7 } },
103995 .{ .type = .f80, .kind = .{ .reg = .st7 } },
103893103996 .{ .type = .i16, .kind = .mem },
103894103997 .unused,
103895103998 .unused,
......@@ -103910,13 +104013,46 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103910104013 }, .{
103911104014 .required_features = .{ .x87, null, null, null },
103912104015 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104016 .dst_constraints = .{ .{ .signed_int = .byte }, .any },
104017 .patterns = &.{
104018 .{ .src = .{ .mem, .none, .none } },
104019 .{ .src = .{ .to_x87, .none, .none } },
104020 },
104021 .extra_temps = .{
104022 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104023 .{ .type = .u16, .kind = .mem },
104024 .{ .type = .i16, .kind = .mem },
104025 .unused,
104026 .unused,
104027 .unused,
104028 .unused,
104029 .unused,
104030 .unused,
104031 .unused,
104032 .unused,
104033 },
104034 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
104035 .each = .{ .once = &.{
104036 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
104037 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104038 .{ ._, ._, .movzx, .dst0d, .tmp1w, ._, ._ },
104039 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104040 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104041 .{ ._, ._, .mov, .tmp1w, .dst0w, ._, ._ },
104042 .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ },
104043 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104044 .{ ._, ._, .movsx, .dst0d, .tmp2b, ._, ._ },
104045 } },
104046 }, .{
104047 .required_features = .{ .sse3, .x87, null, null },
104048 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
103913104049 .dst_constraints = .{ .{ .unsigned_int = .byte }, .any },
103914104050 .patterns = &.{
103915104051 .{ .src = .{ .mem, .none, .none } },
103916104052 .{ .src = .{ .to_x87, .none, .none } },
103917104053 },
103918104054 .extra_temps = .{
103919 .{ .type = .f64, .kind = .{ .reg = .st7 } },
104055 .{ .type = .f80, .kind = .{ .reg = .st7 } },
103920104056 .{ .type = .i16, .kind = .mem },
103921104057 .unused,
103922104058 .unused,
......@@ -103935,7 +104071,40 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103935104071 .{ ._, ._, .movzx, .dst0d, .tmp1b, ._, ._ },
103936104072 } },
103937104073 }, .{
103938 .required_features = .{ .x87, .slow_incdec, null, null },
104074 .required_features = .{ .x87, null, null, null },
104075 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104076 .dst_constraints = .{ .{ .unsigned_int = .byte }, .any },
104077 .patterns = &.{
104078 .{ .src = .{ .mem, .none, .none } },
104079 .{ .src = .{ .to_x87, .none, .none } },
104080 },
104081 .extra_temps = .{
104082 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104083 .{ .type = .u16, .kind = .mem },
104084 .{ .type = .i16, .kind = .mem },
104085 .unused,
104086 .unused,
104087 .unused,
104088 .unused,
104089 .unused,
104090 .unused,
104091 .unused,
104092 .unused,
104093 },
104094 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
104095 .each = .{ .once = &.{
104096 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
104097 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104098 .{ ._, ._, .movzx, .dst0d, .tmp1w, ._, ._ },
104099 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104100 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104101 .{ ._, ._, .mov, .tmp1w, .dst0w, ._, ._ },
104102 .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ },
104103 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104104 .{ ._, ._, .movzx, .dst0d, .tmp2b, ._, ._ },
104105 } },
104106 }, .{
104107 .required_features = .{ .sse3, .x87, .slow_incdec, null },
103939104108 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
103940104109 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
103941104110 .patterns = &.{
......@@ -103946,7 +104115,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103946104115 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
103947104116 .{ .type = .f80, .kind = .{ .reg = .st7 } },
103948104117 .{ .type = .i16, .kind = .mem },
103949 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
104118 .{ .type = .i8, .kind = .{ .rc = .general_purpose } },
103950104119 .unused,
103951104120 .unused,
103952104121 .unused,
......@@ -103968,7 +104137,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103968104137 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
103969104138 } },
103970104139 }, .{
103971 .required_features = .{ .x87, null, null, null },
104140 .required_features = .{ .sse3, .x87, null, null },
103972104141 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
103973104142 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
103974104143 .patterns = &.{
......@@ -103979,7 +104148,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103979104148 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
103980104149 .{ .type = .f80, .kind = .{ .reg = .st7 } },
103981104150 .{ .type = .i16, .kind = .mem },
103982 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
104151 .{ .type = .i8, .kind = .{ .rc = .general_purpose } },
103983104152 .unused,
103984104153 .unused,
103985104154 .unused,
......@@ -104003,6 +104172,84 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104003104172 }, .{
104004104173 .required_features = .{ .x87, .slow_incdec, null, null },
104005104174 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104175 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
104176 .patterns = &.{
104177 .{ .src = .{ .to_mem, .none, .none } },
104178 },
104179 .extra_temps = .{
104180 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
104181 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104182 .{ .type = .u16, .kind = .mem },
104183 .{ .type = .i8, .kind = .{ .rc = .general_purpose } },
104184 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104185 .{ .type = .i16, .kind = .mem },
104186 .unused,
104187 .unused,
104188 .unused,
104189 .unused,
104190 .unused,
104191 },
104192 .dst_temps = .{ .mem, .unused },
104193 .clobbers = .{ .eflags = true },
104194 .each = .{ .once = &.{
104195 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
104196 .{ ._, ._, .mov, .tmp1p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104197 .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ },
104198 .{ ._, ._, .movsx, .tmp3d, .tmp2w, ._, ._ },
104199 .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ },
104200 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104201 .{ ._, ._, .mov, .tmp2w, .tmp3w, ._, ._ },
104202 .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ },
104203 .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ },
104204 .{ ._, ._, .movsx, .tmp3d, .tmp5b, ._, ._ },
104205 .{ ._, ._, .mov, .memia(.dst0b, .tmp1, .add_unaligned_size), .tmp3b, ._, ._ },
104206 .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 16), ._, ._ },
104207 .{ ._, ._, .add, .tmp1p, .si(1), ._, ._ },
104208 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104209 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104210 } },
104211 }, .{
104212 .required_features = .{ .x87, null, null, null },
104213 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104214 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
104215 .patterns = &.{
104216 .{ .src = .{ .to_mem, .none, .none } },
104217 },
104218 .extra_temps = .{
104219 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
104220 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104221 .{ .type = .u16, .kind = .mem },
104222 .{ .type = .i8, .kind = .{ .rc = .general_purpose } },
104223 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104224 .{ .type = .i16, .kind = .mem },
104225 .unused,
104226 .unused,
104227 .unused,
104228 .unused,
104229 .unused,
104230 },
104231 .dst_temps = .{ .mem, .unused },
104232 .clobbers = .{ .eflags = true },
104233 .each = .{ .once = &.{
104234 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
104235 .{ ._, ._, .mov, .tmp1p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104236 .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ },
104237 .{ ._, ._, .movsx, .tmp3d, .tmp2w, ._, ._ },
104238 .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ },
104239 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104240 .{ ._, ._, .mov, .tmp2w, .tmp3w, ._, ._ },
104241 .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ },
104242 .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ },
104243 .{ ._, ._, .movsx, .tmp3d, .tmp5b, ._, ._ },
104244 .{ ._, ._, .mov, .memia(.dst0b, .tmp1, .add_unaligned_size), .tmp3b, ._, ._ },
104245 .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 16), ._, ._ },
104246 .{ ._, ._c, .in, .tmp1p, ._, ._, ._ },
104247 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
104248 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104249 } },
104250 }, .{
104251 .required_features = .{ .sse3, .x87, .slow_incdec, null },
104252 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104006104253 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any },
104007104254 .patterns = &.{
104008104255 .{ .src = .{ .to_mem, .none, .none } },
......@@ -104034,7 +104281,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104034104281 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104035104282 } },
104036104283 }, .{
104037 .required_features = .{ .x87, null, null, null },
104284 .required_features = .{ .sse3, .x87, null, null },
104038104285 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104039104286 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any },
104040104287 .patterns = &.{
......@@ -104066,8 +104313,86 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104066104313 .{ ._, ._c, .in, .tmp1p, ._, ._, ._ },
104067104314 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
104068104315 } },
104316 }, .{
104317 .required_features = .{ .x87, .slow_incdec, null, null },
104318 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104319 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any },
104320 .patterns = &.{
104321 .{ .src = .{ .to_mem, .none, .none } },
104322 },
104323 .extra_temps = .{
104324 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
104325 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104326 .{ .type = .u16, .kind = .mem },
104327 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
104328 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104329 .{ .type = .i16, .kind = .mem },
104330 .unused,
104331 .unused,
104332 .unused,
104333 .unused,
104334 .unused,
104335 },
104336 .dst_temps = .{ .mem, .unused },
104337 .clobbers = .{ .eflags = true },
104338 .each = .{ .once = &.{
104339 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
104340 .{ ._, ._, .mov, .tmp1p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104341 .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ },
104342 .{ ._, ._, .movsx, .tmp3d, .tmp2w, ._, ._ },
104343 .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ },
104344 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104345 .{ ._, ._, .mov, .tmp2w, .tmp3w, ._, ._ },
104346 .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ },
104347 .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ },
104348 .{ ._, ._, .movzx, .tmp3d, .tmp5b, ._, ._ },
104349 .{ ._, ._, .mov, .memia(.dst0b, .tmp1, .add_unaligned_size), .tmp3b, ._, ._ },
104350 .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 16), ._, ._ },
104351 .{ ._, ._, .add, .tmp1p, .si(1), ._, ._ },
104352 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104353 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104354 } },
104069104355 }, .{
104070104356 .required_features = .{ .x87, null, null, null },
104357 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104358 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any },
104359 .patterns = &.{
104360 .{ .src = .{ .to_mem, .none, .none } },
104361 },
104362 .extra_temps = .{
104363 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
104364 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104365 .{ .type = .u16, .kind = .mem },
104366 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
104367 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104368 .{ .type = .i16, .kind = .mem },
104369 .unused,
104370 .unused,
104371 .unused,
104372 .unused,
104373 .unused,
104374 },
104375 .dst_temps = .{ .mem, .unused },
104376 .clobbers = .{ .eflags = true },
104377 .each = .{ .once = &.{
104378 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
104379 .{ ._, ._, .mov, .tmp1p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104380 .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ },
104381 .{ ._, ._, .movsx, .tmp3d, .tmp2w, ._, ._ },
104382 .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ },
104383 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104384 .{ ._, ._, .mov, .tmp2w, .tmp3w, ._, ._ },
104385 .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ },
104386 .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ },
104387 .{ ._, ._, .movzx, .tmp3d, .tmp5b, ._, ._ },
104388 .{ ._, ._, .mov, .memia(.dst0b, .tmp1, .add_unaligned_size), .tmp3b, ._, ._ },
104389 .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 16), ._, ._ },
104390 .{ ._, ._c, .in, .tmp1p, ._, ._, ._ },
104391 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
104392 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104393 } },
104394 }, .{
104395 .required_features = .{ .sse3, .x87, null, null },
104071104396 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104072104397 .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any },
104073104398 .patterns = &.{
......@@ -104094,6 +104419,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104094104419 } },
104095104420 }, .{
104096104421 .required_features = .{ .x87, null, null, null },
104422 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104423 .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any },
104424 .patterns = &.{
104425 .{ .src = .{ .mem, .none, .none } },
104426 .{ .src = .{ .to_x87, .none, .none } },
104427 },
104428 .extra_temps = .{
104429 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104430 .{ .type = .u16, .kind = .mem },
104431 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
104432 .unused,
104433 .unused,
104434 .unused,
104435 .unused,
104436 .unused,
104437 .unused,
104438 .unused,
104439 .unused,
104440 },
104441 .dst_temps = .{ .mem, .unused },
104442 .each = .{ .once = &.{
104443 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
104444 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104445 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
104446 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104447 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104448 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
104449 .{ ._, .fi_p, .st, .dst0w, ._, ._, ._ },
104450 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104451 } },
104452 }, .{
104453 .required_features = .{ .sse3, .x87, null, null },
104097104454 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104098104455 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
104099104456 .patterns = &.{
......@@ -104124,6 +104481,41 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104124104481 }, .{
104125104482 .required_features = .{ .x87, null, null, null },
104126104483 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104484 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
104485 .patterns = &.{
104486 .{ .src = .{ .to_mem, .none, .none } },
104487 },
104488 .extra_temps = .{
104489 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104490 .{ .type = .u16, .kind = .mem },
104491 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104492 .unused,
104493 .unused,
104494 .unused,
104495 .unused,
104496 .unused,
104497 .unused,
104498 .unused,
104499 .unused,
104500 },
104501 .dst_temps = .{ .mem, .unused },
104502 .clobbers = .{ .eflags = true },
104503 .each = .{ .once = &.{
104504 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104505 .{ ._, ._, .movzx, .tmp0d, .tmp1w, ._, ._ },
104506 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104507 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104508 .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ },
104509 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104510 .{ .@"0:", .f_, .ld, .memsia(.src0t, .@"8", .tmp0, .add_unaligned_size), ._, ._, ._ },
104511 .{ ._, .fi_p, .st, .memia(.dst0w, .tmp0, .add_unaligned_size), ._, ._, ._ },
104512 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
104513 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104514 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104515 } },
104516 }, .{
104517 .required_features = .{ .sse3, .x87, null, null },
104518 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104127104519 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any },
104128104520 .patterns = &.{
104129104521 .{ .src = .{ .to_mem, .none, .none } },
......@@ -104154,6 +104546,43 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104154104546 } },
104155104547 }, .{
104156104548 .required_features = .{ .x87, null, null, null },
104549 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104550 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any },
104551 .patterns = &.{
104552 .{ .src = .{ .to_mem, .none, .none } },
104553 },
104554 .extra_temps = .{
104555 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104556 .{ .type = .u16, .kind = .mem },
104557 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
104558 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104559 .{ .type = .i32, .kind = .mem },
104560 .unused,
104561 .unused,
104562 .unused,
104563 .unused,
104564 .unused,
104565 .unused,
104566 },
104567 .dst_temps = .{ .mem, .unused },
104568 .clobbers = .{ .eflags = true },
104569 .each = .{ .once = &.{
104570 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104571 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104572 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
104573 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104574 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104575 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
104576 .{ .@"0:", .f_, .ld, .memsia(.src0t, .@"8", .tmp0, .add_unaligned_size), ._, ._, ._ },
104577 .{ ._, .fi_p, .st, .tmp4d, ._, ._, ._ },
104578 .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ },
104579 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ },
104580 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
104581 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104582 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104583 } },
104584 }, .{
104585 .required_features = .{ .sse3, .x87, null, null },
104157104586 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104158104587 .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any },
104159104588 .patterns = &.{
......@@ -104180,6 +104609,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104180104609 } },
104181104610 }, .{
104182104611 .required_features = .{ .x87, null, null, null },
104612 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104613 .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any },
104614 .patterns = &.{
104615 .{ .src = .{ .mem, .none, .none } },
104616 .{ .src = .{ .to_x87, .none, .none } },
104617 },
104618 .extra_temps = .{
104619 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104620 .{ .type = .u16, .kind = .mem },
104621 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
104622 .unused,
104623 .unused,
104624 .unused,
104625 .unused,
104626 .unused,
104627 .unused,
104628 .unused,
104629 .unused,
104630 },
104631 .dst_temps = .{ .mem, .unused },
104632 .each = .{ .once = &.{
104633 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
104634 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104635 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
104636 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104637 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104638 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
104639 .{ ._, .fi_p, .st, .dst0d, ._, ._, ._ },
104640 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104641 } },
104642 }, .{
104643 .required_features = .{ .sse3, .x87, null, null },
104183104644 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104184104645 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
104185104646 .patterns = &.{
......@@ -104210,6 +104671,41 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104210104671 }, .{
104211104672 .required_features = .{ .x87, null, null, null },
104212104673 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104674 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
104675 .patterns = &.{
104676 .{ .src = .{ .to_mem, .none, .none } },
104677 },
104678 .extra_temps = .{
104679 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104680 .{ .type = .u16, .kind = .mem },
104681 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104682 .unused,
104683 .unused,
104684 .unused,
104685 .unused,
104686 .unused,
104687 .unused,
104688 .unused,
104689 .unused,
104690 },
104691 .dst_temps = .{ .mem, .unused },
104692 .clobbers = .{ .eflags = true },
104693 .each = .{ .once = &.{
104694 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104695 .{ ._, ._, .movzx, .tmp0d, .tmp1w, ._, ._ },
104696 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104697 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104698 .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ },
104699 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104700 .{ .@"0:", .f_, .ld, .memsia(.src0t, .@"4", .tmp0, .add_unaligned_size), ._, ._, ._ },
104701 .{ ._, .fi_p, .st, .memia(.dst0d, .tmp0, .add_unaligned_size), ._, ._, ._ },
104702 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
104703 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104704 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104705 } },
104706 }, .{
104707 .required_features = .{ .sse3, .x87, null, null },
104708 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104213104709 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
104214104710 .patterns = &.{
104215104711 .{ .src = .{ .to_mem, .none, .none } },
......@@ -104240,6 +104736,43 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104240104736 } },
104241104737 }, .{
104242104738 .required_features = .{ .x87, null, null, null },
104739 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104740 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
104741 .patterns = &.{
104742 .{ .src = .{ .to_mem, .none, .none } },
104743 },
104744 .extra_temps = .{
104745 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104746 .{ .type = .u16, .kind = .mem },
104747 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
104748 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104749 .{ .type = .i64, .kind = .mem },
104750 .unused,
104751 .unused,
104752 .unused,
104753 .unused,
104754 .unused,
104755 .unused,
104756 },
104757 .dst_temps = .{ .mem, .unused },
104758 .clobbers = .{ .eflags = true },
104759 .each = .{ .once = &.{
104760 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104761 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104762 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
104763 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104764 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104765 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
104766 .{ .@"0:", .f_, .ld, .memsia(.src0t, .@"4", .tmp0, .add_unaligned_size), ._, ._, ._ },
104767 .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ },
104768 .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ },
104769 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ },
104770 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
104771 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104772 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104773 } },
104774 }, .{
104775 .required_features = .{ .sse3, .x87, null, null },
104243104776 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104244104777 .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any },
104245104778 .patterns = &.{
......@@ -104265,7 +104798,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104265104798 .{ ._, .fi_p, .stt, .dst0q, ._, ._, ._ },
104266104799 } },
104267104800 }, .{
104268 .required_features = .{ .@"64bit", .x87, null, null },
104801 .required_features = .{ .x87, null, null, null },
104802 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104803 .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any },
104804 .patterns = &.{
104805 .{ .src = .{ .mem, .none, .none } },
104806 .{ .src = .{ .to_x87, .none, .none } },
104807 },
104808 .extra_temps = .{
104809 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104810 .{ .type = .u16, .kind = .mem },
104811 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
104812 .unused,
104813 .unused,
104814 .unused,
104815 .unused,
104816 .unused,
104817 .unused,
104818 .unused,
104819 .unused,
104820 },
104821 .dst_temps = .{ .mem, .unused },
104822 .each = .{ .once = &.{
104823 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
104824 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104825 .{ ._, ._, .movzx, .tmp2d, .tmp1w, ._, ._ },
104826 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104827 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104828 .{ ._, ._, .mov, .tmp1w, .tmp2w, ._, ._ },
104829 .{ ._, .fi_p, .st, .dst0q, ._, ._, ._ },
104830 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104831 } },
104832 }, .{
104833 .required_features = .{ .@"64bit", .sse3, .x87, null },
104269104834 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104270104835 .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any },
104271104836 .patterns = &.{
......@@ -104301,7 +104866,49 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104301104866 .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ },
104302104867 } },
104303104868 }, .{
104304 .required_features = .{ .x87, null, null, null },
104869 .required_features = .{ .@"64bit", .x87, null, null },
104870 .src_constraints = .{ .{ .float = .tbyte }, .any, .any },
104871 .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any },
104872 .patterns = &.{
104873 .{ .src = .{ .mem, .none, .none } },
104874 .{ .src = .{ .to_x87, .none, .none } },
104875 },
104876 .extra_temps = .{
104877 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104878 .{ .type = .f80, .kind = .{ .reg = .st6 } },
104879 .{ .type = .u16, .kind = .mem },
104880 .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } },
104881 .{ .type = .i64, .kind = .mem },
104882 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
104883 .unused,
104884 .unused,
104885 .unused,
104886 .unused,
104887 .unused,
104888 },
104889 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
104890 .clobbers = .{ .eflags = true },
104891 .each = .{ .once = &.{
104892 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
104893 .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ },
104894 .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ },
104895 .{ ._, ._, .movzx, .dst0d, .tmp2w, ._, ._ },
104896 .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ },
104897 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104898 .{ ._, ._, .mov, .tmp2w, .dst0w, ._, ._ },
104899 .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ },
104900 .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ },
104901 .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ },
104902 .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ },
104903 .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ },
104904 .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ },
104905 .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ },
104906 .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ },
104907 .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ },
104908 .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ },
104909 } },
104910 }, .{
104911 .required_features = .{ .sse3, .x87, null, null },
104305104912 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104306104913 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any },
104307104914 .patterns = &.{
......@@ -104329,6 +104936,41 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104329104936 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
104330104937 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104331104938 } },
104939 }, .{
104940 .required_features = .{ .x87, null, null, null },
104941 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
104942 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any },
104943 .patterns = &.{
104944 .{ .src = .{ .to_mem, .none, .none } },
104945 },
104946 .extra_temps = .{
104947 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
104948 .{ .type = .u16, .kind = .mem },
104949 .{ .type = .f80, .kind = .{ .reg = .st7 } },
104950 .unused,
104951 .unused,
104952 .unused,
104953 .unused,
104954 .unused,
104955 .unused,
104956 .unused,
104957 .unused,
104958 },
104959 .dst_temps = .{ .mem, .unused },
104960 .clobbers = .{ .eflags = true },
104961 .each = .{ .once = &.{
104962 .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ },
104963 .{ ._, ._, .movzx, .tmp0d, .tmp1w, ._, ._ },
104964 .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ },
104965 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104966 .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ },
104967 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
104968 .{ .@"0:", .f_, .ld, .memsia(.src0t, .@"2", .tmp0, .add_unaligned_size), ._, ._, ._ },
104969 .{ ._, .fi_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
104970 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
104971 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
104972 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
104973 } },
104332104974 }, .{
104333104975 .required_features = .{ .@"64bit", .avx, null, null },
104334104976 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any },
......@@ -116834,8 +117476,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116834117476 switch (reduce.operation) {
116835117477 .And, .Or, .Xor => {},
116836117478 .Min, .Max => break :fallback try cg.airReduce(inst),
116837 .Add => if (cg.floatBits(res_ty)) |_| break :fallback try cg.airReduce(inst),
116838 .Mul => break :fallback try cg.airReduce(inst),
117479 .Add, .Mul => if (cg.floatBits(res_ty)) |_| break :fallback try cg.airReduce(inst),
116839117480 }
116840117481 var ops = try cg.tempsFromOperands(inst, .{reduce.operand});
116841117482 var res: [1]Temp = undefined;
......@@ -118473,12 +119114,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118473119114 .unused,
118474119115 .unused,
118475119116 },
118476 .dst_temps = .{ .{ .rc = .sse }, .unused },
119117 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
118477119118 .each = .{ .once = &.{
118478119119 .{ ._, .vp_d, .srl, .tmp0x, .src0x, .ui(16), ._ },
118479 .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ },
118480 .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ },
118481 .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ },
119120 .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ },
119121 .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ },
119122 .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ },
118482119123 } },
118483119124 }, .{
118484119125 .required_features = .{ .sse2, null, null, null },
......@@ -119584,12 +120225,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
119584120225 .unused,
119585120226 .unused,
119586120227 },
119587 .dst_temps = .{ .{ .rc = .sse }, .unused },
120228 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
119588120229 .each = .{ .once = &.{
119589120230 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
119590 .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ },
119591 .{ ._, .vp_d, .srl, .tmp0x, .src0x, .ui(16), ._ },
119592 .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ },
120231 .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ },
120232 .{ ._, .vp_d, .srl, .dst0x, .src0x, .ui(16), ._ },
120233 .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ },
119593120234 } },
119594120235 }, .{
119595120236 .required_features = .{ .sse2, null, null, null },
......@@ -120479,19 +121120,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
120479121120 .unused,
120480121121 .unused,
120481121122 },
120482 .dst_temps = .{ .{ .rc = .sse }, .unused },
121123 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
120483121124 .each = .{ .once = &.{
120484121125 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
120485 .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ },
120486 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
120487 .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ },
121126 .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ },
121127 .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b01_01_01_01), ._ },
121128 .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ },
120488121129 } },
120489121130 }, .{
120490121131 .required_features = .{ .sse2, null, null, null },
120491121132 .dst_constraints = .{ .{ .int = .dword }, .any },
120492121133 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
120493121134 .patterns = &.{
120494 .{ .src = .{ .to_mut_sse, .none, .none } },
121135 .{ .src = .{ .to_sse, .none, .none } },
120495121136 },
120496121137 .extra_temps = .{
120497121138 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
......@@ -120506,7 +121147,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
120506121147 .unused,
120507121148 .unused,
120508121149 },
120509 .dst_temps = .{ .{ .ref = .src0 }, .unused },
121150 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
120510121151 .each = .{ .once = &.{
120511121152 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
120512121153 .{ ._, .p_, mir_tag, .tmp0x, .src0x, ._, ._ },
......@@ -121203,12 +121844,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
121203121844 .unused,
121204121845 .unused,
121205121846 },
121206 .dst_temps = .{ .{ .rc = .sse }, .unused },
121847 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
121207121848 .each = .{ .once = &.{
121208121849 .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ },
121209 .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ },
121210 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
121211 .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ },
121850 .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ },
121851 .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10_11_10), ._ },
121852 .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ },
121212121853 } },
121213121854 }, .{
121214121855 .required_features = .{ .avx, null, null, null },
......@@ -121230,12 +121871,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
121230121871 .unused,
121231121872 .unused,
121232121873 },
121233 .dst_temps = .{ .{ .rc = .sse }, .unused },
121874 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
121234121875 .each = .{ .once = &.{
121235121876 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
121236 .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ },
121237 .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ },
121238 .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ },
121877 .{ ._, .v_ps, mir_tag, .tmp0x, .src0x, .tmp0x, ._ },
121878 .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ },
121879 .{ ._, .v_ps, mir_tag, .dst0x, .tmp0x, .dst0x, ._ },
121239121880 } },
121240121881 }, .{
121241121882 .required_features = .{ .avx512f, null, null, null },
......@@ -122425,12 +123066,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
122425123066 .unused,
122426123067 .unused,
122427123068 },
122428 .dst_temps = .{ .{ .rc = .sse }, .unused },
123069 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
122429123070 .each = .{ .once = &.{
122430123071 .{ ._, .vp_d, .srl, .tmp0x, .src0x, .ui(16), ._ },
122431 .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ },
122432 .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ },
122433 .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ },
123072 .{ ._, .vp_b, .add, .tmp0x, .src0x, .tmp0x, ._ },
123073 .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ },
123074 .{ ._, .vp_b, .add, .dst0x, .tmp0x, .dst0x, ._ },
122434123075 } },
122435123076 }, .{
122436123077 .required_features = .{ .sse2, null, null, null },
......@@ -123125,7 +123766,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
123125123766 }, .{
123126123767 .required_features = .{ .avx, null, null, null },
123127123768 .dst_constraints = .{ .{ .int = .byte }, .any },
123128 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
123769 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
123129123770 .patterns = &.{
123130123771 .{ .src = .{ .to_mem, .none, .none } },
123131123772 },
......@@ -123171,7 +123812,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
123171123812 },
123172123813 .extra_temps = .{
123173123814 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
123174 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
123815 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } },
123175123816 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
123176123817 .unused,
123177123818 .unused,
......@@ -123405,12 +124046,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
123405124046 .unused,
123406124047 .unused,
123407124048 },
123408 .dst_temps = .{ .{ .rc = .sse }, .unused },
124049 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
123409124050 .each = .{ .once = &.{
123410124051 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
123411 .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ },
123412 .{ ._, .vp_d, .srl, .tmp0x, .src0x, .ui(16), ._ },
123413 .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ },
124052 .{ ._, .vp_w, .add, .tmp0x, .src0x, .tmp0x, ._ },
124053 .{ ._, .vp_d, .srl, .dst0x, .src0x, .ui(16), ._ },
124054 .{ ._, .vp_w, .add, .dst0x, .tmp0x, .dst0x, ._ },
123414124055 } },
123415124056 }, .{
123416124057 .required_features = .{ .sse2, null, null, null },
......@@ -123947,7 +124588,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
123947124588 }, .{
123948124589 .required_features = .{ .avx, null, null, null },
123949124590 .dst_constraints = .{ .{ .int = .word }, .any },
123950 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
124591 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
123951124592 .patterns = &.{
123952124593 .{ .src = .{ .to_mem, .none, .none } },
123953124594 },
......@@ -124191,19 +124832,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
124191124832 .unused,
124192124833 .unused,
124193124834 },
124194 .dst_temps = .{ .{ .rc = .sse }, .unused },
124835 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
124195124836 .each = .{ .once = &.{
124196124837 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
124197 .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ },
124198 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
124199 .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ },
124838 .{ ._, .vp_d, .add, .tmp0x, .src0x, .tmp0x, ._ },
124839 .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b01_01_01_01), ._ },
124840 .{ ._, .vp_d, .add, .dst0x, .tmp0x, .dst0x, ._ },
124200124841 } },
124201124842 }, .{
124202124843 .required_features = .{ .sse2, null, null, null },
124203124844 .dst_constraints = .{ .{ .int = .dword }, .any },
124204124845 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
124205124846 .patterns = &.{
124206 .{ .src = .{ .to_mut_sse, .none, .none } },
124847 .{ .src = .{ .to_sse, .none, .none } },
124207124848 },
124208124849 .extra_temps = .{
124209124850 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
......@@ -124218,7 +124859,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
124218124859 .unused,
124219124860 .unused,
124220124861 },
124221 .dst_temps = .{ .{ .ref = .src0 }, .unused },
124862 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
124222124863 .each = .{ .once = &.{
124223124864 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
124224124865 .{ ._, .p_d, .add, .tmp0x, .src0x, ._, ._ },
......@@ -124588,7 +125229,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
124588125229 }, .{
124589125230 .required_features = .{ .avx, null, null, null },
124590125231 .dst_constraints = .{ .{ .int = .dword }, .any },
124591 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
125232 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
124592125233 .patterns = &.{
124593125234 .{ .src = .{ .to_mem, .none, .none } },
124594125235 },
......@@ -124828,12 +125469,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
124828125469 .unused,
124829125470 .unused,
124830125471 },
124831 .dst_temps = .{ .{ .rc = .sse }, .unused },
125472 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
124832125473 .each = .{ .once = &.{
124833125474 .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ },
124834 .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ },
124835 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
124836 .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ },
125475 .{ ._, .vp_q, .add, .tmp0x, .src0x, .tmp0x, ._ },
125476 .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10_11_10), ._ },
125477 .{ ._, .vp_q, .add, .dst0x, .tmp0x, .dst0x, ._ },
124837125478 } },
124838125479 }, .{
124839125480 .required_features = .{ .avx, null, null, null },
......@@ -124855,12 +125496,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
124855125496 .unused,
124856125497 .unused,
124857125498 },
124858 .dst_temps = .{ .{ .rc = .sse }, .unused },
125499 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
124859125500 .each = .{ .once = &.{
124860125501 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
124861 .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ },
124862 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
124863 .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ },
125502 .{ ._, .vp_q, .add, .tmp0x, .src0x, .tmp0x, ._ },
125503 .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10_11_10), ._ },
125504 .{ ._, .vp_q, .add, .dst0x, .tmp0x, .dst0x, ._ },
124864125505 } },
124865125506 }, .{
124866125507 .required_features = .{ .avx512f, null, null, null },
......@@ -125091,7 +125732,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
125091125732 }, .{
125092125733 .required_features = .{ .avx, null, null, null },
125093125734 .dst_constraints = .{ .{ .int = .qword }, .any },
125094 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any },
125735 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any },
125095125736 .patterns = &.{
125096125737 .{ .src = .{ .to_mem, .none, .none } },
125097125738 },
......@@ -125268,7 +125909,3013 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
125268125909 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
125269125910 } },
125270125911 } },
125271 .Mul => unreachable,
125912 .Mul => comptime &.{ .{
125913 .required_features = .{ .sse, null, null, null },
125914 .dst_constraints = .{ .{ .int = .byte }, .any },
125915 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
125916 .patterns = &.{
125917 .{ .src = .{ .mut_mem, .none, .none } },
125918 .{ .src = .{ .mut_gpr, .none, .none } },
125919 .{ .src = .{ .to_mut_sse, .none, .none } },
125920 },
125921 .dst_temps = .{ .{ .ref = .src0 }, .unused },
125922 .each = .{ .once = &.{} },
125923 }, .{
125924 .dst_constraints = .{ .{ .int = .byte }, .any },
125925 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
125926 .patterns = &.{
125927 .{ .src = .{ .mut_mem, .none, .none } },
125928 .{ .src = .{ .to_mut_gpr, .none, .none } },
125929 },
125930 .dst_temps = .{ .{ .ref = .src0 }, .unused },
125931 .each = .{ .once = &.{} },
125932 }, .{
125933 .required_features = .{ .avx, null, null, null },
125934 .dst_constraints = .{ .{ .int = .byte }, .any },
125935 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any },
125936 .patterns = &.{
125937 .{ .src = .{ .to_sse, .none, .none } },
125938 },
125939 .extra_temps = .{
125940 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
125941 .unused,
125942 .unused,
125943 .unused,
125944 .unused,
125945 .unused,
125946 .unused,
125947 .unused,
125948 .unused,
125949 .unused,
125950 .unused,
125951 },
125952 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
125953 .each = .{ .once = &.{
125954 .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ },
125955 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
125956 } },
125957 }, .{
125958 .required_features = .{ .sse2, null, null, null },
125959 .dst_constraints = .{ .{ .int = .byte }, .any },
125960 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any },
125961 .patterns = &.{
125962 .{ .src = .{ .to_mut_sse, .none, .none } },
125963 },
125964 .extra_temps = .{
125965 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
125966 .unused,
125967 .unused,
125968 .unused,
125969 .unused,
125970 .unused,
125971 .unused,
125972 .unused,
125973 .unused,
125974 .unused,
125975 .unused,
125976 },
125977 .dst_temps = .{ .{ .ref = .src0 }, .unused },
125978 .each = .{ .once = &.{
125979 .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ },
125980 .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ },
125981 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
125982 } },
125983 }, .{
125984 .required_features = .{ .avx, null, null, null },
125985 .dst_constraints = .{ .{ .int = .byte }, .any },
125986 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any },
125987 .patterns = &.{
125988 .{ .src = .{ .to_sse, .none, .none } },
125989 },
125990 .extra_temps = .{
125991 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
125992 .unused,
125993 .unused,
125994 .unused,
125995 .unused,
125996 .unused,
125997 .unused,
125998 .unused,
125999 .unused,
126000 .unused,
126001 .unused,
126002 },
126003 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126004 .each = .{ .once = &.{
126005 .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ },
126006 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
126007 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
126008 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126009 } },
126010 }, .{
126011 .required_features = .{ .sse2, null, null, null },
126012 .dst_constraints = .{ .{ .int = .byte }, .any },
126013 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any },
126014 .patterns = &.{
126015 .{ .src = .{ .to_mut_sse, .none, .none } },
126016 },
126017 .extra_temps = .{
126018 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126019 .unused,
126020 .unused,
126021 .unused,
126022 .unused,
126023 .unused,
126024 .unused,
126025 .unused,
126026 .unused,
126027 .unused,
126028 .unused,
126029 },
126030 .dst_temps = .{ .{ .ref = .src0 }, .unused },
126031 .each = .{ .once = &.{
126032 .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ },
126033 .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ },
126034 .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ },
126035 .{ ._, ._dqa, .mov, .dst0x, .tmp0x, ._, ._ },
126036 .{ ._, .p_d, .srl, .dst0x, .ui(16), ._, ._ },
126037 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
126038 } },
126039 }, .{
126040 .required_features = .{ .avx, null, null, null },
126041 .dst_constraints = .{ .{ .int = .byte }, .any },
126042 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any },
126043 .patterns = &.{
126044 .{ .src = .{ .to_sse, .none, .none } },
126045 },
126046 .extra_temps = .{
126047 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126048 .unused,
126049 .unused,
126050 .unused,
126051 .unused,
126052 .unused,
126053 .unused,
126054 .unused,
126055 .unused,
126056 .unused,
126057 .unused,
126058 },
126059 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126060 .each = .{ .once = &.{
126061 .{ ._, .vp_d, .srl, .tmp0x, .src0x, .ui(16), ._ },
126062 .{ ._, .vp_w, .mull, .tmp0x, .src0x, .tmp0x, ._ },
126063 .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ },
126064 .{ ._, .vp_w, .mull, .dst0x, .tmp0x, .dst0x, ._ },
126065 } },
126066 }, .{
126067 .required_features = .{ .sse2, null, null, null },
126068 .dst_constraints = .{ .{ .int = .byte }, .any },
126069 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any },
126070 .patterns = &.{
126071 .{ .src = .{ .to_mut_sse, .none, .none } },
126072 },
126073 .extra_temps = .{
126074 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126075 .unused,
126076 .unused,
126077 .unused,
126078 .unused,
126079 .unused,
126080 .unused,
126081 .unused,
126082 .unused,
126083 .unused,
126084 .unused,
126085 },
126086 .dst_temps = .{ .{ .ref = .src0 }, .unused },
126087 .each = .{ .once = &.{
126088 .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ },
126089 .{ ._, .p_d, .srl, .tmp0x, .ui(16), ._, ._ },
126090 .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ },
126091 .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ },
126092 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
126093 } },
126094 }, .{
126095 .required_features = .{ .avx, null, null, null },
126096 .dst_constraints = .{ .{ .int = .byte }, .any },
126097 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any },
126098 .patterns = &.{
126099 .{ .src = .{ .to_sse, .none, .none } },
126100 },
126101 .extra_temps = .{
126102 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126103 .unused,
126104 .unused,
126105 .unused,
126106 .unused,
126107 .unused,
126108 .unused,
126109 .unused,
126110 .unused,
126111 .unused,
126112 .unused,
126113 },
126114 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126115 .each = .{ .once = &.{
126116 .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ },
126117 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
126118 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
126119 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126120 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
126121 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126122 } },
126123 }, .{
126124 .required_features = .{ .sse2, null, null, null },
126125 .dst_constraints = .{ .{ .int = .byte }, .any },
126126 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any },
126127 .patterns = &.{
126128 .{ .src = .{ .to_sse, .none, .none } },
126129 },
126130 .extra_temps = .{
126131 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126132 .unused,
126133 .unused,
126134 .unused,
126135 .unused,
126136 .unused,
126137 .unused,
126138 .unused,
126139 .unused,
126140 .unused,
126141 .unused,
126142 },
126143 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126144 .each = .{ .once = &.{
126145 .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ },
126146 .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ },
126147 .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ },
126148 .{ ._, .p_d, .shuf, .dst0x, .tmp0x, .ui(0b01_01_01_01), ._ },
126149 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
126150 .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ },
126151 .{ ._, .p_d, .srl, .tmp0x, .ui(16), ._, ._ },
126152 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
126153 } },
126154 }, .{
126155 .required_features = .{ .avx, null, null, null },
126156 .dst_constraints = .{ .{ .int = .byte }, .any },
126157 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any },
126158 .patterns = &.{
126159 .{ .src = .{ .mem, .none, .none } },
126160 .{ .src = .{ .to_sse, .none, .none } },
126161 },
126162 .extra_temps = .{
126163 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
126164 .{ .type = .vector_8_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126165 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126166 .unused,
126167 .unused,
126168 .unused,
126169 .unused,
126170 .unused,
126171 .unused,
126172 .unused,
126173 .unused,
126174 },
126175 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126176 .each = .{ .once = &.{
126177 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126178 .{ ._, .v_q, .mov, .tmp2x, .lea(.tmp0q), ._, ._ },
126179 .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ },
126180 .{ ._, .vp_b, .sub, .dst0x, .dst0x, .tmp2x, ._ },
126181 .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ },
126182 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126183 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126184 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126185 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126186 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126187 } },
126188 }, .{
126189 .required_features = .{ .sse2, null, null, null },
126190 .dst_constraints = .{ .{ .int = .byte }, .any },
126191 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any },
126192 .patterns = &.{
126193 .{ .src = .{ .mem, .none, .none } },
126194 .{ .src = .{ .to_sse, .none, .none } },
126195 },
126196 .extra_temps = .{
126197 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
126198 .{ .type = .vector_8_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126199 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126200 .unused,
126201 .unused,
126202 .unused,
126203 .unused,
126204 .unused,
126205 .unused,
126206 .unused,
126207 .unused,
126208 },
126209 .dst_temps = .{ .{ .rc = .sse }, .unused },
126210 .each = .{ .once = &.{
126211 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126212 .{ ._, ._q, .mov, .tmp2x, .lea(.tmp0q), ._, ._ },
126213 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
126214 .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ },
126215 .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ },
126216 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126217 .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ },
126218 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126219 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126220 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126221 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126222 .{ ._, .p_d, .srl, .tmp2x, .ui(16), ._, ._ },
126223 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126224 } },
126225 }, .{
126226 .required_features = .{ .avx, null, null, null },
126227 .dst_constraints = .{ .{ .int = .byte }, .any },
126228 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126229 .patterns = &.{
126230 .{ .src = .{ .to_sse, .none, .none } },
126231 },
126232 .extra_temps = .{
126233 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126234 .unused,
126235 .unused,
126236 .unused,
126237 .unused,
126238 .unused,
126239 .unused,
126240 .unused,
126241 .unused,
126242 .unused,
126243 .unused,
126244 },
126245 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126246 .each = .{ .once = &.{
126247 .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ },
126248 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
126249 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
126250 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126251 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
126252 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126253 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
126254 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126255 } },
126256 }, .{
126257 .required_features = .{ .sse2, null, null, null },
126258 .dst_constraints = .{ .{ .int = .byte }, .any },
126259 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126260 .patterns = &.{
126261 .{ .src = .{ .to_sse, .none, .none } },
126262 },
126263 .extra_temps = .{
126264 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126265 .unused,
126266 .unused,
126267 .unused,
126268 .unused,
126269 .unused,
126270 .unused,
126271 .unused,
126272 .unused,
126273 .unused,
126274 .unused,
126275 },
126276 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126277 .each = .{ .once = &.{
126278 .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ },
126279 .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ },
126280 .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ },
126281 .{ ._, .p_d, .shuf, .dst0x, .tmp0x, .ui(0b11_10_11_10), ._ },
126282 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
126283 .{ ._, .p_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
126284 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
126285 .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ },
126286 .{ ._, .p_d, .srl, .tmp0x, .ui(16), ._, ._ },
126287 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
126288 } },
126289 }, .{
126290 .required_features = .{ .avx, null, null, null },
126291 .dst_constraints = .{ .{ .int = .byte }, .any },
126292 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126293 .patterns = &.{
126294 .{ .src = .{ .to_sse, .none, .none } },
126295 },
126296 .extra_temps = .{
126297 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
126298 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126299 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126300 .unused,
126301 .unused,
126302 .unused,
126303 .unused,
126304 .unused,
126305 .unused,
126306 .unused,
126307 .unused,
126308 },
126309 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126310 .each = .{ .once = &.{
126311 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126312 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
126313 .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ },
126314 .{ ._, .vp_b, .sub, .dst0x, .dst0x, .tmp2x, ._ },
126315 .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ },
126316 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126317 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126318 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126319 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126320 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126321 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126322 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126323 } },
126324 }, .{
126325 .required_features = .{ .sse2, null, null, null },
126326 .dst_constraints = .{ .{ .int = .byte }, .any },
126327 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126328 .patterns = &.{
126329 .{ .src = .{ .mem, .none, .none } },
126330 .{ .src = .{ .to_sse, .none, .none } },
126331 },
126332 .extra_temps = .{
126333 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
126334 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126335 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126336 .unused,
126337 .unused,
126338 .unused,
126339 .unused,
126340 .unused,
126341 .unused,
126342 .unused,
126343 .unused,
126344 },
126345 .dst_temps = .{ .{ .rc = .sse }, .unused },
126346 .each = .{ .once = &.{
126347 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126348 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
126349 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
126350 .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ },
126351 .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ },
126352 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126353 .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ },
126354 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126355 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126356 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126357 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126358 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126359 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126360 .{ ._, .p_d, .srl, .tmp2x, .ui(16), ._, ._ },
126361 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126362 } },
126363 }, .{
126364 .required_features = .{ .avx2, null, null, null },
126365 .dst_constraints = .{ .{ .int = .byte }, .any },
126366 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
126367 .patterns = &.{
126368 .{ .src = .{ .to_sse, .none, .none } },
126369 },
126370 .extra_temps = .{
126371 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126372 .unused,
126373 .unused,
126374 .unused,
126375 .unused,
126376 .unused,
126377 .unused,
126378 .unused,
126379 .unused,
126380 .unused,
126381 .unused,
126382 },
126383 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126384 .each = .{ .once = &.{
126385 .{ ._, .vp_w, .srl, .tmp0y, .src0y, .ui(8), ._ },
126386 .{ ._, .vp_w, .mull, .dst0y, .src0y, .tmp0y, ._ },
126387 .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
126388 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126389 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
126390 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126391 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
126392 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126393 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
126394 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126395 } },
126396 }, .{
126397 .required_features = .{ .avx, null, null, null },
126398 .dst_constraints = .{ .{ .int = .byte }, .any },
126399 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
126400 .patterns = &.{
126401 .{ .src = .{ .to_sse, .none, .none } },
126402 },
126403 .extra_temps = .{
126404 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126405 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126406 .unused,
126407 .unused,
126408 .unused,
126409 .unused,
126410 .unused,
126411 .unused,
126412 .unused,
126413 .unused,
126414 .unused,
126415 },
126416 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126417 .each = .{ .once = &.{
126418 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
126419 .{ ._, .vp_w, .srl, .tmp1x, .src0x, .ui(8), ._ },
126420 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp1x, ._ },
126421 .{ ._, .vp_w, .srl, .tmp1x, .tmp0x, .ui(8), ._ },
126422 .{ ._, .vp_w, .mull, .tmp0x, .tmp0x, .tmp1x, ._ },
126423 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126424 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
126425 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126426 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
126427 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126428 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
126429 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126430 } },
126431 }, .{
126432 .required_features = .{ .avx2, null, null, null },
126433 .dst_constraints = .{ .{ .int = .byte }, .any },
126434 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
126435 .patterns = &.{
126436 .{ .src = .{ .mem, .none, .none } },
126437 .{ .src = .{ .to_sse, .none, .none } },
126438 },
126439 .extra_temps = .{
126440 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
126441 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126442 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126443 .unused,
126444 .unused,
126445 .unused,
126446 .unused,
126447 .unused,
126448 .unused,
126449 .unused,
126450 .unused,
126451 },
126452 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126453 .each = .{ .once = &.{
126454 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126455 .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ },
126456 .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ },
126457 .{ ._, .vp_b, .sub, .dst0y, .dst0y, .tmp2y, ._ },
126458 .{ ._, .vp_w, .srl, .tmp2y, .dst0y, .ui(8), ._ },
126459 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
126460 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
126461 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126462 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126463 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126464 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126465 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126466 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126467 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126468 } },
126469 }, .{
126470 .required_features = .{ .avx, null, null, null },
126471 .dst_constraints = .{ .{ .int = .byte }, .any },
126472 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
126473 .patterns = &.{
126474 .{ .src = .{ .mem, .none, .none } },
126475 .{ .src = .{ .to_sse, .none, .none } },
126476 },
126477 .extra_temps = .{
126478 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
126479 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126480 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126481 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126482 .unused,
126483 .unused,
126484 .unused,
126485 .unused,
126486 .unused,
126487 .unused,
126488 .unused,
126489 },
126490 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126491 .each = .{ .once = &.{
126492 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126493 .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ },
126494 .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ },
126495 .{ ._, .vp_b, .sub, .dst0x, .tmp3x, .tmp2x, ._ },
126496 .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ },
126497 .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ },
126498 .{ ._, .vp_b, .sub, .tmp2x, .tmp3x, .tmp2x, ._ },
126499 .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ },
126500 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp3x, ._ },
126501 .{ ._, .vp_w, .srl, .tmp3x, .tmp2x, .ui(8), ._ },
126502 .{ ._, .vp_w, .mull, .tmp2x, .tmp2x, .tmp3x, ._ },
126503 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126504 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126505 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126506 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126507 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126508 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126509 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126510 } },
126511 }, .{
126512 .required_features = .{ .avx512f, null, null, null },
126513 .dst_constraints = .{ .{ .int = .byte }, .any },
126514 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any },
126515 .patterns = &.{
126516 .{ .src = .{ .to_mem, .none, .none } },
126517 },
126518 .extra_temps = .{
126519 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126520 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126521 .unused,
126522 .unused,
126523 .unused,
126524 .unused,
126525 .unused,
126526 .unused,
126527 .unused,
126528 .unused,
126529 .unused,
126530 },
126531 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126532 .each = .{ .once = &.{
126533 .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ },
126534 .{ ._, .v_dqa, .mov, .tmp0y, .memd(.src0y, 32), ._, ._ },
126535 .{ ._, .vp_w, .srl, .tmp1y, .dst0y, .ui(8), ._ },
126536 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ },
126537 .{ ._, .vp_w, .srl, .tmp1y, .tmp0y, .ui(8), ._ },
126538 .{ ._, .vp_w, .mull, .tmp0y, .tmp0y, .tmp1y, ._ },
126539 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },
126540 .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
126541 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126542 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
126543 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126544 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
126545 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126546 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
126547 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
126548 } },
126549 }, .{
126550 .required_features = .{ .avx512f, null, null, null },
126551 .dst_constraints = .{ .{ .int = .byte }, .any },
126552 .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any },
126553 .patterns = &.{
126554 .{ .src = .{ .to_mem, .none, .none } },
126555 },
126556 .extra_temps = .{
126557 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
126558 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126559 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126560 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126561 .unused,
126562 .unused,
126563 .unused,
126564 .unused,
126565 .unused,
126566 .unused,
126567 .unused,
126568 },
126569 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
126570 .each = .{ .once = &.{
126571 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126572 .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ },
126573 .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
126574 .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ },
126575 .{ ._, .vp_b, .sub, .tmp2y, .tmp3y, .tmp2y, ._ },
126576 .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ },
126577 .{ ._, .vp_b, .sub, .dst0y, .tmp3y, .dst0y, ._ },
126578 .{ ._, .vp_w, .srl, .tmp3y, .tmp2y, .ui(8), ._ },
126579 .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ },
126580 .{ ._, .vp_w, .srl, .tmp3y, .dst0y, .ui(8), ._ },
126581 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ },
126582 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
126583 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
126584 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126585 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126586 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126587 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126588 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126589 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126590 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126591 } },
126592 }, .{
126593 .required_features = .{ .avx2, null, null, null },
126594 .dst_constraints = .{ .{ .int = .byte }, .any },
126595 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
126596 .patterns = &.{
126597 .{ .src = .{ .to_mem, .none, .none } },
126598 },
126599 .extra_temps = .{
126600 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126601 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126602 .unused,
126603 .unused,
126604 .unused,
126605 .unused,
126606 .unused,
126607 .unused,
126608 .unused,
126609 .unused,
126610 .unused,
126611 },
126612 .dst_temps = .{ .{ .rc = .sse }, .unused },
126613 .each = .{ .once = &.{
126614 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_unaligned_size), ._, ._ },
126615 .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
126616 .{ ._, .vp_w, .srl, .tmp1y, .dst0y, .ui(8), ._ },
126617 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ },
126618 .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memi(.src0x, .tmp0), ._, ._ },
126619 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ },
126620 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
126621 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126622 .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
126623 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126624 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
126625 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126626 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b01_01_01_01), ._ },
126627 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126628 .{ ._, .vp_d, .srl, .tmp1x, .dst0x, .ui(16), ._ },
126629 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126630 } },
126631 }, .{
126632 .required_features = .{ .avx, null, null, null },
126633 .dst_constraints = .{ .{ .int = .byte }, .any },
126634 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126635 .patterns = &.{
126636 .{ .src = .{ .to_mem, .none, .none } },
126637 },
126638 .extra_temps = .{
126639 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126640 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126641 .unused,
126642 .unused,
126643 .unused,
126644 .unused,
126645 .unused,
126646 .unused,
126647 .unused,
126648 .unused,
126649 .unused,
126650 },
126651 .dst_temps = .{ .{ .rc = .sse }, .unused },
126652 .each = .{ .once = &.{
126653 .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ },
126654 .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
126655 .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ },
126656 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126657 .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
126658 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126659 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
126660 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126661 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
126662 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126663 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b01_01_01_01), ._ },
126664 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126665 .{ ._, .vp_d, .srl, .tmp1x, .dst0x, .ui(16), ._ },
126666 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
126667 } },
126668 }, .{
126669 .required_features = .{ .sse4_1, null, null, null },
126670 .dst_constraints = .{ .{ .int = .byte }, .any },
126671 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126672 .patterns = &.{
126673 .{ .src = .{ .to_mem, .none, .none } },
126674 },
126675 .extra_temps = .{
126676 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126677 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126678 .unused,
126679 .unused,
126680 .unused,
126681 .unused,
126682 .unused,
126683 .unused,
126684 .unused,
126685 .unused,
126686 .unused,
126687 },
126688 .dst_temps = .{ .{ .rc = .sse }, .unused },
126689 .each = .{ .once = &.{
126690 .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ },
126691 .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
126692 .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ },
126693 .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ },
126694 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126695 .{ .@"0:", .p_w, .movzxb, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
126696 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126697 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
126698 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126699 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
126700 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126701 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b01_01_01_01), ._ },
126702 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126703 .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ },
126704 .{ ._, .p_d, .srl, .tmp1x, .ui(16), ._, ._ },
126705 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126706 } },
126707 }, .{
126708 .required_features = .{ .sse2, null, null, null },
126709 .dst_constraints = .{ .{ .int = .byte }, .any },
126710 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126711 .patterns = &.{
126712 .{ .src = .{ .to_mem, .none, .none } },
126713 },
126714 .extra_temps = .{
126715 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126716 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126717 .unused,
126718 .unused,
126719 .unused,
126720 .unused,
126721 .unused,
126722 .unused,
126723 .unused,
126724 .unused,
126725 .unused,
126726 },
126727 .dst_temps = .{ .{ .rc = .sse }, .unused },
126728 .each = .{ .once = &.{
126729 .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ },
126730 .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
126731 .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ },
126732 .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ },
126733 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126734 .{ .@"0:", ._q, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
126735 .{ ._, .p_, .unpcklbw, .tmp1x, .tmp1x, ._, ._ },
126736 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126737 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
126738 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126739 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
126740 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126741 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b01_01_01_01), ._ },
126742 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126743 .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ },
126744 .{ ._, .p_d, .srl, .tmp1x, .ui(16), ._, ._ },
126745 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
126746 } },
126747 }, .{
126748 .required_features = .{ .avx512f, null, null, null },
126749 .dst_constraints = .{ .{ .int = .byte }, .any },
126750 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any },
126751 .patterns = &.{
126752 .{ .src = .{ .to_mem, .none, .none } },
126753 },
126754 .extra_temps = .{
126755 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126756 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126757 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126758 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126759 .unused,
126760 .unused,
126761 .unused,
126762 .unused,
126763 .unused,
126764 .unused,
126765 .unused,
126766 },
126767 .dst_temps = .{ .{ .rc = .sse }, .unused },
126768 .each = .{ .once = &.{
126769 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126770 .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ },
126771 .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
126772 .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ },
126773 .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
126774 .{ ._, .vp_b, .sub, .tmp2y, .tmp3y, .tmp2y, ._ },
126775 .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ },
126776 .{ ._, .vp_b, .sub, .dst0y, .tmp3y, .dst0y, ._ },
126777 .{ ._, .vp_w, .srl, .tmp3y, .tmp2y, .ui(8), ._ },
126778 .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ },
126779 .{ ._, .vp_w, .srl, .tmp3y, .dst0y, .ui(8), ._ },
126780 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ },
126781 .{ .@"0:", .vp_w, .movzxb, .tmp3y, .memid(.src0x, .tmp0, 16), ._, ._ },
126782 .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ },
126783 .{ ._, .vp_w, .movzxb, .tmp3y, .memi(.src0x, .tmp0), ._, ._ },
126784 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ },
126785 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
126786 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126787 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
126788 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
126789 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126790 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126791 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126792 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126793 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126794 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126795 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126796 } },
126797 }, .{
126798 .required_features = .{ .avx2, null, null, null },
126799 .dst_constraints = .{ .{ .int = .byte }, .any },
126800 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
126801 .patterns = &.{
126802 .{ .src = .{ .to_mem, .none, .none } },
126803 },
126804 .extra_temps = .{
126805 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126806 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126807 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126808 .unused,
126809 .unused,
126810 .unused,
126811 .unused,
126812 .unused,
126813 .unused,
126814 .unused,
126815 .unused,
126816 },
126817 .dst_temps = .{ .{ .rc = .sse }, .unused },
126818 .each = .{ .once = &.{
126819 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126820 .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ },
126821 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
126822 .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
126823 .{ ._, .vp_b, .sub, .dst0y, .dst0y, .tmp2y, ._ },
126824 .{ ._, .vp_w, .srl, .tmp2y, .dst0y, .ui(8), ._ },
126825 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
126826 .{ .@"0:", .vp_w, .movzxb, .tmp2y, .memi(.src0x, .tmp0), ._, ._ },
126827 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
126828 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
126829 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126830 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
126831 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126832 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126833 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126834 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126835 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126836 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126837 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126838 } },
126839 }, .{
126840 .required_features = .{ .avx, null, null, null },
126841 .dst_constraints = .{ .{ .int = .byte }, .any },
126842 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any },
126843 .patterns = &.{
126844 .{ .src = .{ .to_mem, .none, .none } },
126845 },
126846 .extra_temps = .{
126847 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126848 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126849 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126850 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
126851 .unused,
126852 .unused,
126853 .unused,
126854 .unused,
126855 .unused,
126856 .unused,
126857 .unused,
126858 },
126859 .dst_temps = .{ .{ .rc = .sse }, .unused },
126860 .each = .{ .once = &.{
126861 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126862 .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ },
126863 .{ ._, ._, .mov, .tmp0d, .sia(-40, .src0, .add_size), ._, ._ },
126864 .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
126865 .{ ._, .vp_b, .sub, .dst0x, .tmp3x, .tmp2x, ._ },
126866 .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ },
126867 .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ },
126868 .{ ._, .vp_b, .sub, .tmp2x, .tmp3x, .tmp2x, ._ },
126869 .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ },
126870 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp3x, ._ },
126871 .{ ._, .vp_w, .srl, .tmp3x, .tmp2x, .ui(8), ._ },
126872 .{ ._, .vp_w, .mull, .tmp2x, .tmp2x, .tmp3x, ._ },
126873 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126874 .{ .@"0:", .vp_w, .movzxb, .tmp2x, .memi(.src0q, .tmp0), ._, ._ },
126875 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126876 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
126877 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126878 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126879 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126880 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126881 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126882 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
126883 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
126884 } },
126885 }, .{
126886 .required_features = .{ .sse4_1, null, null, null },
126887 .dst_constraints = .{ .{ .int = .byte }, .any },
126888 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126889 .patterns = &.{
126890 .{ .src = .{ .to_mem, .none, .none } },
126891 },
126892 .extra_temps = .{
126893 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126894 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126895 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126896 .unused,
126897 .unused,
126898 .unused,
126899 .unused,
126900 .unused,
126901 .unused,
126902 .unused,
126903 .unused,
126904 },
126905 .dst_temps = .{ .{ .rc = .sse }, .unused },
126906 .each = .{ .once = &.{
126907 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126908 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
126909 .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ },
126910 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
126911 .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
126912 .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ },
126913 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126914 .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ },
126915 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126916 .{ .@"0:", .p_w, .movzxb, .tmp2x, .memi(.src0q, .tmp0), ._, ._ },
126917 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126918 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
126919 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126920 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126921 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126922 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126923 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126924 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126925 .{ ._, .p_d, .srl, .tmp2x, .ui(16), ._, ._ },
126926 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126927 } },
126928 }, .{
126929 .required_features = .{ .sse2, null, null, null },
126930 .dst_constraints = .{ .{ .int = .byte }, .any },
126931 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any },
126932 .patterns = &.{
126933 .{ .src = .{ .to_mem, .none, .none } },
126934 },
126935 .extra_temps = .{
126936 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126937 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
126938 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
126939 .unused,
126940 .unused,
126941 .unused,
126942 .unused,
126943 .unused,
126944 .unused,
126945 .unused,
126946 .unused,
126947 },
126948 .dst_temps = .{ .{ .rc = .sse }, .unused },
126949 .each = .{ .once = &.{
126950 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
126951 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
126952 .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ },
126953 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
126954 .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
126955 .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ },
126956 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126957 .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ },
126958 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126959 .{ .@"0:", ._q, .mov, .tmp2x, .memi(.src0q, .tmp0), ._, ._ },
126960 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
126961 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126962 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
126963 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
126964 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
126965 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126966 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
126967 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126968 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
126969 .{ ._, .p_d, .srl, .tmp2x, .ui(16), ._, ._ },
126970 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
126971 } },
126972 }, .{
126973 .required_features = .{ .slow_incdec, null, null, null },
126974 .dst_constraints = .{ .{ .int = .byte }, .any },
126975 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any },
126976 .patterns = &.{
126977 .{ .src = .{ .to_mem, .none, .none } },
126978 },
126979 .extra_temps = .{
126980 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
126981 .unused,
126982 .unused,
126983 .unused,
126984 .unused,
126985 .unused,
126986 .unused,
126987 .unused,
126988 .unused,
126989 .unused,
126990 .unused,
126991 },
126992 .dst_temps = .{ .{ .reg = .al }, .unused },
126993 .clobbers = .{ .eflags = true },
126994 .each = .{ .once = &.{
126995 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ },
126996 .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ },
126997 .{ .@"0:", ._, .mul, .memi(.src0b, .tmp0), ._, ._, ._ },
126998 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
126999 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127000 } },
127001 }, .{
127002 .dst_constraints = .{ .{ .int = .byte }, .any },
127003 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any },
127004 .patterns = &.{
127005 .{ .src = .{ .to_mem, .none, .none } },
127006 },
127007 .extra_temps = .{
127008 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127009 .unused,
127010 .unused,
127011 .unused,
127012 .unused,
127013 .unused,
127014 .unused,
127015 .unused,
127016 .unused,
127017 .unused,
127018 .unused,
127019 },
127020 .dst_temps = .{ .{ .reg = .al }, .unused },
127021 .clobbers = .{ .eflags = true },
127022 .each = .{ .once = &.{
127023 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ },
127024 .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ },
127025 .{ .@"0:", ._, .mul, .memi(.src0b, .tmp0), ._, ._, ._ },
127026 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
127027 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
127028 } },
127029 }, .{
127030 .required_features = .{ .sse, null, null, null },
127031 .dst_constraints = .{ .{ .int = .word }, .any },
127032 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
127033 .patterns = &.{
127034 .{ .src = .{ .mut_mem, .none, .none } },
127035 .{ .src = .{ .mut_gpr, .none, .none } },
127036 .{ .src = .{ .to_mut_sse, .none, .none } },
127037 },
127038 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127039 .each = .{ .once = &.{} },
127040 }, .{
127041 .dst_constraints = .{ .{ .int = .word }, .any },
127042 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
127043 .patterns = &.{
127044 .{ .src = .{ .mut_mem, .none, .none } },
127045 .{ .src = .{ .to_mut_gpr, .none, .none } },
127046 },
127047 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127048 .each = .{ .once = &.{} },
127049 }, .{
127050 .required_features = .{ .avx, null, null, null },
127051 .dst_constraints = .{ .{ .int = .word }, .any },
127052 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any },
127053 .patterns = &.{
127054 .{ .src = .{ .to_sse, .none, .none } },
127055 },
127056 .extra_temps = .{
127057 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127058 .unused,
127059 .unused,
127060 .unused,
127061 .unused,
127062 .unused,
127063 .unused,
127064 .unused,
127065 .unused,
127066 .unused,
127067 .unused,
127068 },
127069 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127070 .each = .{ .once = &.{
127071 .{ ._, .vp_d, .srl, .tmp0x, .src0x, .ui(16), ._ },
127072 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
127073 } },
127074 }, .{
127075 .required_features = .{ .sse2, null, null, null },
127076 .dst_constraints = .{ .{ .int = .word }, .any },
127077 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any },
127078 .patterns = &.{
127079 .{ .src = .{ .to_mut_sse, .none, .none } },
127080 },
127081 .extra_temps = .{
127082 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127083 .unused,
127084 .unused,
127085 .unused,
127086 .unused,
127087 .unused,
127088 .unused,
127089 .unused,
127090 .unused,
127091 .unused,
127092 .unused,
127093 },
127094 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127095 .each = .{ .once = &.{
127096 .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ },
127097 .{ ._, .p_d, .srl, .tmp0x, .ui(16), ._, ._ },
127098 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
127099 } },
127100 }, .{
127101 .required_features = .{ .avx, null, null, null },
127102 .dst_constraints = .{ .{ .int = .word }, .any },
127103 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any },
127104 .patterns = &.{
127105 .{ .src = .{ .to_sse, .none, .none } },
127106 },
127107 .extra_temps = .{
127108 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127109 .unused,
127110 .unused,
127111 .unused,
127112 .unused,
127113 .unused,
127114 .unused,
127115 .unused,
127116 .unused,
127117 .unused,
127118 .unused,
127119 },
127120 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127121 .each = .{ .once = &.{
127122 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
127123 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
127124 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
127125 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127126 } },
127127 }, .{
127128 .required_features = .{ .sse2, null, null, null },
127129 .dst_constraints = .{ .{ .int = .word }, .any },
127130 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any },
127131 .patterns = &.{
127132 .{ .src = .{ .to_mut_sse, .none, .none } },
127133 },
127134 .extra_temps = .{
127135 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127136 .unused,
127137 .unused,
127138 .unused,
127139 .unused,
127140 .unused,
127141 .unused,
127142 .unused,
127143 .unused,
127144 .unused,
127145 .unused,
127146 },
127147 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127148 .each = .{ .once = &.{
127149 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
127150 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
127151 .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ },
127152 .{ ._, .p_d, .srl, .tmp0x, .ui(16), ._, ._ },
127153 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
127154 } },
127155 }, .{
127156 .required_features = .{ .avx, null, null, null },
127157 .dst_constraints = .{ .{ .int = .word }, .any },
127158 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any },
127159 .patterns = &.{
127160 .{ .src = .{ .to_sse, .none, .none } },
127161 },
127162 .extra_temps = .{
127163 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127164 .unused,
127165 .unused,
127166 .unused,
127167 .unused,
127168 .unused,
127169 .unused,
127170 .unused,
127171 .unused,
127172 .unused,
127173 .unused,
127174 },
127175 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127176 .each = .{ .once = &.{
127177 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
127178 .{ ._, .vp_w, .mull, .tmp0x, .src0x, .tmp0x, ._ },
127179 .{ ._, .vp_d, .srl, .dst0x, .src0x, .ui(16), ._ },
127180 .{ ._, .vp_w, .mull, .dst0x, .tmp0x, .dst0x, ._ },
127181 } },
127182 }, .{
127183 .required_features = .{ .sse2, null, null, null },
127184 .dst_constraints = .{ .{ .int = .word }, .any },
127185 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any },
127186 .patterns = &.{
127187 .{ .src = .{ .to_mut_sse, .none, .none } },
127188 },
127189 .extra_temps = .{
127190 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127191 .unused,
127192 .unused,
127193 .unused,
127194 .unused,
127195 .unused,
127196 .unused,
127197 .unused,
127198 .unused,
127199 .unused,
127200 .unused,
127201 },
127202 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127203 .each = .{ .once = &.{
127204 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
127205 .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ },
127206 .{ ._, .p_d, .srl, .dst0x, .ui(16), ._, ._ },
127207 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
127208 } },
127209 }, .{
127210 .required_features = .{ .avx, null, null, null },
127211 .dst_constraints = .{ .{ .int = .word }, .any },
127212 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
127213 .patterns = &.{
127214 .{ .src = .{ .to_sse, .none, .none } },
127215 },
127216 .extra_temps = .{
127217 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127218 .unused,
127219 .unused,
127220 .unused,
127221 .unused,
127222 .unused,
127223 .unused,
127224 .unused,
127225 .unused,
127226 .unused,
127227 .unused,
127228 },
127229 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127230 .each = .{ .once = &.{
127231 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
127232 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
127233 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
127234 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127235 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
127236 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127237 } },
127238 }, .{
127239 .required_features = .{ .sse2, null, null, null },
127240 .dst_constraints = .{ .{ .int = .word }, .any },
127241 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
127242 .patterns = &.{
127243 .{ .src = .{ .to_mut_sse, .none, .none } },
127244 },
127245 .extra_temps = .{
127246 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127247 .unused,
127248 .unused,
127249 .unused,
127250 .unused,
127251 .unused,
127252 .unused,
127253 .unused,
127254 .unused,
127255 .unused,
127256 .unused,
127257 },
127258 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127259 .each = .{ .once = &.{
127260 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
127261 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
127262 .{ ._, .p_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
127263 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
127264 .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ },
127265 .{ ._, .p_d, .srl, .tmp0x, .ui(16), ._, ._ },
127266 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
127267 } },
127268 }, .{
127269 .required_features = .{ .avx, null, null, null },
127270 .dst_constraints = .{ .{ .int = .word }, .any },
127271 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
127272 .patterns = &.{
127273 .{ .src = .{ .mem, .none, .none } },
127274 .{ .src = .{ .to_sse, .none, .none } },
127275 },
127276 .extra_temps = .{
127277 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
127278 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127279 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127280 .unused,
127281 .unused,
127282 .unused,
127283 .unused,
127284 .unused,
127285 .unused,
127286 .unused,
127287 .unused,
127288 },
127289 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127290 .each = .{ .once = &.{
127291 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127292 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
127293 .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ },
127294 .{ ._, .vp_w, .sub, .dst0x, .dst0x, .tmp2x, ._ },
127295 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127296 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127297 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127298 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127299 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
127300 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127301 } },
127302 }, .{
127303 .required_features = .{ .sse2, null, null, null },
127304 .dst_constraints = .{ .{ .int = .word }, .any },
127305 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
127306 .patterns = &.{
127307 .{ .src = .{ .mem, .none, .none } },
127308 .{ .src = .{ .to_sse, .none, .none } },
127309 },
127310 .extra_temps = .{
127311 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
127312 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127313 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127314 .unused,
127315 .unused,
127316 .unused,
127317 .unused,
127318 .unused,
127319 .unused,
127320 .unused,
127321 .unused,
127322 },
127323 .dst_temps = .{ .{ .rc = .sse }, .unused },
127324 .each = .{ .once = &.{
127325 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127326 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
127327 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
127328 .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ },
127329 .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ },
127330 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127331 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
127332 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127333 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
127334 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
127335 .{ ._, .p_d, .srl, .tmp2x, .ui(16), ._, ._ },
127336 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
127337 } },
127338 }, .{
127339 .required_features = .{ .avx2, null, null, null },
127340 .dst_constraints = .{ .{ .int = .word }, .any },
127341 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
127342 .patterns = &.{
127343 .{ .src = .{ .to_sse, .none, .none } },
127344 },
127345 .extra_temps = .{
127346 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127347 .unused,
127348 .unused,
127349 .unused,
127350 .unused,
127351 .unused,
127352 .unused,
127353 .unused,
127354 .unused,
127355 .unused,
127356 .unused,
127357 },
127358 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127359 .each = .{ .once = &.{
127360 .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ },
127361 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
127362 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
127363 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127364 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
127365 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127366 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
127367 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127368 } },
127369 }, .{
127370 .required_features = .{ .avx, null, null, null },
127371 .dst_constraints = .{ .{ .int = .word }, .any },
127372 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
127373 .patterns = &.{
127374 .{ .src = .{ .to_sse, .none, .none } },
127375 },
127376 .extra_temps = .{
127377 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127378 .unused,
127379 .unused,
127380 .unused,
127381 .unused,
127382 .unused,
127383 .unused,
127384 .unused,
127385 .unused,
127386 .unused,
127387 .unused,
127388 },
127389 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127390 .each = .{ .once = &.{
127391 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
127392 .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ },
127393 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
127394 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127395 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
127396 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127397 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
127398 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127399 } },
127400 }, .{
127401 .required_features = .{ .avx2, null, null, null },
127402 .dst_constraints = .{ .{ .int = .word }, .any },
127403 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
127404 .patterns = &.{
127405 .{ .src = .{ .to_sse, .none, .none } },
127406 },
127407 .extra_temps = .{
127408 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
127409 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127410 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127411 .unused,
127412 .unused,
127413 .unused,
127414 .unused,
127415 .unused,
127416 .unused,
127417 .unused,
127418 .unused,
127419 },
127420 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127421 .each = .{ .once = &.{
127422 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127423 .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ },
127424 .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ },
127425 .{ ._, .vp_w, .sub, .dst0y, .dst0y, .tmp2y, ._ },
127426 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
127427 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127428 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127429 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127430 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127431 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127432 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
127433 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127434 } },
127435 }, .{
127436 .required_features = .{ .avx, null, null, null },
127437 .dst_constraints = .{ .{ .int = .word }, .any },
127438 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
127439 .patterns = &.{
127440 .{ .src = .{ .to_sse, .none, .none } },
127441 },
127442 .extra_temps = .{
127443 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
127444 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127445 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127446 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127447 .unused,
127448 .unused,
127449 .unused,
127450 .unused,
127451 .unused,
127452 .unused,
127453 .unused,
127454 },
127455 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127456 .each = .{ .once = &.{
127457 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127458 .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ },
127459 .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ },
127460 .{ ._, .vp_w, .sub, .dst0x, .tmp3x, .tmp2x, ._ },
127461 .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ },
127462 .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ },
127463 .{ ._, .vp_w, .sub, .tmp2x, .tmp3x, .tmp2x, ._ },
127464 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127465 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127466 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127467 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127468 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127469 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
127470 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127471 } },
127472 }, .{
127473 .required_features = .{ .avx512f, null, null, null },
127474 .dst_constraints = .{ .{ .int = .word }, .any },
127475 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any },
127476 .patterns = &.{
127477 .{ .src = .{ .to_mem, .none, .none } },
127478 },
127479 .extra_temps = .{
127480 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127481 .unused,
127482 .unused,
127483 .unused,
127484 .unused,
127485 .unused,
127486 .unused,
127487 .unused,
127488 .unused,
127489 .unused,
127490 .unused,
127491 },
127492 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127493 .each = .{ .once = &.{
127494 .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ },
127495 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .memd(.src0y, 32), ._ },
127496 .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
127497 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127498 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
127499 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127500 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b01_01_01_01), ._ },
127501 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127502 .{ ._, .vp_d, .srl, .tmp0x, .dst0x, .ui(16), ._ },
127503 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
127504 } },
127505 }, .{
127506 .required_features = .{ .avx512f, null, null, null },
127507 .dst_constraints = .{ .{ .int = .word }, .any },
127508 .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .word } }, .any, .any },
127509 .patterns = &.{
127510 .{ .src = .{ .to_mem, .none, .none } },
127511 },
127512 .extra_temps = .{
127513 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
127514 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127515 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127516 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127517 .unused,
127518 .unused,
127519 .unused,
127520 .unused,
127521 .unused,
127522 .unused,
127523 .unused,
127524 },
127525 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127526 .each = .{ .once = &.{
127527 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127528 .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ },
127529 .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
127530 .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ },
127531 .{ ._, .vp_w, .sub, .tmp2y, .tmp3y, .tmp2y, ._ },
127532 .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ },
127533 .{ ._, .vp_w, .sub, .dst0y, .tmp3y, .dst0y, ._ },
127534 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
127535 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
127536 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127537 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127538 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127539 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127540 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127541 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
127542 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127543 } },
127544 }, .{
127545 .required_features = .{ .avx2, null, null, null },
127546 .dst_constraints = .{ .{ .int = .word }, .any },
127547 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
127548 .patterns = &.{
127549 .{ .src = .{ .to_mem, .none, .none } },
127550 },
127551 .extra_temps = .{
127552 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127553 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127554 .unused,
127555 .unused,
127556 .unused,
127557 .unused,
127558 .unused,
127559 .unused,
127560 .unused,
127561 .unused,
127562 .unused,
127563 },
127564 .dst_temps = .{ .{ .rc = .sse }, .unused },
127565 .each = .{ .once = &.{
127566 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ },
127567 .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
127568 .{ .@"0:", .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
127569 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
127570 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127571 .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
127572 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
127573 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
127574 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
127575 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b01_01_01_01), ._ },
127576 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
127577 .{ ._, .vp_d, .srl, .tmp1x, .dst0x, .ui(16), ._ },
127578 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
127579 } },
127580 }, .{
127581 .required_features = .{ .avx, null, null, null },
127582 .dst_constraints = .{ .{ .int = .word }, .any },
127583 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
127584 .patterns = &.{
127585 .{ .src = .{ .to_mem, .none, .none } },
127586 },
127587 .extra_temps = .{
127588 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127589 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127590 .unused,
127591 .unused,
127592 .unused,
127593 .unused,
127594 .unused,
127595 .unused,
127596 .unused,
127597 .unused,
127598 .unused,
127599 },
127600 .dst_temps = .{ .{ .rc = .sse }, .unused },
127601 .each = .{ .once = &.{
127602 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
127603 .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
127604 .{ .@"0:", .vp_w, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
127605 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
127606 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127607 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
127608 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
127609 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b01_01_01_01), ._ },
127610 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
127611 .{ ._, .vp_d, .srl, .tmp1x, .dst0x, .ui(16), ._ },
127612 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ },
127613 } },
127614 }, .{
127615 .required_features = .{ .sse2, null, null, null },
127616 .dst_constraints = .{ .{ .int = .word }, .any },
127617 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
127618 .patterns = &.{
127619 .{ .src = .{ .to_mem, .none, .none } },
127620 },
127621 .extra_temps = .{
127622 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127623 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127624 .unused,
127625 .unused,
127626 .unused,
127627 .unused,
127628 .unused,
127629 .unused,
127630 .unused,
127631 .unused,
127632 .unused,
127633 },
127634 .dst_temps = .{ .{ .rc = .sse }, .unused },
127635 .each = .{ .once = &.{
127636 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
127637 .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
127638 .{ .@"0:", .p_w, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
127639 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
127640 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127641 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
127642 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
127643 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b01_01_01_01), ._ },
127644 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
127645 .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ },
127646 .{ ._, .p_d, .srl, .tmp1x, .ui(16), ._, ._ },
127647 .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ },
127648 } },
127649 }, .{
127650 .required_features = .{ .avx512f, null, null, null },
127651 .dst_constraints = .{ .{ .int = .word }, .any },
127652 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any },
127653 .patterns = &.{
127654 .{ .src = .{ .to_mem, .none, .none } },
127655 },
127656 .extra_temps = .{
127657 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127658 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127659 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127660 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127661 .unused,
127662 .unused,
127663 .unused,
127664 .unused,
127665 .unused,
127666 .unused,
127667 .unused,
127668 },
127669 .dst_temps = .{ .{ .rc = .sse }, .unused },
127670 .each = .{ .once = &.{
127671 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127672 .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ },
127673 .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
127674 .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ },
127675 .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
127676 .{ ._, .vp_w, .sub, .tmp2y, .tmp3y, .tmp2y, ._ },
127677 .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ },
127678 .{ ._, .vp_w, .sub, .dst0y, .tmp3y, .dst0y, ._ },
127679 .{ .@"0:", .vp_w, .mull, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ },
127680 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
127681 .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ },
127682 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127683 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
127684 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
127685 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127686 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127687 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127688 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127689 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127690 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
127691 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127692 } },
127693 }, .{
127694 .required_features = .{ .avx2, null, null, null },
127695 .dst_constraints = .{ .{ .int = .word }, .any },
127696 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
127697 .patterns = &.{
127698 .{ .src = .{ .to_mem, .none, .none } },
127699 },
127700 .extra_temps = .{
127701 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127702 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127703 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127704 .unused,
127705 .unused,
127706 .unused,
127707 .unused,
127708 .unused,
127709 .unused,
127710 .unused,
127711 .unused,
127712 },
127713 .dst_temps = .{ .{ .rc = .sse }, .unused },
127714 .each = .{ .once = &.{
127715 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127716 .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ },
127717 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ },
127718 .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
127719 .{ ._, .vp_w, .sub, .dst0y, .dst0y, .tmp2y, ._ },
127720 .{ .@"0:", .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
127721 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
127722 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127723 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
127724 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127725 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127726 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127727 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127728 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127729 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
127730 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127731 } },
127732 }, .{
127733 .required_features = .{ .avx, null, null, null },
127734 .dst_constraints = .{ .{ .int = .word }, .any },
127735 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any },
127736 .patterns = &.{
127737 .{ .src = .{ .to_mem, .none, .none } },
127738 },
127739 .extra_temps = .{
127740 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127741 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127742 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127743 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
127744 .unused,
127745 .unused,
127746 .unused,
127747 .unused,
127748 .unused,
127749 .unused,
127750 .unused,
127751 },
127752 .dst_temps = .{ .{ .rc = .sse }, .unused },
127753 .each = .{ .once = &.{
127754 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127755 .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ },
127756 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
127757 .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
127758 .{ ._, .vp_w, .sub, .dst0x, .tmp3x, .tmp2x, ._ },
127759 .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ },
127760 .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ },
127761 .{ ._, .vp_w, .sub, .tmp2x, .tmp3x, .tmp2x, ._ },
127762 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127763 .{ .@"0:", .vp_w, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
127764 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
127765 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127766 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127767 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127768 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127769 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127770 .{ ._, .vp_d, .srl, .tmp2x, .dst0x, .ui(16), ._ },
127771 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
127772 } },
127773 }, .{
127774 .required_features = .{ .sse2, null, null, null },
127775 .dst_constraints = .{ .{ .int = .word }, .any },
127776 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any },
127777 .patterns = &.{
127778 .{ .src = .{ .to_mem, .none, .none } },
127779 },
127780 .extra_temps = .{
127781 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127782 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
127783 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
127784 .unused,
127785 .unused,
127786 .unused,
127787 .unused,
127788 .unused,
127789 .unused,
127790 .unused,
127791 .unused,
127792 },
127793 .dst_temps = .{ .{ .rc = .sse }, .unused },
127794 .each = .{ .once = &.{
127795 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
127796 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
127797 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
127798 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
127799 .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
127800 .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ },
127801 .{ .@"0:", .p_w, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
127802 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
127803 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127804 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
127805 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
127806 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b01_01_01_01), ._ },
127807 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
127808 .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ },
127809 .{ ._, .p_d, .srl, .tmp2x, .ui(16), ._, ._ },
127810 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
127811 } },
127812 }, .{
127813 .dst_constraints = .{ .{ .int = .word }, .any },
127814 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any },
127815 .patterns = &.{
127816 .{ .src = .{ .to_mem, .none, .none } },
127817 },
127818 .extra_temps = .{
127819 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
127820 .unused,
127821 .unused,
127822 .unused,
127823 .unused,
127824 .unused,
127825 .unused,
127826 .unused,
127827 .unused,
127828 .unused,
127829 .unused,
127830 },
127831 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
127832 .clobbers = .{ .eflags = true },
127833 .each = .{ .once = &.{
127834 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ },
127835 .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ },
127836 .{ .@"0:", .i_, .mul, .dst0w, .memi(.src0w, .tmp0), ._, ._ },
127837 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
127838 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
127839 } },
127840 }, .{
127841 .required_features = .{ .sse, null, null, null },
127842 .dst_constraints = .{ .{ .int = .dword }, .any },
127843 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
127844 .patterns = &.{
127845 .{ .src = .{ .mut_mem, .none, .none } },
127846 .{ .src = .{ .mut_gpr, .none, .none } },
127847 .{ .src = .{ .to_mut_sse, .none, .none } },
127848 },
127849 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127850 .each = .{ .once = &.{} },
127851 }, .{
127852 .dst_constraints = .{ .{ .int = .dword }, .any },
127853 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
127854 .patterns = &.{
127855 .{ .src = .{ .mut_mem, .none, .none } },
127856 .{ .src = .{ .to_mut_gpr, .none, .none } },
127857 },
127858 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127859 .each = .{ .once = &.{} },
127860 }, .{
127861 .required_features = .{ .avx, null, null, null },
127862 .dst_constraints = .{ .{ .int = .dword }, .any },
127863 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any },
127864 .patterns = &.{
127865 .{ .src = .{ .to_sse, .none, .none } },
127866 },
127867 .extra_temps = .{
127868 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
127869 .unused,
127870 .unused,
127871 .unused,
127872 .unused,
127873 .unused,
127874 .unused,
127875 .unused,
127876 .unused,
127877 .unused,
127878 .unused,
127879 },
127880 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127881 .each = .{ .once = &.{
127882 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
127883 .{ ._, .vp_udq, .mul, .dst0x, .src0x, .tmp0x, ._ },
127884 } },
127885 }, .{
127886 .required_features = .{ .sse2, null, null, null },
127887 .dst_constraints = .{ .{ .int = .dword }, .any },
127888 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any },
127889 .patterns = &.{
127890 .{ .src = .{ .to_mut_sse, .none, .none } },
127891 },
127892 .extra_temps = .{
127893 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
127894 .unused,
127895 .unused,
127896 .unused,
127897 .unused,
127898 .unused,
127899 .unused,
127900 .unused,
127901 .unused,
127902 .unused,
127903 .unused,
127904 },
127905 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127906 .each = .{ .once = &.{
127907 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b01_01_01_01), ._ },
127908 .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ },
127909 } },
127910 }, .{
127911 .required_features = .{ .avx, null, null, null },
127912 .dst_constraints = .{ .{ .int = .dword }, .any },
127913 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
127914 .patterns = &.{
127915 .{ .src = .{ .to_sse, .none, .none } },
127916 },
127917 .extra_temps = .{
127918 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
127919 .unused,
127920 .unused,
127921 .unused,
127922 .unused,
127923 .unused,
127924 .unused,
127925 .unused,
127926 .unused,
127927 .unused,
127928 .unused,
127929 },
127930 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127931 .each = .{ .once = &.{
127932 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_11_01_01), ._ },
127933 .{ ._, .vp_udq, .mul, .dst0x, .src0x, .tmp0x, ._ },
127934 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
127935 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ },
127936 } },
127937 }, .{
127938 .required_features = .{ .sse2, null, null, null },
127939 .dst_constraints = .{ .{ .int = .dword }, .any },
127940 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
127941 .patterns = &.{
127942 .{ .src = .{ .to_mut_sse, .none, .none } },
127943 },
127944 .extra_temps = .{
127945 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
127946 .unused,
127947 .unused,
127948 .unused,
127949 .unused,
127950 .unused,
127951 .unused,
127952 .unused,
127953 .unused,
127954 .unused,
127955 .unused,
127956 },
127957 .dst_temps = .{ .{ .ref = .src0 }, .unused },
127958 .each = .{ .once = &.{
127959 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_11_01_01), ._ },
127960 .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ },
127961 .{ ._, .p_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
127962 .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ },
127963 } },
127964 }, .{
127965 .required_features = .{ .avx, null, null, null },
127966 .dst_constraints = .{ .{ .int = .dword }, .any },
127967 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
127968 .patterns = &.{
127969 .{ .src = .{ .to_sse, .none, .none } },
127970 },
127971 .extra_temps = .{
127972 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
127973 .unused,
127974 .unused,
127975 .unused,
127976 .unused,
127977 .unused,
127978 .unused,
127979 .unused,
127980 .unused,
127981 .unused,
127982 .unused,
127983 },
127984 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
127985 .each = .{ .once = &.{
127986 .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
127987 .{ ._, .vp_udq, .mul, .tmp0x, .src0x, .tmp0x, ._ },
127988 .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b01_01_01_01), ._ },
127989 .{ ._, .vp_udq, .mul, .dst0x, .tmp0x, .dst0x, ._ },
127990 } },
127991 }, .{
127992 .required_features = .{ .sse2, null, null, null },
127993 .dst_constraints = .{ .{ .int = .dword }, .any },
127994 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
127995 .patterns = &.{
127996 .{ .src = .{ .to_sse, .none, .none } },
127997 },
127998 .extra_temps = .{
127999 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128000 .unused,
128001 .unused,
128002 .unused,
128003 .unused,
128004 .unused,
128005 .unused,
128006 .unused,
128007 .unused,
128008 .unused,
128009 .unused,
128010 },
128011 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
128012 .each = .{ .once = &.{
128013 .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10_11_10), ._ },
128014 .{ ._, .p_udq, .mul, .tmp0x, .src0x, ._, ._ },
128015 .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b01_01_01_01), ._ },
128016 .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ },
128017 } },
128018 }, .{
128019 .required_features = .{ .avx2, null, null, null },
128020 .dst_constraints = .{ .{ .int = .dword }, .any },
128021 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
128022 .patterns = &.{
128023 .{ .src = .{ .to_sse, .none, .none } },
128024 },
128025 .extra_temps = .{
128026 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128027 .unused,
128028 .unused,
128029 .unused,
128030 .unused,
128031 .unused,
128032 .unused,
128033 .unused,
128034 .unused,
128035 .unused,
128036 .unused,
128037 },
128038 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
128039 .each = .{ .once = &.{
128040 .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ },
128041 .{ ._, .vp_d, .mull, .dst0x, .src0x, .tmp0x, ._ },
128042 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ },
128043 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ },
128044 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
128045 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ },
128046 } },
128047 }, .{
128048 .required_features = .{ .avx, null, null, null },
128049 .dst_constraints = .{ .{ .int = .dword }, .any },
128050 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
128051 .patterns = &.{
128052 .{ .src = .{ .to_sse, .none, .none } },
128053 },
128054 .extra_temps = .{
128055 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128056 .unused,
128057 .unused,
128058 .unused,
128059 .unused,
128060 .unused,
128061 .unused,
128062 .unused,
128063 .unused,
128064 .unused,
128065 .unused,
128066 },
128067 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
128068 .each = .{ .once = &.{
128069 .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ },
128070 .{ ._, .vp_d, .mull, .dst0x, .src0x, .tmp0x, ._ },
128071 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ },
128072 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ },
128073 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
128074 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ },
128075 } },
128076 }, .{
128077 .required_features = .{ .avx2, null, null, null },
128078 .dst_constraints = .{ .{ .int = .dword }, .any },
128079 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
128080 .patterns = &.{
128081 .{ .src = .{ .mem, .none, .none } },
128082 .{ .src = .{ .to_sse, .none, .none } },
128083 },
128084 .extra_temps = .{
128085 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128086 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128087 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128088 .unused,
128089 .unused,
128090 .unused,
128091 .unused,
128092 .unused,
128093 .unused,
128094 .unused,
128095 .unused,
128096 },
128097 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
128098 .each = .{ .once = &.{
128099 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128100 .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ },
128101 .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ },
128102 .{ ._, .vp_d, .sub, .dst0y, .dst0y, .tmp2y, ._ },
128103 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
128104 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ },
128105 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128106 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128107 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128108 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128109 } },
128110 }, .{
128111 .required_features = .{ .avx, null, null, null },
128112 .dst_constraints = .{ .{ .int = .dword }, .any },
128113 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
128114 .patterns = &.{
128115 .{ .src = .{ .mem, .none, .none } },
128116 .{ .src = .{ .to_sse, .none, .none } },
128117 },
128118 .extra_temps = .{
128119 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128120 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128121 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128122 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128123 .unused,
128124 .unused,
128125 .unused,
128126 .unused,
128127 .unused,
128128 .unused,
128129 .unused,
128130 },
128131 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
128132 .each = .{ .once = &.{
128133 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128134 .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ },
128135 .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ },
128136 .{ ._, .vp_d, .sub, .dst0x, .tmp3x, .tmp2x, ._ },
128137 .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ },
128138 .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ },
128139 .{ ._, .vp_d, .sub, .tmp2x, .tmp3x, .tmp2x, ._ },
128140 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ },
128141 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128142 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128143 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128144 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128145 } },
128146 }, .{
128147 .required_features = .{ .avx512f, null, null, null },
128148 .dst_constraints = .{ .{ .int = .dword }, .any },
128149 .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any },
128150 .patterns = &.{
128151 .{ .src = .{ .to_mem, .none, .none } },
128152 },
128153 .extra_temps = .{
128154 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128155 .unused,
128156 .unused,
128157 .unused,
128158 .unused,
128159 .unused,
128160 .unused,
128161 .unused,
128162 .unused,
128163 .unused,
128164 .unused,
128165 },
128166 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
128167 .each = .{ .once = &.{
128168 .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ },
128169 .{ ._, .vp_d, .mull, .dst0y, .dst0y, .memd(.src0y, 32), ._ },
128170 .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ },
128171 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp0x, ._ },
128172 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ },
128173 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ },
128174 .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10_11_10), ._ },
128175 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ },
128176 } },
128177 }, .{
128178 .required_features = .{ .avx512f, null, null, null },
128179 .dst_constraints = .{ .{ .int = .dword }, .any },
128180 .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any },
128181 .patterns = &.{
128182 .{ .src = .{ .to_mem, .none, .none } },
128183 },
128184 .extra_temps = .{
128185 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128186 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128187 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128188 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128189 .unused,
128190 .unused,
128191 .unused,
128192 .unused,
128193 .unused,
128194 .unused,
128195 .unused,
128196 },
128197 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
128198 .each = .{ .once = &.{
128199 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128200 .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ },
128201 .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
128202 .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ },
128203 .{ ._, .vp_d, .sub, .tmp2y, .tmp3y, .tmp2y, ._ },
128204 .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ },
128205 .{ ._, .vp_d, .sub, .dst0y, .tmp3y, .dst0y, ._ },
128206 .{ ._, .vp_d, .mull, .dst0y, .dst0y, .tmp2y, ._ },
128207 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
128208 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ },
128209 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128210 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128211 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128212 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128213 } },
128214 }, .{
128215 .required_features = .{ .avx2, null, null, null },
128216 .dst_constraints = .{ .{ .int = .dword }, .any },
128217 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
128218 .patterns = &.{
128219 .{ .src = .{ .to_mem, .none, .none } },
128220 },
128221 .extra_temps = .{
128222 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128223 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128224 .unused,
128225 .unused,
128226 .unused,
128227 .unused,
128228 .unused,
128229 .unused,
128230 .unused,
128231 .unused,
128232 .unused,
128233 },
128234 .dst_temps = .{ .{ .rc = .sse }, .unused },
128235 .each = .{ .once = &.{
128236 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ },
128237 .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ },
128238 .{ .@"0:", .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
128239 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
128240 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128241 .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ },
128242 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp1x, ._ },
128243 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ },
128244 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ },
128245 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
128246 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ },
128247 } },
128248 }, .{
128249 .required_features = .{ .avx, null, null, null },
128250 .dst_constraints = .{ .{ .int = .dword }, .any },
128251 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
128252 .patterns = &.{
128253 .{ .src = .{ .to_mem, .none, .none } },
128254 },
128255 .extra_temps = .{
128256 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128257 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128258 .unused,
128259 .unused,
128260 .unused,
128261 .unused,
128262 .unused,
128263 .unused,
128264 .unused,
128265 .unused,
128266 .unused,
128267 },
128268 .dst_temps = .{ .{ .rc = .sse }, .unused },
128269 .each = .{ .once = &.{
128270 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
128271 .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
128272 .{ .@"0:", .vp_d, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
128273 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
128274 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128275 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ },
128276 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ },
128277 .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
128278 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ },
128279 } },
128280 }, .{
128281 .required_features = .{ .sse4_1, null, null, null },
128282 .dst_constraints = .{ .{ .int = .dword }, .any },
128283 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
128284 .patterns = &.{
128285 .{ .src = .{ .to_mem, .none, .none } },
128286 },
128287 .extra_temps = .{
128288 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128289 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128290 .unused,
128291 .unused,
128292 .unused,
128293 .unused,
128294 .unused,
128295 .unused,
128296 .unused,
128297 .unused,
128298 .unused,
128299 },
128300 .dst_temps = .{ .{ .rc = .sse }, .unused },
128301 .each = .{ .once = &.{
128302 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
128303 .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
128304 .{ .@"0:", .p_d, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
128305 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
128306 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128307 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ },
128308 .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ },
128309 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
128310 .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ },
128311 } },
128312 }, .{
128313 .required_features = .{ .sse2, null, null, null },
128314 .dst_constraints = .{ .{ .int = .dword }, .any },
128315 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
128316 .patterns = &.{
128317 .{ .src = .{ .to_mem, .none, .none } },
128318 },
128319 .extra_temps = .{
128320 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128321 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128322 .unused,
128323 .unused,
128324 .unused,
128325 .unused,
128326 .unused,
128327 .unused,
128328 .unused,
128329 .unused,
128330 .unused,
128331 },
128332 .dst_temps = .{ .{ .rc = .sse }, .unused },
128333 .each = .{ .once = &.{
128334 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
128335 .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ },
128336 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ },
128337 .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ },
128338 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
128339 .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ },
128340 .{ ._, .p_d, .shuf, .tmp1x, .tmp1x, .ui(0b11_11_01_01), ._ },
128341 .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ },
128342 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
128343 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128344 .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10_11_10), ._ },
128345 .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ },
128346 } },
128347 }, .{
128348 .required_features = .{ .avx512f, null, null, null },
128349 .dst_constraints = .{ .{ .int = .dword }, .any },
128350 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any },
128351 .patterns = &.{
128352 .{ .src = .{ .to_mem, .none, .none } },
128353 },
128354 .extra_temps = .{
128355 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128356 .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128357 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128358 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128359 .unused,
128360 .unused,
128361 .unused,
128362 .unused,
128363 .unused,
128364 .unused,
128365 .unused,
128366 },
128367 .dst_temps = .{ .{ .rc = .sse }, .unused },
128368 .each = .{ .once = &.{
128369 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128370 .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ },
128371 .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ },
128372 .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ },
128373 .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
128374 .{ ._, .vp_d, .sub, .tmp2y, .tmp3y, .tmp2y, ._ },
128375 .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ },
128376 .{ ._, .vp_d, .sub, .dst0y, .tmp3y, .dst0y, ._ },
128377 .{ .@"0:", .vp_d, .mull, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ },
128378 .{ ._, .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
128379 .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ },
128380 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128381 .{ ._, .vp_d, .mull, .dst0y, .dst0y, .tmp2y, ._ },
128382 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
128383 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ },
128384 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128385 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128386 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128387 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128388 } },
128389 }, .{
128390 .required_features = .{ .avx2, null, null, null },
128391 .dst_constraints = .{ .{ .int = .dword }, .any },
128392 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
128393 .patterns = &.{
128394 .{ .src = .{ .to_mem, .none, .none } },
128395 },
128396 .extra_temps = .{
128397 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128398 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128399 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128400 .unused,
128401 .unused,
128402 .unused,
128403 .unused,
128404 .unused,
128405 .unused,
128406 .unused,
128407 .unused,
128408 },
128409 .dst_temps = .{ .{ .rc = .sse }, .unused },
128410 .each = .{ .once = &.{
128411 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128412 .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ },
128413 .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ },
128414 .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
128415 .{ ._, .vp_d, .sub, .dst0y, .dst0y, .tmp2y, ._ },
128416 .{ .@"0:", .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ },
128417 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
128418 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128419 .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ },
128420 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ },
128421 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128422 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128423 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128424 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128425 } },
128426 }, .{
128427 .required_features = .{ .avx, null, null, null },
128428 .dst_constraints = .{ .{ .int = .dword }, .any },
128429 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any },
128430 .patterns = &.{
128431 .{ .src = .{ .to_mem, .none, .none } },
128432 },
128433 .extra_temps = .{
128434 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128435 .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128436 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128437 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
128438 .unused,
128439 .unused,
128440 .unused,
128441 .unused,
128442 .unused,
128443 .unused,
128444 .unused,
128445 },
128446 .dst_temps = .{ .{ .rc = .sse }, .unused },
128447 .each = .{ .once = &.{
128448 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128449 .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ },
128450 .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ },
128451 .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ },
128452 .{ ._, .vp_d, .sub, .dst0x, .tmp3x, .tmp2x, ._ },
128453 .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ },
128454 .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ },
128455 .{ ._, .vp_d, .sub, .tmp2x, .tmp3x, .tmp2x, ._ },
128456 .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ },
128457 .{ .@"0:", .vp_d, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ },
128458 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
128459 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128460 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128461 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128462 .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128463 .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ },
128464 } },
128465 }, .{
128466 .required_features = .{ .sse4_1, null, null, null },
128467 .dst_constraints = .{ .{ .int = .dword }, .any },
128468 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
128469 .patterns = &.{
128470 .{ .src = .{ .to_mem, .none, .none } },
128471 },
128472 .extra_temps = .{
128473 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128474 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128475 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128476 .unused,
128477 .unused,
128478 .unused,
128479 .unused,
128480 .unused,
128481 .unused,
128482 .unused,
128483 .unused,
128484 },
128485 .dst_temps = .{ .{ .rc = .sse }, .unused },
128486 .each = .{ .once = &.{
128487 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128488 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
128489 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
128490 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
128491 .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
128492 .{ ._, .p_d, .sub, .dst0x, .tmp2x, ._, ._ },
128493 .{ .@"0:", .p_d, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ },
128494 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
128495 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128496 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128497 .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ },
128498 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128499 .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ },
128500 } },
128501 }, .{
128502 .required_features = .{ .sse2, null, null, null },
128503 .dst_constraints = .{ .{ .int = .dword }, .any },
128504 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any },
128505 .patterns = &.{
128506 .{ .src = .{ .to_mem, .none, .none } },
128507 },
128508 .extra_temps = .{
128509 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128510 .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } },
128511 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
128512 .unused,
128513 .unused,
128514 .unused,
128515 .unused,
128516 .unused,
128517 .unused,
128518 .unused,
128519 .unused,
128520 },
128521 .dst_temps = .{ .{ .rc = .sse }, .unused },
128522 .each = .{ .once = &.{
128523 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
128524 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
128525 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
128526 .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ },
128527 .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ },
128528 .{ ._, .p_d, .sub, .dst0x, .tmp2x, ._, ._ },
128529 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ },
128530 .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ },
128531 .{ .@"0:", ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ },
128532 .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ },
128533 .{ ._, .p_d, .shuf, .tmp2x, .tmp2x, .ui(0b11_11_01_01), ._ },
128534 .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ },
128535 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
128536 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128537 .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10_11_10), ._ },
128538 .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ },
128539 } },
128540 }, .{
128541 .dst_constraints = .{ .{ .int = .dword }, .any },
128542 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any },
128543 .patterns = &.{
128544 .{ .src = .{ .to_mem, .none, .none } },
128545 },
128546 .extra_temps = .{
128547 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128548 .unused,
128549 .unused,
128550 .unused,
128551 .unused,
128552 .unused,
128553 .unused,
128554 .unused,
128555 .unused,
128556 .unused,
128557 .unused,
128558 },
128559 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
128560 .clobbers = .{ .eflags = true },
128561 .each = .{ .once = &.{
128562 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ },
128563 .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ },
128564 .{ .@"0:", .i_, .mul, .dst0d, .memi(.src0d, .tmp0), ._, ._ },
128565 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
128566 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128567 } },
128568 }, .{
128569 .required_features = .{ .sse, null, null, null },
128570 .dst_constraints = .{ .{ .int = .qword }, .any },
128571 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
128572 .patterns = &.{
128573 .{ .src = .{ .mut_mem, .none, .none } },
128574 .{ .src = .{ .mut_gpr, .none, .none } },
128575 .{ .src = .{ .to_mut_sse, .none, .none } },
128576 },
128577 .dst_temps = .{ .{ .ref = .src0 }, .unused },
128578 .each = .{ .once = &.{} },
128579 }, .{
128580 .required_features = .{ .@"64bit", null, null, null },
128581 .dst_constraints = .{ .{ .int = .qword }, .any },
128582 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
128583 .patterns = &.{
128584 .{ .src = .{ .mut_mem, .none, .none } },
128585 .{ .src = .{ .to_mut_gpr, .none, .none } },
128586 },
128587 .dst_temps = .{ .{ .ref = .src0 }, .unused },
128588 .each = .{ .once = &.{} },
128589 }, .{
128590 .required_features = .{ .@"64bit", null, null, null },
128591 .dst_constraints = .{ .{ .int = .qword }, .any },
128592 .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any },
128593 .patterns = &.{
128594 .{ .src = .{ .to_mem, .none, .none } },
128595 },
128596 .extra_temps = .{
128597 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
128598 .unused,
128599 .unused,
128600 .unused,
128601 .unused,
128602 .unused,
128603 .unused,
128604 .unused,
128605 .unused,
128606 .unused,
128607 .unused,
128608 },
128609 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
128610 .clobbers = .{ .eflags = true },
128611 .each = .{ .once = &.{
128612 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
128613 .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ },
128614 .{ .@"0:", .i_, .mul, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
128615 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
128616 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128617 } },
128618 }, .{
128619 .required_features = .{ .sse, null, null, null },
128620 .dst_constraints = .{ .{ .int = .xword }, .any },
128621 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
128622 .patterns = &.{
128623 .{ .src = .{ .mut_mem, .none, .none } },
128624 .{ .src = .{ .mut_gpr, .none, .none } },
128625 .{ .src = .{ .to_mut_sse, .none, .none } },
128626 },
128627 .dst_temps = .{ .{ .ref = .src0 }, .unused },
128628 .each = .{ .once = &.{} },
128629 }, .{
128630 .dst_constraints = .{ .{ .int = .xword }, .any },
128631 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
128632 .patterns = &.{
128633 .{ .src = .{ .to_mut_mem, .none, .none } },
128634 },
128635 .dst_temps = .{ .{ .ref = .src0 }, .unused },
128636 .each = .{ .once = &.{} },
128637 }, .{
128638 .required_features = .{ .avx, null, null, null },
128639 .dst_constraints = .{ .{ .int = .yword }, .any },
128640 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
128641 .patterns = &.{
128642 .{ .src = .{ .mut_mem, .none, .none } },
128643 .{ .src = .{ .mut_gpr, .none, .none } },
128644 .{ .src = .{ .to_mut_sse, .none, .none } },
128645 },
128646 .dst_temps = .{ .{ .ref = .src0 }, .unused },
128647 .each = .{ .once = &.{} },
128648 }, .{
128649 .dst_constraints = .{ .{ .int = .yword }, .any },
128650 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
128651 .patterns = &.{
128652 .{ .src = .{ .to_mut_mem, .none, .none } },
128653 },
128654 .dst_temps = .{ .{ .ref = .src0 }, .unused },
128655 .each = .{ .once = &.{} },
128656 }, .{
128657 .dst_constraints = .{ .any_int, .any },
128658 .src_constraints = .{ .{ .vec_len = 1 }, .any, .any },
128659 .patterns = &.{
128660 .{ .src = .{ .to_mut_mem, .none, .none } },
128661 },
128662 .dst_temps = .{ .{ .ref = .src0 }, .unused },
128663 .each = .{ .once = &.{} },
128664 }, .{
128665 .required_features = .{ .@"64bit", .bmi2, .adx, null },
128666 .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any },
128667 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
128668 .patterns = &.{
128669 .{ .src = .{ .to_mem, .none, .none } },
128670 },
128671 .extra_temps = .{
128672 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128673 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128674 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128675 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128676 .{ .type = .u64, .kind = .{ .reg = .rdx } },
128677 .{ .type = .isize, .kind = .{ .reg = .rcx } },
128678 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128679 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128680 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128681 .unused,
128682 .unused,
128683 },
128684 .dst_temps = .{ .mem, .unused },
128685 .clobbers = .{ .eflags = true },
128686 .each = .{ .once = &.{
128687 .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ },
128688 .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ },
128689 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
128690 .{ ._, ._, .mov, .tmp3d, .sia(-8, .dst0, .add_size), ._, ._ },
128691 .{ .@"1:", ._, .xor, .tmp4d, .tmp4d, ._, ._ },
128692 .{ ._, ._, .@"or", .tmp4q, .leai(.tmp1q, .tmp3), ._, ._ },
128693 .{ ._, ._z, .j, .@"3f", ._, ._, ._ },
128694 .{ ._, ._, .lea, .tmp5p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ },
128695 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
128696 .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp2q, .tmp5), ._ },
128697 .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ },
128698 .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_elem_size, -8), .tmp7q, ._, ._ },
128699 .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ },
128700 .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_elem_size), ._, ._ },
128701 .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ },
128702 .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ },
128703 .{ ._, ._mp, .j, .@"2b", ._, ._, ._ },
128704 .{ .@"3:", ._, .mov, .memi(.dst0q, .tmp3), .tmp4q, ._, ._ },
128705 .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
128706 .{ ._, ._, .sub, .tmp3d, .si(8), ._, ._ },
128707 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
128708 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
128709 .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ },
128710 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128711 } },
128712 }, .{
128713 .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null },
128714 .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any },
128715 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
128716 .patterns = &.{
128717 .{ .src = .{ .to_mem, .none, .none } },
128718 },
128719 .extra_temps = .{
128720 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128721 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128722 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128723 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128724 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128725 .{ .type = .u64, .kind = .{ .reg = .rdx } },
128726 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
128727 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128728 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128729 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128730 .unused,
128731 },
128732 .dst_temps = .{ .mem, .unused },
128733 .clobbers = .{ .eflags = true },
128734 .each = .{ .once = &.{
128735 .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ },
128736 .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ },
128737 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
128738 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
128739 .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ },
128740 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
128741 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
128742 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
128743 .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ },
128744 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
128745 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ },
128746 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
128747 .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ },
128748 .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ },
128749 .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ },
128750 .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ },
128751 .{ .@"3:", ._x, .mul, .tmp9q, .tmp8q, .leasi(.tmp2q, .@"8", .tmp4), ._ },
128752 .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ },
128753 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ },
128754 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
128755 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
128756 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
128757 .{ ._, ._, .sub, .tmp3d, .si(1), ._, ._ },
128758 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
128759 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
128760 .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ },
128761 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128762 } },
128763 }, .{
128764 .required_features = .{ .@"64bit", .bmi2, null, null },
128765 .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any },
128766 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
128767 .patterns = &.{
128768 .{ .src = .{ .to_mem, .none, .none } },
128769 },
128770 .extra_temps = .{
128771 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128772 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128773 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128774 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128775 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128776 .{ .type = .u64, .kind = .{ .reg = .rdx } },
128777 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
128778 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128779 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128780 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128781 .unused,
128782 },
128783 .dst_temps = .{ .mem, .unused },
128784 .clobbers = .{ .eflags = true },
128785 .each = .{ .once = &.{
128786 .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ },
128787 .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ },
128788 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
128789 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
128790 .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ },
128791 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
128792 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
128793 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
128794 .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ },
128795 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
128796 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ },
128797 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
128798 .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ },
128799 .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ },
128800 .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ },
128801 .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ },
128802 .{ .@"3:", ._x, .mul, .tmp9q, .tmp8q, .leasi(.tmp2q, .@"8", .tmp4), ._ },
128803 .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ },
128804 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ },
128805 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
128806 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
128807 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
128808 .{ ._, ._c, .de, .tmp3d, ._, ._, ._ },
128809 .{ ._, ._ns, .j, .@"1b", ._, ._, ._ },
128810 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
128811 .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ },
128812 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128813 } },
128814 }, .{
128815 .required_features = .{ .@"64bit", .slow_incdec, null, null },
128816 .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any },
128817 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
128818 .patterns = &.{
128819 .{ .src = .{ .to_mem, .none, .none } },
128820 },
128821 .extra_temps = .{
128822 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128823 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128824 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128825 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128826 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128827 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128828 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
128829 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128830 .{ .type = .u64, .kind = .{ .reg = .rax } },
128831 .{ .type = .u64, .kind = .{ .reg = .rdx } },
128832 .unused,
128833 },
128834 .dst_temps = .{ .mem, .unused },
128835 .clobbers = .{ .eflags = true },
128836 .each = .{ .once = &.{
128837 .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ },
128838 .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ },
128839 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
128840 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
128841 .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ },
128842 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
128843 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
128844 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
128845 .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ },
128846 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
128847 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ },
128848 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
128849 .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ },
128850 .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ },
128851 .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ },
128852 .{ .@"3:", ._, .mov, .tmp8q, .tmp5q, ._, ._ },
128853 .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp4), ._, ._, ._ },
128854 .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ },
128855 .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ },
128856 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ },
128857 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
128858 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
128859 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
128860 .{ ._, ._, .sub, .tmp3d, .si(1), ._, ._ },
128861 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
128862 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
128863 .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ },
128864 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128865 } },
128866 }, .{
128867 .required_features = .{ .@"64bit", null, null, null },
128868 .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any },
128869 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
128870 .patterns = &.{
128871 .{ .src = .{ .to_mem, .none, .none } },
128872 },
128873 .extra_temps = .{
128874 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128875 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128876 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
128877 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128878 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
128879 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128880 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
128881 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
128882 .{ .type = .u64, .kind = .{ .reg = .rax } },
128883 .{ .type = .u64, .kind = .{ .reg = .rdx } },
128884 .unused,
128885 },
128886 .dst_temps = .{ .mem, .unused },
128887 .clobbers = .{ .eflags = true },
128888 .each = .{ .once = &.{
128889 .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ },
128890 .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ },
128891 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
128892 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
128893 .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ },
128894 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
128895 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
128896 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
128897 .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ },
128898 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
128899 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ },
128900 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
128901 .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ },
128902 .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ },
128903 .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ },
128904 .{ .@"3:", ._, .mov, .tmp8q, .tmp5q, ._, ._ },
128905 .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp4), ._, ._, ._ },
128906 .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ },
128907 .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ },
128908 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ },
128909 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
128910 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
128911 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
128912 .{ ._, ._c, .de, .tmp3d, ._, ._, ._ },
128913 .{ ._, ._ns, .j, .@"1b", ._, ._, ._ },
128914 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
128915 .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ },
128916 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
128917 } },
128918 } },
125272128919 }) catch |err| switch (err) {
125273128920 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
125274128921 @tagName(air_tag),
......@@ -141408,14 +145055,14 @@ fn airRepeat(self: *CodeGen, inst: Air.Inst.Index) !void {
141408145055}
141409145056
141410145057fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
145058 @setEvalBranchQuota(1_100);
141411145059 const pt = self.pt;
141412145060 const zcu = pt.zcu;
141413145061 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
141414145062 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
141415145063 const clobbers_len: u31 = @truncate(extra.data.flags);
141416145064 var extra_i: usize = extra.end;
141417 const outputs: []const Air.Inst.Ref =
141418 @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
145065 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
141419145066 extra_i += outputs.len;
141420145067 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);
141421145068 extra_i += inputs.len;
src/arch/x86_64/Encoding.zig+31-17
......@@ -336,7 +336,7 @@ pub const Mnemonic = enum {
336336 fcom, fcomi, fcomip, fcomp, fcompp, fcos,
337337 fdecstp, fdiv, fdivp, fdivr, fdivrp, ffree,
338338 fiadd, ficom, ficomp, fidiv, fidivr, fild, fimul, fincstp, finit,
339 fist, fistp, fisttp, fisub, fisubr,
339 fist, fistp, fisub, fisubr,
340340 fld, fld1, fldcw, fldenv, fldl2e, fldl2t, fldlg2, fldln2, fldpi, fldz,
341341 fmul, fmulp,
342342 fnclex, fninit, fnop, fnsave, fnstcw, fnstenv, fnstsw,
......@@ -349,19 +349,18 @@ pub const Mnemonic = enum {
349349 // MMX
350350 emms, movd, movq,
351351 packssdw, packsswb, packuswb,
352 paddb, paddd, paddq, paddsb, paddsw, paddusb, paddusw, paddw,
352 paddb, paddd, paddsb, paddsw, paddusb, paddusw, paddw,
353353 pand, pandn, por, pxor,
354354 pcmpeqb, pcmpeqd, pcmpeqw,
355355 pcmpgtb, pcmpgtd, pcmpgtw,
356 pmulhw, pmullw,
356 pmaddwd, pmulhw, pmullw,
357357 pslld, psllq, psllw,
358358 psrad, psraw,
359359 psrld, psrlq, psrlw,
360 psubb, psubd, psubq, psubsb, psubsw, psubusb, psubusw, psubw,
360 psubb, psubd, psubsb, psubsw, psubusb, psubusw, psubw,
361361 // SSE
362362 addps, addss,
363 andps,
364 andnps,
363 andnps, andps,
365364 cmpps, cmpss, comiss,
366365 cvtpi2ps, cvtps2pi, cvtsi2ss, cvtss2si, cvttps2pi, cvttss2si,
367366 divps, divss,
......@@ -374,9 +373,11 @@ pub const Mnemonic = enum {
374373 movss, movups,
375374 mulps, mulss,
376375 orps,
376 pavgb, pavgw,
377377 pextrw, pinsrw,
378 pmaxsw, pmaxub, pminsw, pminub, pmovmskb,
378 pmaxsw, pmaxub, pminsw, pminub, pmovmskb, pmulhuw,
379379 prefetchit0, prefetchit1, prefetchnta, prefetcht0, prefetcht1, prefetcht2, prefetchw, prefetchwt1,
380 psadbw, pshufw,
380381 shufps,
381382 sqrtps, sqrtss,
382383 stmxcsr,
......@@ -397,15 +398,16 @@ pub const Mnemonic = enum {
397398 maxpd, maxsd,
398399 minpd, minsd,
399400 movapd,
400 movdqa, movdqu,
401 movdq2q, movdqa, movdqu,
401402 movhpd, movlpd,
402 movmskpd,
403 movmskpd, movq2dq,
403404 //movsd,
404405 movupd,
405406 mulpd, mulsd,
406407 orpd,
408 paddq, pmuludq,
407409 pshufd, pshufhw, pshuflw,
408 pslldq, psrldq,
410 pslldq, psrldq, psubq,
409411 punpckhbw, punpckhdq, punpckhqdq, punpckhwd,
410412 punpcklbw, punpckldq, punpcklqdq, punpcklwd,
411413 shufpd,
......@@ -414,9 +416,17 @@ pub const Mnemonic = enum {
414416 ucomisd, unpckhpd, unpcklpd,
415417 xorpd,
416418 // SSE3
417 addsubpd, addsubps, haddpd, haddps, lddqu, movddup, movshdup, movsldup,
419 addsubpd, addsubps,
420 fisttp,
421 haddpd, haddps,
422 hsubpd, hsubps,
423 lddqu,
424 movddup, movshdup, movsldup,
418425 // SSSE3
419 pabsb, pabsd, pabsw, palignr, pshufb,
426 pabsb, pabsd, pabsw, palignr,
427 phaddw, phaddsw, phaddd, phsubw, phsubsw, phsubd,
428 pmaddubsw, pmulhrsw, pshufb,
429 psignb, psignd, psignw,
420430 // SSE4.1
421431 blendpd, blendps, blendvpd, blendvps,
422432 dppd, dpps,
......@@ -430,7 +440,7 @@ pub const Mnemonic = enum {
430440 pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,
431441 pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,
432442 pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,
433 pmulld,
443 pmuldq, pmulld,
434444 ptest,
435445 roundpd, roundps, roundsd, roundss,
436446 // SSE4.2
......@@ -458,7 +468,7 @@ pub const Mnemonic = enum {
458468 vdppd, vdpps,
459469 vextractf128, vextractps,
460470 vgf2p8affineinvqb, vgf2p8affineqb, vgf2p8mulb,
461 vhaddpd, vhaddps,
471 vhaddpd, vhaddps, vhsubpd, vhsubps,
462472 vinsertf128, vinsertps,
463473 vlddqu, vldmxcsr,
464474 vmaskmovpd, vmaskmovps,
......@@ -480,21 +490,24 @@ pub const Mnemonic = enum {
480490 vpabsb, vpabsd, vpabsw,
481491 vpackssdw, vpacksswb, vpackusdw, vpackuswb,
482492 vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
483 vpalignr, vpand, vpandn,
493 vpalignr, vpand, vpandn, vpavgb, vpavgw,
484494 vpblendvb, vpblendw, vpclmulqdq,
485495 vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw,
486496 vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw,
497 vphaddw, vphaddsw, vphaddd, vphsubw, vphsubsw, vphsubd,
487498 vperm2f128, vpermilpd, vpermilps,
488499 vpextrb, vpextrd, vpextrq, vpextrw,
489500 vpinsrb, vpinsrd, vpinsrq, vpinsrw,
490501 vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw,
491502 vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw,
503 vpmaddubsw,
492504 vpmovmskb,
493505 vpmovsxbd, vpmovsxbq, vpmovsxbw, vpmovsxdq, vpmovsxwd, vpmovsxwq,
494506 vpmovzxbd, vpmovzxbq, vpmovzxbw, vpmovzxdq, vpmovzxwd, vpmovzxwq,
495 vpmulhw, vpmulld, vpmullw,
507 vpmuldq, vpmulhrsw, vpmulhw, vpmulld, vpmullw, vpmuludq,
496508 vpor,
497509 vpshufb, vpshufd, vpshufhw, vpshuflw,
510 vpsignb, vpsignd, vpsignw,
498511 vpslld, vpslldq, vpsllq, vpsllw,
499512 vpsrad, vpsraq, vpsraw,
500513 vpsrld, vpsrldq, vpsrlq, vpsrlw,
......@@ -779,7 +792,7 @@ pub const Op = enum {
779792 pub fn isImmediate(op: Op) bool {
780793 // zig fmt: off
781794 return switch (op) {
782 .imm8, .imm16, .imm32, .imm64,
795 .imm8, .imm16, .imm32, .imm64,
783796 .imm8s, .imm16s, .imm32s,
784797 .rel8, .rel16, .rel32,
785798 .unity,
......@@ -986,6 +999,7 @@ pub const Feature = enum {
986999 sse,
9871000 sse2,
9881001 sse3,
1002 @"sse3 x87",
9891003 sse4_1,
9901004 sse4_2,
9911005 ssse3,
src/arch/x86_64/Lower.zig+1-1
......@@ -567,7 +567,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
567567}
568568
569569fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
570 @setEvalBranchQuota(2_500);
570 @setEvalBranchQuota(2_600);
571571 const fixes = switch (inst.ops) {
572572 .none => inst.data.none.fixes,
573573 .inst => inst.data.inst.fixes,
src/arch/x86_64/Mir.zig+8
......@@ -386,7 +386,10 @@ pub const Inst = struct {
386386 /// Packed ___ Quadword
387387 p_q,
388388 /// Packed ___ Double Quadword
389 /// Packed ___ Doubleword to Quadword
389390 p_dq,
391 /// Packed ___ Unsigned Doubleword to Quadword
392 p_udq,
390393 /// ___ Aligned Packed Integer Values
391394 _dqa,
392395 /// ___ Unaligned Packed Integer Values
......@@ -446,7 +449,10 @@ pub const Inst = struct {
446449 /// VEX-Encoded Packed ___ Quadword
447450 vp_q,
448451 /// VEX-Encoded Packed ___ Double Quadword
452 /// VEX-Encoded Packed ___ Doubleword to Quadword
449453 vp_dq,
454 /// VEX-Encoded Packed ___ Unsigned Doubleword to Quadword
455 vp_udq,
450456 /// VEX-Encoded ___ Scalar Single-Precision Values
451457 v_ss,
452458 /// VEX-Encoded ___ Packed Single-Precision Values
......@@ -663,6 +669,8 @@ pub const Inst = struct {
663669 /// Multiply scalar single-precision floating-point values
664670 /// Multiply packed double-precision floating-point values
665671 /// Multiply scalar double-precision floating-point values
672 /// Multiply packed unsigned doubleword integers
673 /// Multiply packed doubleword integers
666674 mul,
667675 /// Two's complement negation
668676 neg,
src/arch/x86_64/encodings.zon+16-4
......@@ -1160,10 +1160,6 @@
11601160 .{ .fistp, .m, .{ .m32 }, .{ 0xdb }, 3, .none, .x87 },
11611161 .{ .fistp, .m, .{ .m64 }, .{ 0xdf }, 7, .none, .x87 },
11621162
1163 .{ .fisttp, .m, .{ .m16 }, .{ 0xdf }, 1, .none, .x87 },
1164 .{ .fisttp, .m, .{ .m32 }, .{ 0xdb }, 1, .none, .x87 },
1165 .{ .fisttp, .m, .{ .m64 }, .{ 0xdd }, 1, .none, .x87 },
1166
11671163 .{ .fld, .m, .{ .m32 }, .{ 0xd9 }, 0, .none, .x87 },
11681164 .{ .fld, .m, .{ .m64 }, .{ 0xdd }, 0, .none, .x87 },
11691165 .{ .fld, .m, .{ .m80 }, .{ 0xdb }, 5, .none, .x87 },
......@@ -1540,6 +1536,8 @@
15401536
15411537 .{ .pmullw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd5 }, 0, .none, .sse2 },
15421538
1539 .{ .pmuludq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf4 }, 0, .none, .sse2 },
1540
15431541 .{ .por, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xeb }, 0, .none, .sse2 },
15441542
15451543 .{ .pshufd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x70 }, 0, .none, .sse2 },
......@@ -1618,6 +1616,10 @@
16181616
16191617 .{ .addsubps, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xd0 }, 0, .none, .sse3 },
16201618
1619 .{ .fisttp, .m, .{ .m16 }, .{ 0xdf }, 1, .none, .@"sse3 x87" },
1620 .{ .fisttp, .m, .{ .m32 }, .{ 0xdb }, 1, .none, .@"sse3 x87" },
1621 .{ .fisttp, .m, .{ .m64 }, .{ 0xdd }, 1, .none, .@"sse3 x87" },
1622
16211623 .{ .haddpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x7c }, 0, .none, .sse3 },
16221624
16231625 .{ .haddps, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0x7c }, 0, .none, .sse3 },
......@@ -1708,6 +1710,8 @@
17081710 .{ .pmovzxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .none, .sse4_1 },
17091711 .{ .pmovzxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .none, .sse4_1 },
17101712
1713 .{ .pmuldq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .none, .sse4_1 },
1714
17111715 .{ .pmulld, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
17121716
17131717 .{ .ptest, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x17 }, 0, .none, .sse4_1 },
......@@ -2166,12 +2170,16 @@
21662170 .{ .vpmovzxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_128_wig, .avx },
21672171 .{ .vpmovzxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_128_wig, .avx },
21682172
2173 .{ .vpmuldq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .vex_128_wig, .avx },
2174
21692175 .{ .vpmulhw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },
21702176
21712177 .{ .vpmulld, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },
21722178
21732179 .{ .vpmullw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd5 }, 0, .vex_128_wig, .avx },
21742180
2181 .{ .vpmuludq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf4 }, 0, .vex_128_wig, .avx },
2182
21752183 .{ .vpor, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },
21762184
21772185 .{ .vpshufb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx },
......@@ -2493,12 +2501,16 @@
24932501 .{ .vpmovzxwq, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_256_wig, .avx2 },
24942502 .{ .vpmovzxdq, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_256_wig, .avx2 },
24952503
2504 .{ .vpmuldq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .vex_256_wig, .avx2 },
2505
24962506 .{ .vpmulhw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 },
24972507
24982508 .{ .vpmulld, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 },
24992509
25002510 .{ .vpmullw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd5 }, 0, .vex_256_wig, .avx2 },
25012511
2512 .{ .vpmuludq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf4 }, 0, .vex_256_wig, .avx2 },
2513
25022514 .{ .vpor, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },
25032515
25042516 .{ .vpshufb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 },
test/behavior/x86_64/unary.zig+8
......@@ -4889,6 +4889,14 @@ test reduceAdd {
48894889 try test_reduce_add.testIntVectors();
48904890}
48914891
4892inline fn reduceMul(comptime Type: type, rhs: Type) @typeInfo(Type).vector.child {
4893 return @reduce(.Mul, rhs);
4894}
4895test reduceMul {
4896 const test_reduce_mul = unary(reduceMul, .{});
4897 try test_reduce_mul.testIntVectors();
4898}
4899
48924900inline fn splat(comptime Type: type, rhs: Type) Type {
48934901 return @splat(rhs[0]);
48944902}