authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-31 21:36:42-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-31 23:09:58-05:00
log4c5abe5ac664a7214d822b09703682a5cac6891c
treed16fa178bebd33c237e33aff0fd445b0950634b4
parentf0ac14ce97c4fa7f10ddd0d74a6dfade0a04cbc1

x86_64: rewrite vector `@intCast`

4 files changed, 4080 insertions(+), 382 deletions(-)

build.zig+1-1
......@@ -450,7 +450,7 @@ pub fn build(b: *std.Build) !void {
450450 .skip_non_native = skip_non_native,
451451 .skip_libc = skip_libc,
452452 .use_llvm = use_llvm,
453 .max_rss = 1 * 1024 * 1024 * 1024,
453 .max_rss = 1.25 * 1024 * 1024 * 1024,
454454 }));
455455
456456 test_modules_step.dependOn(tests.addModuleTests(b, .{
src/arch/x86_64/CodeGen.zig+3991-340
......@@ -2414,7 +2414,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24142414}
24152415
24162416fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2417 @setEvalBranchQuota(4_100);
2417 @setEvalBranchQuota(4_500);
24182418 const pt = cg.pt;
24192419 const zcu = pt.zcu;
24202420 const ip = &zcu.intern_pool;
......@@ -21900,7 +21900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2190021900 .patterns = &.{
2190121901 .{ .src = .{ .to_sse, .none } },
2190221902 },
21903 .dst_temps = .{.{ .rc = .sse }},
21903 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2190421904 .each = .{ .once = &.{
2190521905 .{ ._, .v_, .cvtps2ph, .dst0q, .src0x, .rm(.{}), ._ },
2190621906 } },
......@@ -21911,7 +21911,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2191121911 .patterns = &.{
2191221912 .{ .src = .{ .to_sse, .none } },
2191321913 },
21914 .dst_temps = .{.{ .rc = .sse }},
21914 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2191521915 .each = .{ .once = &.{
2191621916 .{ ._, .v_, .cvtps2ph, .dst0q, .src0x, .rm(.{}), ._ },
2191721917 } },
......@@ -21922,7 +21922,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2192221922 .patterns = &.{
2192321923 .{ .src = .{ .to_sse, .none } },
2192421924 },
21925 .dst_temps = .{.{ .rc = .sse }},
21925 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2192621926 .each = .{ .once = &.{
2192721927 .{ ._, .v_, .cvtps2ph, .dst0x, .src0y, .rm(.{}), ._ },
2192821928 } },
......@@ -22259,7 +22259,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2225922259 .patterns = &.{
2226022260 .{ .src = .{ .to_sse, .none } },
2226122261 },
22262 .dst_temps = .{.{ .rc = .sse }},
22262 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2226322263 .each = .{ .once = &.{
2226422264 .{ ._, .v_ss, .cvtsd2, .dst0x, .src0x, .src0q, ._ },
2226522265 } },
......@@ -22307,7 +22307,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2230722307 .{ .src = .{ .mem, .none } },
2230822308 .{ .src = .{ .to_sse, .none } },
2230922309 },
22310 .dst_temps = .{.{ .rc = .sse }},
22310 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2231122311 .each = .{ .once = &.{
2231222312 .{ ._, .v_ps, .cvtpd2, .dst0x, .src0x, ._, ._ },
2231322313 } },
......@@ -22319,7 +22319,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2231922319 .{ .src = .{ .mem, .none } },
2232022320 .{ .src = .{ .to_sse, .none } },
2232122321 },
22322 .dst_temps = .{.{ .rc = .sse }},
22322 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2232322323 .each = .{ .once = &.{
2232422324 .{ ._, ._ps, .cvtpd2, .dst0x, .src0x, ._, ._ },
2232522325 } },
......@@ -22331,7 +22331,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2233122331 .{ .src = .{ .mem, .none } },
2233222332 .{ .src = .{ .to_sse, .none } },
2233322333 },
22334 .dst_temps = .{.{ .rc = .sse }},
22334 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2233522335 .each = .{ .once = &.{
2233622336 .{ ._, .v_ps, .cvtpd2, .dst0x, .src0y, ._, ._ },
2233722337 } },
......@@ -23220,7 +23220,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2322023220 .patterns = &.{
2322123221 .{ .src = .{ .to_sse, .none } },
2322223222 },
23223 .dst_temps = .{.{ .rc = .sse }},
23223 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2322423224 .each = .{ .once = &.{
2322523225 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
2322623226 } },
......@@ -23232,7 +23232,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2323223232 .{ .src = .{ .mem, .none } },
2323323233 .{ .src = .{ .to_sse, .none } },
2323423234 },
23235 .dst_temps = .{.{ .rc = .sse }},
23235 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2323623236 .each = .{ .once = &.{
2323723237 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
2323823238 } },
......@@ -23244,7 +23244,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2324423244 .{ .src = .{ .mem, .none } },
2324523245 .{ .src = .{ .to_sse, .none } },
2324623246 },
23247 .dst_temps = .{.{ .rc = .sse }},
23247 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2324823248 .each = .{ .once = &.{
2324923249 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
2325023250 } },
......@@ -23397,7 +23397,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2339723397 .patterns = &.{
2339823398 .{ .src = .{ .to_sse, .none } },
2339923399 },
23400 .dst_temps = .{.{ .rc = .sse }},
23400 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2340123401 .each = .{ .once = &.{
2340223402 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
2340323403 .{ ._, .v_sd, .cvtss2, .dst0x, .dst0x, .dst0d, ._ },
......@@ -23410,7 +23410,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2341023410 .{ .src = .{ .mem, .none } },
2341123411 .{ .src = .{ .to_sse, .none } },
2341223412 },
23413 .dst_temps = .{.{ .rc = .sse }},
23413 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2341423414 .each = .{ .once = &.{
2341523415 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
2341623416 .{ ._, .v_pd, .cvtps2, .dst0x, .dst0q, ._, ._ },
......@@ -23423,7 +23423,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2342323423 .{ .src = .{ .mem, .none } },
2342423424 .{ .src = .{ .to_sse, .none } },
2342523425 },
23426 .dst_temps = .{.{ .rc = .sse }},
23426 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2342723427 .each = .{ .once = &.{
2342823428 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
2342923429 .{ ._, .v_pd, .cvtps2, .dst0y, .dst0x, ._, ._ },
......@@ -23610,7 +23610,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2361023610 .{ .src = .{ .to_sse, .none } },
2361123611 },
2361223612 .extra_temps = .{
23613 .{ .type = .f32, .kind = .{ .rc = .sse } },
23613 .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
2361423614 .{ .type = .f32, .kind = .mem },
2361523615 .{ .type = .f80, .kind = .{ .reg = .st7 } },
2361623616 .unused,
......@@ -23879,7 +23879,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2387923879 .patterns = &.{
2388023880 .{ .src = .{ .to_sse, .none } },
2388123881 },
23882 .dst_temps = .{.{ .rc = .sse }},
23882 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2388323883 .each = .{ .once = &.{
2388423884 .{ ._, .v_sd, .cvtss2, .dst0x, .src0x, .src0d, ._ },
2388523885 } },
......@@ -23927,7 +23927,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2392723927 .{ .src = .{ .mem, .none } },
2392823928 .{ .src = .{ .to_sse, .none } },
2392923929 },
23930 .dst_temps = .{.{ .rc = .sse }},
23930 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2393123931 .each = .{ .once = &.{
2393223932 .{ ._, .v_pd, .cvtps2, .dst0x, .src0q, ._, ._ },
2393323933 } },
......@@ -23939,7 +23939,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2393923939 .{ .src = .{ .mem, .none } },
2394023940 .{ .src = .{ .to_sse, .none } },
2394123941 },
23942 .dst_temps = .{.{ .rc = .sse }},
23942 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2394323943 .each = .{ .once = &.{
2394423944 .{ ._, ._pd, .cvtps2, .dst0x, .src0q, ._, ._ },
2394523945 } },
......@@ -23951,7 +23951,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2395123951 .{ .src = .{ .mem, .none } },
2395223952 .{ .src = .{ .to_sse, .none } },
2395323953 },
23954 .dst_temps = .{.{ .rc = .sse }},
23954 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2395523955 .each = .{ .once = &.{
2395623956 .{ ._, .v_pd, .cvtps2, .dst0y, .src0x, ._, ._ },
2395723957 } },
......@@ -24536,34 +24536,29 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2453624536 };
2453724537 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
2453824538 },
24539 .intcast => |air_tag| if (use_old) try cg.airIntCast(inst) else fallback: {
24539 .intcast => |air_tag| if (use_old) try cg.airIntCast(inst) else {
2454024540 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
2454124541 const dst_ty = ty_op.ty.toType();
24542 if (dst_ty.isVector(zcu)) break :fallback try cg.airIntCast(inst);
2454324542 const src_ty = cg.typeOf(ty_op.operand);
2454424543 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
2454524544 var res: [1]Temp = undefined;
2454624545 cg.select(&res, &.{dst_ty}, &ops, if (dst_ty.scalarType(zcu).abiSize(zcu) <= src_ty.scalarType(zcu).abiSize(zcu)) comptime &.{ .{
2454724546 .dst_constraints = .{.{ .int = .dword }},
2454824547 .patterns = &.{
24549 .{ .src = .{ .mem, .none } },
24550 .{ .src = .{ .to_gpr, .none } },
24548 .{ .src = .{ .mut_mem, .none } },
24549 .{ .src = .{ .to_mut_gpr, .none } },
2455124550 },
24552 .dst_temps = .{.{ .rc = .general_purpose }},
24553 .each = .{ .once = &.{
24554 .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ },
24555 } },
24551 .dst_temps = .{.{ .ref = .src0 }},
24552 .each = .{ .once = &.{} },
2455624553 }, .{
2455724554 .required_features = .{ .@"64bit", null, null, null },
2455824555 .dst_constraints = .{.{ .int = .qword }},
2455924556 .patterns = &.{
24560 .{ .src = .{ .mem, .none } },
24561 .{ .src = .{ .to_gpr, .none } },
24557 .{ .src = .{ .mut_mem, .none } },
24558 .{ .src = .{ .to_mut_gpr, .none } },
2456224559 },
24563 .dst_temps = .{.{ .rc = .general_purpose }},
24564 .each = .{ .once = &.{
24565 .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ },
24566 } },
24560 .dst_temps = .{.{ .ref = .src0 }},
24561 .each = .{ .once = &.{} },
2456724562 }, .{
2456824563 .dst_constraints = .{.{ .int = .byte }},
2456924564 .patterns = &.{
......@@ -24603,87 +24598,3690 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2460324598 } },
2460424599 }, .{
2460524600 .required_features = .{ .@"64bit", null, null, null },
24606 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
24601 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
24602 .patterns = &.{
24603 .{ .src = .{ .to_mem, .none } },
24604 },
24605 .extra_temps = .{
24606 .{ .type = .usize, .kind = .{ .reg = .rsi } },
24607 .{ .type = .usize, .kind = .{ .reg = .rdi } },
24608 .{ .type = .u32, .kind = .{ .reg = .ecx } },
24609 .unused,
24610 .unused,
24611 .unused,
24612 .unused,
24613 .unused,
24614 .unused,
24615 },
24616 .dst_temps = .{.mem},
24617 .each = .{ .once = &.{
24618 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
24619 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24620 .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ },
24621 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
24622 } },
24623 }, .{
24624 .required_features = .{ .sse, null, null, null },
24625 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },
24626 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .byte } }},
24627 .patterns = &.{
24628 .{ .src = .{ .mut_mem, .none } },
24629 .{ .src = .{ .to_mut_sse, .none } },
24630 },
24631 .dst_temps = .{.{ .ref = .src0 }},
24632 .each = .{ .once = &.{} },
24633 }, .{
24634 .required_features = .{ .avx, null, null, null },
24635 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any },
24636 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .byte } }},
24637 .patterns = &.{
24638 .{ .src = .{ .mut_mem, .none } },
24639 .{ .src = .{ .to_mut_sse, .none } },
24640 },
24641 .dst_temps = .{.{ .ref = .src0 }},
24642 .each = .{ .once = &.{} },
24643 }, .{
24644 .required_features = .{ .avx, null, null, null },
24645 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
24646 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }},
24647 .patterns = &.{
24648 .{ .src = .{ .to_sse, .none } },
24649 },
24650 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24651 .each = .{ .once = &.{
24652 .{ ._, .vp_b, .ackssw, .dst0x, .src0x, .dst0x, ._ },
24653 } },
24654 }, .{
24655 .required_features = .{ .avx, null, null, null },
24656 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
24657 .dst_constraints = .{.{ .scalar_int = .{ .of = .qword, .is = .byte } }},
24658 .patterns = &.{
24659 .{ .src = .{ .to_sse, .none } },
24660 },
24661 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24662 .each = .{ .once = &.{
24663 .{ ._, .vp_b, .ackusw, .dst0x, .src0x, .dst0x, ._ },
24664 } },
24665 }, .{
24666 .required_features = .{ .sse2, null, null, null },
24667 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
24668 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }},
24669 .patterns = &.{
24670 .{ .src = .{ .to_mut_sse, .none } },
24671 },
24672 .dst_temps = .{.{ .ref = .src0 }},
24673 .each = .{ .once = &.{
24674 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
24675 } },
24676 }, .{
24677 .required_features = .{ .sse2, null, null, null },
24678 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
24679 .dst_constraints = .{.{ .scalar_int = .{ .of = .qword, .is = .byte } }},
24680 .patterns = &.{
24681 .{ .src = .{ .to_mut_sse, .none } },
24682 },
24683 .dst_temps = .{.{ .ref = .src0 }},
24684 .each = .{ .once = &.{
24685 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
24686 } },
24687 }, .{
24688 .required_features = .{ .avx2, null, null, null },
24689 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any },
24690 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }},
24691 .patterns = &.{
24692 .{ .src = .{ .to_sse, .none } },
24693 },
24694 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24695 .each = .{ .once = &.{
24696 .{ ._, .vp_b, .ackssw, .dst0y, .src0y, .dst0y, ._ },
24697 } },
24698 }, .{
24699 .required_features = .{ .avx2, null, null, null },
24700 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any },
24701 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .byte } }},
24702 .patterns = &.{
24703 .{ .src = .{ .to_sse, .none } },
24704 },
24705 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24706 .each = .{ .once = &.{
24707 .{ ._, .vp_b, .ackusw, .dst0y, .src0y, .dst0y, ._ },
24708 } },
24709 }, .{
24710 .required_features = .{ .slow_incdec, null, null, null },
24711 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
24712 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24713 .patterns = &.{
24714 .{ .src = .{ .to_mem, .none } },
24715 },
24716 .extra_temps = .{
24717 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24718 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24719 .unused,
24720 .unused,
24721 .unused,
24722 .unused,
24723 .unused,
24724 .unused,
24725 .unused,
24726 },
24727 .dst_temps = .{.mem},
24728 .each = .{ .once = &.{
24729 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24730 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
24731 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
24732 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
24733 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
24734 } },
24735 }, .{
24736 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
24737 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24738 .patterns = &.{
24739 .{ .src = .{ .to_mem, .none } },
24740 },
24741 .extra_temps = .{
24742 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24743 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24744 .unused,
24745 .unused,
24746 .unused,
24747 .unused,
24748 .unused,
24749 .unused,
24750 .unused,
24751 },
24752 .dst_temps = .{.mem},
24753 .each = .{ .once = &.{
24754 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24755 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
24756 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
24757 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
24758 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
24759 } },
24760 }, .{
24761 .required_features = .{ .avx, null, null, null },
24762 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
24763 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }},
24764 .patterns = &.{
24765 .{ .src = .{ .to_sse, .none } },
24766 },
24767 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24768 .each = .{ .once = &.{
24769 .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ },
24770 .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ },
24771 } },
24772 }, .{
24773 .required_features = .{ .avx, null, null, null },
24774 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
24775 .dst_constraints = .{.{ .scalar_int = .{ .of = .dword, .is = .byte } }},
24776 .patterns = &.{
24777 .{ .src = .{ .to_sse, .none } },
24778 },
24779 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24780 .each = .{ .once = &.{
24781 .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ },
24782 .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ },
24783 } },
24784 }, .{
24785 .required_features = .{ .sse2, null, null, null },
24786 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
24787 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }},
24788 .patterns = &.{
24789 .{ .src = .{ .to_mut_sse, .none } },
24790 },
24791 .dst_temps = .{.{ .ref = .src0 }},
24792 .each = .{ .once = &.{
24793 .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ },
24794 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
24795 } },
24796 }, .{
24797 .required_features = .{ .sse4_1, null, null, null },
24798 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
24799 .dst_constraints = .{.{ .scalar_int = .{ .of = .dword, .is = .byte } }},
24800 .patterns = &.{
24801 .{ .src = .{ .to_mut_sse, .none } },
24802 },
24803 .dst_temps = .{.{ .ref = .src0 }},
24804 .each = .{ .once = &.{
24805 .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ },
24806 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
24807 } },
24808 }, .{
24809 .required_features = .{ .avx2, null, null, null },
24810 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any },
24811 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }},
24812 .patterns = &.{
24813 .{ .src = .{ .to_sse, .none } },
24814 },
24815 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24816 .each = .{ .once = &.{
24817 .{ ._, .vp_w, .ackssd, .dst0y, .src0y, .dst0y, ._ },
24818 .{ ._, .vp_b, .ackssw, .dst0y, .dst0y, .dst0y, ._ },
24819 } },
24820 }, .{
24821 .required_features = .{ .avx2, null, null, null },
24822 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any },
24823 .dst_constraints = .{.{ .scalar_int = .{ .of = .qword, .is = .byte } }},
24824 .patterns = &.{
24825 .{ .src = .{ .to_sse, .none } },
24826 },
24827 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
24828 .each = .{ .once = &.{
24829 .{ ._, .vp_w, .ackusd, .dst0y, .src0y, .dst0y, ._ },
24830 .{ ._, .vp_b, .ackusw, .dst0y, .dst0y, .dst0y, ._ },
24831 } },
24832 }, .{
24833 .required_features = .{ .slow_incdec, null, null, null },
24834 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any },
24835 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24836 .patterns = &.{
24837 .{ .src = .{ .to_mem, .none } },
24838 },
24839 .extra_temps = .{
24840 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24841 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24842 .unused,
24843 .unused,
24844 .unused,
24845 .unused,
24846 .unused,
24847 .unused,
24848 .unused,
24849 },
24850 .dst_temps = .{.mem},
24851 .each = .{ .once = &.{
24852 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24853 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ },
24854 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
24855 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
24856 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
24857 } },
24858 }, .{
24859 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any },
24860 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24861 .patterns = &.{
24862 .{ .src = .{ .to_mem, .none } },
24863 },
24864 .extra_temps = .{
24865 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24866 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24867 .unused,
24868 .unused,
24869 .unused,
24870 .unused,
24871 .unused,
24872 .unused,
24873 .unused,
24874 },
24875 .dst_temps = .{.mem},
24876 .each = .{ .once = &.{
24877 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24878 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ },
24879 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
24880 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
24881 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
24882 } },
24883 }, .{
24884 .required_features = .{ .slow_incdec, null, null, null },
24885 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any },
24886 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24887 .patterns = &.{
24888 .{ .src = .{ .to_mem, .none } },
24889 },
24890 .extra_temps = .{
24891 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24892 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24893 .unused,
24894 .unused,
24895 .unused,
24896 .unused,
24897 .unused,
24898 .unused,
24899 .unused,
24900 },
24901 .dst_temps = .{.mem},
24902 .each = .{ .once = &.{
24903 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24904 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ },
24905 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
24906 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
24907 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
24908 } },
24909 }, .{
24910 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any },
24911 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24912 .patterns = &.{
24913 .{ .src = .{ .to_mem, .none } },
24914 },
24915 .extra_temps = .{
24916 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24917 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24918 .unused,
24919 .unused,
24920 .unused,
24921 .unused,
24922 .unused,
24923 .unused,
24924 .unused,
24925 },
24926 .dst_temps = .{.mem},
24927 .each = .{ .once = &.{
24928 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24929 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ },
24930 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
24931 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
24932 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
24933 } },
24934 }, .{
24935 .required_features = .{ .slow_incdec, null, null, null },
24936 .src_constraints = .{ .any_scalar_int, .any },
24937 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24938 .patterns = &.{
24939 .{ .src = .{ .to_mem, .none } },
24940 },
24941 .extra_temps = .{
24942 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24943 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
24944 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24945 .unused,
24946 .unused,
24947 .unused,
24948 .unused,
24949 .unused,
24950 .unused,
24951 },
24952 .dst_temps = .{.mem},
24953 .each = .{ .once = &.{
24954 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24955 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
24956 .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ },
24957 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ },
24958 .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ },
24959 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
24960 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
24961 } },
24962 }, .{
24963 .src_constraints = .{ .any_scalar_int, .any },
24964 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }},
24965 .patterns = &.{
24966 .{ .src = .{ .to_mem, .none } },
24967 },
24968 .extra_temps = .{
24969 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24970 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
24971 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
24972 .unused,
24973 .unused,
24974 .unused,
24975 .unused,
24976 .unused,
24977 .unused,
24978 },
24979 .dst_temps = .{.mem},
24980 .each = .{ .once = &.{
24981 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
24982 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
24983 .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ },
24984 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ },
24985 .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ },
24986 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
24987 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
24988 } },
24989 }, .{
24990 .required_features = .{ .sse, null, null, null },
24991 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
24992 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .word } }},
24993 .patterns = &.{
24994 .{ .src = .{ .mut_mem, .none } },
24995 .{ .src = .{ .to_mut_sse, .none } },
24996 },
24997 .dst_temps = .{.{ .ref = .src0 }},
24998 .each = .{ .once = &.{} },
24999 }, .{
25000 .required_features = .{ .avx, null, null, null },
25001 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any },
25002 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .word } }},
25003 .patterns = &.{
25004 .{ .src = .{ .mut_mem, .none } },
25005 .{ .src = .{ .to_mut_sse, .none } },
25006 },
25007 .dst_temps = .{.{ .ref = .src0 }},
25008 .each = .{ .once = &.{} },
25009 }, .{
25010 .required_features = .{ .avx, null, null, null },
25011 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
25012 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .qword, .is = .word } }},
25013 .patterns = &.{
25014 .{ .src = .{ .to_sse, .none } },
25015 },
25016 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
25017 .each = .{ .once = &.{
25018 .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ },
25019 } },
25020 }, .{
25021 .required_features = .{ .avx, null, null, null },
25022 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
25023 .dst_constraints = .{.{ .scalar_int = .{ .of = .qword, .is = .word } }},
25024 .patterns = &.{
25025 .{ .src = .{ .to_sse, .none } },
25026 },
25027 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
25028 .each = .{ .once = &.{
25029 .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ },
25030 } },
25031 }, .{
25032 .required_features = .{ .sse2, null, null, null },
25033 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
25034 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .qword, .is = .word } }},
25035 .patterns = &.{
25036 .{ .src = .{ .to_mut_sse, .none } },
25037 },
25038 .dst_temps = .{.{ .ref = .src0 }},
25039 .each = .{ .once = &.{
25040 .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ },
25041 } },
25042 }, .{
25043 .required_features = .{ .sse4_1, null, null, null },
25044 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
25045 .dst_constraints = .{.{ .scalar_int = .{ .of = .qword, .is = .word } }},
25046 .patterns = &.{
25047 .{ .src = .{ .to_mut_sse, .none } },
25048 },
25049 .dst_temps = .{.{ .ref = .src0 }},
25050 .each = .{ .once = &.{
25051 .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ },
25052 } },
25053 }, .{
25054 .required_features = .{ .avx2, null, null, null },
25055 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any },
25056 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .word } }},
25057 .patterns = &.{
25058 .{ .src = .{ .to_sse, .none } },
25059 },
25060 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
25061 .each = .{ .once = &.{
25062 .{ ._, .vp_w, .ackssd, .dst0y, .src0y, .dst0y, ._ },
25063 } },
25064 }, .{
25065 .required_features = .{ .avx2, null, null, null },
25066 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any },
25067 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .word } }},
25068 .patterns = &.{
25069 .{ .src = .{ .to_sse, .none } },
25070 },
25071 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
25072 .each = .{ .once = &.{
25073 .{ ._, .vp_w, .ackusd, .dst0y, .src0y, .dst0y, ._ },
25074 } },
25075 }, .{
25076 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any },
25077 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .word, .is = .word } }},
25078 .patterns = &.{
25079 .{ .src = .{ .to_mem, .none } },
25080 },
25081 .extra_temps = .{
25082 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25083 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
25084 .unused,
25085 .unused,
25086 .unused,
25087 .unused,
25088 .unused,
25089 .unused,
25090 .unused,
25091 },
25092 .dst_temps = .{.mem},
25093 .each = .{ .once = &.{
25094 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25095 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
25096 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
25097 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
25098 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25099 } },
25100 }, .{
25101 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any },
25102 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .word, .is = .word } }},
25103 .patterns = &.{
25104 .{ .src = .{ .to_mem, .none } },
25105 },
25106 .extra_temps = .{
25107 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25108 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
25109 .unused,
25110 .unused,
25111 .unused,
25112 .unused,
25113 .unused,
25114 .unused,
25115 .unused,
25116 },
25117 .dst_temps = .{.mem},
25118 .each = .{ .once = &.{
25119 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25120 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ },
25121 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
25122 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
25123 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25124 } },
25125 }, .{
25126 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any },
25127 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .word, .is = .word } }},
25128 .patterns = &.{
25129 .{ .src = .{ .to_mem, .none } },
25130 },
25131 .extra_temps = .{
25132 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25133 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
25134 .unused,
25135 .unused,
25136 .unused,
25137 .unused,
25138 .unused,
25139 .unused,
25140 .unused,
25141 },
25142 .dst_temps = .{.mem},
25143 .each = .{ .once = &.{
25144 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25145 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ },
25146 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
25147 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
25148 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25149 } },
25150 }, .{
25151 .src_constraints = .{ .any_scalar_int, .any },
25152 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .word, .is = .word } }},
25153 .patterns = &.{
25154 .{ .src = .{ .to_mem, .none } },
25155 },
25156 .extra_temps = .{
25157 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25158 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25159 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
25160 .unused,
25161 .unused,
25162 .unused,
25163 .unused,
25164 .unused,
25165 .unused,
25166 },
25167 .dst_temps = .{.mem},
25168 .each = .{ .once = &.{
25169 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25170 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
25171 .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ },
25172 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ },
25173 .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ },
25174 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
25175 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25176 } },
25177 }, .{
25178 .required_features = .{ .sse, null, null, null },
25179 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
25180 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .dword } }},
25181 .patterns = &.{
25182 .{ .src = .{ .mut_mem, .none } },
25183 .{ .src = .{ .to_mut_sse, .none } },
25184 },
25185 .dst_temps = .{.{ .ref = .src0 }},
25186 .each = .{ .once = &.{} },
25187 }, .{
25188 .required_features = .{ .avx, null, null, null },
25189 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any },
25190 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .dword } }},
25191 .patterns = &.{
25192 .{ .src = .{ .mut_mem, .none } },
25193 .{ .src = .{ .to_mut_sse, .none } },
25194 },
25195 .dst_temps = .{.{ .ref = .src0 }},
25196 .each = .{ .once = &.{} },
25197 }, .{
25198 .required_features = .{ .avx, null, null, null },
25199 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any },
25200 .dst_constraints = .{.{ .scalar_int = .{ .of = .qword, .is = .dword } }},
25201 .patterns = &.{
25202 .{ .src = .{ .mem, .none } },
25203 .{ .src = .{ .to_sse, .none } },
25204 },
25205 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
25206 .each = .{ .once = &.{
25207 .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b10_00_10_00), ._ },
25208 } },
25209 }, .{
25210 .required_features = .{ .sse2, null, null, null },
25211 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any },
25212 .dst_constraints = .{.{ .scalar_int = .{ .of = .qword, .is = .dword } }},
25213 .patterns = &.{
25214 .{ .src = .{ .mem, .none } },
25215 .{ .src = .{ .to_sse, .none } },
25216 },
25217 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
25218 .each = .{ .once = &.{
25219 .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b10_00_10_00), ._ },
25220 } },
25221 }, .{
25222 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any },
25223 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }},
25224 .patterns = &.{
25225 .{ .src = .{ .to_mem, .none } },
25226 },
25227 .extra_temps = .{
25228 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25229 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25230 .unused,
25231 .unused,
25232 .unused,
25233 .unused,
25234 .unused,
25235 .unused,
25236 .unused,
25237 },
25238 .dst_temps = .{.mem},
25239 .each = .{ .once = &.{
25240 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25241 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ },
25242 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
25243 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
25244 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25245 } },
25246 }, .{
25247 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any },
25248 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }},
25249 .patterns = &.{
25250 .{ .src = .{ .to_mem, .none } },
25251 },
25252 .extra_temps = .{
25253 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25254 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25255 .unused,
25256 .unused,
25257 .unused,
25258 .unused,
25259 .unused,
25260 .unused,
25261 .unused,
25262 },
25263 .dst_temps = .{.mem},
25264 .each = .{ .once = &.{
25265 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25266 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ },
25267 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
25268 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
25269 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25270 } },
25271 }, .{
25272 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any },
25273 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }},
25274 .patterns = &.{
25275 .{ .src = .{ .to_mem, .none } },
25276 },
25277 .extra_temps = .{
25278 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25279 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25280 .unused,
25281 .unused,
25282 .unused,
25283 .unused,
25284 .unused,
25285 .unused,
25286 .unused,
25287 },
25288 .dst_temps = .{.mem},
25289 .each = .{ .once = &.{
25290 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25291 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ },
25292 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
25293 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
25294 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25295 } },
25296 }, .{
25297 .src_constraints = .{ .any_scalar_int, .any },
25298 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }},
25299 .patterns = &.{
25300 .{ .src = .{ .to_mem, .none } },
25301 },
25302 .extra_temps = .{
25303 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25304 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25305 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25306 .unused,
25307 .unused,
25308 .unused,
25309 .unused,
25310 .unused,
25311 .unused,
25312 },
25313 .dst_temps = .{.mem},
25314 .each = .{ .once = &.{
25315 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25316 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
25317 .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ },
25318 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ },
25319 .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ },
25320 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
25321 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25322 } },
25323 }, .{
25324 .required_features = .{ .sse, null, null, null },
25325 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any },
25326 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
25327 .patterns = &.{
25328 .{ .src = .{ .mut_mem, .none } },
25329 .{ .src = .{ .to_mut_sse, .none } },
25330 },
25331 .dst_temps = .{.{ .ref = .src0 }},
25332 .each = .{ .once = &.{} },
25333 }, .{
25334 .required_features = .{ .avx, null, null, null },
25335 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any },
25336 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .qword } }},
25337 .patterns = &.{
25338 .{ .src = .{ .mut_mem, .none } },
25339 .{ .src = .{ .to_mut_sse, .none } },
25340 },
25341 .dst_temps = .{.{ .ref = .src0 }},
25342 .each = .{ .once = &.{} },
25343 }, .{
25344 .required_features = .{ .@"64bit", null, null, null },
25345 .src_constraints = .{ .any_scalar_int, .any },
25346 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }},
25347 .patterns = &.{
25348 .{ .src = .{ .to_mem, .none } },
25349 },
25350 .extra_temps = .{
25351 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
25352 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25353 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
25354 .unused,
25355 .unused,
25356 .unused,
25357 .unused,
25358 .unused,
25359 .unused,
25360 },
25361 .dst_temps = .{.mem},
25362 .each = .{ .once = &.{
25363 .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ },
25364 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
25365 .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp1), ._, ._ },
25366 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ },
25367 .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ },
25368 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
25369 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
25370 } },
25371 }, .{
25372 .required_features = .{ .sse, null, null, null },
25373 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any },
25374 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
25375 .patterns = &.{
25376 .{ .src = .{ .mut_mem, .none } },
25377 .{ .src = .{ .to_mut_sse, .none } },
25378 },
25379 .dst_temps = .{.{ .ref = .src0 }},
25380 .each = .{ .once = &.{} },
25381 }, .{
25382 .required_features = .{ .avx, null, null, null },
25383 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any },
25384 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .xword } }},
25385 .patterns = &.{
25386 .{ .src = .{ .mut_mem, .none } },
25387 .{ .src = .{ .to_mut_sse, .none } },
25388 },
25389 .dst_temps = .{.{ .ref = .src0 }},
25390 .each = .{ .once = &.{} },
25391 }, .{
25392 .required_features = .{ .avx, null, null, null },
25393 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .yword } }, .any },
25394 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .yword } }},
25395 .patterns = &.{
25396 .{ .src = .{ .mut_mem, .none } },
25397 .{ .src = .{ .to_mut_sse, .none } },
25398 },
25399 .dst_temps = .{.{ .ref = .src0 }},
25400 .each = .{ .once = &.{} },
25401 }, .{
25402 .required_features = .{ .@"64bit", .slow_incdec, null, null },
25403 .src_constraints = .{ .any_scalar_int, .any },
25404 .dst_constraints = .{.any_scalar_int},
25405 .patterns = &.{
25406 .{ .src = .{ .to_mem, .none } },
25407 },
25408 .extra_temps = .{
25409 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25410 .{ .type = .usize, .kind = .{ .reg = .rsi } },
25411 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25412 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25413 .unused,
25414 .unused,
25415 .unused,
25416 .unused,
25417 .unused,
25418 },
25419 .dst_temps = .{.mem},
25420 .clobbers = .{ .eflags = true },
25421 .each = .{ .once = &.{
25422 .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ },
25423 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
25424 .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ },
25425 .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ },
25426 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
25427 .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ },
25428 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
25429 .{ ._, ._a, .j, .@"0b", ._, ._, ._ },
25430 } },
25431 }, .{
25432 .required_features = .{ .@"64bit", null, null, null },
25433 .src_constraints = .{ .any_scalar_int, .any },
25434 .dst_constraints = .{.any_scalar_int},
25435 .patterns = &.{
25436 .{ .src = .{ .to_mem, .none } },
25437 },
25438 .extra_temps = .{
25439 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
25440 .{ .type = .usize, .kind = .{ .reg = .rsi } },
25441 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25442 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25443 .unused,
25444 .unused,
25445 .unused,
25446 .unused,
25447 .unused,
25448 },
25449 .dst_temps = .{.mem},
25450 .clobbers = .{ .eflags = true },
25451 .each = .{ .once = &.{
25452 .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ },
25453 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
25454 .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ },
25455 .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ },
25456 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
25457 .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ },
25458 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
25459 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
25460 } },
25461 } } else comptime &.{ .{
25462 .src_constraints = .{ .{ .signed_int = .byte }, .any },
25463 .dst_constraints = .{.{ .signed_int = .dword }},
25464 .patterns = &.{
25465 .{ .src = .{ .mem, .none } },
25466 .{ .src = .{ .to_gpr, .none } },
25467 },
25468 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25469 .each = .{ .once = &.{
25470 .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ },
25471 } },
25472 }, .{
25473 .src_constraints = .{ .{ .int = .byte }, .any },
25474 .dst_constraints = .{.{ .int = .dword }},
25475 .patterns = &.{
25476 .{ .src = .{ .mem, .none } },
25477 .{ .src = .{ .to_gpr, .none } },
25478 },
25479 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25480 .each = .{ .once = &.{
25481 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
25482 } },
25483 }, .{
25484 .required_features = .{ .@"64bit", null, null, null },
25485 .src_constraints = .{ .{ .signed_int = .byte }, .any },
25486 .dst_constraints = .{.{ .signed_int = .qword }},
25487 .patterns = &.{
25488 .{ .src = .{ .mem, .none } },
25489 .{ .src = .{ .to_gpr, .none } },
25490 },
25491 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25492 .each = .{ .once = &.{
25493 .{ ._, ._, .movsx, .dst0q, .src0b, ._, ._ },
25494 } },
25495 }, .{
25496 .required_features = .{ .@"64bit", null, null, null },
25497 .src_constraints = .{ .{ .int = .byte }, .any },
25498 .dst_constraints = .{.{ .int = .qword }},
25499 .patterns = &.{
25500 .{ .src = .{ .mem, .none } },
25501 .{ .src = .{ .to_gpr, .none } },
25502 },
25503 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25504 .each = .{ .once = &.{
25505 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
25506 } },
25507 }, .{
25508 .required_features = .{ .@"64bit", null, null, null },
25509 .src_constraints = .{ .{ .signed_int = .byte }, .any },
25510 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
25511 .patterns = &.{
25512 .{ .src = .{ .mem, .none } },
25513 .{ .src = .{ .to_gpr, .none } },
25514 },
25515 .extra_temps = .{
25516 .{ .type = .i64, .kind = .{ .reg = .rax } },
25517 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25518 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25519 .unused,
25520 .unused,
25521 .unused,
25522 .unused,
25523 .unused,
25524 .unused,
25525 },
25526 .dst_temps = .{.mem},
25527 .clobbers = .{ .eflags = true },
25528 .each = .{ .once = &.{
25529 .{ ._, ._, .movsx, .tmp0q, .src0b, ._, ._ },
25530 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25531 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25532 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
25533 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25534 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25535 } },
25536 }, .{
25537 .required_features = .{ .@"64bit", null, null, null },
25538 .src_constraints = .{ .{ .int = .byte }, .any },
25539 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
25540 .patterns = &.{
25541 .{ .src = .{ .mem, .none } },
25542 .{ .src = .{ .to_gpr, .none } },
25543 },
25544 .extra_temps = .{
25545 .{ .type = .u64, .kind = .{ .reg = .rax } },
25546 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25547 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25548 .unused,
25549 .unused,
25550 .unused,
25551 .unused,
25552 .unused,
25553 .unused,
25554 },
25555 .dst_temps = .{.mem},
25556 .clobbers = .{ .eflags = true },
25557 .each = .{ .once = &.{
25558 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
25559 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25560 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25561 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25562 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25563 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25564 } },
25565 }, .{
25566 .src_constraints = .{ .{ .signed_int = .byte }, .any },
25567 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .dword, .is = .dword } }},
25568 .patterns = &.{
25569 .{ .src = .{ .mem, .none } },
25570 .{ .src = .{ .to_gpr, .none } },
25571 },
25572 .extra_temps = .{
25573 .{ .type = .i32, .kind = .{ .reg = .eax } },
25574 .{ .type = .usize, .kind = .{ .reg = .edi } },
25575 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25576 .unused,
25577 .unused,
25578 .unused,
25579 .unused,
25580 .unused,
25581 .unused,
25582 },
25583 .dst_temps = .{.mem},
25584 .clobbers = .{ .eflags = true },
25585 .each = .{ .once = &.{
25586 .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ },
25587 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25588 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25589 .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ },
25590 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
25591 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25592 } },
25593 }, .{
25594 .src_constraints = .{ .{ .int = .byte }, .any },
25595 .dst_constraints = .{.{ .remainder_int = .{ .of = .dword, .is = .dword } }},
25596 .patterns = &.{
25597 .{ .src = .{ .mem, .none } },
25598 .{ .src = .{ .to_gpr, .none } },
25599 },
25600 .extra_temps = .{
25601 .{ .type = .u32, .kind = .{ .reg = .eax } },
25602 .{ .type = .usize, .kind = .{ .reg = .edi } },
25603 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25604 .unused,
25605 .unused,
25606 .unused,
25607 .unused,
25608 .unused,
25609 .unused,
25610 },
25611 .dst_temps = .{.mem},
25612 .clobbers = .{ .eflags = true },
25613 .each = .{ .once = &.{
25614 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
25615 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25616 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25617 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25618 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
25619 .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ },
25620 } },
25621 }, .{
25622 .src_constraints = .{ .{ .signed_int = .word }, .any },
25623 .dst_constraints = .{.{ .signed_int = .dword }},
25624 .patterns = &.{
25625 .{ .src = .{ .mem, .none } },
25626 .{ .src = .{ .to_gpr, .none } },
25627 },
25628 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25629 .each = .{ .once = &.{
25630 .{ ._, ._, .movsx, .dst0d, .src0w, ._, ._ },
25631 } },
25632 }, .{
25633 .src_constraints = .{ .{ .int = .word }, .any },
25634 .dst_constraints = .{.{ .int = .dword }},
25635 .patterns = &.{
25636 .{ .src = .{ .mem, .none } },
25637 .{ .src = .{ .to_gpr, .none } },
25638 },
25639 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25640 .each = .{ .once = &.{
25641 .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ },
25642 } },
25643 }, .{
25644 .required_features = .{ .@"64bit", null, null, null },
25645 .src_constraints = .{ .{ .signed_int = .word }, .any },
25646 .dst_constraints = .{.{ .signed_int = .qword }},
25647 .patterns = &.{
25648 .{ .src = .{ .mem, .none } },
25649 .{ .src = .{ .to_gpr, .none } },
25650 },
25651 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25652 .each = .{ .once = &.{
25653 .{ ._, ._, .movsx, .dst0q, .src0w, ._, ._ },
25654 } },
25655 }, .{
25656 .required_features = .{ .@"64bit", null, null, null },
25657 .src_constraints = .{ .{ .int = .word }, .any },
25658 .dst_constraints = .{.{ .int = .qword }},
25659 .patterns = &.{
25660 .{ .src = .{ .mem, .none } },
25661 .{ .src = .{ .to_gpr, .none } },
25662 },
25663 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25664 .each = .{ .once = &.{
25665 .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ },
25666 } },
25667 }, .{
25668 .required_features = .{ .@"64bit", null, null, null },
25669 .src_constraints = .{ .{ .signed_int = .word }, .any },
25670 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
25671 .patterns = &.{
25672 .{ .src = .{ .mem, .none } },
25673 .{ .src = .{ .to_gpr, .none } },
25674 },
25675 .extra_temps = .{
25676 .{ .type = .i64, .kind = .{ .reg = .rax } },
25677 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25678 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25679 .unused,
25680 .unused,
25681 .unused,
25682 .unused,
25683 .unused,
25684 .unused,
25685 },
25686 .dst_temps = .{.mem},
25687 .clobbers = .{ .eflags = true },
25688 .each = .{ .once = &.{
25689 .{ ._, ._, .movsx, .tmp0q, .src0w, ._, ._ },
25690 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25691 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25692 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
25693 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25694 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25695 } },
25696 }, .{
25697 .required_features = .{ .@"64bit", null, null, null },
25698 .src_constraints = .{ .{ .int = .word }, .any },
25699 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
25700 .patterns = &.{
25701 .{ .src = .{ .mem, .none } },
25702 .{ .src = .{ .to_gpr, .none } },
25703 },
25704 .extra_temps = .{
25705 .{ .type = .u64, .kind = .{ .reg = .rax } },
25706 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25707 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25708 .unused,
25709 .unused,
25710 .unused,
25711 .unused,
25712 .unused,
25713 .unused,
25714 },
25715 .dst_temps = .{.mem},
25716 .clobbers = .{ .eflags = true },
25717 .each = .{ .once = &.{
25718 .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ },
25719 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25720 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25721 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25722 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25723 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25724 } },
25725 }, .{
25726 .src_constraints = .{ .{ .signed_int = .word }, .any },
25727 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .dword, .is = .dword } }},
25728 .patterns = &.{
25729 .{ .src = .{ .mem, .none } },
25730 .{ .src = .{ .to_gpr, .none } },
25731 },
25732 .extra_temps = .{
25733 .{ .type = .i32, .kind = .{ .reg = .eax } },
25734 .{ .type = .usize, .kind = .{ .reg = .edi } },
25735 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25736 .unused,
25737 .unused,
25738 .unused,
25739 .unused,
25740 .unused,
25741 .unused,
25742 },
25743 .dst_temps = .{.mem},
25744 .clobbers = .{ .eflags = true },
25745 .each = .{ .once = &.{
25746 .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ },
25747 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25748 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25749 .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ },
25750 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
25751 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25752 } },
25753 }, .{
25754 .src_constraints = .{ .{ .int = .word }, .any },
25755 .dst_constraints = .{.{ .remainder_int = .{ .of = .dword, .is = .dword } }},
25756 .patterns = &.{
25757 .{ .src = .{ .mem, .none } },
25758 .{ .src = .{ .to_gpr, .none } },
25759 },
25760 .extra_temps = .{
25761 .{ .type = .u32, .kind = .{ .reg = .eax } },
25762 .{ .type = .usize, .kind = .{ .reg = .edi } },
25763 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25764 .unused,
25765 .unused,
25766 .unused,
25767 .unused,
25768 .unused,
25769 .unused,
25770 },
25771 .dst_temps = .{.mem},
25772 .clobbers = .{ .eflags = true },
25773 .each = .{ .once = &.{
25774 .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ },
25775 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25776 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25777 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25778 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
25779 .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ },
25780 } },
25781 }, .{
25782 .required_features = .{ .@"64bit", null, null, null },
25783 .src_constraints = .{ .{ .signed_int = .dword }, .any },
25784 .dst_constraints = .{.{ .signed_int = .qword }},
25785 .patterns = &.{
25786 .{ .src = .{ .mem, .none } },
25787 .{ .src = .{ .to_gpr, .none } },
25788 },
25789 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25790 .each = .{ .once = &.{
25791 .{ ._, ._d, .movsx, .dst0q, .src0d, ._, ._ },
25792 } },
25793 }, .{
25794 .required_features = .{ .@"64bit", null, null, null },
25795 .src_constraints = .{ .{ .int = .dword }, .any },
25796 .dst_constraints = .{.{ .int = .qword }},
25797 .patterns = &.{
25798 .{ .src = .{ .mem, .none } },
25799 .{ .src = .{ .to_gpr, .none } },
25800 },
25801 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }},
25802 .each = .{ .once = &.{
25803 .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ },
25804 } },
25805 }, .{
25806 .required_features = .{ .@"64bit", null, null, null },
25807 .src_constraints = .{ .{ .signed_int = .dword }, .any },
25808 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
25809 .patterns = &.{
25810 .{ .src = .{ .mem, .none } },
25811 .{ .src = .{ .to_gpr, .none } },
25812 },
25813 .extra_temps = .{
25814 .{ .type = .i64, .kind = .{ .reg = .rax } },
25815 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25816 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25817 .unused,
25818 .unused,
25819 .unused,
25820 .unused,
25821 .unused,
25822 .unused,
25823 },
25824 .dst_temps = .{.mem},
25825 .clobbers = .{ .eflags = true },
25826 .each = .{ .once = &.{
25827 .{ ._, ._d, .movsx, .tmp0q, .src0d, ._, ._ },
25828 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25829 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25830 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
25831 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25832 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25833 } },
25834 }, .{
25835 .required_features = .{ .@"64bit", null, null, null },
25836 .src_constraints = .{ .{ .int = .dword }, .any },
25837 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
25838 .patterns = &.{
25839 .{ .src = .{ .mem, .none } },
25840 .{ .src = .{ .to_gpr, .none } },
25841 },
25842 .extra_temps = .{
25843 .{ .type = .u64, .kind = .{ .reg = .rax } },
25844 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25845 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25846 .unused,
25847 .unused,
25848 .unused,
25849 .unused,
25850 .unused,
25851 .unused,
25852 },
25853 .dst_temps = .{.mem},
25854 .clobbers = .{ .eflags = true },
25855 .each = .{ .once = &.{
25856 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
25857 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25858 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25859 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25860 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25861 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25862 } },
25863 }, .{
25864 .src_constraints = .{ .{ .signed_int = .dword }, .any },
25865 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .dword, .is = .dword } }},
25866 .patterns = &.{
25867 .{ .src = .{ .mem, .none } },
25868 .{ .src = .{ .to_gpr, .none } },
25869 },
25870 .extra_temps = .{
25871 .{ .type = .i32, .kind = .{ .reg = .eax } },
25872 .{ .type = .usize, .kind = .{ .reg = .edi } },
25873 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25874 .unused,
25875 .unused,
25876 .unused,
25877 .unused,
25878 .unused,
25879 .unused,
25880 },
25881 .dst_temps = .{.mem},
25882 .clobbers = .{ .eflags = true },
25883 .each = .{ .once = &.{
25884 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
25885 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25886 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25887 .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ },
25888 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
25889 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25890 } },
25891 }, .{
25892 .src_constraints = .{ .{ .int = .dword }, .any },
25893 .dst_constraints = .{.{ .remainder_int = .{ .of = .dword, .is = .dword } }},
25894 .patterns = &.{
25895 .{ .src = .{ .mem, .none } },
25896 .{ .src = .{ .to_gpr, .none } },
25897 },
25898 .extra_temps = .{
25899 .{ .type = .u32, .kind = .{ .reg = .eax } },
25900 .{ .type = .usize, .kind = .{ .reg = .edi } },
25901 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25902 .unused,
25903 .unused,
25904 .unused,
25905 .unused,
25906 .unused,
25907 .unused,
25908 },
25909 .dst_temps = .{.mem},
25910 .clobbers = .{ .eflags = true },
25911 .each = .{ .once = &.{
25912 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
25913 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25914 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25915 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25916 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
25917 .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ },
25918 } },
25919 }, .{
25920 .required_features = .{ .@"64bit", null, null, null },
25921 .src_constraints = .{ .{ .signed_int = .qword }, .any },
25922 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
25923 .patterns = &.{
25924 .{ .src = .{ .mem, .none } },
25925 .{ .src = .{ .to_gpr, .none } },
25926 },
25927 .extra_temps = .{
25928 .{ .type = .i64, .kind = .{ .reg = .rax } },
25929 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25930 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25931 .unused,
25932 .unused,
25933 .unused,
25934 .unused,
25935 .unused,
25936 .unused,
25937 },
25938 .dst_temps = .{.mem},
25939 .clobbers = .{ .eflags = true },
25940 .each = .{ .once = &.{
25941 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
25942 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25943 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25944 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
25945 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25946 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25947 } },
25948 }, .{
25949 .required_features = .{ .@"64bit", null, null, null },
25950 .src_constraints = .{ .{ .int = .qword }, .any },
25951 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
25952 .patterns = &.{
25953 .{ .src = .{ .mem, .none } },
25954 .{ .src = .{ .to_gpr, .none } },
25955 },
25956 .extra_temps = .{
25957 .{ .type = .u64, .kind = .{ .reg = .rax } },
25958 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25959 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25960 .unused,
25961 .unused,
25962 .unused,
25963 .unused,
25964 .unused,
25965 .unused,
25966 },
25967 .dst_temps = .{.mem},
25968 .clobbers = .{ .eflags = true },
25969 .each = .{ .once = &.{
25970 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
25971 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25972 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25973 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25974 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25975 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
25976 } },
25977 }, .{
25978 .required_features = .{ .@"64bit", null, null, null },
25979 .src_constraints = .{ .{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
25980 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
25981 .patterns = &.{
25982 .{ .src = .{ .to_mem, .none } },
25983 },
25984 .extra_temps = .{
25985 .{ .type = .usize, .kind = .{ .reg = .rsi } },
25986 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25987 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25988 .{ .type = .i64, .kind = .{ .reg = .rax } },
25989 .unused,
25990 .unused,
25991 .unused,
25992 .unused,
25993 .unused,
25994 },
25995 .dst_temps = .{.mem},
25996 .clobbers = .{ .eflags = true },
25997 .each = .{ .once = &.{
25998 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
25999 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
26000 .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
26001 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
26002 .{ ._, ._sq, .lod, ._, ._, ._, ._ },
26003 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
26004 .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ },
26005 .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ },
26006 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
26007 } },
26008 }, .{
26009 .required_features = .{ .@"64bit", null, null, null },
26010 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any },
26011 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
26012 .patterns = &.{
26013 .{ .src = .{ .to_mem, .none } },
26014 },
26015 .extra_temps = .{
26016 .{ .type = .usize, .kind = .{ .reg = .rsi } },
26017 .{ .type = .usize, .kind = .{ .reg = .rdi } },
26018 .{ .type = .u32, .kind = .{ .reg = .ecx } },
26019 .{ .type = .u64, .kind = .{ .reg = .rax } },
26020 .unused,
26021 .unused,
26022 .unused,
26023 .unused,
26024 .unused,
26025 },
26026 .dst_temps = .{.mem},
26027 .clobbers = .{ .eflags = true },
26028 .each = .{ .once = &.{
26029 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
26030 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
26031 .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ },
26032 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
26033 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
26034 .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ },
26035 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
26036 } },
26037 }, .{
26038 .required_features = .{ .avx, null, null, null },
26039 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any },
26040 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .word } }},
26041 .patterns = &.{
26042 .{ .src = .{ .mem, .none } },
26043 .{ .src = .{ .to_sse, .none } },
26044 },
26045 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26046 .each = .{ .once = &.{
26047 .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ },
26048 } },
26049 }, .{
26050 .required_features = .{ .avx, null, null, null },
26051 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any },
26052 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .word } }},
26053 .patterns = &.{
26054 .{ .src = .{ .mem, .none } },
26055 .{ .src = .{ .to_sse, .none } },
26056 },
26057 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26058 .each = .{ .once = &.{
26059 .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ },
26060 } },
26061 }, .{
26062 .required_features = .{ .sse4_1, null, null, null },
26063 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any },
26064 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .word } }},
26065 .patterns = &.{
26066 .{ .src = .{ .mem, .none } },
26067 .{ .src = .{ .to_sse, .none } },
26068 },
26069 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26070 .each = .{ .once = &.{
26071 .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ },
26072 } },
26073 }, .{
26074 .required_features = .{ .sse4_1, null, null, null },
26075 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any },
26076 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .word } }},
26077 .patterns = &.{
26078 .{ .src = .{ .mem, .none } },
26079 .{ .src = .{ .to_sse, .none } },
26080 },
26081 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26082 .each = .{ .once = &.{
26083 .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ },
26084 } },
26085 }, .{
26086 .required_features = .{ .sse2, null, null, null },
26087 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any },
26088 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .word } }},
26089 .patterns = &.{
26090 .{ .src = .{ .to_mut_sse, .none } },
26091 },
26092 .extra_temps = .{
26093 .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } },
26094 .unused,
26095 .unused,
26096 .unused,
26097 .unused,
26098 .unused,
26099 .unused,
26100 .unused,
26101 .unused,
26102 },
26103 .dst_temps = .{.{ .ref = .src0 }},
26104 .each = .{ .once = &.{
26105 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
26106 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },
26107 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
26108 } },
26109 }, .{
26110 .required_features = .{ .sse2, null, null, null },
26111 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any },
26112 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .word } }},
26113 .patterns = &.{
26114 .{ .src = .{ .to_mut_sse, .none } },
26115 },
26116 .extra_temps = .{
26117 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
26118 .unused,
26119 .unused,
26120 .unused,
26121 .unused,
26122 .unused,
26123 .unused,
26124 .unused,
26125 .unused,
26126 },
26127 .dst_temps = .{.{ .ref = .src0 }},
26128 .each = .{ .once = &.{
26129 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
26130 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
26131 } },
26132 }, .{
26133 .required_features = .{ .avx2, null, null, null },
26134 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any },
26135 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .yword, .is = .word } }},
26136 .patterns = &.{
26137 .{ .src = .{ .mem, .none } },
26138 .{ .src = .{ .to_sse, .none } },
26139 },
26140 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26141 .each = .{ .once = &.{
26142 .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ },
26143 } },
26144 }, .{
26145 .required_features = .{ .avx2, null, null, null },
26146 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },
26147 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .word } }},
26148 .patterns = &.{
26149 .{ .src = .{ .mem, .none } },
26150 .{ .src = .{ .to_sse, .none } },
26151 },
26152 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26153 .each = .{ .once = &.{
26154 .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ },
26155 } },
26156 }, .{
26157 .required_features = .{ .avx2, null, null, null },
26158 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any },
26159 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }},
26160 .patterns = &.{
26161 .{ .src = .{ .mem, .none } },
26162 },
26163 .extra_temps = .{
26164 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26165 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
26166 .unused,
26167 .unused,
26168 .unused,
26169 .unused,
26170 .unused,
26171 .unused,
26172 .unused,
26173 },
26174 .dst_temps = .{.mem},
26175 .each = .{ .once = &.{
26176 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26177 .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ },
26178 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_size), .tmp1y, ._, ._ },
26179 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
26180 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26181 } },
26182 }, .{
26183 .required_features = .{ .avx2, null, null, null },
26184 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any },
26185 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }},
26186 .patterns = &.{
26187 .{ .src = .{ .mem, .none } },
26188 },
26189 .extra_temps = .{
26190 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26191 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
26192 .unused,
26193 .unused,
26194 .unused,
26195 .unused,
26196 .unused,
26197 .unused,
26198 .unused,
26199 },
26200 .dst_temps = .{.mem},
26201 .each = .{ .once = &.{
26202 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26203 .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ },
26204 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_size), .tmp1y, ._, ._ },
26205 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
26206 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26207 } },
26208 }, .{
26209 .required_features = .{ .avx, null, null, null },
26210 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any },
26211 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }},
26212 .patterns = &.{
26213 .{ .src = .{ .mem, .none } },
26214 },
26215 .extra_temps = .{
26216 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26217 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
26218 .unused,
26219 .unused,
26220 .unused,
26221 .unused,
26222 .unused,
26223 .unused,
26224 .unused,
26225 },
26226 .dst_temps = .{.mem},
26227 .each = .{ .once = &.{
26228 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26229 .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
26230 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
26231 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
26232 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26233 } },
26234 }, .{
26235 .required_features = .{ .avx, null, null, null },
26236 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any },
26237 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }},
26238 .patterns = &.{
26239 .{ .src = .{ .mem, .none } },
26240 },
26241 .extra_temps = .{
26242 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26243 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
26244 .unused,
26245 .unused,
26246 .unused,
26247 .unused,
26248 .unused,
26249 .unused,
26250 .unused,
26251 },
26252 .dst_temps = .{.mem},
26253 .each = .{ .once = &.{
26254 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26255 .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
26256 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
26257 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
26258 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26259 } },
26260 }, .{
26261 .required_features = .{ .sse4_1, null, null, null },
26262 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any },
26263 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }},
26264 .patterns = &.{
26265 .{ .src = .{ .mem, .none } },
26266 },
26267 .extra_temps = .{
26268 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26269 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
26270 .unused,
26271 .unused,
26272 .unused,
26273 .unused,
26274 .unused,
26275 .unused,
26276 .unused,
26277 },
26278 .dst_temps = .{.mem},
26279 .each = .{ .once = &.{
26280 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26281 .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
26282 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
26283 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
26284 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26285 } },
26286 }, .{
26287 .required_features = .{ .sse4_1, null, null, null },
26288 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any },
26289 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }},
26290 .patterns = &.{
26291 .{ .src = .{ .mem, .none } },
26292 },
26293 .extra_temps = .{
26294 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26295 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
26296 .unused,
26297 .unused,
26298 .unused,
26299 .unused,
26300 .unused,
26301 .unused,
26302 .unused,
26303 },
26304 .dst_temps = .{.mem},
26305 .each = .{ .once = &.{
26306 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26307 .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
26308 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
26309 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
26310 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26311 } },
26312 }, .{
26313 .required_features = .{ .slow_incdec, null, null, null },
26314 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
26315 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }},
26316 .patterns = &.{
26317 .{ .src = .{ .to_mem, .none } },
26318 },
26319 .extra_temps = .{
26320 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26321 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
26322 .unused,
26323 .unused,
26324 .unused,
26325 .unused,
26326 .unused,
26327 .unused,
26328 .unused,
26329 },
26330 .dst_temps = .{.mem},
26331 .each = .{ .once = &.{
26332 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26333 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26334 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
26335 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
26336 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26337 } },
26338 }, .{
26339 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
26340 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }},
26341 .patterns = &.{
26342 .{ .src = .{ .to_mem, .none } },
26343 },
26344 .extra_temps = .{
26345 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26346 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
26347 .unused,
26348 .unused,
26349 .unused,
26350 .unused,
26351 .unused,
26352 .unused,
26353 .unused,
26354 },
26355 .dst_temps = .{.mem},
26356 .each = .{ .once = &.{
26357 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26358 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26359 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
26360 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
26361 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
26362 } },
26363 }, .{
26364 .required_features = .{ .slow_incdec, null, null, null },
26365 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
26366 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .word, .is = .word } }},
26367 .patterns = &.{
26368 .{ .src = .{ .to_mem, .none } },
26369 },
26370 .extra_temps = .{
26371 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26372 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
26373 .unused,
26374 .unused,
26375 .unused,
26376 .unused,
26377 .unused,
26378 .unused,
26379 .unused,
26380 },
26381 .dst_temps = .{.mem},
26382 .each = .{ .once = &.{
26383 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26384 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26385 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
26386 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
26387 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26388 } },
26389 }, .{
26390 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
26391 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .word, .is = .word } }},
26392 .patterns = &.{
26393 .{ .src = .{ .to_mem, .none } },
26394 },
26395 .extra_temps = .{
26396 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26397 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
26398 .unused,
26399 .unused,
26400 .unused,
26401 .unused,
26402 .unused,
26403 .unused,
26404 .unused,
26405 },
26406 .dst_temps = .{.mem},
26407 .each = .{ .once = &.{
26408 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26409 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26410 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
26411 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
26412 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
26413 } },
26414 }, .{
26415 .required_features = .{ .avx, null, null, null },
26416 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any },
26417 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }},
26418 .patterns = &.{
26419 .{ .src = .{ .mem, .none } },
26420 .{ .src = .{ .to_sse, .none } },
26421 },
26422 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26423 .each = .{ .once = &.{
26424 .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ },
26425 } },
26426 }, .{
26427 .required_features = .{ .avx, null, null, null },
26428 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any },
26429 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .dword } }},
26430 .patterns = &.{
26431 .{ .src = .{ .mem, .none } },
26432 .{ .src = .{ .to_sse, .none } },
26433 },
26434 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26435 .each = .{ .once = &.{
26436 .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ },
26437 } },
26438 }, .{
26439 .required_features = .{ .sse4_1, null, null, null },
26440 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any },
26441 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }},
26442 .patterns = &.{
26443 .{ .src = .{ .mem, .none } },
26444 .{ .src = .{ .to_sse, .none } },
26445 },
26446 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26447 .each = .{ .once = &.{
26448 .{ ._, .p_d, .movsxb, .dst0x, .src0d, ._, ._ },
26449 } },
26450 }, .{
26451 .required_features = .{ .sse4_1, null, null, null },
26452 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any },
26453 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .dword } }},
26454 .patterns = &.{
26455 .{ .src = .{ .mem, .none } },
26456 .{ .src = .{ .to_sse, .none } },
26457 },
26458 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26459 .each = .{ .once = &.{
26460 .{ ._, .p_d, .movzxb, .dst0x, .src0d, ._, ._ },
26461 } },
26462 }, .{
26463 .required_features = .{ .sse2, null, null, null },
26464 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any },
26465 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }},
26466 .patterns = &.{
26467 .{ .src = .{ .to_mut_sse, .none } },
26468 },
26469 .extra_temps = .{
26470 .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } },
26471 .unused,
26472 .unused,
26473 .unused,
26474 .unused,
26475 .unused,
26476 .unused,
26477 .unused,
26478 .unused,
26479 },
26480 .dst_temps = .{.{ .ref = .src0 }},
26481 .each = .{ .once = &.{
26482 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
26483 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },
26484 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
26485 .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ },
26486 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
26487 } },
26488 }, .{
26489 .required_features = .{ .sse2, null, null, null },
26490 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any },
26491 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .dword } }},
26492 .patterns = &.{
26493 .{ .src = .{ .to_mut_sse, .none } },
26494 },
26495 .extra_temps = .{
26496 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
26497 .unused,
26498 .unused,
26499 .unused,
26500 .unused,
26501 .unused,
26502 .unused,
26503 .unused,
26504 .unused,
26505 },
26506 .dst_temps = .{.{ .ref = .src0 }},
26507 .each = .{ .once = &.{
26508 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
26509 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
26510 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
26511 } },
26512 }, .{
26513 .required_features = .{ .avx2, null, null, null },
26514 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any },
26515 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }},
26516 .patterns = &.{
26517 .{ .src = .{ .mem, .none } },
26518 .{ .src = .{ .to_sse, .none } },
26519 },
26520 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26521 .each = .{ .once = &.{
26522 .{ ._, .vp_d, .movsxb, .dst0y, .src0q, ._, ._ },
26523 } },
26524 }, .{
26525 .required_features = .{ .avx2, null, null, null },
26526 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any },
26527 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .dword } }},
26528 .patterns = &.{
26529 .{ .src = .{ .mem, .none } },
26530 .{ .src = .{ .to_sse, .none } },
26531 },
26532 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26533 .each = .{ .once = &.{
26534 .{ ._, .vp_d, .movzxb, .dst0y, .src0q, ._, ._ },
26535 } },
26536 }, .{
26537 .required_features = .{ .avx2, null, null, null },
26538 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any },
26539 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }},
26540 .patterns = &.{
26541 .{ .src = .{ .mem, .none } },
26542 },
26543 .extra_temps = .{
26544 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26545 .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } },
26546 .unused,
26547 .unused,
26548 .unused,
26549 .unused,
26550 .unused,
26551 .unused,
26552 .unused,
26553 },
26554 .dst_temps = .{.mem},
26555 .each = .{ .once = &.{
26556 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26557 .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memia(.src0q, .tmp0, .add_size), ._, ._ },
26558 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_size), .tmp1y, ._, ._ },
26559 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
26560 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26561 } },
26562 }, .{
26563 .required_features = .{ .avx2, null, null, null },
26564 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any },
26565 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }},
26566 .patterns = &.{
26567 .{ .src = .{ .mem, .none } },
26568 },
26569 .extra_temps = .{
26570 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26571 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
26572 .unused,
26573 .unused,
26574 .unused,
26575 .unused,
26576 .unused,
26577 .unused,
26578 .unused,
26579 },
26580 .dst_temps = .{.mem},
26581 .each = .{ .once = &.{
26582 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26583 .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ },
26584 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_size), .tmp1y, ._, ._ },
26585 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
26586 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26587 } },
26588 }, .{
26589 .required_features = .{ .avx, null, null, null },
26590 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any },
26591 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }},
26592 .patterns = &.{
26593 .{ .src = .{ .mem, .none } },
26594 },
26595 .extra_temps = .{
26596 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26597 .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } },
26598 .unused,
26599 .unused,
26600 .unused,
26601 .unused,
26602 .unused,
26603 .unused,
26604 .unused,
26605 },
26606 .dst_temps = .{.mem},
26607 .each = .{ .once = &.{
26608 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26609 .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
26610 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
26611 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
26612 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26613 } },
26614 }, .{
26615 .required_features = .{ .avx, null, null, null },
26616 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any },
26617 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }},
26618 .patterns = &.{
26619 .{ .src = .{ .mem, .none } },
26620 },
26621 .extra_temps = .{
26622 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26623 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
26624 .unused,
26625 .unused,
26626 .unused,
26627 .unused,
26628 .unused,
26629 .unused,
26630 .unused,
26631 },
26632 .dst_temps = .{.mem},
26633 .each = .{ .once = &.{
26634 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26635 .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
26636 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
26637 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
26638 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26639 } },
26640 }, .{
26641 .required_features = .{ .sse4_1, null, null, null },
26642 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any },
26643 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }},
26644 .patterns = &.{
26645 .{ .src = .{ .mem, .none } },
26646 },
26647 .extra_temps = .{
26648 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26649 .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } },
26650 .unused,
26651 .unused,
26652 .unused,
26653 .unused,
26654 .unused,
26655 .unused,
26656 .unused,
26657 },
26658 .dst_temps = .{.mem},
26659 .each = .{ .once = &.{
26660 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26661 .{ .@"0:", .p_d, .movsxb, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
26662 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
26663 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
26664 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26665 } },
26666 }, .{
26667 .required_features = .{ .sse4_1, null, null, null },
26668 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any },
26669 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }},
26670 .patterns = &.{
26671 .{ .src = .{ .mem, .none } },
26672 },
26673 .extra_temps = .{
26674 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26675 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
26676 .unused,
26677 .unused,
26678 .unused,
26679 .unused,
26680 .unused,
26681 .unused,
26682 .unused,
26683 },
26684 .dst_temps = .{.mem},
26685 .each = .{ .once = &.{
26686 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26687 .{ .@"0:", .p_d, .movzxb, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
26688 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
26689 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
26690 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26691 } },
26692 }, .{
26693 .required_features = .{ .slow_incdec, null, null, null },
26694 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
26695 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }},
26696 .patterns = &.{
26697 .{ .src = .{ .to_mem, .none } },
26698 },
26699 .extra_temps = .{
26700 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26701 .{ .type = .i32, .kind = .{ .rc = .general_purpose } },
26702 .unused,
26703 .unused,
26704 .unused,
26705 .unused,
26706 .unused,
26707 .unused,
26708 .unused,
26709 },
26710 .dst_temps = .{.mem},
26711 .each = .{ .once = &.{
26712 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26713 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26714 .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
26715 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
26716 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26717 } },
26718 }, .{
26719 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
26720 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }},
26721 .patterns = &.{
26722 .{ .src = .{ .to_mem, .none } },
26723 },
26724 .extra_temps = .{
26725 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26726 .{ .type = .i32, .kind = .{ .rc = .general_purpose } },
26727 .unused,
26728 .unused,
26729 .unused,
26730 .unused,
26731 .unused,
26732 .unused,
26733 .unused,
26734 },
26735 .dst_temps = .{.mem},
26736 .each = .{ .once = &.{
26737 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26738 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26739 .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
26740 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
26741 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
26742 } },
26743 }, .{
26744 .required_features = .{ .slow_incdec, null, null, null },
26745 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
26746 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }},
26747 .patterns = &.{
26748 .{ .src = .{ .to_mem, .none } },
26749 },
26750 .extra_temps = .{
26751 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26752 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
26753 .unused,
26754 .unused,
26755 .unused,
26756 .unused,
26757 .unused,
26758 .unused,
26759 .unused,
26760 },
26761 .dst_temps = .{.mem},
26762 .each = .{ .once = &.{
26763 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26764 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26765 .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
26766 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
26767 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26768 } },
26769 }, .{
26770 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
26771 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }},
26772 .patterns = &.{
26773 .{ .src = .{ .to_mem, .none } },
26774 },
26775 .extra_temps = .{
26776 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26777 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
26778 .unused,
26779 .unused,
26780 .unused,
26781 .unused,
26782 .unused,
26783 .unused,
26784 .unused,
26785 },
26786 .dst_temps = .{.mem},
26787 .each = .{ .once = &.{
26788 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26789 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
26790 .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
26791 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
26792 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
26793 } },
26794 }, .{
26795 .required_features = .{ .avx, null, null, null },
26796 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any },
26797 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
26798 .patterns = &.{
26799 .{ .src = .{ .mem, .none } },
26800 .{ .src = .{ .to_sse, .none } },
26801 },
26802 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26803 .each = .{ .once = &.{
26804 .{ ._, .vp_q, .movsxb, .dst0x, .src0w, ._, ._ },
26805 } },
26806 }, .{
26807 .required_features = .{ .avx, null, null, null },
26808 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any },
26809 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
26810 .patterns = &.{
26811 .{ .src = .{ .mem, .none } },
26812 .{ .src = .{ .to_sse, .none } },
26813 },
26814 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26815 .each = .{ .once = &.{
26816 .{ ._, .vp_q, .movzxb, .dst0x, .src0w, ._, ._ },
26817 } },
26818 }, .{
26819 .required_features = .{ .sse4_1, null, null, null },
26820 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any },
26821 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
26822 .patterns = &.{
26823 .{ .src = .{ .mem, .none } },
26824 .{ .src = .{ .to_sse, .none } },
26825 },
26826 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26827 .each = .{ .once = &.{
26828 .{ ._, .p_q, .movsxb, .dst0x, .src0w, ._, ._ },
26829 } },
26830 }, .{
26831 .required_features = .{ .sse4_1, null, null, null },
26832 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any },
26833 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
26834 .patterns = &.{
26835 .{ .src = .{ .mem, .none } },
26836 .{ .src = .{ .to_sse, .none } },
26837 },
26838 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26839 .each = .{ .once = &.{
26840 .{ ._, .p_q, .movzxb, .dst0x, .src0w, ._, ._ },
26841 } },
26842 }, .{
26843 .required_features = .{ .sse2, null, null, null },
26844 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any },
26845 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
26846 .patterns = &.{
26847 .{ .src = .{ .to_mut_sse, .none } },
26848 },
26849 .extra_temps = .{
26850 .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } },
26851 .unused,
26852 .unused,
26853 .unused,
26854 .unused,
26855 .unused,
26856 .unused,
26857 .unused,
26858 .unused,
26859 },
26860 .dst_temps = .{.{ .ref = .src0 }},
26861 .each = .{ .once = &.{
26862 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
26863 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },
26864 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
26865 .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ },
26866 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
26867 .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ },
26868 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
26869 } },
26870 }, .{
26871 .required_features = .{ .sse2, null, null, null },
26872 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any },
26873 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
26874 .patterns = &.{
26875 .{ .src = .{ .to_mut_sse, .none } },
26876 },
26877 .extra_temps = .{
26878 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
26879 .unused,
26880 .unused,
26881 .unused,
26882 .unused,
26883 .unused,
26884 .unused,
26885 .unused,
26886 .unused,
26887 },
26888 .dst_temps = .{.{ .ref = .src0 }},
26889 .each = .{ .once = &.{
26890 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
26891 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
26892 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
26893 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
26894 } },
26895 }, .{
26896 .required_features = .{ .avx2, null, null, null },
26897 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any },
26898 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }},
26899 .patterns = &.{
26900 .{ .src = .{ .mem, .none } },
26901 .{ .src = .{ .to_sse, .none } },
26902 },
26903 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26904 .each = .{ .once = &.{
26905 .{ ._, .vp_q, .movsxb, .dst0y, .src0d, ._, ._ },
26906 } },
26907 }, .{
26908 .required_features = .{ .avx2, null, null, null },
26909 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any },
26910 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .qword } }},
26911 .patterns = &.{
26912 .{ .src = .{ .mem, .none } },
26913 .{ .src = .{ .to_sse, .none } },
26914 },
26915 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
26916 .each = .{ .once = &.{
26917 .{ ._, .vp_q, .movzxb, .dst0y, .src0d, ._, ._ },
26918 } },
26919 }, .{
26920 .required_features = .{ .avx2, null, null, null },
26921 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any },
26922 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }},
26923 .patterns = &.{
26924 .{ .src = .{ .mem, .none } },
26925 },
26926 .extra_temps = .{
26927 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26928 .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } },
26929 .unused,
26930 .unused,
26931 .unused,
26932 .unused,
26933 .unused,
26934 .unused,
26935 .unused,
26936 },
26937 .dst_temps = .{.mem},
26938 .each = .{ .once = &.{
26939 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26940 .{ .@"0:", .vp_q, .movsxb, .tmp1y, .memia(.src0d, .tmp0, .add_size), ._, ._ },
26941 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"8", .tmp0, .add_size), .tmp1y, ._, ._ },
26942 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
26943 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26944 } },
26945 }, .{
26946 .required_features = .{ .avx2, null, null, null },
26947 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any },
26948 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }},
26949 .patterns = &.{
26950 .{ .src = .{ .mem, .none } },
26951 },
26952 .extra_temps = .{
26953 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26954 .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } },
26955 .unused,
26956 .unused,
26957 .unused,
26958 .unused,
26959 .unused,
26960 .unused,
26961 .unused,
26962 },
26963 .dst_temps = .{.mem},
26964 .each = .{ .once = &.{
26965 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26966 .{ .@"0:", .vp_q, .movzxb, .tmp1y, .memia(.src0d, .tmp0, .add_size), ._, ._ },
26967 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"8", .tmp0, .add_size), .tmp1y, ._, ._ },
26968 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
26969 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26970 } },
26971 }, .{
26972 .required_features = .{ .avx, null, null, null },
26973 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any },
26974 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }},
26975 .patterns = &.{
26976 .{ .src = .{ .mem, .none } },
26977 },
26978 .extra_temps = .{
26979 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26980 .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } },
26981 .unused,
26982 .unused,
26983 .unused,
26984 .unused,
26985 .unused,
26986 .unused,
26987 .unused,
26988 },
26989 .dst_temps = .{.mem},
26990 .each = .{ .once = &.{
26991 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26992 .{ .@"0:", .vp_q, .movsxb, .tmp1x, .memia(.src0w, .tmp0, .add_size), ._, ._ },
26993 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_size), .tmp1x, ._, ._ },
26994 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
26995 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26996 } },
26997 }, .{
26998 .required_features = .{ .avx, null, null, null },
26999 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any },
27000 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }},
27001 .patterns = &.{
27002 .{ .src = .{ .mem, .none } },
27003 },
27004 .extra_temps = .{
27005 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27006 .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } },
27007 .unused,
27008 .unused,
27009 .unused,
27010 .unused,
27011 .unused,
27012 .unused,
27013 .unused,
27014 },
27015 .dst_temps = .{.mem},
27016 .each = .{ .once = &.{
27017 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27018 .{ .@"0:", .vp_q, .movzxb, .tmp1x, .memia(.src0w, .tmp0, .add_size), ._, ._ },
27019 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_size), .tmp1x, ._, ._ },
27020 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
27021 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27022 } },
27023 }, .{
27024 .required_features = .{ .sse4_1, null, null, null },
27025 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any },
27026 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }},
27027 .patterns = &.{
27028 .{ .src = .{ .mem, .none } },
27029 },
27030 .extra_temps = .{
27031 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27032 .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } },
27033 .unused,
27034 .unused,
27035 .unused,
27036 .unused,
27037 .unused,
27038 .unused,
27039 .unused,
27040 },
27041 .dst_temps = .{.mem},
27042 .each = .{ .once = &.{
27043 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27044 .{ .@"0:", .p_q, .movsxb, .tmp1x, .memia(.src0w, .tmp0, .add_size), ._, ._ },
27045 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_size), .tmp1x, ._, ._ },
27046 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
27047 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27048 } },
27049 }, .{
27050 .required_features = .{ .sse4_1, null, null, null },
27051 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any },
27052 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }},
27053 .patterns = &.{
27054 .{ .src = .{ .mem, .none } },
27055 },
27056 .extra_temps = .{
27057 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27058 .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } },
27059 .unused,
27060 .unused,
27061 .unused,
27062 .unused,
27063 .unused,
27064 .unused,
27065 .unused,
27066 },
27067 .dst_temps = .{.mem},
27068 .each = .{ .once = &.{
27069 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27070 .{ .@"0:", .p_q, .movzxb, .tmp1x, .memia(.src0w, .tmp0, .add_size), ._, ._ },
27071 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_size), .tmp1x, ._, ._ },
27072 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
27073 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27074 } },
27075 }, .{
27076 .required_features = .{ .@"64bit", .slow_incdec, null, null },
27077 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
27078 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }},
27079 .patterns = &.{
27080 .{ .src = .{ .to_mem, .none } },
27081 },
27082 .extra_temps = .{
27083 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27084 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
27085 .unused,
27086 .unused,
27087 .unused,
27088 .unused,
27089 .unused,
27090 .unused,
27091 .unused,
27092 },
27093 .dst_temps = .{.mem},
27094 .each = .{ .once = &.{
27095 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27096 .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27097 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
27098 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
27099 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27100 } },
27101 }, .{
27102 .required_features = .{ .@"64bit", null, null, null },
27103 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
27104 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }},
27105 .patterns = &.{
27106 .{ .src = .{ .to_mem, .none } },
27107 },
27108 .extra_temps = .{
27109 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27110 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
27111 .unused,
27112 .unused,
27113 .unused,
27114 .unused,
27115 .unused,
27116 .unused,
27117 .unused,
27118 },
27119 .dst_temps = .{.mem},
27120 .each = .{ .once = &.{
27121 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27122 .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27123 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
27124 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
27125 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
27126 } },
27127 }, .{
27128 .required_features = .{ .slow_incdec, null, null, null },
27129 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
27130 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }},
27131 .patterns = &.{
27132 .{ .src = .{ .to_mem, .none } },
27133 },
27134 .extra_temps = .{
27135 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27136 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
27137 .unused,
27138 .unused,
27139 .unused,
27140 .unused,
27141 .unused,
27142 .unused,
27143 .unused,
27144 },
27145 .dst_temps = .{.mem},
27146 .each = .{ .once = &.{
27147 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27148 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27149 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
27150 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
27151 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27152 } },
27153 }, .{
27154 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
27155 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }},
27156 .patterns = &.{
27157 .{ .src = .{ .to_mem, .none } },
27158 },
27159 .extra_temps = .{
27160 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27161 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
27162 .unused,
27163 .unused,
27164 .unused,
27165 .unused,
27166 .unused,
27167 .unused,
27168 .unused,
27169 },
27170 .dst_temps = .{.mem},
27171 .each = .{ .once = &.{
27172 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27173 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27174 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
27175 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
27176 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
27177 } },
27178 }, .{
27179 .required_features = .{ .avx, null, null, null },
27180 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
27181 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
27182 .patterns = &.{
27183 .{ .src = .{ .mem, .none } },
27184 .{ .src = .{ .to_sse, .none } },
27185 },
27186 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27187 .each = .{ .once = &.{
27188 .{ ._, .vp_q, .movsxb, .dst0x, .src0w, ._, ._ },
27189 .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ },
27190 } },
27191 }, .{
27192 .required_features = .{ .avx, null, null, null },
27193 .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any },
27194 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
27195 .patterns = &.{
27196 .{ .src = .{ .mem, .none } },
27197 .{ .src = .{ .to_sse, .none } },
27198 },
27199 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27200 .each = .{ .once = &.{
27201 .{ ._, .vp_q, .movzxb, .dst0x, .src0w, ._, ._ },
27202 .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ },
27203 } },
27204 }, .{
27205 .required_features = .{ .sse4_1, null, null, null },
27206 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
27207 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
27208 .patterns = &.{
27209 .{ .src = .{ .mem, .none } },
27210 .{ .src = .{ .to_sse, .none } },
27211 },
27212 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27213 .each = .{ .once = &.{
27214 .{ ._, .p_q, .movsxb, .dst0x, .src0w, ._, ._ },
27215 .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ },
27216 } },
27217 }, .{
27218 .required_features = .{ .sse4_1, null, null, null },
27219 .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any },
27220 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
27221 .patterns = &.{
27222 .{ .src = .{ .mem, .none } },
27223 .{ .src = .{ .to_sse, .none } },
27224 },
27225 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27226 .each = .{ .once = &.{
27227 .{ ._, .p_q, .movzxb, .dst0x, .src0w, ._, ._ },
27228 .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ },
27229 } },
27230 }, .{
27231 .required_features = .{ .sse2, null, null, null },
27232 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
27233 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
27234 .patterns = &.{
27235 .{ .src = .{ .to_mut_sse, .none } },
27236 },
27237 .extra_temps = .{
27238 .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } },
27239 .unused,
27240 .unused,
27241 .unused,
27242 .unused,
27243 .unused,
27244 .unused,
27245 .unused,
27246 .unused,
27247 },
27248 .dst_temps = .{.{ .ref = .src0 }},
27249 .each = .{ .once = &.{
27250 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
27251 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },
27252 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
27253 .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ },
27254 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
27255 .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ },
27256 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
27257 .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ },
27258 .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ },
27259 } },
27260 }, .{
27261 .required_features = .{ .sse2, null, null, null },
27262 .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any },
27263 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
27264 .patterns = &.{
27265 .{ .src = .{ .to_mut_sse, .none } },
27266 },
27267 .extra_temps = .{
27268 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
27269 .unused,
27270 .unused,
27271 .unused,
27272 .unused,
27273 .unused,
27274 .unused,
27275 .unused,
27276 .unused,
27277 },
27278 .dst_temps = .{.{ .ref = .src0 }},
27279 .each = .{ .once = &.{
27280 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
27281 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
27282 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
27283 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
27284 .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ },
27285 } },
27286 }, .{
27287 .required_features = .{ .@"64bit", .slow_incdec, null, null },
27288 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
27289 .dst_constraints = .{.any_scalar_signed_int},
27290 .patterns = &.{
27291 .{ .src = .{ .to_mem, .none } },
27292 },
27293 .extra_temps = .{
27294 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27295 .{ .type = .usize, .kind = .{ .reg = .rdi } },
27296 .{ .type = .i64, .kind = .{ .reg = .rax } },
27297 .{ .type = .u32, .kind = .{ .reg = .ecx } },
27298 .unused,
27299 .unused,
27300 .unused,
27301 .unused,
27302 .unused,
27303 },
27304 .dst_temps = .{.mem},
27305 .each = .{ .once = &.{
27306 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27307 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
27308 .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27309 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
27310 .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ },
27311 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
27312 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
27313 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
27314 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27315 } },
27316 }, .{
27317 .required_features = .{ .@"64bit", null, null, null },
27318 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any },
27319 .dst_constraints = .{.any_scalar_signed_int},
27320 .patterns = &.{
27321 .{ .src = .{ .to_mem, .none } },
27322 },
27323 .extra_temps = .{
27324 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27325 .{ .type = .usize, .kind = .{ .reg = .rdi } },
27326 .{ .type = .i64, .kind = .{ .reg = .rax } },
27327 .{ .type = .u32, .kind = .{ .reg = .ecx } },
27328 .unused,
27329 .unused,
27330 .unused,
27331 .unused,
27332 .unused,
27333 },
27334 .dst_temps = .{.mem},
27335 .each = .{ .once = &.{
27336 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27337 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
27338 .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27339 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
27340 .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ },
27341 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
27342 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
27343 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
27344 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
27345 } },
27346 }, .{
27347 .required_features = .{ .@"64bit", .slow_incdec, null, null },
27348 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
27349 .dst_constraints = .{.any_scalar_int},
27350 .patterns = &.{
27351 .{ .src = .{ .to_mem, .none } },
27352 },
27353 .extra_temps = .{
27354 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27355 .{ .type = .usize, .kind = .{ .reg = .rdi } },
27356 .{ .type = .u64, .kind = .{ .reg = .rax } },
27357 .{ .type = .u32, .kind = .{ .reg = .ecx } },
27358 .unused,
27359 .unused,
27360 .unused,
27361 .unused,
27362 .unused,
27363 },
27364 .dst_temps = .{.mem},
27365 .each = .{ .once = &.{
27366 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27367 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
27368 .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27369 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
27370 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
27371 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
27372 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
27373 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
27374 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27375 } },
27376 }, .{
27377 .required_features = .{ .@"64bit", null, null, null },
27378 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
27379 .dst_constraints = .{.any_scalar_int},
27380 .patterns = &.{
27381 .{ .src = .{ .to_mem, .none } },
27382 },
27383 .extra_temps = .{
27384 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27385 .{ .type = .usize, .kind = .{ .reg = .rdi } },
27386 .{ .type = .u64, .kind = .{ .reg = .rax } },
27387 .{ .type = .u32, .kind = .{ .reg = .ecx } },
27388 .unused,
27389 .unused,
27390 .unused,
27391 .unused,
27392 .unused,
27393 },
27394 .dst_temps = .{.mem},
27395 .each = .{ .once = &.{
27396 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27397 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
27398 .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
27399 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
27400 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
27401 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
27402 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
27403 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
27404 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
27405 } },
27406 }, .{
27407 .required_features = .{ .avx, null, null, null },
27408 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any },
27409 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }},
27410 .patterns = &.{
27411 .{ .src = .{ .mem, .none } },
27412 .{ .src = .{ .to_sse, .none } },
27413 },
27414 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27415 .each = .{ .once = &.{
27416 .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ },
27417 } },
27418 }, .{
27419 .required_features = .{ .avx, null, null, null },
27420 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any },
27421 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .dword } }},
27422 .patterns = &.{
27423 .{ .src = .{ .mem, .none } },
27424 .{ .src = .{ .to_sse, .none } },
27425 },
27426 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27427 .each = .{ .once = &.{
27428 .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ },
27429 } },
27430 }, .{
27431 .required_features = .{ .sse4_1, null, null, null },
27432 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any },
27433 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }},
27434 .patterns = &.{
27435 .{ .src = .{ .mem, .none } },
27436 .{ .src = .{ .to_sse, .none } },
27437 },
27438 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27439 .each = .{ .once = &.{
27440 .{ ._, .p_d, .movsxw, .dst0x, .src0q, ._, ._ },
27441 } },
27442 }, .{
27443 .required_features = .{ .sse4_1, null, null, null },
27444 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any },
27445 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .dword } }},
27446 .patterns = &.{
27447 .{ .src = .{ .mem, .none } },
27448 .{ .src = .{ .to_sse, .none } },
27449 },
27450 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27451 .each = .{ .once = &.{
27452 .{ ._, .p_d, .movzxw, .dst0x, .src0q, ._, ._ },
27453 } },
27454 }, .{
27455 .required_features = .{ .sse2, null, null, null },
27456 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any },
27457 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }},
27458 .patterns = &.{
27459 .{ .src = .{ .to_mut_sse, .none } },
27460 },
27461 .extra_temps = .{
27462 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
27463 .unused,
27464 .unused,
27465 .unused,
27466 .unused,
27467 .unused,
27468 .unused,
27469 .unused,
27470 .unused,
27471 },
27472 .dst_temps = .{.{ .ref = .src0 }},
27473 .each = .{ .once = &.{
27474 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
27475 .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ },
27476 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
27477 } },
27478 }, .{
27479 .required_features = .{ .sse2, null, null, null },
27480 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any },
27481 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .dword } }},
27482 .patterns = &.{
27483 .{ .src = .{ .to_mut_sse, .none } },
27484 },
27485 .extra_temps = .{
27486 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
27487 .unused,
27488 .unused,
27489 .unused,
27490 .unused,
27491 .unused,
27492 .unused,
27493 .unused,
27494 .unused,
27495 },
27496 .dst_temps = .{.{ .ref = .src0 }},
27497 .each = .{ .once = &.{
27498 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
27499 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
27500 } },
27501 }, .{
27502 .required_features = .{ .avx2, null, null, null },
27503 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any },
27504 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }},
27505 .patterns = &.{
27506 .{ .src = .{ .mem, .none } },
27507 .{ .src = .{ .to_sse, .none } },
27508 },
27509 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27510 .each = .{ .once = &.{
27511 .{ ._, .vp_d, .movsxw, .dst0y, .src0x, ._, ._ },
27512 } },
27513 }, .{
27514 .required_features = .{ .avx2, null, null, null },
27515 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
27516 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .dword } }},
27517 .patterns = &.{
27518 .{ .src = .{ .mem, .none } },
27519 .{ .src = .{ .to_sse, .none } },
27520 },
27521 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27522 .each = .{ .once = &.{
27523 .{ ._, .vp_d, .movzxw, .dst0y, .src0x, ._, ._ },
27524 } },
27525 }, .{
27526 .required_features = .{ .avx2, null, null, null },
27527 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any },
27528 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }},
27529 .patterns = &.{
27530 .{ .src = .{ .mem, .none } },
27531 },
27532 .extra_temps = .{
27533 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27534 .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } },
27535 .unused,
27536 .unused,
27537 .unused,
27538 .unused,
27539 .unused,
27540 .unused,
27541 .unused,
27542 },
27543 .dst_temps = .{.mem},
27544 .each = .{ .once = &.{
27545 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27546 .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ },
27547 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_size), .tmp1y, ._, ._ },
27548 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27549 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27550 } },
27551 }, .{
27552 .required_features = .{ .avx2, null, null, null },
27553 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any },
27554 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }},
27555 .patterns = &.{
27556 .{ .src = .{ .mem, .none } },
27557 },
27558 .extra_temps = .{
27559 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27560 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
27561 .unused,
27562 .unused,
27563 .unused,
27564 .unused,
27565 .unused,
27566 .unused,
27567 .unused,
27568 },
27569 .dst_temps = .{.mem},
27570 .each = .{ .once = &.{
27571 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27572 .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ },
27573 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_size), .tmp1y, ._, ._ },
27574 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27575 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27576 } },
27577 }, .{
27578 .required_features = .{ .avx, null, null, null },
27579 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any },
27580 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }},
27581 .patterns = &.{
27582 .{ .src = .{ .mem, .none } },
27583 },
27584 .extra_temps = .{
27585 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27586 .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } },
27587 .unused,
27588 .unused,
27589 .unused,
27590 .unused,
27591 .unused,
27592 .unused,
27593 .unused,
27594 },
27595 .dst_temps = .{.mem},
27596 .each = .{ .once = &.{
27597 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27598 .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
27599 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
27600 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27601 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27602 } },
27603 }, .{
27604 .required_features = .{ .avx, null, null, null },
27605 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any },
27606 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }},
27607 .patterns = &.{
27608 .{ .src = .{ .mem, .none } },
27609 },
27610 .extra_temps = .{
27611 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27612 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
27613 .unused,
27614 .unused,
27615 .unused,
27616 .unused,
27617 .unused,
27618 .unused,
27619 .unused,
27620 },
27621 .dst_temps = .{.mem},
27622 .each = .{ .once = &.{
27623 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27624 .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
27625 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
27626 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27627 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27628 } },
27629 }, .{
27630 .required_features = .{ .sse4_1, null, null, null },
27631 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any },
27632 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }},
27633 .patterns = &.{
27634 .{ .src = .{ .mem, .none } },
27635 },
27636 .extra_temps = .{
27637 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27638 .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } },
27639 .unused,
27640 .unused,
27641 .unused,
27642 .unused,
27643 .unused,
27644 .unused,
27645 .unused,
27646 },
27647 .dst_temps = .{.mem},
27648 .each = .{ .once = &.{
27649 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27650 .{ .@"0:", .p_d, .movsxw, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
27651 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
27652 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27653 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27654 } },
27655 }, .{
27656 .required_features = .{ .sse4_1, null, null, null },
27657 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any },
27658 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }},
27659 .patterns = &.{
27660 .{ .src = .{ .mem, .none } },
27661 },
27662 .extra_temps = .{
27663 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27664 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
27665 .unused,
27666 .unused,
27667 .unused,
27668 .unused,
27669 .unused,
27670 .unused,
27671 .unused,
27672 },
27673 .dst_temps = .{.mem},
27674 .each = .{ .once = &.{
27675 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27676 .{ .@"0:", .p_d, .movzxw, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
27677 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
27678 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27679 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27680 } },
27681 }, .{
27682 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
27683 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }},
27684 .patterns = &.{
27685 .{ .src = .{ .to_mem, .none } },
27686 },
27687 .extra_temps = .{
27688 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27689 .{ .type = .i32, .kind = .{ .rc = .general_purpose } },
27690 .unused,
27691 .unused,
27692 .unused,
27693 .unused,
27694 .unused,
27695 .unused,
27696 .unused,
27697 },
27698 .dst_temps = .{.mem},
27699 .each = .{ .once = &.{
27700 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27701 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
27702 .{ ._, ._, .mov, .memsia(.dst0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
27703 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
27704 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27705 } },
27706 }, .{
27707 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
27708 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }},
27709 .patterns = &.{
27710 .{ .src = .{ .to_mem, .none } },
27711 },
27712 .extra_temps = .{
27713 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27714 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
27715 .unused,
27716 .unused,
27717 .unused,
27718 .unused,
27719 .unused,
27720 .unused,
27721 .unused,
27722 },
27723 .dst_temps = .{.mem},
27724 .each = .{ .once = &.{
27725 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27726 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
27727 .{ ._, ._, .mov, .memsia(.dst0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
27728 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
27729 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27730 } },
27731 }, .{
27732 .required_features = .{ .avx, null, null, null },
27733 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any },
27734 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
27735 .patterns = &.{
27736 .{ .src = .{ .mem, .none } },
27737 .{ .src = .{ .to_sse, .none } },
27738 },
27739 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27740 .each = .{ .once = &.{
27741 .{ ._, .vp_q, .movsxw, .dst0x, .src0d, ._, ._ },
27742 } },
27743 }, .{
27744 .required_features = .{ .avx, null, null, null },
27745 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any },
27746 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
27747 .patterns = &.{
27748 .{ .src = .{ .mem, .none } },
27749 .{ .src = .{ .to_sse, .none } },
27750 },
27751 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27752 .each = .{ .once = &.{
27753 .{ ._, .vp_q, .movzxw, .dst0x, .src0d, ._, ._ },
27754 } },
27755 }, .{
27756 .required_features = .{ .sse4_1, null, null, null },
27757 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any },
27758 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
27759 .patterns = &.{
27760 .{ .src = .{ .mem, .none } },
27761 .{ .src = .{ .to_sse, .none } },
27762 },
27763 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27764 .each = .{ .once = &.{
27765 .{ ._, .p_q, .movsxw, .dst0x, .src0d, ._, ._ },
27766 } },
27767 }, .{
27768 .required_features = .{ .sse4_1, null, null, null },
27769 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any },
27770 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
27771 .patterns = &.{
27772 .{ .src = .{ .mem, .none } },
27773 .{ .src = .{ .to_sse, .none } },
27774 },
27775 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27776 .each = .{ .once = &.{
27777 .{ ._, .p_q, .movzxw, .dst0x, .src0d, ._, ._ },
27778 } },
27779 }, .{
27780 .required_features = .{ .sse2, null, null, null },
27781 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any },
27782 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
27783 .patterns = &.{
27784 .{ .src = .{ .to_mut_sse, .none } },
27785 },
27786 .extra_temps = .{
27787 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
27788 .unused,
27789 .unused,
27790 .unused,
27791 .unused,
27792 .unused,
27793 .unused,
27794 .unused,
27795 .unused,
27796 },
27797 .dst_temps = .{.{ .ref = .src0 }},
27798 .each = .{ .once = &.{
27799 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
27800 .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ },
27801 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
27802 .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ },
27803 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
27804 } },
27805 }, .{
27806 .required_features = .{ .sse2, null, null, null },
27807 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any },
27808 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
27809 .patterns = &.{
27810 .{ .src = .{ .to_mut_sse, .none } },
27811 },
27812 .extra_temps = .{
27813 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
27814 .unused,
27815 .unused,
27816 .unused,
27817 .unused,
27818 .unused,
27819 .unused,
27820 .unused,
27821 .unused,
27822 },
27823 .dst_temps = .{.{ .ref = .src0 }},
27824 .each = .{ .once = &.{
27825 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
27826 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
27827 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
27828 } },
27829 }, .{
27830 .required_features = .{ .avx2, null, null, null },
27831 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any },
27832 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }},
27833 .patterns = &.{
27834 .{ .src = .{ .mem, .none } },
27835 .{ .src = .{ .to_sse, .none } },
27836 },
27837 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27838 .each = .{ .once = &.{
27839 .{ ._, .vp_q, .movsxw, .dst0y, .src0q, ._, ._ },
27840 } },
27841 }, .{
27842 .required_features = .{ .avx2, null, null, null },
27843 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any },
27844 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .qword } }},
27845 .patterns = &.{
27846 .{ .src = .{ .mem, .none } },
27847 .{ .src = .{ .to_sse, .none } },
27848 },
27849 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
27850 .each = .{ .once = &.{
27851 .{ ._, .vp_q, .movzxw, .dst0y, .src0q, ._, ._ },
27852 } },
27853 }, .{
27854 .required_features = .{ .avx2, null, null, null },
27855 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any },
27856 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }},
27857 .patterns = &.{
27858 .{ .src = .{ .mem, .none } },
27859 },
27860 .extra_temps = .{
27861 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27862 .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } },
27863 .unused,
27864 .unused,
27865 .unused,
27866 .unused,
27867 .unused,
27868 .unused,
27869 .unused,
27870 },
27871 .dst_temps = .{.mem},
27872 .each = .{ .once = &.{
27873 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27874 .{ .@"0:", .vp_q, .movsxw, .tmp1y, .memia(.src0q, .tmp0, .add_size), ._, ._ },
27875 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_size), .tmp1y, ._, ._ },
27876 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27877 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27878 } },
27879 }, .{
27880 .required_features = .{ .avx2, null, null, null },
27881 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any },
27882 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }},
27883 .patterns = &.{
27884 .{ .src = .{ .mem, .none } },
27885 },
27886 .extra_temps = .{
27887 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27888 .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } },
27889 .unused,
27890 .unused,
27891 .unused,
27892 .unused,
27893 .unused,
27894 .unused,
27895 .unused,
27896 },
27897 .dst_temps = .{.mem},
27898 .each = .{ .once = &.{
27899 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27900 .{ .@"0:", .vp_q, .movzxw, .tmp1y, .memia(.src0q, .tmp0, .add_size), ._, ._ },
27901 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_size), .tmp1y, ._, ._ },
27902 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27903 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27904 } },
27905 }, .{
27906 .required_features = .{ .avx, null, null, null },
27907 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any },
27908 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }},
27909 .patterns = &.{
27910 .{ .src = .{ .mem, .none } },
27911 },
27912 .extra_temps = .{
27913 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27914 .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } },
27915 .unused,
27916 .unused,
27917 .unused,
27918 .unused,
27919 .unused,
27920 .unused,
27921 .unused,
27922 },
27923 .dst_temps = .{.mem},
27924 .each = .{ .once = &.{
27925 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27926 .{ .@"0:", .vp_q, .movsxw, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
27927 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
27928 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
27929 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27930 } },
27931 }, .{
27932 .required_features = .{ .avx, null, null, null },
27933 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any },
27934 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }},
27935 .patterns = &.{
27936 .{ .src = .{ .mem, .none } },
27937 },
27938 .extra_temps = .{
27939 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27940 .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } },
27941 .unused,
27942 .unused,
27943 .unused,
27944 .unused,
27945 .unused,
27946 .unused,
27947 .unused,
27948 },
27949 .dst_temps = .{.mem},
27950 .each = .{ .once = &.{
27951 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27952 .{ .@"0:", .vp_q, .movzxw, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
27953 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
27954 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
27955 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27956 } },
27957 }, .{
27958 .required_features = .{ .sse4_1, null, null, null },
27959 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any },
27960 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }},
27961 .patterns = &.{
27962 .{ .src = .{ .mem, .none } },
27963 },
27964 .extra_temps = .{
27965 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27966 .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } },
27967 .unused,
27968 .unused,
27969 .unused,
27970 .unused,
27971 .unused,
27972 .unused,
27973 .unused,
27974 },
27975 .dst_temps = .{.mem},
27976 .each = .{ .once = &.{
27977 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27978 .{ .@"0:", .p_q, .movsxw, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
27979 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
27980 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
27981 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27982 } },
27983 }, .{
27984 .required_features = .{ .sse4_1, null, null, null },
27985 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any },
27986 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }},
27987 .patterns = &.{
27988 .{ .src = .{ .mem, .none } },
27989 },
27990 .extra_temps = .{
27991 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27992 .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } },
27993 .unused,
27994 .unused,
27995 .unused,
27996 .unused,
27997 .unused,
27998 .unused,
27999 .unused,
28000 },
28001 .dst_temps = .{.mem},
28002 .each = .{ .once = &.{
28003 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28004 .{ .@"0:", .p_q, .movzxw, .tmp1x, .memia(.src0d, .tmp0, .add_size), ._, ._ },
28005 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_size), .tmp1x, ._, ._ },
28006 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
28007 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
28008 } },
28009 }, .{
28010 .required_features = .{ .@"64bit", null, null, null },
28011 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
28012 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }},
28013 .patterns = &.{
28014 .{ .src = .{ .to_mem, .none } },
28015 },
28016 .extra_temps = .{
28017 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28018 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
28019 .unused,
28020 .unused,
28021 .unused,
28022 .unused,
28023 .unused,
28024 .unused,
28025 .unused,
28026 },
28027 .dst_temps = .{.mem},
28028 .each = .{ .once = &.{
28029 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28030 .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
28031 .{ ._, ._, .mov, .memsia(.dst0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
28032 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
28033 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
28034 } },
28035 }, .{
28036 .required_features = .{ .@"64bit", null, null, null },
28037 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
28038 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }},
28039 .patterns = &.{
28040 .{ .src = .{ .to_mem, .none } },
28041 },
28042 .extra_temps = .{
28043 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28044 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
28045 .unused,
28046 .unused,
28047 .unused,
28048 .unused,
28049 .unused,
28050 .unused,
28051 .unused,
28052 },
28053 .dst_temps = .{.mem},
28054 .each = .{ .once = &.{
28055 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28056 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
28057 .{ ._, ._, .mov, .memsia(.dst0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
28058 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
28059 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
28060 } },
28061 }, .{
28062 .required_features = .{ .avx, null, null, null },
28063 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
28064 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
28065 .patterns = &.{
28066 .{ .src = .{ .mem, .none } },
28067 .{ .src = .{ .to_sse, .none } },
28068 },
28069 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28070 .each = .{ .once = &.{
28071 .{ ._, .vp_q, .movsxw, .dst0x, .src0d, ._, ._ },
28072 .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ },
28073 } },
28074 }, .{
28075 .required_features = .{ .avx, null, null, null },
28076 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any },
28077 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
28078 .patterns = &.{
28079 .{ .src = .{ .mem, .none } },
28080 .{ .src = .{ .to_sse, .none } },
28081 },
28082 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28083 .each = .{ .once = &.{
28084 .{ ._, .vp_q, .movzxw, .dst0x, .src0d, ._, ._ },
28085 .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ },
28086 } },
28087 }, .{
28088 .required_features = .{ .sse4_1, null, null, null },
28089 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
28090 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
28091 .patterns = &.{
28092 .{ .src = .{ .mem, .none } },
28093 .{ .src = .{ .to_sse, .none } },
28094 },
28095 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28096 .each = .{ .once = &.{
28097 .{ ._, .p_q, .movsxw, .dst0x, .src0d, ._, ._ },
28098 .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ },
28099 } },
28100 }, .{
28101 .required_features = .{ .sse4_1, null, null, null },
28102 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any },
28103 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
28104 .patterns = &.{
28105 .{ .src = .{ .mem, .none } },
28106 .{ .src = .{ .to_sse, .none } },
28107 },
28108 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28109 .each = .{ .once = &.{
28110 .{ ._, .p_q, .movzxw, .dst0x, .src0d, ._, ._ },
28111 .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ },
28112 } },
28113 }, .{
28114 .required_features = .{ .sse2, null, null, null },
28115 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
28116 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
28117 .patterns = &.{
28118 .{ .src = .{ .to_mut_sse, .none } },
28119 },
28120 .extra_temps = .{
28121 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
28122 .unused,
28123 .unused,
28124 .unused,
28125 .unused,
28126 .unused,
28127 .unused,
28128 .unused,
28129 .unused,
28130 },
28131 .dst_temps = .{.{ .ref = .src0 }},
28132 .each = .{ .once = &.{
28133 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
28134 .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ },
28135 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
28136 .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ },
28137 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
28138 .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ },
28139 .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ },
28140 } },
28141 }, .{
28142 .required_features = .{ .sse2, null, null, null },
28143 .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any },
28144 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
28145 .patterns = &.{
28146 .{ .src = .{ .to_mut_sse, .none } },
28147 },
28148 .extra_temps = .{
28149 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
28150 .unused,
28151 .unused,
28152 .unused,
28153 .unused,
28154 .unused,
28155 .unused,
28156 .unused,
28157 .unused,
28158 },
28159 .dst_temps = .{.{ .ref = .src0 }},
28160 .each = .{ .once = &.{
28161 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
28162 .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ },
28163 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
28164 .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ },
28165 } },
28166 }, .{
28167 .required_features = .{ .@"64bit", null, null, null },
28168 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any },
28169 .dst_constraints = .{.any_scalar_signed_int},
28170 .patterns = &.{
28171 .{ .src = .{ .to_mem, .none } },
28172 },
28173 .extra_temps = .{
28174 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28175 .{ .type = .usize, .kind = .{ .reg = .rdi } },
28176 .{ .type = .i64, .kind = .{ .reg = .rax } },
28177 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28178 .unused,
28179 .unused,
28180 .unused,
28181 .unused,
28182 .unused,
28183 },
28184 .dst_temps = .{.mem},
28185 .each = .{ .once = &.{
28186 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28187 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
28188 .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
28189 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
28190 .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ },
28191 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
28192 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28193 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
28194 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
28195 } },
28196 }, .{
28197 .required_features = .{ .@"64bit", null, null, null },
28198 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
28199 .dst_constraints = .{.any_scalar_int},
2460728200 .patterns = &.{
2460828201 .{ .src = .{ .to_mem, .none } },
2460928202 },
2461028203 .extra_temps = .{
24611 .{ .type = .usize, .kind = .{ .reg = .rsi } },
28204 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
2461228205 .{ .type = .usize, .kind = .{ .reg = .rdi } },
28206 .{ .type = .u64, .kind = .{ .reg = .rax } },
2461328207 .{ .type = .u32, .kind = .{ .reg = .ecx } },
2461428208 .unused,
2461528209 .unused,
2461628210 .unused,
2461728211 .unused,
2461828212 .unused,
24619 .unused,
2462028213 },
2462128214 .dst_temps = .{.mem},
24622 .clobbers = .{ .eflags = true },
2462328215 .each = .{ .once = &.{
24624 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
24625 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24626 .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ },
24627 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
28216 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28217 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
28218 .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
28219 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
28220 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
28221 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
28222 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28223 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
28224 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2462828225 } },
24629 } } else comptime &.{ .{
24630 .src_constraints = .{ .{ .signed_int = .byte }, .any },
24631 .dst_constraints = .{.{ .signed_int = .dword }},
28226 }, .{
28227 .required_features = .{ .avx, null, null, null },
28228 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any },
28229 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
2463228230 .patterns = &.{
2463328231 .{ .src = .{ .mem, .none } },
24634 .{ .src = .{ .to_gpr, .none } },
28232 .{ .src = .{ .to_sse, .none } },
2463528233 },
24636 .dst_temps = .{.{ .rc = .general_purpose }},
28234 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2463728235 .each = .{ .once = &.{
24638 .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ },
28236 .{ ._, .vp_q, .movsxd, .dst0x, .src0q, ._, ._ },
2463928237 } },
2464028238 }, .{
24641 .src_constraints = .{ .{ .int = .byte }, .any },
24642 .dst_constraints = .{.{ .int = .dword }},
28239 .required_features = .{ .avx, null, null, null },
28240 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any },
28241 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
2464328242 .patterns = &.{
2464428243 .{ .src = .{ .mem, .none } },
24645 .{ .src = .{ .to_gpr, .none } },
28244 .{ .src = .{ .to_sse, .none } },
2464628245 },
24647 .dst_temps = .{.{ .rc = .general_purpose }},
28246 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2464828247 .each = .{ .once = &.{
24649 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
28248 .{ ._, .vp_q, .movzxd, .dst0x, .src0q, ._, ._ },
2465028249 } },
2465128250 }, .{
24652 .required_features = .{ .@"64bit", null, null, null },
24653 .src_constraints = .{ .{ .signed_int = .byte }, .any },
24654 .dst_constraints = .{.{ .signed_int = .qword }},
28251 .required_features = .{ .sse4_1, null, null, null },
28252 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any },
28253 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
2465528254 .patterns = &.{
2465628255 .{ .src = .{ .mem, .none } },
24657 .{ .src = .{ .to_gpr, .none } },
28256 .{ .src = .{ .to_sse, .none } },
2465828257 },
24659 .dst_temps = .{.{ .rc = .general_purpose }},
28258 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2466028259 .each = .{ .once = &.{
24661 .{ ._, ._, .movsx, .dst0q, .src0b, ._, ._ },
28260 .{ ._, .p_q, .movsxd, .dst0x, .src0q, ._, ._ },
2466228261 } },
2466328262 }, .{
24664 .required_features = .{ .@"64bit", null, null, null },
24665 .src_constraints = .{ .{ .int = .byte }, .any },
24666 .dst_constraints = .{.{ .int = .qword }},
28263 .required_features = .{ .sse4_1, null, null, null },
28264 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any },
28265 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
2466728266 .patterns = &.{
2466828267 .{ .src = .{ .mem, .none } },
24669 .{ .src = .{ .to_gpr, .none } },
28268 .{ .src = .{ .to_sse, .none } },
2467028269 },
24671 .dst_temps = .{.{ .rc = .general_purpose }},
28270 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2467228271 .each = .{ .once = &.{
24673 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
28272 .{ ._, .p_q, .movzxd, .dst0x, .src0q, ._, ._ },
2467428273 } },
2467528274 }, .{
24676 .required_features = .{ .@"64bit", null, null, null },
24677 .src_constraints = .{ .{ .signed_int = .byte }, .any },
24678 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
28275 .required_features = .{ .sse2, null, null, null },
28276 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any },
28277 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }},
2467928278 .patterns = &.{
24680 .{ .src = .{ .mem, .none } },
24681 .{ .src = .{ .to_gpr, .none } },
28279 .{ .src = .{ .to_mut_sse, .none } },
2468228280 },
2468328281 .extra_temps = .{
24684 .{ .type = .i64, .kind = .{ .reg = .rax } },
24685 .{ .type = .usize, .kind = .{ .reg = .rdi } },
24686 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28282 .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } },
28283 .unused,
28284 .unused,
2468728285 .unused,
2468828286 .unused,
2468928287 .unused,
......@@ -24691,28 +28289,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2469128289 .unused,
2469228290 .unused,
2469328291 },
24694 .dst_temps = .{.mem},
24695 .clobbers = .{ .eflags = true },
28292 .dst_temps = .{.{ .ref = .src0 }},
2469628293 .each = .{ .once = &.{
24697 .{ ._, ._, .movsx, .tmp0q, .src0b, ._, ._ },
24698 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24699 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
24700 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
24701 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
24702 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28294 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
28295 .{ ._, .p_d, .cmpgt, .tmp0x, .src0x, ._, ._ },
28296 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
2470328297 } },
2470428298 }, .{
24705 .required_features = .{ .@"64bit", null, null, null },
24706 .src_constraints = .{ .{ .int = .byte }, .any },
24707 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
28299 .required_features = .{ .sse2, null, null, null },
28300 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any },
28301 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .qword } }},
2470828302 .patterns = &.{
24709 .{ .src = .{ .mem, .none } },
24710 .{ .src = .{ .to_gpr, .none } },
28303 .{ .src = .{ .to_mut_sse, .none } },
2471128304 },
2471228305 .extra_temps = .{
24713 .{ .type = .u64, .kind = .{ .reg = .rax } },
24714 .{ .type = .usize, .kind = .{ .reg = .rdi } },
24715 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28306 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
28307 .unused,
28308 .unused,
2471628309 .unused,
2471728310 .unused,
2471828311 .unused,
......@@ -24720,27 +28313,46 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2472028313 .unused,
2472128314 .unused,
2472228315 },
24723 .dst_temps = .{.mem},
24724 .clobbers = .{ .eflags = true },
28316 .dst_temps = .{.{ .ref = .src0 }},
2472528317 .each = .{ .once = &.{
24726 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
24727 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24728 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
24729 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
24730 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
24731 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28318 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
28319 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
2473228320 } },
2473328321 }, .{
24734 .src_constraints = .{ .{ .signed_int = .byte }, .any },
24735 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .dword, .is = .dword } }},
28322 .required_features = .{ .avx2, null, null, null },
28323 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any },
28324 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }},
28325 .patterns = &.{
28326 .{ .src = .{ .mem, .none } },
28327 .{ .src = .{ .to_sse, .none } },
28328 },
28329 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28330 .each = .{ .once = &.{
28331 .{ ._, .vp_q, .movsxd, .dst0y, .src0x, ._, ._ },
28332 } },
28333 }, .{
28334 .required_features = .{ .avx2, null, null, null },
28335 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
28336 .dst_constraints = .{.{ .scalar_int = .{ .of = .yword, .is = .qword } }},
28337 .patterns = &.{
28338 .{ .src = .{ .mem, .none } },
28339 .{ .src = .{ .to_sse, .none } },
28340 },
28341 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28342 .each = .{ .once = &.{
28343 .{ ._, .vp_q, .movzxd, .dst0y, .src0x, ._, ._ },
28344 } },
28345 }, .{
28346 .required_features = .{ .avx2, null, null, null },
28347 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any },
28348 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }},
2473628349 .patterns = &.{
2473728350 .{ .src = .{ .mem, .none } },
24738 .{ .src = .{ .to_gpr, .none } },
2473928351 },
2474028352 .extra_temps = .{
24741 .{ .type = .i32, .kind = .{ .reg = .eax } },
24742 .{ .type = .usize, .kind = .{ .reg = .edi } },
24743 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28353 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28354 .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } },
28355 .unused,
2474428356 .unused,
2474528357 .unused,
2474628358 .unused,
......@@ -24749,26 +28361,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2474928361 .unused,
2475028362 },
2475128363 .dst_temps = .{.mem},
24752 .clobbers = .{ .eflags = true },
2475328364 .each = .{ .once = &.{
24754 .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ },
24755 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24756 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
24757 .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ },
24758 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
24759 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28365 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28366 .{ .@"0:", .vp_q, .movsxd, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ },
28367 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_size), .tmp1y, ._, ._ },
28368 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
28369 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2476028370 } },
2476128371 }, .{
24762 .src_constraints = .{ .{ .int = .byte }, .any },
24763 .dst_constraints = .{.{ .remainder_int = .{ .of = .dword, .is = .dword } }},
28372 .required_features = .{ .avx2, null, null, null },
28373 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any },
28374 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }},
2476428375 .patterns = &.{
2476528376 .{ .src = .{ .mem, .none } },
24766 .{ .src = .{ .to_gpr, .none } },
2476728377 },
2476828378 .extra_temps = .{
24769 .{ .type = .u32, .kind = .{ .reg = .eax } },
24770 .{ .type = .usize, .kind = .{ .reg = .edi } },
24771 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28379 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28380 .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } },
28381 .unused,
2477228382 .unused,
2477328383 .unused,
2477428384 .unused,
......@@ -24777,73 +28387,76 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2477728387 .unused,
2477828388 },
2477928389 .dst_temps = .{.mem},
24780 .clobbers = .{ .eflags = true },
2478128390 .each = .{ .once = &.{
24782 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
24783 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24784 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
24785 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
24786 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
24787 .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ },
28391 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28392 .{ .@"0:", .vp_q, .movzxd, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ },
28393 .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_size), .tmp1y, ._, ._ },
28394 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
28395 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2478828396 } },
2478928397 }, .{
24790 .src_constraints = .{ .{ .signed_int = .word }, .any },
24791 .dst_constraints = .{.{ .signed_int = .dword }},
28398 .required_features = .{ .avx, null, null, null },
28399 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any },
28400 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }},
2479228401 .patterns = &.{
2479328402 .{ .src = .{ .mem, .none } },
24794 .{ .src = .{ .to_gpr, .none } },
2479528403 },
24796 .dst_temps = .{.{ .rc = .general_purpose }},
24797 .each = .{ .once = &.{
24798 .{ ._, ._, .movsx, .dst0d, .src0w, ._, ._ },
24799 } },
24800 }, .{
24801 .src_constraints = .{ .{ .int = .word }, .any },
24802 .dst_constraints = .{.{ .int = .dword }},
24803 .patterns = &.{
24804 .{ .src = .{ .mem, .none } },
24805 .{ .src = .{ .to_gpr, .none } },
28404 .extra_temps = .{
28405 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28406 .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } },
28407 .unused,
28408 .unused,
28409 .unused,
28410 .unused,
28411 .unused,
28412 .unused,
28413 .unused,
2480628414 },
24807 .dst_temps = .{.{ .rc = .general_purpose }},
28415 .dst_temps = .{.mem},
2480828416 .each = .{ .once = &.{
24809 .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ },
28417 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28418 .{ .@"0:", .vp_q, .movsxd, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
28419 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
28420 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
28421 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2481028422 } },
2481128423 }, .{
24812 .required_features = .{ .@"64bit", null, null, null },
24813 .src_constraints = .{ .{ .signed_int = .word }, .any },
24814 .dst_constraints = .{.{ .signed_int = .qword }},
28424 .required_features = .{ .avx, null, null, null },
28425 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any },
28426 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }},
2481528427 .patterns = &.{
2481628428 .{ .src = .{ .mem, .none } },
24817 .{ .src = .{ .to_gpr, .none } },
2481828429 },
24819 .dst_temps = .{.{ .rc = .general_purpose }},
24820 .each = .{ .once = &.{
24821 .{ ._, ._, .movsx, .dst0q, .src0w, ._, ._ },
24822 } },
24823 }, .{
24824 .required_features = .{ .@"64bit", null, null, null },
24825 .src_constraints = .{ .{ .int = .word }, .any },
24826 .dst_constraints = .{.{ .int = .qword }},
24827 .patterns = &.{
24828 .{ .src = .{ .mem, .none } },
24829 .{ .src = .{ .to_gpr, .none } },
28430 .extra_temps = .{
28431 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28432 .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } },
28433 .unused,
28434 .unused,
28435 .unused,
28436 .unused,
28437 .unused,
28438 .unused,
28439 .unused,
2483028440 },
24831 .dst_temps = .{.{ .rc = .general_purpose }},
28441 .dst_temps = .{.mem},
2483228442 .each = .{ .once = &.{
24833 .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ },
28443 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28444 .{ .@"0:", .vp_q, .movzxd, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
28445 .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
28446 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
28447 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2483428448 } },
2483528449 }, .{
24836 .required_features = .{ .@"64bit", null, null, null },
24837 .src_constraints = .{ .{ .signed_int = .word }, .any },
24838 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
28450 .required_features = .{ .sse4_1, null, null, null },
28451 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any },
28452 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }},
2483928453 .patterns = &.{
2484028454 .{ .src = .{ .mem, .none } },
24841 .{ .src = .{ .to_gpr, .none } },
2484228455 },
2484328456 .extra_temps = .{
24844 .{ .type = .i64, .kind = .{ .reg = .rax } },
24845 .{ .type = .usize, .kind = .{ .reg = .rdi } },
24846 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28457 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28458 .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } },
28459 .unused,
2484728460 .unused,
2484828461 .unused,
2484928462 .unused,
......@@ -24852,27 +28465,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2485228465 .unused,
2485328466 },
2485428467 .dst_temps = .{.mem},
24855 .clobbers = .{ .eflags = true },
2485628468 .each = .{ .once = &.{
24857 .{ ._, ._, .movsx, .tmp0q, .src0w, ._, ._ },
24858 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24859 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
24860 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
24861 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
24862 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28469 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28470 .{ .@"0:", .p_q, .movsxd, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
28471 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
28472 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
28473 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2486328474 } },
2486428475 }, .{
24865 .required_features = .{ .@"64bit", null, null, null },
24866 .src_constraints = .{ .{ .int = .word }, .any },
24867 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
28476 .required_features = .{ .sse4_1, null, null, null },
28477 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .dword } }, .any },
28478 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }},
2486828479 .patterns = &.{
2486928480 .{ .src = .{ .mem, .none } },
24870 .{ .src = .{ .to_gpr, .none } },
2487128481 },
2487228482 .extra_temps = .{
24873 .{ .type = .u64, .kind = .{ .reg = .rax } },
24874 .{ .type = .usize, .kind = .{ .reg = .rdi } },
24875 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28483 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28484 .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } },
28485 .unused,
2487628486 .unused,
2487728487 .unused,
2487828488 .unused,
......@@ -24881,26 +28491,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2488128491 .unused,
2488228492 },
2488328493 .dst_temps = .{.mem},
24884 .clobbers = .{ .eflags = true },
2488528494 .each = .{ .once = &.{
24886 .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ },
24887 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24888 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
24889 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
24890 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
24891 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28495 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28496 .{ .@"0:", .p_q, .movzxd, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ },
28497 .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_size), .tmp1x, ._, ._ },
28498 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
28499 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2489228500 } },
2489328501 }, .{
24894 .src_constraints = .{ .{ .signed_int = .word }, .any },
24895 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .dword, .is = .dword } }},
28502 .required_features = .{ .@"64bit", null, null, null },
28503 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
28504 .dst_constraints = .{.{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }},
2489628505 .patterns = &.{
24897 .{ .src = .{ .mem, .none } },
24898 .{ .src = .{ .to_gpr, .none } },
28506 .{ .src = .{ .to_mem, .none } },
2489928507 },
2490028508 .extra_temps = .{
24901 .{ .type = .i32, .kind = .{ .reg = .eax } },
24902 .{ .type = .usize, .kind = .{ .reg = .edi } },
24903 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28509 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28510 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
28511 .unused,
2490428512 .unused,
2490528513 .unused,
2490628514 .unused,
......@@ -24909,26 +28517,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2490928517 .unused,
2491028518 },
2491128519 .dst_temps = .{.mem},
24912 .clobbers = .{ .eflags = true },
2491328520 .each = .{ .once = &.{
24914 .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ },
24915 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24916 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
24917 .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ },
24918 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
24919 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28521 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28522 .{ .@"0:", ._, .movsxd, .tmp1q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
28523 .{ ._, ._, .mov, .memsia(.dst0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
28524 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
28525 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2492028526 } },
2492128527 }, .{
24922 .src_constraints = .{ .{ .int = .word }, .any },
24923 .dst_constraints = .{.{ .remainder_int = .{ .of = .dword, .is = .dword } }},
28528 .required_features = .{ .@"64bit", null, null, null },
28529 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any },
28530 .dst_constraints = .{.{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }},
2492428531 .patterns = &.{
24925 .{ .src = .{ .mem, .none } },
24926 .{ .src = .{ .to_gpr, .none } },
28532 .{ .src = .{ .to_mem, .none } },
2492728533 },
2492828534 .extra_temps = .{
24929 .{ .type = .u32, .kind = .{ .reg = .eax } },
24930 .{ .type = .usize, .kind = .{ .reg = .edi } },
24931 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28535 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28536 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
28537 .unused,
2493228538 .unused,
2493328539 .unused,
2493428540 .unused,
......@@ -24937,51 +28543,76 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2493728543 .unused,
2493828544 },
2493928545 .dst_temps = .{.mem},
24940 .clobbers = .{ .eflags = true },
2494128546 .each = .{ .once = &.{
24942 .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ },
24943 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24944 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
24945 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
24946 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
24947 .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ },
28547 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28548 .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
28549 .{ ._, ._, .mov, .memsia(.dst0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
28550 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
28551 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2494828552 } },
2494928553 }, .{
24950 .required_features = .{ .@"64bit", null, null, null },
24951 .src_constraints = .{ .{ .signed_int = .dword }, .any },
24952 .dst_constraints = .{.{ .signed_int = .qword }},
28554 .required_features = .{ .avx, null, null, null },
28555 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
28556 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
2495328557 .patterns = &.{
2495428558 .{ .src = .{ .mem, .none } },
24955 .{ .src = .{ .to_gpr, .none } },
28559 .{ .src = .{ .to_sse, .none } },
2495628560 },
24957 .dst_temps = .{.{ .rc = .general_purpose }},
28561 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2495828562 .each = .{ .once = &.{
24959 .{ ._, ._d, .movsx, .dst0q, .src0d, ._, ._ },
28563 .{ ._, .vp_q, .movsxd, .dst0x, .src0q, ._, ._ },
28564 .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ },
2496028565 } },
2496128566 }, .{
24962 .required_features = .{ .@"64bit", null, null, null },
24963 .src_constraints = .{ .{ .int = .dword }, .any },
24964 .dst_constraints = .{.{ .int = .qword }},
28567 .required_features = .{ .avx, null, null, null },
28568 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any },
28569 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
2496528570 .patterns = &.{
2496628571 .{ .src = .{ .mem, .none } },
24967 .{ .src = .{ .to_gpr, .none } },
28572 .{ .src = .{ .to_sse, .none } },
2496828573 },
24969 .dst_temps = .{.{ .rc = .general_purpose }},
28574 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
2497028575 .each = .{ .once = &.{
24971 .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ },
28576 .{ ._, .vp_q, .movzxd, .dst0x, .src0q, ._, ._ },
28577 .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ },
2497228578 } },
2497328579 }, .{
24974 .required_features = .{ .@"64bit", null, null, null },
24975 .src_constraints = .{ .{ .signed_int = .dword }, .any },
24976 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
28580 .required_features = .{ .sse4_1, null, null, null },
28581 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
28582 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
2497728583 .patterns = &.{
2497828584 .{ .src = .{ .mem, .none } },
24979 .{ .src = .{ .to_gpr, .none } },
28585 .{ .src = .{ .to_sse, .none } },
28586 },
28587 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28588 .each = .{ .once = &.{
28589 .{ ._, .p_q, .movsxd, .dst0x, .src0q, ._, ._ },
28590 .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ },
28591 } },
28592 }, .{
28593 .required_features = .{ .sse4_1, null, null, null },
28594 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any },
28595 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
28596 .patterns = &.{
28597 .{ .src = .{ .mem, .none } },
28598 .{ .src = .{ .to_sse, .none } },
28599 },
28600 .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }},
28601 .each = .{ .once = &.{
28602 .{ ._, .p_q, .movzxd, .dst0x, .src0q, ._, ._ },
28603 .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ },
28604 } },
28605 }, .{
28606 .required_features = .{ .sse2, null, null, null },
28607 .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
28608 .dst_constraints = .{.{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }},
28609 .patterns = &.{
28610 .{ .src = .{ .to_mut_sse, .none } },
2498028611 },
2498128612 .extra_temps = .{
24982 .{ .type = .i64, .kind = .{ .reg = .rax } },
24983 .{ .type = .usize, .kind = .{ .reg = .rdi } },
24984 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28613 .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } },
28614 .unused,
28615 .unused,
2498528616 .unused,
2498628617 .unused,
2498728618 .unused,
......@@ -24989,28 +28620,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2498928620 .unused,
2499028621 .unused,
2499128622 },
24992 .dst_temps = .{.mem},
24993 .clobbers = .{ .eflags = true },
28623 .dst_temps = .{.{ .ref = .src0 }},
2499428624 .each = .{ .once = &.{
24995 .{ ._, ._d, .movsx, .tmp0q, .src0d, ._, ._ },
24996 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
24997 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
24998 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
24999 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25000 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28625 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
28626 .{ ._, .p_d, .cmpgt, .tmp0x, .src0x, ._, ._ },
28627 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
28628 .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ },
28629 .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ },
2500128630 } },
2500228631 }, .{
25003 .required_features = .{ .@"64bit", null, null, null },
25004 .src_constraints = .{ .{ .int = .dword }, .any },
25005 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
28632 .required_features = .{ .sse2, null, null, null },
28633 .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any },
28634 .dst_constraints = .{.{ .scalar_int = .{ .of = .xword, .is = .xword } }},
2500628635 .patterns = &.{
25007 .{ .src = .{ .mem, .none } },
25008 .{ .src = .{ .to_gpr, .none } },
28636 .{ .src = .{ .to_mut_sse, .none } },
2500928637 },
2501028638 .extra_temps = .{
25011 .{ .type = .u64, .kind = .{ .reg = .rax } },
25012 .{ .type = .usize, .kind = .{ .reg = .rdi } },
25013 .{ .type = .u32, .kind = .{ .reg = .ecx } },
28639 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
28640 .unused,
28641 .unused,
2501428642 .unused,
2501528643 .unused,
2501628644 .unused,
......@@ -25018,86 +28646,85 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2501828646 .unused,
2501928647 .unused,
2502028648 },
25021 .dst_temps = .{.mem},
25022 .clobbers = .{ .eflags = true },
28649 .dst_temps = .{.{ .ref = .src0 }},
2502328650 .each = .{ .once = &.{
25024 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
25025 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25026 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25027 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25028 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
25029 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28651 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
28652 .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ },
28653 .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ },
2503028654 } },
2503128655 }, .{
25032 .src_constraints = .{ .{ .signed_int = .dword }, .any },
25033 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .dword, .is = .dword } }},
28656 .required_features = .{ .@"64bit", null, null, null },
28657 .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
28658 .dst_constraints = .{.any_scalar_signed_int},
2503428659 .patterns = &.{
25035 .{ .src = .{ .mem, .none } },
25036 .{ .src = .{ .to_gpr, .none } },
28660 .{ .src = .{ .to_mem, .none } },
2503728661 },
2503828662 .extra_temps = .{
25039 .{ .type = .i32, .kind = .{ .reg = .eax } },
25040 .{ .type = .usize, .kind = .{ .reg = .edi } },
28663 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28664 .{ .type = .usize, .kind = .{ .reg = .rdi } },
28665 .{ .type = .i64, .kind = .{ .reg = .rax } },
2504128666 .{ .type = .u32, .kind = .{ .reg = .ecx } },
2504228667 .unused,
2504328668 .unused,
2504428669 .unused,
2504528670 .unused,
2504628671 .unused,
25047 .unused,
2504828672 },
2504928673 .dst_temps = .{.mem},
25050 .clobbers = .{ .eflags = true },
2505128674 .each = .{ .once = &.{
25052 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
25053 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25054 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25055 .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ },
25056 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
28675 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28676 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
28677 .{ .@"0:", ._, .movsxd, .tmp2q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
28678 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
28679 .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ },
28680 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
2505728681 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28682 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
28683 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2505828684 } },
2505928685 }, .{
25060 .src_constraints = .{ .{ .int = .dword }, .any },
25061 .dst_constraints = .{.{ .remainder_int = .{ .of = .dword, .is = .dword } }},
28686 .required_features = .{ .@"64bit", null, null, null },
28687 .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any },
28688 .dst_constraints = .{.any_scalar_int},
2506228689 .patterns = &.{
25063 .{ .src = .{ .mem, .none } },
25064 .{ .src = .{ .to_gpr, .none } },
28690 .{ .src = .{ .to_mem, .none } },
2506528691 },
2506628692 .extra_temps = .{
25067 .{ .type = .u32, .kind = .{ .reg = .eax } },
25068 .{ .type = .usize, .kind = .{ .reg = .edi } },
28693 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
28694 .{ .type = .usize, .kind = .{ .reg = .rdi } },
28695 .{ .type = .u64, .kind = .{ .reg = .rax } },
2506928696 .{ .type = .u32, .kind = .{ .reg = .ecx } },
2507028697 .unused,
2507128698 .unused,
2507228699 .unused,
2507328700 .unused,
2507428701 .unused,
25075 .unused,
2507628702 },
2507728703 .dst_temps = .{.mem},
25078 .clobbers = .{ .eflags = true },
2507928704 .each = .{ .once = &.{
25080 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
25081 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25082 .{ ._, ._sd, .sto, ._, ._, ._, ._ },
25083 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25084 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ },
25085 .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ },
28705 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
28706 .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ },
28707 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
28708 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
28709 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
28710 .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ },
28711 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28712 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
28713 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2508628714 } },
2508728715 }, .{
25088 .required_features = .{ .@"64bit", null, null, null },
25089 .src_constraints = .{ .{ .signed_int = .qword }, .any },
25090 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
28716 .required_features = .{ .@"64bit", .slow_incdec, null, null },
28717 .src_constraints = .{ .any_scalar_signed_int, .any },
28718 .dst_constraints = .{.any_scalar_signed_int},
2509128719 .patterns = &.{
25092 .{ .src = .{ .mem, .none } },
25093 .{ .src = .{ .to_gpr, .none } },
28720 .{ .src = .{ .to_mem, .none } },
2509428721 },
2509528722 .extra_temps = .{
25096 .{ .type = .i64, .kind = .{ .reg = .rax } },
28723 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
28724 .{ .type = .usize, .kind = .{ .reg = .rsi } },
2509728725 .{ .type = .usize, .kind = .{ .reg = .rdi } },
2509828726 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25099 .unused,
25100 .unused,
28727 .{ .type = .i64, .kind = .{ .reg = .rax } },
2510128728 .unused,
2510228729 .unused,
2510328730 .unused,
......@@ -25106,27 +28733,32 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2510628733 .dst_temps = .{.mem},
2510728734 .clobbers = .{ .eflags = true },
2510828735 .each = .{ .once = &.{
25109 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
25110 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
28736 .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ },
28737 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
28738 .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ },
28739 .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
28740 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
28741 .{ ._, ._sq, .lod, ._, ._, ._, ._ },
2511128742 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25112 .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ },
25113 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
28743 .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ },
28744 .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ },
2511428745 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28746 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
28747 .{ ._, ._a, .j, .@"0b", ._, ._, ._ },
2511528748 } },
2511628749 }, .{
2511728750 .required_features = .{ .@"64bit", null, null, null },
25118 .src_constraints = .{ .{ .int = .qword }, .any },
25119 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
28751 .src_constraints = .{ .any_scalar_signed_int, .any },
28752 .dst_constraints = .{.any_scalar_signed_int},
2512028753 .patterns = &.{
25121 .{ .src = .{ .mem, .none } },
25122 .{ .src = .{ .to_gpr, .none } },
28754 .{ .src = .{ .to_mem, .none } },
2512328755 },
2512428756 .extra_temps = .{
25125 .{ .type = .u64, .kind = .{ .reg = .rax } },
28757 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
28758 .{ .type = .usize, .kind = .{ .reg = .rsi } },
2512628759 .{ .type = .usize, .kind = .{ .reg = .rdi } },
2512728760 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25128 .unused,
25129 .unused,
28761 .{ .type = .i64, .kind = .{ .reg = .rax } },
2513028762 .unused,
2513128763 .unused,
2513228764 .unused,
......@@ -25135,26 +28767,32 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2513528767 .dst_temps = .{.mem},
2513628768 .clobbers = .{ .eflags = true },
2513728769 .each = .{ .once = &.{
25138 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
25139 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
28770 .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ },
28771 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
28772 .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ },
28773 .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
28774 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
28775 .{ ._, ._sq, .lod, ._, ._, ._, ._ },
2514028776 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25141 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
25142 .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ },
28777 .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ },
28778 .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ },
2514328779 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28780 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
28781 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
2514428782 } },
2514528783 }, .{
25146 .required_features = .{ .@"64bit", null, null, null },
25147 .src_constraints = .{ .{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
25148 .dst_constraints = .{.{ .signed_remainder_int = .{ .of = .qword, .is = .qword } }},
28784 .required_features = .{ .@"64bit", .slow_incdec, null, null },
28785 .src_constraints = .{ .any_scalar_int, .any },
28786 .dst_constraints = .{.any_scalar_int},
2514928787 .patterns = &.{
2515028788 .{ .src = .{ .to_mem, .none } },
2515128789 },
2515228790 .extra_temps = .{
28791 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
2515328792 .{ .type = .usize, .kind = .{ .reg = .rsi } },
2515428793 .{ .type = .usize, .kind = .{ .reg = .rdi } },
2515528794 .{ .type = .u32, .kind = .{ .reg = .ecx } },
25156 .{ .type = .i64, .kind = .{ .reg = .rax } },
25157 .unused,
28795 .{ .type = .u64, .kind = .{ .reg = .rax } },
2515828796 .unused,
2515928797 .unused,
2516028798 .unused,
......@@ -25163,24 +28801,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2516328801 .dst_temps = .{.mem},
2516428802 .clobbers = .{ .eflags = true },
2516528803 .each = .{ .once = &.{
25166 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
25167 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25168 .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
28804 .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ },
28805 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
28806 .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ },
28807 .{ .@"0:", ._, .mov, .tmp3d, .sa(.src0, .add_elem_size_div_8), ._, ._ },
2516928808 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
25170 .{ ._, ._sq, .lod, ._, ._, ._, ._ },
25171 .{ ._, ._sq, .sto, ._, ._, ._, ._ },
25172 .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ },
25173 .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ },
28809 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
28810 .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ },
2517428811 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28812 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
28813 .{ ._, ._a, .j, .@"0b", ._, ._, ._ },
2517528814 } },
2517628815 }, .{
2517728816 .required_features = .{ .@"64bit", null, null, null },
25178 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any },
25179 .dst_constraints = .{.{ .remainder_int = .{ .of = .qword, .is = .qword } }},
28817 .src_constraints = .{ .any_scalar_int, .any },
28818 .dst_constraints = .{.any_scalar_int},
2518028819 .patterns = &.{
2518128820 .{ .src = .{ .to_mem, .none } },
2518228821 },
2518328822 .extra_temps = .{
28823 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
2518428824 .{ .type = .usize, .kind = .{ .reg = .rsi } },
2518528825 .{ .type = .usize, .kind = .{ .reg = .rdi } },
2518628826 .{ .type = .u32, .kind = .{ .reg = .ecx } },
......@@ -25189,18 +28829,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2518928829 .unused,
2519028830 .unused,
2519128831 .unused,
25192 .unused,
2519328832 },
2519428833 .dst_temps = .{.mem},
2519528834 .clobbers = .{ .eflags = true },
2519628835 .each = .{ .once = &.{
25197 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
25198 .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ },
25199 .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ },
28836 .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ },
28837 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
28838 .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ },
28839 .{ .@"0:", ._, .mov, .tmp3d, .sa(.src0, .add_elem_size_div_8), ._, ._ },
2520028840 .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ },
25201 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
25202 .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ },
28841 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
28842 .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ },
2520328843 .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ },
28844 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
28845 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
2520428846 } },
2520528847 } }) catch |err| switch (err) {
2520628848 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
......@@ -38527,7 +42169,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
3852742169 .reserved_frame,
3852842170 .air_ref,
3852942171 => unreachable, // unmodifiable destination
38530 .register => |reg| try self.genSetReg(reg, ty, src_mcv, opts),
42172 .register => |dst_reg| try self.genSetReg(dst_reg, ty, src_mcv, opts),
3853142173 .register_offset => |dst_reg_off| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) {
3853242174 .none,
3853342175 .unreach,
......@@ -38677,7 +42319,8 @@ fn genSetReg(
3867742319 const pt = self.pt;
3867842320 const zcu = pt.zcu;
3867942321 const abi_size: u32 = @intCast(ty.abiSize(zcu));
38680 if (ty.bitSize(zcu) > dst_reg.bitSize())
42322 const dst_alias = registerAlias(dst_reg, abi_size);
42323 if (ty.bitSize(zcu) > dst_alias.bitSize())
3868142324 return self.fail("genSetReg called with a value larger than dst_reg", .{});
3868242325 switch (src_mcv) {
3868342326 .none,
......@@ -38733,13 +42376,13 @@ fn genSetReg(
3873342376 } else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) {
3873442377 try self.asmRegisterImmediate(
3873542378 .{ ._, .mov },
38736 registerAlias(dst_reg, abi_size),
42379 dst_alias,
3873742380 .s(@intCast(@as(i64, @bitCast(imm)))),
3873842381 );
3873942382 } else {
3874042383 try self.asmRegisterImmediate(
3874142384 .{ ._, .mov },
38742 registerAlias(dst_reg, abi_size),
42385 dst_alias,
3874342386 .u(imm),
3874442387 );
3874542388 }
......@@ -38748,12 +42391,12 @@ fn genSetReg(
3874842391 .general_purpose => switch (src_reg.class()) {
3874942392 .general_purpose => try self.asmRegisterRegister(
3875042393 .{ ._, .mov },
38751 registerAlias(dst_reg, abi_size),
42394 dst_alias,
3875242395 registerAlias(src_reg, abi_size),
3875342396 ),
3875442397 .segment => try self.asmRegisterRegister(
3875542398 .{ ._, .mov },
38756 registerAlias(dst_reg, abi_size),
42399 dst_alias,
3875742400 src_reg,
3875842401 ),
3875942402 .x87, .mmx, .ip, .cr, .dr => unreachable,
......@@ -38780,7 +42423,7 @@ fn genSetReg(
3878042423 .base = .{ .frame = frame_index },
3878142424 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
3878242425 }, src_reg.to128());
38783 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
42426 try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
3878442427 .base = .{ .frame = frame_index },
3878542428 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
3878642429 });
......@@ -38917,7 +42560,7 @@ fn genSetReg(
3891742560 else => unreachable,
3891842561 },
3891942562 }) orelse return self.fail("TODO implement genSetReg for {}", .{ty.fmt(pt)}),
38920 registerAlias(dst_reg, abi_size),
42563 dst_alias,
3892142564 registerAlias(src_reg, abi_size),
3892242565 ),
3892342566 },
......@@ -38978,7 +42621,7 @@ fn genSetReg(
3897842621 ),
3897942622 .lea_frame => .{ .move = .{ ._, .lea } },
3898042623 else => unreachable,
38981 }).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {
42624 }).read(self, dst_alias, switch (src_mcv) {
3898242625 .register_offset, .indirect => |reg_off| .{
3898342626 .base = .{ .reg = reg_off.reg.to64() },
3898442627 .mod = .{ .rm = .{
......@@ -39043,7 +42686,7 @@ fn genSetReg(
3904342686 ty,
3904442687 dst_reg.class(),
3904542688 ty.abiAlignment(zcu).check(@as(u32, @bitCast(small_addr))),
39046 )).read(self, registerAlias(dst_reg, abi_size), .{
42689 )).read(self, dst_alias, .{
3904742690 .base = .{ .reg = .ds },
3904842691 .mod = .{ .rm = .{
3904942692 .size = self.memSize(ty),
......@@ -39053,7 +42696,7 @@ fn genSetReg(
3905342696 .load_symbol => |sym_off| switch (dst_reg.class()) {
3905442697 .general_purpose => {
3905542698 assert(sym_off.off == 0);
39056 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
42699 try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
3905742700 .base = .{ .reloc = sym_off.sym_index },
3905842701 .mod = .{ .rm = .{
3905942702 .size = self.memSize(ty),
......@@ -39071,7 +42714,7 @@ fn genSetReg(
3907142714 .tag = .mov,
3907242715 .ops = .direct_reloc,
3907342716 .data = .{ .rx = .{
39074 .r1 = registerAlias(dst_reg, abi_size),
42717 .r1 = dst_alias,
3907542718 .payload = try self.addExtra(bits.SymbolOffset{ .sym_index = sym_index }),
3907642719 } },
3907742720 });
......@@ -39088,7 +42731,7 @@ fn genSetReg(
3908842731 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3908942732 defer self.register_manager.unlockReg(addr_lock);
3909042733
39091 try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, registerAlias(dst_reg, abi_size), .{
42734 try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, dst_alias, .{
3909242735 .base = .{ .reg = addr_reg.to64() },
3909342736 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
3909442737 });
......@@ -39613,7 +43256,7 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
3961343256
3961443257 if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
3961543258
39616 if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and
43259 if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and
3961743260 dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
3961843261
3961943262 const abi_size = dst_ty.abiSize(zcu);
......@@ -44201,8 +47844,8 @@ const Temp = struct {
4420147844 const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
4420247845 const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
4420347846 const result: MCValue = .{ .register_pair = .{
44204 first_temp_tracking.short.register,
44205 second_temp_tracking.short.register,
47847 first_temp_tracking.short.register.to64(),
47848 second_temp_tracking.short.register.to64(),
4420647849 } };
4420747850 const result_temp_index = cg.next_temp_index;
4420847851 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
......@@ -45654,7 +49297,7 @@ const Temp = struct {
4565449297 .temp => |temp_index| {
4565549298 const temp_tracking = temp_index.tracking(cg);
4565649299 tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });
45657 cg.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
49300 cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(temp_tracking.short));
4565849301 assert(cg.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
4565949302 },
4566049303 }
......@@ -46630,12 +50273,14 @@ const Select = struct {
4663050273 ptr_bit_size,
4663150274 size,
4663250275 delta_size,
50276 delta_elem_size,
4663350277 size_sub_elem_size,
4663450278 unaligned_size,
4663550279 bit_size,
4663650280 src0_bit_size,
4663750281 len,
4663850282 elem_limbs,
50283 elem_size,
4663950284 src0_elem_size,
4664050285 src0_elem_size_times_src1,
4664150286 log2_src0_elem_size,
......@@ -46656,6 +50301,8 @@ const Select = struct {
4665650301 const sub_size_div_4: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"4" };
4665750302 const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" };
4665850303 const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" };
50304 const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" };
50305 const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" };
4665950306 const add_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .size_sub_elem_size, .op = .mul, .rhs = .@"1" };
4666050307 const add_unaligned_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" };
4666150308 const sub_unaligned_size: Adjust = .{ .sign = .neg, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" };
......@@ -46670,6 +50317,7 @@ const Select = struct {
4667050317 const add_2_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"2" };
4667150318 const add_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"1" };
4667250319 const sub_len: Adjust = .{ .sign = .neg, .lhs = .len, .op = .mul, .rhs = .@"1" };
50320 const add_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .elem_size, .op = .div, .rhs = .@"8" };
4667350321 const add_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" };
4667450322 const add_2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"2" };
4667550323 const add_4_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"4" };
......@@ -47141,6 +50789,8 @@ const Select = struct {
4714150789 .size => @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)),
4714250790 .delta_size => @intCast(@as(SignedImm, @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu))) -
4714350791 @as(SignedImm, @intCast(op.index.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)))),
50792 .delta_elem_size => @intCast(@as(SignedImm, @intCast(op.base.ref.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))) -
50793 @as(SignedImm, @intCast(op.index.ref.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)))),
4714450794 .size_sub_elem_size => {
4714550795 const ty = op.base.ref.deref(s).typeOf(s.cg);
4714650796 break :lhs @intCast(ty.abiSize(s.cg.pt.zcu) - ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu));
......@@ -47153,6 +50803,7 @@ const Select = struct {
4715350803 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
4715450804 @divExact(op.base.size.bitSize(s.cg.target), 8),
4715550805 )),
50806 .elem_size => @intCast(op.base.ref.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)),
4715650807 .src0_elem_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)),
4715750808 .src0_elem_size_times_src1 => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) *
4715850809 Select.Operand.Ref.src1.deref(s).tracking(s.cg).short.immediate),
test/behavior/maximum_minimum.zig+3-4
......@@ -203,8 +203,8 @@ test "@min/@max notices vector bounds" {
203203 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
204204 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
205205 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
206 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
207206 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
207 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
208208
209209 var x: @Vector(2, u16) = .{ 140, 40 };
210210 const y: @Vector(2, u64) = .{ 5, 100 };
......@@ -257,8 +257,8 @@ test "@min/@max notices bounds from vector types" {
257257 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
258258 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
259259 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
260 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
261260 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
261 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
262262
263263 var x: @Vector(2, u16) = .{ 30, 67 };
264264 var y: @Vector(2, u32) = .{ 20, 500 };
......@@ -301,8 +301,7 @@ test "@min/@max notices bounds from vector types when element of comptime-known
301301 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
302302 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
303303 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
304 if (builtin.zig_backend == .stage2_x86_64 and
305 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest;
304 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
306305
307306 var x: @Vector(2, u32) = .{ 1_000_000, 12345 };
308307 _ = &x;
test/behavior/x86_64/math.zig+85-37
......@@ -24,27 +24,32 @@ inline fn runtime(comptime Type: type, comptime value: Type) Type {
2424 }.variable;
2525}
2626
27fn Scalar(comptime Type: type) type {
28 return switch (@typeInfo(Type)) {
29 else => Type,
30 .vector => |info| info.child,
31 };
32}
33// inline to avoid a runtime `@splat`
34inline fn splat(comptime Type: type, scalar: Scalar(Type)) Type {
35 return switch (@typeInfo(Type)) {
36 else => scalar,
37 .vector => @splat(scalar),
38 };
39}
2740fn sign(rhs: anytype) switch (@typeInfo(@TypeOf(rhs))) {
2841 else => bool,
2942 .vector => |vector| @Vector(vector.len, bool),
3043} {
31 switch (@typeInfo(@TypeOf(rhs))) {
32 else => {
33 const I = @Type(.{ .int = .{
34 .signedness = .unsigned,
35 .bits = @bitSizeOf(@TypeOf(rhs)),
36 } });
37 return @as(I, @bitCast(rhs)) & @as(I, 1) << (@bitSizeOf(I) - 1) != 0;
38 },
39 .vector => |vector| {
40 const I = @Type(.{ .int = .{
41 .signedness = .unsigned,
42 .bits = @bitSizeOf(vector.child),
43 } });
44 const V = @Vector(vector.len, I);
45 return @as(V, @bitCast(rhs)) & @as(V, @splat(@as(I, 1) << (@bitSizeOf(I) - 1))) != @as(V, @splat(0));
46 },
47 }
44 const ScalarInt = @Type(.{ .int = .{
45 .signedness = .unsigned,
46 .bits = @bitSizeOf(Scalar(@TypeOf(rhs))),
47 } });
48 const VectorInt = switch (@typeInfo(@TypeOf(rhs))) {
49 else => ScalarInt,
50 .vector => |vector| @Vector(vector.len, ScalarInt),
51 };
52 return @as(VectorInt, @bitCast(rhs)) & splat(VectorInt, @as(ScalarInt, 1) << @bitSizeOf(ScalarInt) - 1) != splat(VectorInt, 0);
4853}
4954fn boolAnd(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
5055 switch (@typeInfo(@TypeOf(lhs))) {
......@@ -1648,12 +1653,12 @@ fn cast(comptime op: anytype, comptime opts: struct { strict: bool = false }) ty
16481653 comptime imm_arg: Type,
16491654 mem_arg: Type,
16501655 ) !void {
1651 const expected = comptime op(Result, Type, imm_arg);
1656 const expected = comptime op(Result, Type, imm_arg, imm_arg);
16521657 var reg_arg = mem_arg;
16531658 _ = .{&reg_arg};
1654 try checkExpected(expected, op(Result, Type, reg_arg), opts.strict);
1655 try checkExpected(expected, op(Result, Type, mem_arg), opts.strict);
1656 try checkExpected(expected, op(Result, Type, imm_arg), opts.strict);
1659 try checkExpected(expected, op(Result, Type, reg_arg, imm_arg), opts.strict);
1660 try checkExpected(expected, op(Result, Type, mem_arg, imm_arg), opts.strict);
1661 try checkExpected(expected, op(Result, Type, imm_arg, imm_arg), opts.strict);
16571662 }
16581663 // noinline for a more helpful stack trace
16591664 noinline fn testArgs(comptime Result: type, comptime Type: type, comptime imm_arg: Type) !void {
......@@ -5218,6 +5223,39 @@ fn cast(comptime op: anytype, comptime opts: struct { strict: bool = false }) ty
52185223 try testArgs(f128, f128, nan(f128));
52195224 }
52205225 fn testIntVectors() !void {
5226 try testArgs(@Vector(1, i8), @Vector(1, i1), .{-1});
5227 try testArgs(@Vector(1, u8), @Vector(1, i1), .{-1});
5228 try testArgs(@Vector(1, i16), @Vector(1, i1), .{-1});
5229 try testArgs(@Vector(1, u16), @Vector(1, i1), .{-1});
5230 try testArgs(@Vector(1, i32), @Vector(1, i1), .{-1});
5231 try testArgs(@Vector(1, u32), @Vector(1, i1), .{-1});
5232 try testArgs(@Vector(1, i64), @Vector(1, i1), .{-1});
5233 try testArgs(@Vector(1, u64), @Vector(1, i1), .{-1});
5234 try testArgs(@Vector(1, i128), @Vector(1, i1), .{-1});
5235 try testArgs(@Vector(1, u128), @Vector(1, i1), .{-1});
5236 try testArgs(@Vector(1, i256), @Vector(1, i1), .{-1});
5237 try testArgs(@Vector(1, u256), @Vector(1, i1), .{-1});
5238 try testArgs(@Vector(1, i512), @Vector(1, i1), .{-1});
5239 try testArgs(@Vector(1, u512), @Vector(1, i1), .{-1});
5240 try testArgs(@Vector(1, i1024), @Vector(1, i1), .{-1});
5241 try testArgs(@Vector(1, u1024), @Vector(1, i1), .{-1});
5242 try testArgs(@Vector(1, i8), @Vector(1, u1), .{1});
5243 try testArgs(@Vector(1, u8), @Vector(1, u1), .{1});
5244 try testArgs(@Vector(1, i16), @Vector(1, u1), .{1});
5245 try testArgs(@Vector(1, u16), @Vector(1, u1), .{1});
5246 try testArgs(@Vector(1, i32), @Vector(1, u1), .{1});
5247 try testArgs(@Vector(1, u32), @Vector(1, u1), .{1});
5248 try testArgs(@Vector(1, i64), @Vector(1, u1), .{1});
5249 try testArgs(@Vector(1, u64), @Vector(1, u1), .{1});
5250 try testArgs(@Vector(1, i128), @Vector(1, u1), .{1});
5251 try testArgs(@Vector(1, u128), @Vector(1, u1), .{1});
5252 try testArgs(@Vector(1, i256), @Vector(1, u1), .{1});
5253 try testArgs(@Vector(1, u256), @Vector(1, u1), .{1});
5254 try testArgs(@Vector(1, i512), @Vector(1, u1), .{1});
5255 try testArgs(@Vector(1, u512), @Vector(1, u1), .{1});
5256 try testArgs(@Vector(1, i1024), @Vector(1, u1), .{1});
5257 try testArgs(@Vector(1, u1024), @Vector(1, u1), .{1});
5258
52215259 try testArgs(@Vector(2, i8), @Vector(2, i1), .{ -1, 0 });
52225260 try testArgs(@Vector(2, u8), @Vector(2, i1), .{ -1, 0 });
52235261 try testArgs(@Vector(2, i16), @Vector(2, i1), .{ -1, 0 });
......@@ -9917,21 +9955,18 @@ test clz {
99179955 try test_clz.testIntVectors();
99189956}
99199957
9920inline fn intCast(comptime Result: type, comptime Type: type, rhs: Type) Result {
9958inline fn intCast(comptime Result: type, comptime Type: type, rhs: Type, comptime ct_rhs: Type) Result {
9959 @setRuntimeSafety(false); // TODO
99219960 const res_info = switch (@typeInfo(Result)) {
99229961 .int => |info| info,
99239962 .vector => |info| @typeInfo(info.child).int,
99249963 else => @compileError(@typeName(Result)),
99259964 };
9926 const rhs_info = switch (@typeInfo(Type)) {
9927 .int => |info| info,
9928 .vector => |info| @typeInfo(info.child).int,
9929 else => @compileError(@typeName(Type)),
9930 };
9965 const rhs_info = @typeInfo(Scalar(Type)).int;
99319966 const min_bits = @min(res_info.bits, rhs_info.bits);
99329967 return @intCast(switch (@as(union(enum) {
9933 shift: std.math.Log2Int(Type),
9934 mask: std.math.Log2IntCeil(Type),
9968 shift: std.math.Log2Int(Scalar(Type)),
9969 mask: std.math.Log2IntCeil(Scalar(Type)),
99359970 }, switch (res_info.signedness) {
99369971 .signed => switch (rhs_info.signedness) {
99379972 .signed => .{ .shift = rhs_info.bits - min_bits },
......@@ -9943,21 +9978,34 @@ inline fn intCast(comptime Result: type, comptime Type: type, rhs: Type) Result
99439978 },
99449979 })) {
99459980 // TODO: if (bits == 0) rhs else rhs >> bits,
9946 .shift => |bits| if (bits == 0)
9947 rhs
9948 else if (rhs < 0)
9949 rhs | std.math.minInt(Type) >> bits
9950 else
9951 rhs & std.math.maxInt(Type) >> bits,
9952 .mask => |bits| if (bits == rhs_info.bits) rhs else rhs & (1 << bits) - 1,
9981 .shift => |bits| if (bits == 0) rhs else switch (@typeInfo(Type)) {
9982 .int => if (ct_rhs < 0)
9983 rhs | std.math.minInt(Type) >> bits
9984 else
9985 rhs & std.math.maxInt(Type) >> bits,
9986 .vector => rhs | @select(
9987 Scalar(Type),
9988 ct_rhs < splat(Type, 0),
9989 splat(Type, std.math.minInt(Scalar(Type)) >> bits),
9990 splat(Type, 0),
9991 ) & ~@select(
9992 Scalar(Type),
9993 ct_rhs >= splat(Type, 0),
9994 splat(Type, std.math.minInt(Scalar(Type)) >> bits),
9995 splat(Type, 0),
9996 ),
9997 else => comptime unreachable,
9998 },
9999 .mask => |bits| if (bits == rhs_info.bits) rhs else rhs & splat(Type, (1 << bits) - 1),
995310000 });
995410001}
995510002test intCast {
995610003 const test_int_cast = cast(intCast, .{});
995710004 try test_int_cast.testInts();
10005 try test_int_cast.testIntVectors();
995810006}
995910007
9960inline fn floatCast(comptime Result: type, comptime Type: type, rhs: Type) Result {
10008inline fn floatCast(comptime Result: type, comptime Type: type, rhs: Type, comptime _: Type) Result {
996110009 return @floatCast(rhs);
996210010}
996310011test floatCast {