authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-12-23 15:53:13-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:42:08-05:00
loga1828ebcda0f684cdd146fa3ab5b86701909ca12
tree9c957480367632eb15112438ea7ff437716bedce
parent73a42953c93273c178a1f8c8067bc732ca933be0

x86_64: demolish the old

8 files changed, 2798 insertions(+), 2127 deletions(-)

src/arch/x86_64/CodeGen.zig+2664-2060
......@@ -465,7 +465,7 @@ pub const MCValue = union(enum) {
465465 } },
466466 } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
467467 .indirect => |reg_off| .{
468 .base = .{ .reg = reg_off.reg },
468 .base = .{ .reg = registerAlias(reg_off.reg, @divExact(function.target.ptrBitWidth(), 8)) },
469469 .mod = .{ .rm = .{
470470 .size = mod_rm.size,
471471 .index = mod_rm.index,
......@@ -986,6 +986,7 @@ pub fn generate(
986986 .air = function.air,
987987 .lower = .{
988988 .bin_file = bin_file,
989 .target = function.target,
989990 .allocator = gpa,
990991 .mir = mir,
991992 .cc = cc,
......@@ -1074,6 +1075,7 @@ pub fn generateLazy(
10741075 .air = function.air,
10751076 .lower = .{
10761077 .bin_file = bin_file,
1078 .target = function.target,
10771079 .allocator = gpa,
10781080 .mir = mir,
10791081 .cc = abi.resolveCallingConvention(.auto, function.target.*),
......@@ -1154,6 +1156,7 @@ fn formatWipMir(
11541156 const mod = comp.root_mod;
11551157 var lower: Lower = .{
11561158 .bin_file = data.self.bin_file,
1159 .target = data.self.target,
11571160 .allocator = data.self.gpa,
11581161 .mir = .{
11591162 .instructions = data.self.mir_instructions.slice(),
......@@ -2514,7 +2517,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25142517 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
25152518 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
25162519 var res: [1]Temp = undefined;
2517 cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) {
2520 cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) {
25182521 else => unreachable,
25192522 .bit_and => .@"and",
25202523 .bit_or => .@"or",
......@@ -2530,7 +2533,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25302533 },
25312534 .dst_temps = .{.{ .rc = .sse }},
25322535 .each = .{ .once = &.{
2533 .{ .vp_, mir_tag, .ydst0, .ysrc0, .ysrc1, .none },
2536 .{ ._, .vp_, mir_tag, .dst0y, .src0y, .src1y, ._ },
25342537 } },
25352538 }, .{
25362539 .required_features = .{ .avx, null },
......@@ -2541,7 +2544,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25412544 },
25422545 .dst_temps = .{.{ .rc = .sse }},
25432546 .each = .{ .once = &.{
2544 .{ .v_pd, mir_tag, .ydst0, .ysrc0, .ysrc1, .none },
2547 .{ ._, .v_pd, mir_tag, .dst0y, .src0y, .src1y, ._ },
25452548 } },
25462549 }, .{
25472550 .required_features = .{ .avx, null },
......@@ -2552,7 +2555,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25522555 },
25532556 .dst_temps = .{.{ .rc = .sse }},
25542557 .each = .{ .once = &.{
2555 .{ .vp_, mir_tag, .xdst0, .xsrc0, .xsrc1, .none },
2558 .{ ._, .vp_, mir_tag, .dst0x, .src0x, .src1x, ._ },
25562559 } },
25572560 }, .{
25582561 .required_features = .{ .sse2, null },
......@@ -2561,9 +2564,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25612564 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
25622565 .{ .src = .{ .mut_xmm, .xmm } },
25632566 },
2564 .dst_temps = .{.{ .src = 0 }},
2567 .dst_temps = .{.{ .ref = .src0 }},
25652568 .each = .{ .once = &.{
2566 .{ .p_, mir_tag, .xdst0, .xsrc1, .none, .none },
2569 .{ ._, .p_, mir_tag, .dst0x, .src1x, ._, ._ },
25672570 } },
25682571 }, .{
25692572 .required_features = .{ .sse, null },
......@@ -2572,9 +2575,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25722575 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
25732576 .{ .src = .{ .mut_xmm, .xmm } },
25742577 },
2575 .dst_temps = .{.{ .src = 0 }},
2578 .dst_temps = .{.{ .ref = .src0 }},
25762579 .each = .{ .once = &.{
2577 .{ ._ps, mir_tag, .xdst0, .xsrc1, .none, .none },
2580 .{ ._, ._ps, mir_tag, .dst0x, .src1x, ._, ._ },
25782581 } },
25792582 }, .{
25802583 .required_features = .{ .mmx, null },
......@@ -2583,12 +2586,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25832586 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
25842587 .{ .src = .{ .mut_mm, .mm } },
25852588 },
2586 .dst_temps = .{.{ .src = 0 }},
2589 .dst_temps = .{.{ .ref = .src0 }},
25872590 .each = .{ .once = &.{
2588 .{ .p_, mir_tag, .rdst0, .rsrc1, .none, .none },
2591 .{ ._, .p_, mir_tag, .dst0q, .src1q, ._, ._ },
25892592 } },
25902593 }, .{
2591 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
2594 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
25922595 .patterns = &.{
25932596 .{ .src = .{ .mut_mem, .imm8 } },
25942597 .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -2601,12 +2604,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26012604 .{ .src = .{ .mut_gpr, .gpr } },
26022605 },
26032606 .clobbers = .{ .eflags = true },
2604 .dst_temps = .{.{ .src = 0 }},
2607 .dst_temps = .{.{ .ref = .src0 }},
26052608 .each = .{ .once = &.{
2606 .{ ._, mir_tag, .dst0b, .src1b, .none, .none },
2609 .{ ._, ._, mir_tag, .dst0b, .src1b, ._, ._ },
26072610 } },
26082611 }, .{
2609 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
2612 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
26102613 .patterns = &.{
26112614 .{ .src = .{ .mut_mem, .imm16 } },
26122615 .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -2619,12 +2622,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26192622 .{ .src = .{ .mut_gpr, .gpr } },
26202623 },
26212624 .clobbers = .{ .eflags = true },
2622 .dst_temps = .{.{ .src = 0 }},
2625 .dst_temps = .{.{ .ref = .src0 }},
26232626 .each = .{ .once = &.{
2624 .{ ._, mir_tag, .dst0w, .src1w, .none, .none },
2627 .{ ._, ._, mir_tag, .dst0w, .src1w, ._, ._ },
26252628 } },
26262629 }, .{
2627 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
2630 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
26282631 .patterns = &.{
26292632 .{ .src = .{ .mut_mem, .imm32 } },
26302633 .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -2637,13 +2640,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26372640 .{ .src = .{ .mut_gpr, .gpr } },
26382641 },
26392642 .clobbers = .{ .eflags = true },
2640 .dst_temps = .{.{ .src = 0 }},
2643 .dst_temps = .{.{ .ref = .src0 }},
26412644 .each = .{ .once = &.{
2642 .{ ._, mir_tag, .edst0, .esrc1, .none, .none },
2645 .{ ._, ._, mir_tag, .dst0d, .src1d, ._, ._ },
26432646 } },
26442647 }, .{
26452648 .required_features = .{ .@"64bit", null },
2646 .constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
2649 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
26472650 .patterns = &.{
26482651 .{ .src = .{ .mut_mem, .simm32 } },
26492652 .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -2656,9 +2659,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26562659 .{ .src = .{ .mut_gpr, .gpr } },
26572660 },
26582661 .clobbers = .{ .eflags = true },
2659 .dst_temps = .{.{ .src = 0 }},
2662 .dst_temps = .{.{ .ref = .src0 }},
26602663 .each = .{ .once = &.{
2661 .{ ._, mir_tag, .rdst0, .rsrc1, .none, .none },
2664 .{ ._, ._, mir_tag, .dst0q, .src1q, ._, ._ },
26622665 } },
26632666 }, .{
26642667 .required_features = .{ .avx2, null },
......@@ -2669,14 +2672,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26692672 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
26702673 .{ .kind = .{ .rc = .sse } },
26712674 .unused,
2675 .unused,
2676 .unused,
2677 .unused,
26722678 },
26732679 .dst_temps = .{.mem},
26742680 .each = .{ .limb = .{
2675 .of = .ysrc0,
2681 .of = .src0y,
26762682 .body = &.{
2677 .{ .v_, .movdqu, .ytmp1, .{ .src_limb = 0 }, .none, .none },
2678 .{ .vp_, mir_tag, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none },
2679 .{ .v_, .movdqu, .{ .dst_limb = 0 }, .ytmp1, .none, .none },
2683 .{ ._, .v_dqu, .mov, .tmp1y, .limb(.src0y), ._, ._ },
2684 .{ ._, .vp_, mir_tag, .tmp1y, .tmp1y, .limb(.src1y), ._ },
2685 .{ ._, .v_dqu, .mov, .limb(.dst0y), .tmp1y, ._, ._ },
26802686 },
26812687 } },
26822688 }, .{
......@@ -2688,14 +2694,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26882694 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
26892695 .{ .kind = .{ .rc = .sse } },
26902696 .unused,
2697 .unused,
2698 .unused,
2699 .unused,
26912700 },
26922701 .dst_temps = .{.mem},
26932702 .each = .{ .limb = .{
2694 .of = .ysrc0,
2703 .of = .src0y,
26952704 .body = &.{
2696 .{ .v_pd, .movu, .ytmp1, .{ .src_limb = 0 }, .none, .none },
2697 .{ .v_pd, mir_tag, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none },
2698 .{ .v_pd, .movu, .{ .dst_limb = 0 }, .ytmp1, .none, .none },
2705 .{ ._, .v_pd, .movu, .tmp1y, .limb(.src0y), ._, ._ },
2706 .{ ._, .v_pd, mir_tag, .tmp1y, .tmp1y, .limb(.src1y), ._ },
2707 .{ ._, .v_pd, .movu, .limb(.dst0y), .tmp1y, ._, ._ },
26992708 },
27002709 } },
27012710 }, .{
......@@ -2707,14 +2716,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27072716 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
27082717 .{ .kind = .{ .rc = .sse } },
27092718 .unused,
2719 .unused,
2720 .unused,
2721 .unused,
27102722 },
27112723 .dst_temps = .{.mem},
27122724 .each = .{ .limb = .{
2713 .of = .xsrc0,
2725 .of = .src0x,
27142726 .body = &.{
2715 .{ .v_, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none },
2716 .{ .vp_, mir_tag, .xtmp1, .xtmp1, .{ .src_limb = 1 }, .none },
2717 .{ .v_, .movdqu, .{ .dst_limb = 0 }, .xtmp1, .none, .none },
2727 .{ ._, .v_dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ },
2728 .{ ._, .vp_, mir_tag, .tmp1x, .tmp1x, .limb(.src1x), ._ },
2729 .{ ._, .v_dqu, .mov, .limb(.dst0x), .tmp1x, ._, ._ },
27182730 },
27192731 } },
27202732 }, .{
......@@ -2726,14 +2738,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27262738 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
27272739 .{ .kind = .{ .rc = .sse } },
27282740 .unused,
2741 .unused,
2742 .unused,
2743 .unused,
27292744 },
27302745 .dst_temps = .{.mem},
27312746 .each = .{ .limb = .{
2732 .of = .xsrc0,
2747 .of = .src0x,
27332748 .body = &.{
2734 .{ ._, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none },
2735 .{ .p_, mir_tag, .xtmp1, .{ .src_limb = 1 }, .none, .none },
2736 .{ ._, .movdqu, .{ .dst_limb = 0 }, .xtmp1, .none, .none },
2749 .{ ._, ._dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ },
2750 .{ ._, .p_, mir_tag, .tmp1x, .limb(.src1x), ._, ._ },
2751 .{ ._, ._dqu, .mov, .limb(.dst0x), .tmp1x, ._, ._ },
27372752 },
27382753 } },
27392754 }, .{
......@@ -2745,14 +2760,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27452760 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
27462761 .{ .kind = .{ .rc = .sse } },
27472762 .unused,
2763 .unused,
2764 .unused,
2765 .unused,
27482766 },
27492767 .dst_temps = .{.mem},
27502768 .each = .{ .limb = .{
2751 .of = .xsrc0,
2769 .of = .src0x,
27522770 .body = &.{
2753 .{ ._ps, .movu, .xtmp1, .{ .src_limb = 0 }, .none, .none },
2754 .{ ._ps, mir_tag, .xtmp1, .{ .src_limb = 1 }, .none, .none },
2755 .{ ._ps, .movu, .{ .dst_limb = 0 }, .xtmp1, .none, .none },
2771 .{ ._, ._ps, .movu, .tmp1x, .limb(.src0x), ._, ._ },
2772 .{ ._, ._ps, mir_tag, .tmp1x, .limb(.src1x), ._, ._ },
2773 .{ ._, ._ps, .movu, .limb(.dst0x), .tmp1x, ._, ._ },
27562774 },
27572775 } },
27582776 }, .{
......@@ -2764,33 +2782,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27642782 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
27652783 .{ .kind = .{ .rc = .mmx } },
27662784 .unused,
2767 },
2768 .dst_temps = .{.mem},
2769 .each = .{ .limb = .{
2770 .of = .rsrc0,
2771 .body = &.{
2772 .{ ._q, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none },
2773 .{ .p_, mir_tag, .rtmp1, .{ .src_limb = 1 }, .none, .none },
2774 .{ ._q, .mov, .{ .dst_limb = 0 }, .rtmp1, .none, .none },
2775 },
2776 } },
2777 }, .{
2778 .required_features = .{ .@"64bit", null },
2779 .patterns = &.{
2780 .{ .src = .{ .to_mem, .to_mem } },
2781 },
2782 .extra_temps = .{
2783 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
2784 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
2785 .unused,
2786 .unused,
27852787 .unused,
27862788 },
27872789 .dst_temps = .{.mem},
27882790 .each = .{ .limb = .{
2789 .of = .rsrc0,
2791 .of = .src0q,
27902792 .body = &.{
2791 .{ ._, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none },
2792 .{ ._, mir_tag, .rtmp1, .{ .src_limb = 1 }, .none, .none },
2793 .{ ._, .mov, .{ .dst_limb = 0 }, .rtmp1, .none, .none },
2793 .{ ._, ._q, .mov, .tmp1q, .limb(.src0q), ._, ._ },
2794 .{ ._, .p_, mir_tag, .tmp1q, .limb(.src1q), ._, ._ },
2795 .{ ._, ._q, .mov, .limb(.dst0q), .tmp1q, ._, ._ },
27942796 },
27952797 } },
27962798 }, .{
......@@ -2799,21 +2801,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27992801 },
28002802 .extra_temps = .{
28012803 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
2802 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
2804 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
2805 .unused,
2806 .unused,
2807 .unused,
28032808 .unused,
28042809 },
28052810 .dst_temps = .{.mem},
28062811 .each = .{ .limb = .{
2807 .of = .esrc0,
2812 .of = .src0p,
28082813 .body = &.{
2809 .{ ._, .mov, .etmp1, .{ .src_limb = 0 }, .none, .none },
2810 .{ ._, mir_tag, .etmp1, .{ .src_limb = 1 }, .none, .none },
2811 .{ ._, .mov, .{ .dst_limb = 0 }, .etmp1, .none, .none },
2814 .{ ._, ._, .mov, .tmp1p, .limb(.src0p), ._, ._ },
2815 .{ ._, ._, mir_tag, .tmp1p, .limb(.src1p), ._, ._ },
2816 .{ ._, ._, .mov, .limb(.dst0p), .tmp1p, ._, ._ },
28122817 },
28132818 } },
28142819 } },
28152820 }) catch |err2| switch (err2) {
2816 error.Select2Failed => return cg.fail("failed to select2 {s} {} {} {}", .{
2821 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
28172822 @tagName(air_tag),
28182823 cg.typeOf(bin_op.lhs).fmt(pt),
28192824 ops[0].tracking(cg),
......@@ -2875,7 +2880,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
28752880 .call_never_tail => try cg.airCall(inst, .never_tail),
28762881 .call_never_inline => try cg.airCall(inst, .never_inline),
28772882
2878 .cmp_vector, .cmp_vector_optimized => if (use_old) try cg.airCmpVector(inst) else fallback: {
2883 .cmp_vector, .cmp_vector_optimized => |air_tag| if (use_old) try cg.airCmpVector(inst) else fallback: {
28792884 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
28802885 const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
28812886 switch (extra.compareOperator()) {
......@@ -2887,7 +2892,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
28872892 switch (extra.compareOperator()) {
28882893 .lt => unreachable,
28892894 .lte => unreachable,
2890 .eq, .neq => |cmp_op| cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (cmp_op) {
2895 .eq, .neq => |cmp_op| cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (cmp_op) {
28912896 else => unreachable,
28922897 .eq => .e,
28932898 .neq => .ne,
......@@ -2895,7 +2900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
28952900 else => unreachable,
28962901 inline .e, .ne => |cc| comptime &.{ .{
28972902 .required_features = .{ .avx2, null },
2898 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
2903 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
28992904 .patterns = &.{
29002905 .{ .src = .{ .ymm, .mem } },
29012906 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
......@@ -2911,11 +2916,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
29112916 .scalar = .byte,
29122917 } } }},
29132918 .each = .{ .once = &.{
2914 .{ .vp_b, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none },
2919 .{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ },
29152920 } },
29162921 }, .{
29172922 .required_features = .{ .avx2, null },
2918 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
2923 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
29192924 .patterns = &.{
29202925 .{ .src = .{ .ymm, .mem } },
29212926 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
......@@ -2931,11 +2936,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
29312936 .scalar = .word,
29322937 } } }},
29332938 .each = .{ .once = &.{
2934 .{ .vp_w, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none },
2939 .{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ },
29352940 } },
29362941 }, .{
29372942 .required_features = .{ .avx2, null },
2938 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
2943 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
29392944 .patterns = &.{
29402945 .{ .src = .{ .ymm, .mem } },
29412946 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
......@@ -2951,11 +2956,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
29512956 .scalar = .dword,
29522957 } } }},
29532958 .each = .{ .once = &.{
2954 .{ .vp_d, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none },
2959 .{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ },
29552960 } },
29562961 }, .{
29572962 .required_features = .{ .avx2, null },
2958 .constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
2963 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
29592964 .patterns = &.{
29602965 .{ .src = .{ .ymm, .mem } },
29612966 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
......@@ -2971,11 +2976,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
29712976 .scalar = .qword,
29722977 } } }},
29732978 .each = .{ .once = &.{
2974 .{ .vp_q, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none },
2979 .{ ._, .vp_q, .cmpeq, .dst0y, .src0y, .src1y, ._ },
29752980 } },
29762981 }, .{
29772982 .required_features = .{ .avx, null },
2978 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
2983 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
29792984 .patterns = &.{
29802985 .{ .src = .{ .xmm, .mem } },
29812986 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
......@@ -2991,11 +2996,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
29912996 .scalar = .byte,
29922997 } } }},
29932998 .each = .{ .once = &.{
2994 .{ .vp_b, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none },
2999 .{ ._, .vp_b, .cmpeq, .dst0x, .src0x, .src1x, ._ },
29953000 } },
29963001 }, .{
29973002 .required_features = .{ .avx, null },
2998 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
3003 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
29993004 .patterns = &.{
30003005 .{ .src = .{ .xmm, .mem } },
30013006 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
......@@ -3011,11 +3016,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30113016 .scalar = .word,
30123017 } } }},
30133018 .each = .{ .once = &.{
3014 .{ .vp_w, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none },
3019 .{ ._, .vp_w, .cmpeq, .dst0x, .src0x, .src1x, ._ },
30153020 } },
30163021 }, .{
30173022 .required_features = .{ .avx, null },
3018 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3023 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
30193024 .patterns = &.{
30203025 .{ .src = .{ .xmm, .mem } },
30213026 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
......@@ -3031,11 +3036,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30313036 .scalar = .dword,
30323037 } } }},
30333038 .each = .{ .once = &.{
3034 .{ .vp_d, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none },
3039 .{ ._, .vp_d, .cmpeq, .dst0x, .src0x, .src1x, ._ },
30353040 } },
30363041 }, .{
30373042 .required_features = .{ .avx, null },
3038 .constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
3043 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
30393044 .patterns = &.{
30403045 .{ .src = .{ .xmm, .mem } },
30413046 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
......@@ -3051,17 +3056,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30513056 .scalar = .qword,
30523057 } } }},
30533058 .each = .{ .once = &.{
3054 .{ .vp_q, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none },
3059 .{ ._, .vp_q, .cmpeq, .dst0x, .src0x, .src1x, ._ },
30553060 } },
30563061 }, .{
30573062 .required_features = .{ .sse2, null },
3058 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3063 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
30593064 .patterns = &.{
30603065 .{ .src = .{ .mut_xmm, .mem } },
30613066 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
30623067 .{ .src = .{ .mut_xmm, .xmm } },
30633068 },
3064 .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{
3069 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
30653070 .kind = .all,
30663071 .inverted = switch (cc) {
30673072 else => unreachable,
......@@ -3071,17 +3076,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30713076 .scalar = .byte,
30723077 } } }},
30733078 .each = .{ .once = &.{
3074 .{ .p_b, .cmpeq, .xdst0, .xsrc1, .none, .none },
3079 .{ ._, .p_b, .cmpeq, .dst0x, .src1x, ._, ._ },
30753080 } },
30763081 }, .{
30773082 .required_features = .{ .sse2, null },
3078 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
3083 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
30793084 .patterns = &.{
30803085 .{ .src = .{ .mut_xmm, .mem } },
30813086 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
30823087 .{ .src = .{ .mut_xmm, .xmm } },
30833088 },
3084 .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{
3089 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
30853090 .kind = .all,
30863091 .inverted = switch (cc) {
30873092 else => unreachable,
......@@ -3091,17 +3096,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30913096 .scalar = .word,
30923097 } } }},
30933098 .each = .{ .once = &.{
3094 .{ .p_w, .cmpeq, .xdst0, .xsrc1, .none, .none },
3099 .{ ._, .p_w, .cmpeq, .dst0x, .src1x, ._, ._ },
30953100 } },
30963101 }, .{
30973102 .required_features = .{ .sse2, null },
3098 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3103 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
30993104 .patterns = &.{
31003105 .{ .src = .{ .mut_xmm, .mem } },
31013106 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
31023107 .{ .src = .{ .mut_xmm, .xmm } },
31033108 },
3104 .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{
3109 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31053110 .kind = .all,
31063111 .inverted = switch (cc) {
31073112 else => unreachable,
......@@ -3111,17 +3116,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31113116 .scalar = .dword,
31123117 } } }},
31133118 .each = .{ .once = &.{
3114 .{ .p_d, .cmpeq, .xdst0, .xsrc1, .none, .none },
3119 .{ ._, .p_d, .cmpeq, .dst0x, .src1x, ._, ._ },
31153120 } },
31163121 }, .{
31173122 .required_features = .{ .sse4_1, null },
3118 .constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
3123 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
31193124 .patterns = &.{
31203125 .{ .src = .{ .mut_xmm, .mem } },
31213126 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
31223127 .{ .src = .{ .mut_xmm, .xmm } },
31233128 },
3124 .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{
3129 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31253130 .kind = .all,
31263131 .inverted = switch (cc) {
31273132 else => unreachable,
......@@ -3131,17 +3136,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31313136 .scalar = .qword,
31323137 } } }},
31333138 .each = .{ .once = &.{
3134 .{ .p_q, .cmpeq, .xdst0, .xsrc1, .none, .none },
3139 .{ ._, .p_q, .cmpeq, .dst0x, .src1x, ._, ._ },
31353140 } },
31363141 }, .{
31373142 .required_features = .{ .mmx, null },
3138 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3143 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
31393144 .patterns = &.{
31403145 .{ .src = .{ .mut_mm, .mem } },
31413146 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
31423147 .{ .src = .{ .mut_mm, .mm } },
31433148 },
3144 .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{
3149 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31453150 .kind = .all,
31463151 .inverted = switch (cc) {
31473152 else => unreachable,
......@@ -3151,17 +3156,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31513156 .scalar = .byte,
31523157 } } }},
31533158 .each = .{ .once = &.{
3154 .{ .p_b, .cmpeq, .rdst0, .rsrc1, .none, .none },
3159 .{ ._, .p_b, .cmpeq, .dst0q, .src1q, ._, ._ },
31553160 } },
31563161 }, .{
31573162 .required_features = .{ .mmx, null },
3158 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
3163 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
31593164 .patterns = &.{
31603165 .{ .src = .{ .mut_mm, .mem } },
31613166 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
31623167 .{ .src = .{ .mut_mm, .mm } },
31633168 },
3164 .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{
3169 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31653170 .kind = .all,
31663171 .inverted = switch (cc) {
31673172 else => unreachable,
......@@ -3171,17 +3176,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31713176 .scalar = .word,
31723177 } } }},
31733178 .each = .{ .once = &.{
3174 .{ .p_w, .cmpeq, .rdst0, .rsrc1, .none, .none },
3179 .{ ._, .p_w, .cmpeq, .dst0q, .src1q, ._, ._ },
31753180 } },
31763181 }, .{
31773182 .required_features = .{ .mmx, null },
3178 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3183 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
31793184 .patterns = &.{
31803185 .{ .src = .{ .mut_mm, .mem } },
31813186 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
31823187 .{ .src = .{ .mut_mm, .mm } },
31833188 },
3184 .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{
3189 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31853190 .kind = .all,
31863191 .inverted = switch (cc) {
31873192 else => unreachable,
......@@ -3191,10 +3196,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31913196 .scalar = .dword,
31923197 } } }},
31933198 .each = .{ .once = &.{
3194 .{ .p_d, .cmpeq, .rdst0, .rsrc1, .none, .none },
3199 .{ ._, .p_d, .cmpeq, .dst0q, .src1q, ._, ._ },
31953200 } },
31963201 }, .{
3197 .constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte } },
3202 .src_constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte } },
31983203 .patterns = &.{
31993204 .{ .src = .{ .mut_mem, .imm8 } },
32003205 .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -3207,19 +3212,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32073212 .{ .src = .{ .mut_gpr, .gpr } },
32083213 },
32093214 .clobbers = .{ .eflags = true },
3210 .dst_temps = .{.{ .src = 0 }},
3215 .dst_temps = .{.{ .ref = .src0 }},
32113216 .each = .{ .once = switch (cc) {
32123217 else => unreachable,
32133218 .e => &.{
3214 .{ ._, .xor, .dst0b, .src1b, .none, .none },
3215 .{ ._, .not, .dst0b, .none, .none, .none },
3219 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
3220 .{ ._, ._, .not, .dst0b, ._, ._, ._ },
32163221 },
32173222 .ne => &.{
3218 .{ ._, .xor, .dst0b, .src1b, .none, .none },
3223 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
32193224 },
32203225 } },
32213226 }, .{
3222 .constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } },
3227 .src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } },
32233228 .patterns = &.{
32243229 .{ .src = .{ .mut_mem, .imm16 } },
32253230 .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -3232,19 +3237,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32323237 .{ .src = .{ .mut_gpr, .gpr } },
32333238 },
32343239 .clobbers = .{ .eflags = true },
3235 .dst_temps = .{.{ .src = 0 }},
3240 .dst_temps = .{.{ .ref = .src0 }},
32363241 .each = .{ .once = switch (cc) {
32373242 else => unreachable,
32383243 .e => &.{
3239 .{ ._, .xor, .dst0w, .src1w, .none, .none },
3240 .{ ._, .not, .dst0w, .none, .none, .none },
3244 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
3245 .{ ._, ._, .not, .dst0w, ._, ._, ._ },
32413246 },
32423247 .ne => &.{
3243 .{ ._, .xor, .dst0w, .src1w, .none, .none },
3248 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
32443249 },
32453250 } },
32463251 }, .{
3247 .constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } },
3252 .src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } },
32483253 .patterns = &.{
32493254 .{ .src = .{ .mut_mem, .imm32 } },
32503255 .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -3257,20 +3262,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32573262 .{ .src = .{ .mut_gpr, .gpr } },
32583263 },
32593264 .clobbers = .{ .eflags = true },
3260 .dst_temps = .{.{ .src = 0 }},
3265 .dst_temps = .{.{ .ref = .src0 }},
32613266 .each = .{ .once = switch (cc) {
32623267 else => unreachable,
32633268 .e => &.{
3264 .{ ._, .xor, .edst0, .esrc1, .none, .none },
3265 .{ ._, .not, .edst0, .none, .none, .none },
3269 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
3270 .{ ._, ._, .not, .dst0d, ._, ._, ._ },
32663271 },
32673272 .ne => &.{
3268 .{ ._, .xor, .edst0, .esrc1, .none, .none },
3273 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
32693274 },
32703275 } },
32713276 }, .{
32723277 .required_features = .{ .@"64bit", null },
3273 .constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } },
3278 .src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } },
32743279 .patterns = &.{
32753280 .{ .src = .{ .mut_mem, .simm32 } },
32763281 .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
......@@ -3283,326 +3288,1665 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32833288 .{ .src = .{ .mut_gpr, .gpr } },
32843289 },
32853290 .clobbers = .{ .eflags = true },
3286 .dst_temps = .{.{ .src = 0 }},
3291 .dst_temps = .{.{ .ref = .src0 }},
32873292 .each = .{ .once = switch (cc) {
32883293 else => unreachable,
32893294 .e => &.{
3290 .{ ._, .xor, .rdst0, .rsrc1, .none, .none },
3291 .{ ._, .not, .rdst0, .none, .none, .none },
3295 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
3296 .{ ._, ._, .not, .dst0q, ._, ._, ._ },
32923297 },
32933298 .ne => &.{
3294 .{ ._, .xor, .rdst0, .rsrc1, .none, .none },
3299 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
32953300 },
32963301 } },
3297 } },
3298 }) catch |err2| switch (err2) {
3299 error.Select2Failed => cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, &.{
3300 .{
3301 .required_features = &.{.avx2},
3302 .scalar = .{ .any_int = .byte },
3303 .loop = .elementwise,
3304 .mir_tag = .{ .vp_b, .cmpeq },
3305 .patterns = &.{
3306 .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3307 .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3308 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } },
3309 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } },
3310 },
3302 }, .{
3303 .src_constraints = .{ .any_bool_vec, .any_bool_vec },
3304 .patterns = &.{
3305 .{ .src = .{ .to_mem, .to_mem } },
33113306 },
3312 .{
3313 .required_features = &.{.avx2},
3314 .scalar = .{ .any_int = .word },
3315 .loop = .elementwise,
3316 .mir_tag = .{ .vp_w, .cmpeq },
3317 .patterns = &.{
3318 .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3319 .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3320 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } },
3321 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } },
3322 },
3307 .clobbers = .{ .eflags = true },
3308 .extra_temps = .{
3309 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3310 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3311 .unused,
3312 .unused,
3313 .unused,
3314 .unused,
33233315 },
3324 .{
3325 .required_features = &.{.avx2},
3326 .scalar = .{ .any_int = .dword },
3327 .loop = .elementwise,
3328 .mir_tag = .{ .vp_d, .cmpeq },
3329 .patterns = &.{
3330 .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3331 .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3332 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } },
3333 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } },
3316 .dst_temps = .{.mem},
3317 .each = .{ .once = switch (cc) {
3318 else => unreachable,
3319 .e => &.{
3320 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3321 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
3322 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
3323 .{ ._, ._, .not, .tmp1p, ._, ._, ._ },
3324 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
3325 .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ },
3326 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
33343327 },
3335 },
3336 .{
3337 .required_features = &.{.avx2},
3338 .scalar = .{ .any_int = .qword },
3339 .loop = .elementwise,
3340 .mir_tag = .{ .vp_q, .cmpeq },
3341 .patterns = &.{
3342 .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3343 .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3344 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } },
3345 .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } },
3328 .ne => &.{
3329 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3330 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
3331 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
3332 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
3333 .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ },
3334 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
33463335 },
3336 } },
3337 }, .{
3338 .required_features = .{ .avx2, null },
3339 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3340 .patterns = &.{
3341 .{ .src = .{ .to_mem, .to_mem } },
33473342 },
3348 .{
3349 .required_features = &.{.avx},
3350 .scalar = .{ .any_int = .byte },
3351 .loop = .elementwise,
3352 .mir_tag = .{ .vp_b, .cmpeq },
3353 .patterns = &.{
3354 .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3355 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3356 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } },
3357 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } },
3358 },
3343 .extra_temps = .{
3344 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3345 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3346 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3347 .{ .kind = .{ .rc = .sse } },
3348 .unused,
3349 .unused,
33593350 },
3360 .{
3361 .required_features = &.{.avx},
3362 .scalar = .{ .any_int = .word },
3363 .loop = .elementwise,
3364 .mir_tag = .{ .vp_w, .cmpeq },
3365 .patterns = &.{
3366 .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3367 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3368 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } },
3369 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } },
3351 .dst_temps = .{.mem},
3352 .each = .{ .limb_and_mask_limb = .{
3353 .of = .src0y,
3354 .of_mask = .dst0b,
3355 .body = switch (cc) {
3356 else => unreachable,
3357 .e => &.{
3358 .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ },
3359 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ },
3360 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3361 .{ ._, ._, .mov, .maskLimb(.dst0d), .tmp2d, ._, ._ },
3362 },
3363 .ne => &.{
3364 .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ },
3365 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ },
3366 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3367 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
3368 .{ ._, ._, .mov, .maskLimb(.dst0d), .tmp2d, ._, ._ },
3369 },
33703370 },
3371 } },
3372 }, .{
3373 .required_features = .{ .avx2, null },
3374 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
3375 .patterns = &.{
3376 .{ .src = .{ .to_mem, .to_mem } },
33713377 },
3372 .{
3373 .required_features = &.{.avx},
3374 .scalar = .{ .any_int = .dword },
3375 .loop = .elementwise,
3376 .mir_tag = .{ .vp_d, .cmpeq },
3377 .patterns = &.{
3378 .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3379 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3380 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } },
3381 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } },
3378 .extra_temps = .{
3379 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3380 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3381 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3382 .{ .kind = .{ .rc = .sse } },
3383 .unused,
3384 .unused,
3385 },
3386 .dst_temps = .{.mem},
3387 .each = .{ .limb_and_mask_limb = .{
3388 .of = .src0y,
3389 .of_mask = .dst0w,
3390 .body = switch (cc) {
3391 else => unreachable,
3392 .e => &.{
3393 .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ },
3394 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ },
3395 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
3396 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3397 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3398 },
3399 .ne => &.{
3400 .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ },
3401 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ },
3402 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
3403 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3404 .{ ._, ._, .not, .tmp2w, ._, ._, ._ },
3405 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3406 },
33823407 },
3408 } },
3409 }, .{
3410 .required_features = .{ .avx2, null },
3411 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3412 .patterns = &.{
3413 .{ .src = .{ .to_mem, .to_mem } },
33833414 },
3384 .{
3385 .required_features = &.{.avx},
3386 .scalar = .{ .any_int = .qword },
3387 .loop = .elementwise,
3388 .mir_tag = .{ .vp_q, .cmpeq },
3389 .patterns = &.{
3390 .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } },
3391 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
3392 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } },
3393 .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } },
3415 .extra_temps = .{
3416 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3417 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3418 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3419 .{ .kind = .{ .rc = .sse } },
3420 .unused,
3421 .unused,
3422 },
3423 .dst_temps = .{.mem},
3424 .each = .{ .limb_and_mask_limb = .{
3425 .of = .src0y,
3426 .of_mask = .dst0d,
3427 .body = switch (cc) {
3428 else => unreachable,
3429 .e => &.{
3430 .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ },
3431 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ },
3432 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3433 .{ ._, ._, .mov, .maskLimb(.dst0b), .tmp2b, ._, ._ },
3434 },
3435 .ne => &.{
3436 .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ },
3437 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ },
3438 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3439 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
3440 .{ ._, ._, .mov, .maskLimb(.dst0b), .tmp2b, ._, ._ },
3441 },
33943442 },
3443 } },
3444 }, .{
3445 .required_features = .{ .avx2, null },
3446 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
3447 .patterns = &.{
3448 .{ .src = .{ .to_mem, .to_mem } },
33953449 },
3396 .{
3397 .required_features = &.{.sse2},
3398 .scalar = .{ .any_int = .byte },
3399 .loop = .elementwise,
3400 .mir_tag = .{ .p_b, .cmpeq },
3401 .patterns = &.{
3402 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } },
3403 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } },
3404 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } },
3450 .extra_temps = .{
3451 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3452 .{ .type = .usize, .kind = .{ .reg = .rcx } },
3453 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3454 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3455 .{ .kind = .{ .rc = .sse } },
3456 .unused,
3457 },
3458 .dst_temps = .{.mem},
3459 .each = .{ .limb_and_mask_limb = .{
3460 .of = .src0y,
3461 .of_mask = .dst0q,
3462 .body = switch (cc) {
3463 else => unreachable,
3464 .e => &.{
3465 .{ ._, .v_dqu, .mov, .tmp4y, .limb(.src0y), ._, ._ },
3466 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .limb(.src1y), ._ },
3467 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
3468 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3469 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3470 },
3471 .ne => &.{
3472 .{ ._, .v_dqu, .mov, .tmp4y, .limb(.src0y), ._, ._ },
3473 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .limb(.src1y), ._ },
3474 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
3475 .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ },
3476 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3477 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3478 },
34053479 },
3480 } },
3481 }, .{
3482 .required_features = .{ .avx, null },
3483 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3484 .patterns = &.{
3485 .{ .src = .{ .to_mem, .to_mem } },
34063486 },
3407 .{
3408 .required_features = &.{.sse2},
3409 .scalar = .{ .any_int = .word },
3410 .loop = .elementwise,
3411 .mir_tag = .{ .p_w, .cmpeq },
3412 .patterns = &.{
3413 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } },
3414 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } },
3415 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } },
3487 .extra_temps = .{
3488 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3489 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3490 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3491 .{ .kind = .{ .rc = .sse } },
3492 .unused,
3493 .unused,
3494 },
3495 .dst_temps = .{.mem},
3496 .each = .{ .limb_and_mask_limb = .{
3497 .of = .src0x,
3498 .of_mask = .dst0b,
3499 .body = switch (cc) {
3500 else => unreachable,
3501 .e => &.{
3502 .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3503 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ },
3504 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3505 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3506 },
3507 .ne => &.{
3508 .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3509 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ },
3510 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3511 .{ ._, ._, .not, .tmp2w, ._, ._, ._ },
3512 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3513 },
34163514 },
3515 } },
3516 }, .{
3517 .required_features = .{ .avx, null },
3518 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
3519 .patterns = &.{
3520 .{ .src = .{ .to_mem, .to_mem } },
34173521 },
3418 .{
3419 .required_features = &.{.sse2},
3420 .scalar = .{ .any_int = .dword },
3421 .loop = .elementwise,
3422 .mir_tag = .{ .p_d, .cmpeq },
3423 .patterns = &.{
3424 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } },
3425 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } },
3426 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } },
3522 .extra_temps = .{
3523 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3524 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3525 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3526 .{ .kind = .{ .rc = .sse } },
3527 .unused,
3528 .unused,
3529 },
3530 .dst_temps = .{.mem},
3531 .each = .{ .limb_and_mask_limb = .{
3532 .of = .src0x,
3533 .of_mask = .dst0w,
3534 .body = switch (cc) {
3535 else => unreachable,
3536 .e => &.{
3537 .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3538 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ },
3539 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
3540 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3541 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ },
3542 },
3543 .ne => &.{
3544 .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3545 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ },
3546 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
3547 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3548 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
3549 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ },
3550 },
34273551 },
3552 } },
3553 }, .{
3554 .required_features = .{ .avx, null },
3555 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3556 .patterns = &.{
3557 .{ .src = .{ .to_mem, .to_mem } },
34283558 },
3429 .{
3430 .required_features = &.{.sse4_1},
3431 .scalar = .{ .any_int = .qword },
3432 .loop = .elementwise,
3433 .mir_tag = .{ .p_q, .cmpeq },
3434 .patterns = &.{
3435 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } },
3436 .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } },
3437 .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } },
3559 .extra_temps = .{
3560 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3561 .{ .type = .usize, .kind = .{ .reg = .rcx } },
3562 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3563 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3564 .{ .kind = .{ .rc = .sse } },
3565 .unused,
3566 },
3567 .dst_temps = .{.mem},
3568 .each = .{ .limb_and_mask_limb = .{
3569 .of = .src0x,
3570 .of_mask = .dst0d,
3571 .body = switch (cc) {
3572 else => unreachable,
3573 .e => &.{
3574 .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3575 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ },
3576 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3577 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3578 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3579 },
3580 .ne => &.{
3581 .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3582 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ },
3583 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3584 .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ },
3585 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3586 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3587 },
34383588 },
3589 } },
3590 }, .{
3591 .required_features = .{ .avx, null },
3592 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
3593 .patterns = &.{
3594 .{ .src = .{ .to_mem, .to_mem } },
34393595 },
3440 .{
3441 .required_features = &.{.mmx},
3442 .scalar = .{ .any_int = .byte },
3443 .loop = .elementwise,
3444 .mir_tag = .{ .p_b, .cmpeq },
3445 .patterns = &.{
3446 .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } },
3447 .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } },
3448 .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } },
3596 .extra_temps = .{
3597 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3598 .{ .type = .usize, .kind = .{ .reg = .rcx } },
3599 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3600 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3601 .{ .kind = .{ .rc = .sse } },
3602 .unused,
3603 },
3604 .dst_temps = .{.mem},
3605 .each = .{ .limb_and_mask_limb = .{
3606 .of = .src0x,
3607 .of_mask = .dst0q,
3608 .body = switch (cc) {
3609 else => unreachable,
3610 .e => &.{
3611 .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3612 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ },
3613 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3614 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3615 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3616 },
3617 .ne => &.{
3618 .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3619 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ },
3620 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3621 .{ ._, ._, .xor, .tmp3b, .i(0b11), ._, ._ },
3622 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3623 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3624 },
34493625 },
3626 } },
3627 }, .{
3628 .required_features = .{ .sse2, null },
3629 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3630 .patterns = &.{
3631 .{ .src = .{ .to_mem, .to_mem } },
34503632 },
3451 .{
3452 .required_features = &.{.mmx},
3453 .scalar = .{ .any_int = .word },
3454 .loop = .elementwise,
3455 .mir_tag = .{ .p_w, .cmpeq },
3456 .patterns = &.{
3457 .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } },
3458 .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } },
3459 .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } },
3633 .extra_temps = .{
3634 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3635 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3636 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3637 .{ .kind = .{ .rc = .sse } },
3638 .unused,
3639 .unused,
3640 },
3641 .dst_temps = .{.mem},
3642 .each = .{ .limb_and_mask_limb = .{
3643 .of = .src0x,
3644 .of_mask = .dst0b,
3645 .body = switch (cc) {
3646 else => unreachable,
3647 .e => &.{
3648 .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3649 .{ ._, .p_b, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ },
3650 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3651 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3652 },
3653 .ne => &.{
3654 .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3655 .{ ._, .p_b, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ },
3656 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3657 .{ ._, ._, .not, .tmp2w, ._, ._, ._ },
3658 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3659 },
34603660 },
3661 } },
3662 }, .{
3663 .required_features = .{ .sse2, null },
3664 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
3665 .patterns = &.{
3666 .{ .src = .{ .to_mem, .to_mem } },
34613667 },
3462 .{
3463 .required_features = &.{.mmx},
3464 .scalar = .{ .any_int = .dword },
3465 .loop = .elementwise,
3466 .mir_tag = .{ .p_d, .cmpeq },
3467 .patterns = &.{
3468 .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } },
3469 .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } },
3470 .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } },
3668 .extra_temps = .{
3669 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3670 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3671 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3672 .{ .kind = .{ .rc = .sse } },
3673 .unused,
3674 .unused,
3675 },
3676 .dst_temps = .{.mem},
3677 .each = .{ .limb_and_mask_limb = .{
3678 .of = .src0x,
3679 .of_mask = .dst0w,
3680 .body = switch (cc) {
3681 else => unreachable,
3682 .e => &.{
3683 .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3684 .{ ._, .p_w, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ },
3685 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
3686 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3687 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ },
3688 },
3689 .ne => &.{
3690 .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ },
3691 .{ ._, .p_w, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ },
3692 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
3693 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3694 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
3695 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ },
3696 },
34713697 },
3698 } },
3699 }, .{
3700 .required_features = .{ .sse2, null },
3701 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3702 .patterns = &.{
3703 .{ .src = .{ .to_mem, .to_mem } },
34723704 },
3473 .{
3474 .scalar = .bool,
3475 .clobbers = .{ .eflags = true },
3476 .invert_result = true,
3477 .loop = .elementwise,
3478 .mir_tag = .{ ._, .xor },
3479 .patterns = &.{
3480 .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } },
3481 .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } },
3482 .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } },
3483 .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } },
3484 .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } },
3705 .extra_temps = .{
3706 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3707 .{ .type = .usize, .kind = .{ .reg = .rcx } },
3708 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3709 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3710 .{ .kind = .{ .rc = .sse } },
3711 .unused,
3712 },
3713 .dst_temps = .{.mem},
3714 .each = .{ .limb_and_mask_limb = .{
3715 .of = .src0x,
3716 .of_mask = .dst0d,
3717 .body = switch (cc) {
3718 else => unreachable,
3719 .e => &.{
3720 .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3721 .{ ._, .p_q, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ },
3722 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3723 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3724 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3725 },
3726 .ne => &.{
3727 .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3728 .{ ._, .p_d, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ },
3729 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3730 .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ },
3731 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3732 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3733 },
34853734 },
3735 } },
3736 }, .{
3737 .required_features = .{ .sse4_1, null },
3738 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
3739 .patterns = &.{
3740 .{ .src = .{ .to_mem, .to_mem } },
34863741 },
3487 .{
3488 .scalar = .{ .any_int = .byte },
3489 .clobbers = .{ .eflags = true },
3490 .loop = .elementwise,
3491 .mir_tag = .{ ._, .cmp },
3492 .patterns = &.{
3493 .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } },
3494 .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } },
3495 .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } },
3742 .extra_temps = .{
3743 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3744 .{ .type = .usize, .kind = .{ .reg = .rcx } },
3745 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3746 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3747 .{ .kind = .{ .rc = .sse } },
3748 .unused,
3749 },
3750 .dst_temps = .{.mem},
3751 .each = .{ .limb_and_mask_limb = .{
3752 .of = .src0x,
3753 .of_mask = .dst0q,
3754 .body = switch (cc) {
3755 else => unreachable,
3756 .e => &.{
3757 .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3758 .{ ._, .p_q, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ },
3759 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3760 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3761 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3762 },
3763 .ne => &.{
3764 .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ },
3765 .{ ._, .p_q, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ },
3766 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3767 .{ ._, ._, .xor, .tmp3b, .i(0b11), ._, ._ },
3768 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3769 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3770 },
34963771 },
3772 } },
3773 }, .{
3774 .required_features = .{ .mmx, null },
3775 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3776 .patterns = &.{
3777 .{ .src = .{ .to_mem, .to_mem } },
34973778 },
3498 .{
3499 .scalar = .{ .any_int = .word },
3500 .clobbers = .{ .eflags = true },
3501 .loop = .elementwise,
3502 .mir_tag = .{ ._, .cmp },
3503 .patterns = &.{
3504 .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } },
3505 .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } },
3506 .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } },
3779 .extra_temps = .{
3780 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3781 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3782 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3783 .{ .kind = .{ .rc = .mmx } },
3784 .unused,
3785 .unused,
3786 },
3787 .dst_temps = .{.mem},
3788 .each = .{ .limb_and_mask_limb = .{
3789 .of = .src0q,
3790 .of_mask = .dst0b,
3791 .body = switch (cc) {
3792 else => unreachable,
3793 .e => &.{
3794 .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ },
3795 .{ ._, .p_b, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ },
3796 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
3797 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3798 },
3799 .ne => &.{
3800 .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ },
3801 .{ ._, .p_b, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ },
3802 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
3803 .{ ._, ._, .not, .tmp2w, ._, ._, ._ },
3804 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ },
3805 },
35073806 },
3807 } },
3808 }, .{
3809 .required_features = .{ .mmx, null },
3810 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
3811 .patterns = &.{
3812 .{ .src = .{ .to_mem, .to_mem } },
35083813 },
3509 .{
3510 .scalar = .{ .any_int = .dword },
3511 .clobbers = .{ .eflags = true },
3512 .loop = .elementwise,
3513 .mir_tag = .{ ._, .cmp },
3514 .patterns = &.{
3515 .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } },
3516 .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } },
3517 .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } },
3814 .extra_temps = .{
3815 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3816 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3817 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3818 .{ .kind = .{ .rc = .mmx } },
3819 .unused,
3820 .unused,
3821 },
3822 .dst_temps = .{.mem},
3823 .each = .{ .limb_and_mask_limb = .{
3824 .of = .src0q,
3825 .of_mask = .dst0w,
3826 .body = switch (cc) {
3827 else => unreachable,
3828 .e => &.{
3829 .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ },
3830 .{ ._, .p_w, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ },
3831 .{ ._, .p_b, .ackssw, .tmp3q, .tmp3q, ._, ._ },
3832 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
3833 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ },
3834 },
3835 .ne => &.{
3836 .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ },
3837 .{ ._, .p_w, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ },
3838 .{ ._, .p_b, .ackssw, .tmp3q, .tmp3q, ._, ._ },
3839 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
3840 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
3841 .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ },
3842 },
35183843 },
3844 } },
3845 }, .{
3846 .required_features = .{ .mmx, null },
3847 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3848 .patterns = &.{
3849 .{ .src = .{ .to_mem, .to_mem } },
35193850 },
3520 .{
3521 .scalar = .{ .any_int = .qword },
3522 .clobbers = .{ .eflags = true },
3523 .loop = .elementwise,
3524 .mir_tag = .{ ._, .cmp },
3525 .patterns = &.{
3526 .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } },
3527 .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } },
3528 .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } },
3851 .extra_temps = .{
3852 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3853 .{ .type = .usize, .kind = .{ .reg = .rcx } },
3854 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3855 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3856 .{ .kind = .{ .rc = .mmx } },
3857 .unused,
3858 },
3859 .dst_temps = .{.mem},
3860 .each = .{ .limb_and_mask_limb = .{
3861 .of = .src0q,
3862 .of_mask = .dst0d,
3863 .body = switch (cc) {
3864 else => unreachable,
3865 .e => &.{
3866 .{ ._, ._dqu, .mov, .tmp4q, .limb(.src0q), ._, ._ },
3867 .{ ._, .p_q, .cmpeq, .tmp4q, .limb(.src1q), ._, ._ },
3868 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4q, ._, ._ },
3869 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3870 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3871 },
3872 .ne => &.{
3873 .{ ._, ._dqu, .mov, .tmp4q, .limb(.src0q), ._, ._ },
3874 .{ ._, .p_q, .cmpeq, .tmp4q, .limb(.src1q), ._, ._ },
3875 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4q, ._, ._ },
3876 .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ },
3877 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3878 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3879 },
35293880 },
3881 } },
3882 }, .{
3883 .required_features = .{ .slow_incdec, null },
3884 .dst_constraints = .{.{ .bool_vec = .byte }},
3885 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3886 .patterns = &.{
3887 .{ .src = .{ .to_mem, .to_mem } },
35303888 },
3889 .extra_temps = .{
3890 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3891 .{ .type = .u8, .kind = .{ .reg = .cl } },
3892 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3893 .unused,
3894 .unused,
3895 .unused,
3896 },
3897 .dst_temps = .{.{ .rc = .general_purpose }},
3898 .each = .{ .once = &.{
3899 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
3900 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3901 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
3902 .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
3903 .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
3904 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
3905 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
3906 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
3907 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
3908 .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ },
3909 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3910 } },
35313911 }, .{
3532 .cc = .e,
3533 .invert_result = switch (cmp_op) {
3534 .eq => false,
3535 .neq => true,
3536 else => unreachable,
3912 .dst_constraints = .{.{ .bool_vec = .byte }},
3913 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3914 .patterns = &.{
3915 .{ .src = .{ .to_mem, .to_mem } },
35373916 },
3538 }) catch |err| switch (err) {
3539 error.SelectFailed => return cg.fail("failed to select", .{}),
3540 else => |e| return e,
3541 },
3542 else => |e| return e,
3543 },
3544 .gte => unreachable,
3545 .gt => unreachable,
3546 }
3547 if (ops[0].index != res[0].index) try ops[0].die(cg);
3548 if (ops[1].index != res[0].index) try ops[1].die(cg);
3549 try res[0].moveTo(inst, cg);
3550 },
3551
3552 .cmp_lt,
3553 .cmp_lt_optimized,
3554 .cmp_lte,
3555 .cmp_lte_optimized,
3556 .cmp_gte,
3557 .cmp_gte_optimized,
3558 .cmp_gt,
3559 .cmp_gt_optimized,
3560 => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
3561 else => unreachable,
3562 .cmp_lt, .cmp_lt_optimized => .lt,
3563 .cmp_lte, .cmp_lte_optimized => .lte,
3564 .cmp_gte, .cmp_gte_optimized => .gte,
3565 .cmp_gt, .cmp_gt_optimized => .gt,
3566 }) else {
3567 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
3568 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
3569 const signedness = if (scalar_ty.isAbiInt(zcu))
3570 scalar_ty.intInfo(zcu).signedness
3571 else
3572 .unsigned;
3573 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
3574 var res: [1]Temp = undefined;
3575 cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (signedness) {
3576 .signed => switch (air_tag) {
3577 else => unreachable,
3578 .cmp_lt, .cmp_lt_optimized => .l,
3579 .cmp_lte, .cmp_lte_optimized => .le,
3580 .cmp_gte, .cmp_gte_optimized => .ge,
3581 .cmp_gt, .cmp_gt_optimized => .g,
3582 },
3583 .unsigned => switch (air_tag) {
3584 else => unreachable,
3585 .cmp_lt, .cmp_lt_optimized => .b,
3586 .cmp_lte, .cmp_lte_optimized => .be,
3587 .cmp_gte, .cmp_gte_optimized => .ae,
3588 .cmp_gt, .cmp_gt_optimized => .a,
3589 },
3590 })) {
3591 else => unreachable,
3592 inline .l, .le, .ge, .g, .b, .be, .ae, .a => |cc| comptime &.{ .{
3593 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3594 .patterns = &.{
3595 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
3596 .{ .src = .{ .imm8, .gpr }, .commute = .{ 0, 1 } },
3597 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
3598 },
3599 .clobbers = .{ .eflags = true },
3600 .dst_temps = .{.{ .cc = cc.commute() }},
3601 .each = .{ .once = &.{
3602 .{ ._, .cmp, .src0b, .src1b, .none, .none },
3603 } },
3604 }, .{
3605 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3917 .extra_temps = .{
3918 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3919 .{ .type = .u8, .kind = .{ .reg = .cl } },
3920 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3921 .unused,
3922 .unused,
3923 .unused,
3924 },
3925 .dst_temps = .{.{ .rc = .general_purpose }},
3926 .each = .{ .once = &.{
3927 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
3928 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3929 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
3930 .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
3931 .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
3932 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
3933 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
3934 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
3935 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
3936 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
3937 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
3938 } },
3939 }, .{
3940 .required_features = .{ .slow_incdec, null },
3941 .dst_constraints = .{.{ .bool_vec = .byte }},
3942 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
3943 .patterns = &.{
3944 .{ .src = .{ .to_mem, .to_mem } },
3945 },
3946 .extra_temps = .{
3947 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3948 .{ .type = .u8, .kind = .{ .reg = .cl } },
3949 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3950 .unused,
3951 .unused,
3952 .unused,
3953 },
3954 .dst_temps = .{.{ .rc = .general_purpose }},
3955 .each = .{ .once = &.{
3956 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
3957 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3958 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
3959 .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
3960 .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
3961 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
3962 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
3963 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
3964 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
3965 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
3966 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3967 } },
3968 }, .{
3969 .dst_constraints = .{.{ .bool_vec = .byte }},
3970 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
3971 .patterns = &.{
3972 .{ .src = .{ .to_mem, .to_mem } },
3973 },
3974 .extra_temps = .{
3975 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3976 .{ .type = .u8, .kind = .{ .reg = .cl } },
3977 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3978 .unused,
3979 .unused,
3980 .unused,
3981 },
3982 .dst_temps = .{.{ .rc = .general_purpose }},
3983 .each = .{ .once = &.{
3984 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
3985 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3986 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
3987 .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
3988 .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
3989 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
3990 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
3991 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
3992 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
3993 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
3994 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3995 } },
3996 }, .{
3997 .required_features = .{ .slow_incdec, null },
3998 .dst_constraints = .{.{ .bool_vec = .byte }},
3999 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4000 .patterns = &.{
4001 .{ .src = .{ .to_mem, .to_mem } },
4002 },
4003 .extra_temps = .{
4004 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4005 .{ .type = .u8, .kind = .{ .reg = .cl } },
4006 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4007 .unused,
4008 .unused,
4009 .unused,
4010 },
4011 .dst_temps = .{.{ .rc = .general_purpose }},
4012 .each = .{ .once = &.{
4013 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4014 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4015 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4016 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4017 .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4018 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4019 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4020 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4021 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4022 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4023 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4024 } },
4025 }, .{
4026 .dst_constraints = .{.{ .bool_vec = .byte }},
4027 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4028 .patterns = &.{
4029 .{ .src = .{ .to_mem, .to_mem } },
4030 },
4031 .extra_temps = .{
4032 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4033 .{ .type = .u8, .kind = .{ .reg = .cl } },
4034 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4035 .unused,
4036 .unused,
4037 .unused,
4038 },
4039 .dst_temps = .{.{ .rc = .general_purpose }},
4040 .each = .{ .once = &.{
4041 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4042 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4043 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4044 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4045 .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4046 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4047 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4048 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4049 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4050 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4051 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4052 } },
4053 }, .{
4054 .required_features = .{ .@"64bit", .slow_incdec },
4055 .dst_constraints = .{.{ .bool_vec = .byte }},
4056 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
4057 .patterns = &.{
4058 .{ .src = .{ .to_mem, .to_mem } },
4059 },
4060 .extra_temps = .{
4061 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4062 .{ .type = .u8, .kind = .{ .reg = .cl } },
4063 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4064 .unused,
4065 .unused,
4066 .unused,
4067 },
4068 .dst_temps = .{.{ .rc = .general_purpose }},
4069 .each = .{ .once = &.{
4070 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4071 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4072 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4073 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4074 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4075 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4076 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4077 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4078 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4079 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4080 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4081 } },
4082 }, .{
4083 .required_features = .{ .@"64bit", null },
4084 .dst_constraints = .{.{ .bool_vec = .byte }},
4085 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
4086 .patterns = &.{
4087 .{ .src = .{ .to_mem, .to_mem } },
4088 },
4089 .extra_temps = .{
4090 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4091 .{ .type = .u8, .kind = .{ .reg = .cl } },
4092 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4093 .unused,
4094 .unused,
4095 .unused,
4096 },
4097 .dst_temps = .{.{ .rc = .general_purpose }},
4098 .each = .{ .once = &.{
4099 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4100 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4101 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4102 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4103 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4104 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4105 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4106 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4107 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4108 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4109 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4110 } },
4111 }, .{
4112 .required_features = .{ .slow_incdec, null },
4113 .dst_constraints = .{.{ .bool_vec = .byte }},
4114 .patterns = &.{
4115 .{ .src = .{ .to_mem, .to_mem } },
4116 },
4117 .extra_temps = .{
4118 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4119 .{ .type = .u8, .kind = .{ .reg = .cl } },
4120 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4121 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4122 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4123 .unused,
4124 },
4125 .dst_temps = .{.{ .rc = .general_purpose }},
4126 .each = .{ .once = &.{
4127 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4128 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4129 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4130 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4131 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4132 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4133 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4134 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4135 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4136 .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ },
4137 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
4138 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4139 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4140 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4141 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4142 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4143 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4144 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
4145 } },
4146 }, .{
4147 .dst_constraints = .{.{ .bool_vec = .byte }},
4148 .patterns = &.{
4149 .{ .src = .{ .to_mem, .to_mem } },
4150 },
4151 .extra_temps = .{
4152 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4153 .{ .type = .u8, .kind = .{ .reg = .cl } },
4154 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4155 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4156 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4157 .unused,
4158 },
4159 .dst_temps = .{.{ .rc = .general_purpose }},
4160 .each = .{ .once = &.{
4161 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4162 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4163 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4164 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4165 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4166 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4167 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4168 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4169 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4170 .{ ._, ._, .dec, .tmp2d, ._, ._, ._ },
4171 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
4172 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4173 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4174 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4175 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4176 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4177 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4178 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
4179 } },
4180 }, .{
4181 .required_features = .{ .slow_incdec, null },
4182 .dst_constraints = .{.{ .bool_vec = .dword }},
4183 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4184 .patterns = &.{
4185 .{ .src = .{ .to_mem, .to_mem } },
4186 },
4187 .extra_temps = .{
4188 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4189 .{ .type = .u8, .kind = .{ .reg = .cl } },
4190 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4191 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4192 .unused,
4193 .unused,
4194 },
4195 .dst_temps = .{.{ .rc = .general_purpose }},
4196 .each = .{ .once = &.{
4197 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4198 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4199 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4200 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4201 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4202 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4203 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4204 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4205 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4206 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4207 .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ },
4208 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4209 } },
4210 }, .{
4211 .dst_constraints = .{.{ .bool_vec = .dword }},
4212 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4213 .patterns = &.{
4214 .{ .src = .{ .to_mem, .to_mem } },
4215 },
4216 .extra_temps = .{
4217 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4218 .{ .type = .u8, .kind = .{ .reg = .cl } },
4219 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4220 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4221 .unused,
4222 .unused,
4223 },
4224 .dst_temps = .{.{ .rc = .general_purpose }},
4225 .each = .{ .once = &.{
4226 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4227 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4228 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4229 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4230 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4231 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4232 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4233 .{ ._, ._l, .sh, .tmp2d, .tmp1d, ._, ._ },
4234 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4235 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4236 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
4237 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
4238 } },
4239 }, .{
4240 .required_features = .{ .slow_incdec, null },
4241 .dst_constraints = .{.{ .bool_vec = .dword }},
4242 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
4243 .patterns = &.{
4244 .{ .src = .{ .to_mem, .to_mem } },
4245 },
4246 .extra_temps = .{
4247 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4248 .{ .type = .u8, .kind = .{ .reg = .cl } },
4249 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4250 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4251 .unused,
4252 .unused,
4253 },
4254 .dst_temps = .{.{ .rc = .general_purpose }},
4255 .each = .{ .once = &.{
4256 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4257 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4258 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4259 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4260 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4261 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4262 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4263 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4264 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4265 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4266 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4267 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4268 } },
4269 }, .{
4270 .dst_constraints = .{.{ .bool_vec = .dword }},
4271 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
4272 .patterns = &.{
4273 .{ .src = .{ .to_mem, .to_mem } },
4274 },
4275 .extra_temps = .{
4276 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4277 .{ .type = .u8, .kind = .{ .reg = .cl } },
4278 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4279 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4280 .unused,
4281 .unused,
4282 },
4283 .dst_temps = .{.{ .rc = .general_purpose }},
4284 .each = .{ .once = &.{
4285 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4286 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4287 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4288 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4289 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4290 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4291 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4292 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4293 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4294 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4295 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4296 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4297 } },
4298 }, .{
4299 .required_features = .{ .slow_incdec, null },
4300 .dst_constraints = .{.{ .bool_vec = .dword }},
4301 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4302 .patterns = &.{
4303 .{ .src = .{ .to_mem, .to_mem } },
4304 },
4305 .extra_temps = .{
4306 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4307 .{ .type = .u8, .kind = .{ .reg = .cl } },
4308 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4309 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4310 .unused,
4311 .unused,
4312 },
4313 .dst_temps = .{.{ .rc = .general_purpose }},
4314 .each = .{ .once = &.{
4315 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4316 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4317 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4318 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4319 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4320 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4321 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4322 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4323 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4324 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4325 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4326 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4327 } },
4328 }, .{
4329 .dst_constraints = .{.{ .bool_vec = .dword }},
4330 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4331 .patterns = &.{
4332 .{ .src = .{ .to_mem, .to_mem } },
4333 },
4334 .extra_temps = .{
4335 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4336 .{ .type = .u8, .kind = .{ .reg = .cl } },
4337 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4338 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4339 .unused,
4340 .unused,
4341 },
4342 .dst_temps = .{.{ .rc = .general_purpose }},
4343 .each = .{ .once = &.{
4344 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4345 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4346 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4347 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4348 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4349 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4350 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4351 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4352 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4353 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4354 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4355 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4356 } },
4357 }, .{
4358 .required_features = .{ .@"64bit", .slow_incdec },
4359 .dst_constraints = .{.{ .bool_vec = .dword }},
4360 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
4361 .patterns = &.{
4362 .{ .src = .{ .to_mem, .to_mem } },
4363 },
4364 .extra_temps = .{
4365 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4366 .{ .type = .u8, .kind = .{ .reg = .cl } },
4367 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4368 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4369 .unused,
4370 .unused,
4371 },
4372 .dst_temps = .{.{ .rc = .general_purpose }},
4373 .each = .{ .once = &.{
4374 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4375 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4376 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4377 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4378 .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4379 .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4380 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4381 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4382 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4383 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4384 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4385 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4386 } },
4387 }, .{
4388 .required_features = .{ .@"64bit", null },
4389 .dst_constraints = .{.{ .bool_vec = .dword }},
4390 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
4391 .patterns = &.{
4392 .{ .src = .{ .to_mem, .to_mem } },
4393 },
4394 .extra_temps = .{
4395 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4396 .{ .type = .u8, .kind = .{ .reg = .cl } },
4397 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4398 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4399 .unused,
4400 .unused,
4401 },
4402 .dst_temps = .{.{ .rc = .general_purpose }},
4403 .each = .{ .once = &.{
4404 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4405 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4406 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4407 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4408 .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4409 .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4410 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4411 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4412 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4413 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4414 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4415 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4416 } },
4417 }, .{
4418 .required_features = .{ .slow_incdec, null },
4419 .dst_constraints = .{.{ .bool_vec = .dword }},
4420 .patterns = &.{
4421 .{ .src = .{ .to_mem, .to_mem } },
4422 },
4423 .extra_temps = .{
4424 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4425 .{ .type = .u8, .kind = .{ .reg = .cl } },
4426 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4427 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4428 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4429 .unused,
4430 },
4431 .dst_temps = .{.{ .rc = .general_purpose }},
4432 .each = .{ .once = &.{
4433 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4434 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4435 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4436 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4437 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4438 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4439 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4440 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4441 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4442 .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ },
4443 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
4444 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4445 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4446 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4447 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4448 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4449 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4450 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4451 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
4452 } },
4453 }, .{
4454 .dst_constraints = .{.{ .bool_vec = .dword }},
4455 .patterns = &.{
4456 .{ .src = .{ .to_mem, .to_mem } },
4457 },
4458 .extra_temps = .{
4459 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4460 .{ .type = .u8, .kind = .{ .reg = .cl } },
4461 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4462 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4463 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4464 .unused,
4465 },
4466 .dst_temps = .{.{ .rc = .general_purpose }},
4467 .each = .{ .once = &.{
4468 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4469 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4470 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4471 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4472 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4473 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4474 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4475 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4476 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4477 .{ ._, ._, .dec, .tmp2d, ._, ._, ._ },
4478 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
4479 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4480 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4481 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4482 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4483 .{ ._, ._, .@"or", .dst0d, .tmp2b, ._, ._ },
4484 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4485 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4486 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
4487 } },
4488 }, .{
4489 .required_features = .{ .@"64bit", .slow_incdec },
4490 .dst_constraints = .{.{ .bool_vec = .qword }},
4491 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4492 .patterns = &.{
4493 .{ .src = .{ .to_mem, .to_mem } },
4494 },
4495 .extra_temps = .{
4496 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4497 .{ .type = .u8, .kind = .{ .reg = .cl } },
4498 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4499 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4500 .unused,
4501 .unused,
4502 },
4503 .dst_temps = .{.{ .rc = .general_purpose }},
4504 .each = .{ .once = &.{
4505 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4506 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4507 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4508 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4509 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4510 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4511 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4512 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4513 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4514 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4515 .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ },
4516 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4517 } },
4518 }, .{
4519 .required_features = .{ .@"64bit", null },
4520 .dst_constraints = .{.{ .bool_vec = .qword }},
4521 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4522 .patterns = &.{
4523 .{ .src = .{ .to_mem, .to_mem } },
4524 },
4525 .extra_temps = .{
4526 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4527 .{ .type = .u8, .kind = .{ .reg = .cl } },
4528 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4529 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4530 .unused,
4531 .unused,
4532 },
4533 .dst_temps = .{.{ .rc = .general_purpose }},
4534 .each = .{ .once = &.{
4535 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4536 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4537 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4538 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4539 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4540 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4541 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4542 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4543 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4544 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4545 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
4546 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
4547 } },
4548 }, .{
4549 .required_features = .{ .@"64bit", .slow_incdec },
4550 .dst_constraints = .{.{ .bool_vec = .qword }},
4551 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
4552 .patterns = &.{
4553 .{ .src = .{ .to_mem, .to_mem } },
4554 },
4555 .extra_temps = .{
4556 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4557 .{ .type = .u8, .kind = .{ .reg = .cl } },
4558 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4559 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4560 .unused,
4561 .unused,
4562 },
4563 .dst_temps = .{.{ .rc = .general_purpose }},
4564 .each = .{ .once = &.{
4565 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4566 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4567 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4568 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4569 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4570 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4571 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4572 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4573 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4574 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4575 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4576 } },
4577 }, .{
4578 .required_features = .{ .@"64bit", null },
4579 .dst_constraints = .{.{ .bool_vec = .qword }},
4580 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
4581 .patterns = &.{
4582 .{ .src = .{ .to_mem, .to_mem } },
4583 },
4584 .extra_temps = .{
4585 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4586 .{ .type = .u8, .kind = .{ .reg = .cl } },
4587 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4588 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4589 .unused,
4590 .unused,
4591 },
4592 .dst_temps = .{.{ .rc = .general_purpose }},
4593 .each = .{ .once = &.{
4594 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4595 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4596 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4597 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4598 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4599 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4600 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4601 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4602 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4603 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4604 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4605 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4606 } },
4607 }, .{
4608 .required_features = .{ .@"64bit", .slow_incdec },
4609 .dst_constraints = .{.{ .bool_vec = .qword }},
4610 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4611 .patterns = &.{
4612 .{ .src = .{ .to_mem, .to_mem } },
4613 },
4614 .extra_temps = .{
4615 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4616 .{ .type = .u8, .kind = .{ .reg = .cl } },
4617 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4618 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4619 .unused,
4620 .unused,
4621 },
4622 .dst_temps = .{.{ .rc = .general_purpose }},
4623 .each = .{ .once = &.{
4624 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4625 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4626 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4627 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4628 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4629 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4630 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4631 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4632 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4633 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4634 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4635 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4636 } },
4637 }, .{
4638 .required_features = .{ .@"64bit", null },
4639 .dst_constraints = .{.{ .bool_vec = .qword }},
4640 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4641 .patterns = &.{
4642 .{ .src = .{ .to_mem, .to_mem } },
4643 },
4644 .extra_temps = .{
4645 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4646 .{ .type = .u8, .kind = .{ .reg = .cl } },
4647 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4648 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4649 .unused,
4650 .unused,
4651 },
4652 .dst_temps = .{.{ .rc = .general_purpose }},
4653 .each = .{ .once = &.{
4654 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4655 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4656 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4657 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4658 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4659 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4660 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4661 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4662 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4663 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4664 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4665 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4666 } },
4667 }, .{
4668 .required_features = .{ .@"64bit", .slow_incdec },
4669 .dst_constraints = .{.{ .bool_vec = .qword }},
4670 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
4671 .patterns = &.{
4672 .{ .src = .{ .to_mem, .to_mem } },
4673 },
4674 .extra_temps = .{
4675 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4676 .{ .type = .u8, .kind = .{ .reg = .cl } },
4677 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4678 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4679 .unused,
4680 .unused,
4681 },
4682 .dst_temps = .{.{ .rc = .general_purpose }},
4683 .each = .{ .once = &.{
4684 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4685 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4686 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4687 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4688 .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4689 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4690 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4691 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4692 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4693 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4694 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
4695 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4696 } },
4697 }, .{
4698 .required_features = .{ .@"64bit", null },
4699 .dst_constraints = .{.{ .bool_vec = .qword }},
4700 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
4701 .patterns = &.{
4702 .{ .src = .{ .to_mem, .to_mem } },
4703 },
4704 .extra_temps = .{
4705 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4706 .{ .type = .u8, .kind = .{ .reg = .cl } },
4707 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4708 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4709 .unused,
4710 .unused,
4711 },
4712 .dst_temps = .{.{ .rc = .general_purpose }},
4713 .each = .{ .once = &.{
4714 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4715 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4716 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4717 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4718 .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4719 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4720 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4721 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4722 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4723 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4724 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
4725 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4726 } },
4727 }, .{
4728 .required_features = .{ .@"64bit", .slow_incdec },
4729 .dst_constraints = .{.{ .bool_vec = .qword }},
4730 .patterns = &.{
4731 .{ .src = .{ .to_mem, .to_mem } },
4732 },
4733 .extra_temps = .{
4734 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4735 .{ .type = .u8, .kind = .{ .reg = .cl } },
4736 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4737 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4738 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4739 .unused,
4740 },
4741 .dst_temps = .{.{ .rc = .general_purpose }},
4742 .each = .{ .once = &.{
4743 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4744 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4745 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4746 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4747 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4748 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4749 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4750 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4751 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4752 .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ },
4753 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
4754 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4755 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4756 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4757 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4758 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4759 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4760 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4761 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
4762 } },
4763 }, .{
4764 .required_features = .{ .@"64bit", null },
4765 .dst_constraints = .{.{ .bool_vec = .qword }},
4766 .patterns = &.{
4767 .{ .src = .{ .to_mem, .to_mem } },
4768 },
4769 .extra_temps = .{
4770 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4771 .{ .type = .u8, .kind = .{ .reg = .cl } },
4772 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4773 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4774 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4775 .unused,
4776 },
4777 .dst_temps = .{.{ .rc = .general_purpose }},
4778 .each = .{ .once = &.{
4779 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4780 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4781 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4782 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4783 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4784 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4785 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4786 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4787 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4788 .{ ._, ._, .dec, .tmp2d, ._, ._, ._ },
4789 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
4790 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4791 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4792 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4793 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4794 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4795 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4796 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4797 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
4798 } },
4799 }, .{
4800 .required_features = .{ .slow_incdec, null },
4801 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4802 .patterns = &.{
4803 .{ .src = .{ .to_mem, .to_mem } },
4804 },
4805 .extra_temps = .{
4806 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4807 .{ .type = .u32, .kind = .{ .reg = .ecx } },
4808 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4809 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4810 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4811 .unused,
4812 },
4813 .dst_temps = .{.mem},
4814 .each = .{ .once = &.{
4815 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4816 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
4817 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4818 .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4819 .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4820 .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4821 .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ },
4822 .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ },
4823 .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ },
4824 .{ ._, ._, .add, .tmp1d, .i(1), ._, ._ },
4825 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ },
4826 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
4827 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
4828 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
4829 .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_access_size), .tmp2p, ._, ._ },
4830 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4831 .{ .@"1:", ._, .add, .tmp0p, .i(1), ._, ._ },
4832 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4833 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ },
4834 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4835 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
4836 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
4837 .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ },
4838 } },
4839 }, .{
4840 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4841 .patterns = &.{
4842 .{ .src = .{ .to_mem, .to_mem } },
4843 },
4844 .extra_temps = .{
4845 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4846 .{ .type = .u32, .kind = .{ .reg = .ecx } },
4847 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4848 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4849 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4850 .unused,
4851 },
4852 .dst_temps = .{.mem},
4853 .each = .{ .once = &.{
4854 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4855 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
4856 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4857 .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4858 .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4859 .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4860 .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ },
4861 .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ },
4862 .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ },
4863 .{ ._, ._, .inc, .tmp1d, ._, ._, ._ },
4864 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ },
4865 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
4866 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
4867 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
4868 .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_access_size), .tmp2p, ._, ._ },
4869 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4870 .{ .@"1:", ._, .inc, .tmp0p, ._, ._, ._ },
4871 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
4872 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ },
4873 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4874 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
4875 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
4876 .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ },
4877 } },
4878 } },
4879 }) catch |err2| switch (err2) {
4880 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
4881 @tagName(air_tag),
4882 cg.typeOf(extra.lhs).fmt(pt),
4883 ops[0].tracking(cg),
4884 ops[1].tracking(cg),
4885 }),
4886 else => |e| return e,
4887 },
4888 .gte => unreachable,
4889 .gt => unreachable,
4890 }
4891 if (ops[0].index != res[0].index) try ops[0].die(cg);
4892 if (ops[1].index != res[0].index) try ops[1].die(cg);
4893 try res[0].moveTo(inst, cg);
4894 },
4895
4896 .cmp_lt,
4897 .cmp_lt_optimized,
4898 .cmp_lte,
4899 .cmp_lte_optimized,
4900 .cmp_gte,
4901 .cmp_gte_optimized,
4902 .cmp_gt,
4903 .cmp_gt_optimized,
4904 => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
4905 else => unreachable,
4906 .cmp_lt, .cmp_lt_optimized => .lt,
4907 .cmp_lte, .cmp_lte_optimized => .lte,
4908 .cmp_gte, .cmp_gte_optimized => .gte,
4909 .cmp_gt, .cmp_gt_optimized => .gt,
4910 }) else {
4911 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
4912 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
4913 const signedness = if (scalar_ty.isAbiInt(zcu))
4914 scalar_ty.intInfo(zcu).signedness
4915 else
4916 .unsigned;
4917 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
4918 var res: [1]Temp = undefined;
4919 cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (signedness) {
4920 .signed => switch (air_tag) {
4921 else => unreachable,
4922 .cmp_lt, .cmp_lt_optimized => .l,
4923 .cmp_lte, .cmp_lte_optimized => .le,
4924 .cmp_gte, .cmp_gte_optimized => .ge,
4925 .cmp_gt, .cmp_gt_optimized => .g,
4926 },
4927 .unsigned => switch (air_tag) {
4928 else => unreachable,
4929 .cmp_lt, .cmp_lt_optimized => .b,
4930 .cmp_lte, .cmp_lte_optimized => .be,
4931 .cmp_gte, .cmp_gte_optimized => .ae,
4932 .cmp_gt, .cmp_gt_optimized => .a,
4933 },
4934 })) {
4935 else => unreachable,
4936 inline .l, .le, .ge, .g, .b, .be, .ae, .a => |cc| comptime &.{ .{
4937 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4938 .patterns = &.{
4939 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
4940 .{ .src = .{ .imm8, .gpr }, .commute = .{ 0, 1 } },
4941 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
4942 },
4943 .clobbers = .{ .eflags = true },
4944 .dst_temps = .{.{ .cc = cc.commute() }},
4945 .each = .{ .once = &.{
4946 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
4947 } },
4948 }, .{
4949 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
36064950 .patterns = &.{
36074951 .{ .src = .{ .mem, .imm8 } },
36084952 .{ .src = .{ .gpr, .imm8 } },
......@@ -3612,10 +4956,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
36124956 .clobbers = .{ .eflags = true },
36134957 .dst_temps = .{.{ .cc = cc }},
36144958 .each = .{ .once = &.{
3615 .{ ._, .cmp, .src0b, .src1b, .none, .none },
4959 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
36164960 } },
36174961 }, .{
3618 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
4962 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
36194963 .patterns = &.{
36204964 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
36214965 .{ .src = .{ .imm16, .gpr }, .commute = .{ 0, 1 } },
......@@ -3624,10 +4968,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
36244968 .clobbers = .{ .eflags = true },
36254969 .dst_temps = .{.{ .cc = cc.commute() }},
36264970 .each = .{ .once = &.{
3627 .{ ._, .cmp, .src0w, .src1w, .none, .none },
4971 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
36284972 } },
36294973 }, .{
3630 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
4974 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
36314975 .patterns = &.{
36324976 .{ .src = .{ .mem, .imm16 } },
36334977 .{ .src = .{ .gpr, .imm16 } },
......@@ -3637,10 +4981,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
36374981 .clobbers = .{ .eflags = true },
36384982 .dst_temps = .{.{ .cc = cc }},
36394983 .each = .{ .once = &.{
3640 .{ ._, .cmp, .src0w, .src1w, .none, .none },
4984 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
36414985 } },
36424986 }, .{
3643 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4987 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
36444988 .patterns = &.{
36454989 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
36464990 .{ .src = .{ .imm32, .gpr }, .commute = .{ 0, 1 } },
......@@ -3649,10 +4993,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
36494993 .clobbers = .{ .eflags = true },
36504994 .dst_temps = .{.{ .cc = cc.commute() }},
36514995 .each = .{ .once = &.{
3652 .{ ._, .cmp, .esrc0, .esrc1, .none, .none },
4996 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
36534997 } },
36544998 }, .{
3655 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4999 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
36565000 .patterns = &.{
36575001 .{ .src = .{ .mem, .imm32 } },
36585002 .{ .src = .{ .gpr, .imm32 } },
......@@ -3662,11 +5006,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
36625006 .clobbers = .{ .eflags = true },
36635007 .dst_temps = .{.{ .cc = cc }},
36645008 .each = .{ .once = &.{
3665 .{ ._, .cmp, .esrc0, .esrc1, .none, .none },
5009 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
36665010 } },
36675011 }, .{
36685012 .required_features = .{ .@"64bit", null },
3669 .constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
5013 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
36705014 .patterns = &.{
36715015 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
36725016 .{ .src = .{ .simm32, .gpr }, .commute = .{ 0, 1 } },
......@@ -3675,11 +5019,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
36755019 .clobbers = .{ .eflags = true },
36765020 .dst_temps = .{.{ .cc = cc.commute() }},
36775021 .each = .{ .once = &.{
3678 .{ ._, .cmp, .rsrc0, .rsrc1, .none, .none },
5022 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
36795023 } },
36805024 }, .{
36815025 .required_features = .{ .@"64bit", null },
3682 .constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
5026 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
36835027 .patterns = &.{
36845028 .{ .src = .{ .mem, .simm32 } },
36855029 .{ .src = .{ .gpr, .simm32 } },
......@@ -3689,60 +5033,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
36895033 .clobbers = .{ .eflags = true },
36905034 .dst_temps = .{.{ .cc = cc }},
36915035 .each = .{ .once = &.{
3692 .{ ._, .cmp, .rsrc0, .rsrc1, .none, .none },
5036 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
36935037 } },
36945038 }, .{
3695 .required_features = .{ .@"64bit", null },
36965039 .patterns = &.{
36975040 .{ .src = .{ .to_mem, .to_mem } },
36985041 },
36995042 .extra_temps = .{
3700 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5043 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
37015044 .{ .type = .bool, .kind = .{ .rc = .general_purpose } },
37025045 .unused,
3703 },
3704 .clobbers = .{ .eflags = true },
3705 .dst_temps = .{.{ .rc = .general_purpose }},
3706 .each = .{ .limb = .{
3707 .of = .rsrc0,
3708 .header = &.{
3709 .{ ._, .xor, .tmp1b, .tmp1b, .none, .none },
3710 },
3711 .body = &.{
3712 .{ ._, .mov, .rtmp0, .{ .src_limb = 0 }, .none, .none },
3713 .{ ._r, .sh, .tmp1b, .{ .simm32 = 1 }, .none, .none },
3714 .{ ._, .sbb, .rtmp0, .{ .src_limb = 1 }, .none, .none },
3715 .{ ._c, .set, .tmp1b, .none, .none, .none },
3716 .{ .fromCondition(cc), .set, .dst0b, .none, .none, .none },
3717 },
3718 } },
3719 }, .{
3720 .patterns = &.{
3721 .{ .src = .{ .to_mem, .to_mem } },
3722 },
3723 .extra_temps = .{
3724 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3725 .{ .type = .bool, .kind = .{ .rc = .general_purpose } },
5046 .unused,
5047 .unused,
37265048 .unused,
37275049 },
37285050 .clobbers = .{ .eflags = true },
37295051 .dst_temps = .{.{ .rc = .general_purpose }},
37305052 .each = .{ .limb = .{
3731 .of = .esrc0,
5053 .of = .src0p,
37325054 .header = &.{
3733 .{ ._, .xor, .tmp1b, .tmp1b, .none, .none },
5055 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
37345056 },
37355057 .body = &.{
3736 .{ ._, .mov, .etmp0, .{ .src_limb = 0 }, .none, .none },
3737 .{ ._r, .sh, .tmp1b, .{ .simm32 = 1 }, .none, .none },
3738 .{ ._, .sbb, .etmp0, .{ .src_limb = 1 }, .none, .none },
3739 .{ ._c, .set, .tmp1b, .none, .none, .none },
3740 .{ .fromCondition(cc), .set, .dst0b, .none, .none, .none },
5058 .{ ._, ._, .mov, .tmp0p, .limb(.src0p), ._, ._ },
5059 .{ ._, ._r, .sh, .tmp1b, .i(1), ._, ._ },
5060 .{ ._, ._, .sbb, .tmp0p, .limb(.src1p), ._, ._ },
5061 .{ ._, ._c, .set, .tmp1b, ._, ._, ._ },
5062 .{ ._, .fromCondition(cc), .set, .dst0b, ._, ._, ._ },
37415063 },
37425064 } },
37435065 } },
37445066 }) catch |err| switch (err) {
3745 error.Select2Failed => return cg.fail("failed to select2 {s} {} {} {}", .{
5067 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
37465068 @tagName(air_tag),
37475069 cg.typeOf(bin_op.lhs).fmt(pt),
37485070 ops[0].tracking(cg),
......@@ -3767,7 +5089,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
37675089 });
37685090 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
37695091 var res: [1]Temp = undefined;
3770 cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (air_tag) {
5092 cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (air_tag) {
37715093 else => unreachable,
37725094 .cmp_eq, .cmp_eq_optimized => .e,
37735095 .cmp_neq, .cmp_neq_optimized => .ne,
......@@ -3775,52 +5097,73 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
37755097 else => unreachable,
37765098 inline .e, .ne => |cc| comptime &.{ .{
37775099 .required_features = .{ .avx2, null },
3778 .constraints = .{ .any_int, .any_int },
5100 .src_constraints = .{ .any_int, .any_int },
37795101 .patterns = &.{
37805102 .{ .src = .{ .ymm, .mem } },
37815103 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
37825104 .{ .src = .{ .ymm, .ymm } },
37835105 },
37845106 .clobbers = .{ .eflags = true },
3785 .extra_temps = .{ .{ .kind = .{ .rc = .sse } }, .unused, .unused },
5107 .extra_temps = .{
5108 .{ .kind = .{ .rc = .sse } },
5109 .unused,
5110 .unused,
5111 .unused,
5112 .unused,
5113 .unused,
5114 },
37865115 .dst_temps = .{.{ .cc = cc }},
37875116 .each = .{ .once = &.{
3788 .{ .vp_, .xor, .ytmp0, .ysrc0, .ysrc1, .none },
3789 .{ .vp_, .@"test", .ytmp0, .ytmp0, .none, .none },
5117 .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ },
5118 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
37905119 } },
37915120 }, .{
37925121 .required_features = .{ .avx, null },
3793 .constraints = .{ .any_int, .any_int },
5122 .src_constraints = .{ .any_int, .any_int },
37945123 .patterns = &.{
37955124 .{ .src = .{ .ymm, .mem } },
37965125 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
37975126 .{ .src = .{ .ymm, .ymm } },
37985127 },
37995128 .clobbers = .{ .eflags = true },
3800 .extra_temps = .{ .{ .kind = .{ .rc = .sse } }, .unused, .unused },
5129 .extra_temps = .{
5130 .{ .kind = .{ .rc = .sse } },
5131 .unused,
5132 .unused,
5133 .unused,
5134 .unused,
5135 .unused,
5136 },
38015137 .dst_temps = .{.{ .cc = cc }},
38025138 .each = .{ .once = &.{
3803 .{ .v_pd, .xor, .ytmp0, .ysrc0, .ysrc1, .none },
3804 .{ .vp_, .@"test", .ytmp0, .ytmp0, .none, .none },
5139 .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ },
5140 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
38055141 } },
38065142 }, .{
38075143 .required_features = .{ .avx, null },
3808 .constraints = .{ .any_int, .any_int },
5144 .src_constraints = .{ .any_int, .any_int },
38095145 .patterns = &.{
38105146 .{ .src = .{ .xmm, .mem } },
38115147 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
38125148 .{ .src = .{ .xmm, .xmm } },
38135149 },
38145150 .clobbers = .{ .eflags = true },
3815 .extra_temps = .{ .{ .kind = .{ .rc = .sse } }, .unused, .unused },
5151 .extra_temps = .{
5152 .{ .kind = .{ .rc = .sse } },
5153 .unused,
5154 .unused,
5155 .unused,
5156 .unused,
5157 .unused,
5158 },
38165159 .dst_temps = .{.{ .cc = cc }},
38175160 .each = .{ .once = &.{
3818 .{ .vp_, .xor, .xtmp0, .xsrc0, .xsrc1, .none },
3819 .{ .vp_, .@"test", .xtmp0, .xtmp0, .none, .none },
5161 .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ },
5162 .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ },
38205163 } },
38215164 }, .{
38225165 .required_features = .{ .sse4_1, null },
3823 .constraints = .{ .any_int, .any_int },
5166 .src_constraints = .{ .any_int, .any_int },
38245167 .patterns = &.{
38255168 .{ .src = .{ .mut_xmm, .mem } },
38265169 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
......@@ -3829,12 +5172,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
38295172 .clobbers = .{ .eflags = true },
38305173 .dst_temps = .{.{ .cc = cc }},
38315174 .each = .{ .once = &.{
3832 .{ .p_, .xor, .xsrc0, .xsrc1, .none, .none },
3833 .{ .p_, .@"test", .xsrc0, .xsrc0, .none, .none },
5175 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
5176 .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ },
38345177 } },
38355178 }, .{
38365179 .required_features = .{ .sse2, null },
3837 .constraints = .{ .any_int, .any_int },
5180 .src_constraints = .{ .any_int, .any_int },
38385181 .patterns = &.{
38395182 .{ .src = .{ .mut_xmm, .mem } },
38405183 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
......@@ -3845,18 +5188,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
38455188 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
38465189 .{ .kind = .{ .rc = .sse } },
38475190 .unused,
5191 .unused,
5192 .unused,
5193 .unused,
38485194 },
38495195 .dst_temps = .{.{ .cc = cc }},
38505196 .each = .{ .once = &.{
3851 .{ .p_, .xor, .xtmp1, .xtmp1, .none, .none },
3852 .{ .p_, .xor, .xsrc0, .xsrc1, .none, .none },
3853 .{ .p_b, .cmpeq, .xtmp1, .xsrc0, .none, .none },
3854 .{ .p_b, .movmsk, .etmp0, .xtmp1, .none, .none },
3855 .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u16) }, .none, .none },
5197 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
5198 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
5199 .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ },
5200 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
5201 .{ ._, ._, .xor, .tmp0d, .i(0xffff), ._, ._ },
38565202 } },
38575203 }, .{
38585204 .required_features = .{ .sse2, .mmx },
3859 .constraints = .{ .any_int, .any_int },
5205 .src_constraints = .{ .any_int, .any_int },
38605206 .patterns = &.{
38615207 .{ .src = .{ .mut_mm, .mem } },
38625208 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
......@@ -3867,17 +5213,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
38675213 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
38685214 .{ .kind = .{ .rc = .mmx } },
38695215 .unused,
5216 .unused,
5217 .unused,
5218 .unused,
38705219 },
38715220 .dst_temps = .{.{ .cc = cc }},
38725221 .each = .{ .once = &.{
3873 .{ .p_, .xor, .rtmp1, .rtmp1, .none, .none },
3874 .{ .p_, .xor, .rsrc0, .rsrc1, .none, .none },
3875 .{ .p_b, .cmpeq, .rtmp1, .rsrc0, .none, .none },
3876 .{ .p_b, .movmsk, .etmp0, .rtmp1, .none, .none },
3877 .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u8) }, .none, .none },
5222 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
5223 .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ },
5224 .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ },
5225 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
5226 .{ ._, ._, .xor, .tmp0d, .i(0xff), ._, ._ },
38785227 } },
38795228 }, .{
3880 .constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
5229 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
38815230 .patterns = &.{
38825231 .{ .src = .{ .mem, .imm8 } },
38835232 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
......@@ -3890,10 +5239,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
38905239 .clobbers = .{ .eflags = true },
38915240 .dst_temps = .{.{ .cc = cc }},
38925241 .each = .{ .once = &.{
3893 .{ ._, .cmp, .src0b, .src1b, .none, .none },
5242 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
38945243 } },
38955244 }, .{
3896 .constraints = .{ .{ .int = .word }, .{ .int = .word } },
5245 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
38975246 .patterns = &.{
38985247 .{ .src = .{ .mem, .imm16 } },
38995248 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
......@@ -3906,10 +5255,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
39065255 .clobbers = .{ .eflags = true },
39075256 .dst_temps = .{.{ .cc = cc }},
39085257 .each = .{ .once = &.{
3909 .{ ._, .cmp, .src0w, .src1w, .none, .none },
5258 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
39105259 } },
39115260 }, .{
3912 .constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
5261 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
39135262 .patterns = &.{
39145263 .{ .src = .{ .mem, .imm32 } },
39155264 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
......@@ -3922,11 +5271,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
39225271 .clobbers = .{ .eflags = true },
39235272 .dst_temps = .{.{ .cc = cc }},
39245273 .each = .{ .once = &.{
3925 .{ ._, .cmp, .esrc0, .esrc1, .none, .none },
5274 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
39265275 } },
39275276 }, .{
39285277 .required_features = .{ .@"64bit", null },
3929 .constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
5278 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
39305279 .patterns = &.{
39315280 .{ .src = .{ .mem, .simm32 } },
39325281 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
......@@ -3939,7 +5288,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
39395288 .clobbers = .{ .eflags = true },
39405289 .dst_temps = .{.{ .cc = cc }},
39415290 .each = .{ .once = &.{
3942 .{ ._, .cmp, .rsrc0, .rsrc1, .none, .none },
5291 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
39435292 } },
39445293 }, .{
39455294 .required_features = .{ .avx2, null },
......@@ -3950,20 +5299,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
39505299 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
39515300 .{ .kind = .{ .rc = .sse } },
39525301 .{ .kind = .{ .rc = .sse } },
5302 .unused,
5303 .unused,
5304 .unused,
39535305 },
39545306 .dst_temps = .{.{ .cc = cc }},
39555307 .each = .{ .limb = .{
3956 .of = .ysrc0,
5308 .of = .src0y,
39575309 .header = &.{
3958 .{ .vp_, .xor, .ytmp2, .ytmp2, .ytmp2, .none },
5310 .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ },
39595311 },
39605312 .body = &.{
3961 .{ .v_, .movdqu, .ytmp1, .{ .src_limb = 0 }, .none, .none },
3962 .{ .vp_, .xor, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none },
3963 .{ .vp_, .@"or", .ytmp2, .ytmp2, .ytmp1, .none },
5313 .{ ._, .v_dqu, .mov, .tmp1y, .limb(.src0y), ._, ._ },
5314 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .limb(.src1y), ._ },
5315 .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .tmp1y, ._ },
39645316 },
39655317 .trailer = &.{
3966 .{ .vp_, .@"test", .ytmp2, .ytmp2, .none, .none },
5318 .{ ._, .vp_, .@"test", .tmp2y, .tmp2y, ._, ._ },
39675319 },
39685320 } },
39695321 }, .{
......@@ -3975,20 +5327,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
39755327 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
39765328 .{ .kind = .{ .rc = .sse } },
39775329 .{ .kind = .{ .rc = .sse } },
5330 .unused,
5331 .unused,
5332 .unused,
39785333 },
39795334 .dst_temps = .{.{ .cc = cc }},
39805335 .each = .{ .limb = .{
3981 .of = .ysrc0,
5336 .of = .src0y,
39825337 .header = &.{
3983 .{ .v_pd, .xor, .ytmp2, .ytmp2, .ytmp2, .none },
5338 .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .tmp2y, ._ },
39845339 },
39855340 .body = &.{
3986 .{ .v_pd, .movu, .ytmp1, .{ .src_limb = 0 }, .none, .none },
3987 .{ .v_pd, .xor, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none },
3988 .{ .v_pd, .@"or", .ytmp2, .ytmp2, .ytmp1, .none },
5341 .{ ._, .v_pd, .movu, .tmp1y, .limb(.src0y), ._, ._ },
5342 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .limb(.src1y), ._ },
5343 .{ ._, .v_pd, .@"or", .tmp2y, .tmp2y, .tmp1y, ._ },
39895344 },
39905345 .trailer = &.{
3991 .{ .vp_, .@"test", .ytmp2, .ytmp2, .none, .none },
5346 .{ ._, .vp_, .@"test", .tmp2y, .tmp2y, ._, ._ },
39925347 },
39935348 } },
39945349 }, .{
......@@ -4000,20 +5355,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40005355 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
40015356 .{ .kind = .{ .rc = .sse } },
40025357 .{ .kind = .{ .rc = .sse } },
5358 .unused,
5359 .unused,
5360 .unused,
40035361 },
40045362 .dst_temps = .{.{ .cc = cc }},
40055363 .each = .{ .limb = .{
4006 .of = .xsrc0,
5364 .of = .src0x,
40075365 .header = &.{
4008 .{ .vp_, .xor, .xtmp2, .xtmp2, .xtmp2, .none },
5366 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
40095367 },
40105368 .body = &.{
4011 .{ .v_, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none },
4012 .{ .vp_, .xor, .xtmp1, .xtmp1, .{ .src_limb = 1 }, .none },
4013 .{ .vp_, .@"or", .xtmp2, .xtmp2, .xtmp1, .none },
5369 .{ ._, .v_dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ },
5370 .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .limb(.src1x), ._ },
5371 .{ ._, .vp_, .@"or", .tmp2x, .tmp2x, .tmp1x, ._ },
40145372 },
40155373 .trailer = &.{
4016 .{ .vp_, .@"test", .xtmp2, .xtmp2, .none, .none },
5374 .{ ._, .vp_, .@"test", .tmp2x, .tmp2x, ._, ._ },
40175375 },
40185376 } },
40195377 }, .{
......@@ -4025,20 +5383,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40255383 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
40265384 .{ .kind = .{ .rc = .sse } },
40275385 .{ .kind = .{ .rc = .sse } },
5386 .unused,
5387 .unused,
5388 .unused,
40285389 },
40295390 .dst_temps = .{.{ .cc = cc }},
40305391 .each = .{ .limb = .{
4031 .of = .xsrc0,
5392 .of = .src0x,
40325393 .header = &.{
4033 .{ .vp_, .xor, .xtmp2, .xtmp2, .xtmp2, .none },
5394 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
40345395 },
40355396 .body = &.{
4036 .{ .v_, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none },
4037 .{ .vp_, .xor, .xtmp1, .xtmp1, .{ .src_limb = 1 }, .none },
4038 .{ .vp_, .@"or", .xtmp2, .xtmp2, .xtmp1, .none },
5397 .{ ._, .v_dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ },
5398 .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .limb(.src1x), ._ },
5399 .{ ._, .vp_, .@"or", .tmp2x, .tmp2x, .tmp1x, ._ },
40395400 },
40405401 .trailer = &.{
4041 .{ .vp_, .@"test", .xtmp2, .xtmp2, .none, .none },
5402 .{ ._, .vp_, .@"test", .tmp2x, .tmp2x, ._, ._ },
40425403 },
40435404 } },
40445405 }, .{
......@@ -4050,20 +5411,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40505411 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
40515412 .{ .kind = .{ .rc = .sse } },
40525413 .{ .kind = .{ .rc = .sse } },
5414 .unused,
5415 .unused,
5416 .unused,
40535417 },
40545418 .dst_temps = .{.{ .cc = cc }},
40555419 .each = .{ .limb = .{
4056 .of = .xsrc0,
5420 .of = .src0x,
40575421 .header = &.{
4058 .{ .p_, .xor, .xtmp2, .xtmp2, .none, .none },
5422 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
40595423 },
40605424 .body = &.{
4061 .{ ._, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none },
4062 .{ .p_, .xor, .xtmp1, .{ .src_limb = 1 }, .none, .none },
4063 .{ .p_, .@"or", .xtmp2, .xtmp1, .none, .none },
5425 .{ ._, ._dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ },
5426 .{ ._, .p_, .xor, .tmp1x, .limb(.src1x), ._, ._ },
5427 .{ ._, .p_, .@"or", .tmp2x, .tmp1x, ._, ._ },
40645428 },
40655429 .trailer = &.{
4066 .{ .p_, .@"test", .xtmp2, .xtmp2, .none, .none },
5430 .{ ._, .p_, .@"test", .tmp2x, .tmp2x, ._, ._ },
40675431 },
40685432 } },
40695433 }, .{
......@@ -4075,23 +5439,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40755439 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
40765440 .{ .kind = .{ .rc = .sse } },
40775441 .{ .kind = .{ .rc = .sse } },
5442 .unused,
5443 .unused,
5444 .unused,
40785445 },
40795446 .dst_temps = .{.{ .cc = cc }},
40805447 .each = .{ .limb = .{
4081 .of = .xsrc0,
5448 .of = .src0x,
40825449 .header = &.{
4083 .{ .p_, .xor, .xtmp2, .xtmp2, .none, .none },
5450 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
40845451 },
40855452 .body = &.{
4086 .{ ._, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none },
4087 .{ .p_, .xor, .xtmp1, .{ .src_limb = 1 }, .none, .none },
4088 .{ .p_, .@"or", .xtmp2, .xtmp1, .none, .none },
5453 .{ ._, ._dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ },
5454 .{ ._, .p_, .xor, .tmp1x, .limb(.src1x), ._, ._ },
5455 .{ ._, .p_, .@"or", .tmp2x, .tmp1x, ._, ._ },
40895456 },
40905457 .trailer = &.{
4091 .{ .p_, .xor, .xtmp1, .xtmp1, .none, .none },
4092 .{ .p_b, .cmpeq, .xtmp2, .xtmp1, .none, .none },
4093 .{ .p_b, .movmsk, .etmp0, .xtmp2, .none, .none },
4094 .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u16) }, .none, .none },
5458 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
5459 .{ ._, .p_b, .cmpeq, .tmp2x, .tmp1x, ._, ._ },
5460 .{ ._, .p_b, .movmsk, .tmp0d, .tmp2x, ._, ._ },
5461 .{ ._, ._, .xor, .tmp0d, .i(0xffff), ._, ._ },
40955462 },
40965463 } },
40975464 }, .{
......@@ -4103,77 +5470,58 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
41035470 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
41045471 .{ .kind = .{ .rc = .mmx } },
41055472 .{ .kind = .{ .rc = .mmx } },
5473 .unused,
5474 .unused,
5475 .unused,
41065476 },
41075477 .dst_temps = .{.{ .cc = cc }},
41085478 .each = .{ .limb = .{
4109 .of = .rsrc0,
5479 .of = .src0q,
41105480 .header = &.{
4111 .{ .p_, .xor, .rtmp2, .rtmp2, .none, .none },
5481 .{ ._, .p_, .xor, .tmp2q, .tmp2q, ._, ._ },
41125482 },
41135483 .body = &.{
4114 .{ ._q, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none },
4115 .{ .p_, .xor, .rtmp1, .{ .src_limb = 1 }, .none, .none },
4116 .{ .p_, .@"or", .rtmp2, .rtmp1, .none, .none },
5484 .{ ._, ._q, .mov, .tmp1q, .limb(.src0q), ._, ._ },
5485 .{ ._, .p_, .xor, .tmp1q, .limb(.src1q), ._, ._ },
5486 .{ ._, .p_, .@"or", .tmp2q, .tmp1q, ._, ._ },
41175487 },
41185488 .trailer = &.{
4119 .{ .p_, .xor, .rtmp1, .rtmp1, .none, .none },
4120 .{ .p_b, .cmpeq, .rtmp2, .rtmp1, .none, .none },
4121 .{ .p_b, .movmsk, .etmp0, .rtmp2, .none, .none },
4122 .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u8) }, .none, .none },
5489 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
5490 .{ ._, .p_b, .cmpeq, .tmp2q, .tmp1q, ._, ._ },
5491 .{ ._, .p_b, .movmsk, .tmp0d, .tmp2q, ._, ._ },
5492 .{ ._, ._, .xor, .tmp0d, .i(0xff), ._, ._ },
41235493 },
41245494 } },
41255495 }, .{
4126 .required_features = .{ .@"64bit", null },
41275496 .patterns = &.{
41285497 .{ .src = .{ .to_mem, .to_mem } },
41295498 },
41305499 .extra_temps = .{
41315500 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4132 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4133 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4134 },
4135 .dst_temps = .{.{ .cc = cc }},
4136 .each = .{ .limb = .{
4137 .of = .rsrc0,
4138 .header = &.{
4139 .{ ._, .xor, .rtmp2, .rtmp2, .none, .none },
4140 },
4141 .body = &.{
4142 .{ ._, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none },
4143 .{ ._, .xor, .rtmp1, .{ .src_limb = 1 }, .none, .none },
4144 .{ ._, .@"or", .rtmp2, .rtmp1, .none, .none },
4145 },
4146 .trailer = &.{
4147 .{ ._, .@"test", .rtmp2, .rtmp2, .none, .none },
4148 },
4149 } },
4150 }, .{
4151 .patterns = &.{
4152 .{ .src = .{ .to_mem, .to_mem } },
4153 },
4154 .extra_temps = .{
41555501 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4156 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4157 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5502 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5503 .unused,
5504 .unused,
5505 .unused,
41585506 },
41595507 .dst_temps = .{.{ .cc = cc }},
41605508 .each = .{ .limb = .{
4161 .of = .esrc0,
5509 .of = .src0p,
41625510 .header = &.{
4163 .{ ._, .xor, .etmp2, .etmp2, .none, .none },
5511 .{ ._, ._, .xor, .tmp2p, .tmp2p, ._, ._ },
41645512 },
41655513 .body = &.{
4166 .{ ._, .mov, .etmp1, .{ .src_limb = 0 }, .none, .none },
4167 .{ ._, .xor, .etmp1, .{ .src_limb = 1 }, .none, .none },
4168 .{ ._, .@"or", .etmp2, .etmp1, .none, .none },
5514 .{ ._, ._, .mov, .tmp1p, .limb(.src0p), ._, ._ },
5515 .{ ._, ._, .xor, .tmp1p, .limb(.src1p), ._, ._ },
5516 .{ ._, ._, .@"or", .tmp2p, .tmp1p, ._, ._ },
41695517 },
41705518 .trailer = &.{
4171 .{ ._, .@"test", .etmp2, .etmp2, .none, .none },
5519 .{ ._, ._, .@"test", .tmp2p, .tmp2p, ._, ._ },
41725520 },
41735521 } },
41745522 } },
41755523 }) catch |err| switch (err) {
4176 error.Select2Failed => return cg.fail("failed to select2 {s} {} {} {}", .{
5524 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
41775525 @tagName(air_tag),
41785526 cg.typeOf(bin_op.lhs).fmt(pt),
41795527 ops[0].tracking(cg),
......@@ -16456,7 +17804,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
1645617804 } },
1645717805 } };
1645817806 } else {
16459 if (mnem_size) |size| if (reg.bitSize() != size.bitSize())
17807 if (mnem_size) |size| if (reg.bitSize() != size.bitSize(self.target))
1646017808 return self.fail("invalid register size: '{s}'", .{op_str});
1646117809 op.* = .{ .reg = reg };
1646217810 }
......@@ -16524,14 +17872,14 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
1652417872 } else if (std.mem.startsWith(u8, op_str, "$")) {
1652517873 if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| {
1652617874 if (mnem_size) |size| {
16527 const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1));
17875 const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize(self.target) - 1));
1652817876 if ((if (s < 0) ~s else s) > max)
1652917877 return self.fail("invalid immediate size: '{s}'", .{op_str});
1653017878 }
1653117879 op.* = .{ .imm = .s(s) };
1653217880 } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| {
1653317881 if (mnem_size) |size| {
16534 const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize());
17882 const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize(self.target));
1653517883 if (u > max)
1653617884 return self.fail("invalid immediate size: '{s}'", .{op_str});
1653717885 }
......@@ -16827,10 +18175,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1682718175 else
1682818176 .{ ._q, .mov } },
1682918177 9...16 => return .{ .move = if (self.hasFeature(.avx))
16830 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16831 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18178 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18179 else if (self.hasFeature(.sse2))
18180 .{ if (aligned) ._dqa else ._dqu, .mov }
18181 else
18182 .{ ._ps, if (aligned) .mova else .movu } },
1683218183 17...32 => if (self.hasFeature(.avx))
16833 return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
18184 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1683418185 else => {},
1683518186 } else switch (abi_size) {
1683618187 4 => return .{ .move = if (self.hasFeature(.avx))
......@@ -16842,12 +18193,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1684218193 else
1684318194 .{ ._sd, .mov } },
1684418195 9...16 => return .{ .move = if (self.hasFeature(.avx))
16845 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
16846 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
16847 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
16848 .{ .v_pd, .mova }
18196 .{ .v_pd, if (aligned) .mova else .movu }
18197 else if (self.hasFeature(.sse2))
18198 .{ ._pd, if (aligned) .mova else .movu }
1684918199 else
16850 .{ .v_pd, .movu } },
18200 .{ ._ps, if (aligned) .mova else .movu } },
18201 17...32 => if (self.hasFeature(.avx))
18202 return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } },
1685118203 else => {},
1685218204 }
1685318205 },
......@@ -16868,8 +18220,11 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1686818220 else
1686918221 .{ ._sd, .mov } },
1687018222 128 => return .{ .move = if (self.hasFeature(.avx))
16871 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16872 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18223 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18224 else if (self.hasFeature(.sse2))
18225 .{ if (aligned) ._dqa else ._dqu, .mov }
18226 else
18227 .{ ._ps, if (aligned) .mova else .movu } },
1687318228 else => {},
1687418229 },
1687518230 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
......@@ -16883,65 +18238,62 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1688318238 .int => switch (ty.childType(zcu).intInfo(zcu).bits) {
1688418239 1...8 => switch (ty.vectorLen(zcu)) {
1688518240 1...16 => return .{ .move = if (self.hasFeature(.avx))
16886 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16887 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18241 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18242 else if (self.hasFeature(.sse2))
18243 .{ if (aligned) ._dqa else ._dqu, .mov }
18244 else
18245 .{ ._ps, if (aligned) .mova else .movu } },
1688818246 17...32 => if (self.hasFeature(.avx))
16889 return .{ .move = if (aligned)
16890 .{ .v_, .movdqa }
16891 else
16892 .{ .v_, .movdqu } },
18247 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1689318248 else => {},
1689418249 },
1689518250 9...16 => switch (ty.vectorLen(zcu)) {
1689618251 1...8 => return .{ .move = if (self.hasFeature(.avx))
16897 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16898 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18252 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18253 else if (self.hasFeature(.sse2))
18254 .{ if (aligned) ._dqa else ._dqu, .mov }
18255 else
18256 .{ ._ps, if (aligned) .mova else .movu } },
1689918257 9...16 => if (self.hasFeature(.avx))
16900 return .{ .move = if (aligned)
16901 .{ .v_, .movdqa }
16902 else
16903 .{ .v_, .movdqu } },
18258 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1690418259 else => {},
1690518260 },
1690618261 17...32 => switch (ty.vectorLen(zcu)) {
1690718262 1...4 => return .{ .move = if (self.hasFeature(.avx))
16908 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16909 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18263 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18264 else if (self.hasFeature(.sse2))
18265 .{ if (aligned) ._dqa else ._dqu, .mov }
18266 else
18267 .{ ._ps, if (aligned) .mova else .movu } },
1691018268 5...8 => if (self.hasFeature(.avx))
16911 return .{ .move = if (aligned)
16912 .{ .v_, .movdqa }
16913 else
16914 .{ .v_, .movdqu } },
18269 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1691518270 else => {},
1691618271 },
1691718272 33...64 => switch (ty.vectorLen(zcu)) {
1691818273 1...2 => return .{ .move = if (self.hasFeature(.avx))
16919 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16920 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18274 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18275 else if (self.hasFeature(.sse2))
18276 .{ if (aligned) ._dqa else ._dqu, .mov }
18277 else
18278 .{ ._ps, if (aligned) .mova else .movu } },
1692118279 3...4 => if (self.hasFeature(.avx))
16922 return .{ .move = if (aligned)
16923 .{ .v_, .movdqa }
16924 else
16925 .{ .v_, .movdqu } },
18280 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1692618281 else => {},
1692718282 },
1692818283 65...128 => switch (ty.vectorLen(zcu)) {
1692918284 1 => return .{ .move = if (self.hasFeature(.avx))
16930 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16931 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18285 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18286 else if (self.hasFeature(.sse2))
18287 .{ if (aligned) ._dqa else ._dqu, .mov }
18288 else
18289 .{ ._ps, if (aligned) .mova else .movu } },
1693218290 2 => if (self.hasFeature(.avx))
16933 return .{ .move = if (aligned)
16934 .{ .v_, .movdqa }
16935 else
16936 .{ .v_, .movdqu } },
18291 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1693718292 else => {},
1693818293 },
1693918294 129...256 => switch (ty.vectorLen(zcu)) {
1694018295 1 => if (self.hasFeature(.avx))
16941 return .{ .move = if (aligned)
16942 .{ .v_, .movdqa }
16943 else
16944 .{ .v_, .movdqu } },
18296 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1694518297 else => {},
1694618298 },
1694718299 else => {},
......@@ -16949,13 +18301,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1694918301 .pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu))
1695018302 switch (ty.vectorLen(zcu)) {
1695118303 1...2 => return .{ .move = if (self.hasFeature(.avx))
16952 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16953 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18304 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18305 else if (self.hasFeature(.sse2))
18306 .{ if (aligned) ._dqa else ._dqu, .mov }
18307 else
18308 .{ ._ps, if (aligned) .mova else .movu } },
1695418309 3...4 => if (self.hasFeature(.avx))
16955 return .{ .move = if (aligned)
16956 .{ .v_, .movdqa }
16957 else
16958 .{ .v_, .movdqu } },
18310 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1695918311 else => {},
1696018312 }
1696118313 else
......@@ -16963,46 +18315,42 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1696318315 .float => switch (ty.childType(zcu).floatBits(self.target.*)) {
1696418316 16 => switch (ty.vectorLen(zcu)) {
1696518317 1...8 => return .{ .move = if (self.hasFeature(.avx))
16966 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
16967 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18318 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18319 else if (self.hasFeature(.sse2))
18320 .{ if (aligned) ._dqa else ._dqu, .mov }
18321 else
18322 .{ ._ps, if (aligned) .mova else .movu } },
1696818323 9...16 => if (self.hasFeature(.avx))
16969 return .{ .move = if (aligned)
16970 .{ .v_, .movdqa }
16971 else
16972 .{ .v_, .movdqu } },
18324 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1697318325 else => {},
1697418326 },
1697518327 32 => switch (ty.vectorLen(zcu)) {
1697618328 1...4 => return .{ .move = if (self.hasFeature(.avx))
16977 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
16978 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
18329 .{ .v_ps, if (aligned) .mova else .movu }
18330 else
18331 .{ ._ps, if (aligned) .mova else .movu } },
1697918332 5...8 => if (self.hasFeature(.avx))
16980 return .{ .move = if (aligned)
16981 .{ .v_ps, .mova }
16982 else
16983 .{ .v_ps, .movu } },
18333 return .{ .move = .{ .v_ps, if (aligned) .mova else .movu } },
1698418334 else => {},
1698518335 },
1698618336 64 => switch (ty.vectorLen(zcu)) {
1698718337 1...2 => return .{ .move = if (self.hasFeature(.avx))
16988 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
16989 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
18338 .{ .v_pd, if (aligned) .mova else .movu }
18339 else
18340 .{ ._pd, if (aligned) .mova else .movu } },
1699018341 3...4 => if (self.hasFeature(.avx))
16991 return .{ .move = if (aligned)
16992 .{ .v_pd, .mova }
16993 else
16994 .{ .v_pd, .movu } },
18342 return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } },
1699518343 else => {},
1699618344 },
1699718345 128 => switch (ty.vectorLen(zcu)) {
1699818346 1 => return .{ .move = if (self.hasFeature(.avx))
16999 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
17000 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
18347 .{ if (aligned) .v_dqa else .v_dqu, .mov }
18348 else if (self.hasFeature(.sse2))
18349 .{ if (aligned) ._dqa else ._dqu, .mov }
18350 else
18351 .{ ._ps, if (aligned) .mova else .movu } },
1700118352 2 => if (self.hasFeature(.avx))
17002 return .{ .move = if (aligned)
17003 .{ .v_, .movdqa }
17004 else
17005 .{ .v_, .movdqu } },
18353 return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } },
1700618354 else => {},
1700718355 },
1700818356 else => {},
......@@ -17211,7 +18559,7 @@ fn genSetReg(
1721118559 src_reg,
1721218560 ),
1721318561 .x87, .mmx, .ip => unreachable,
17214 .sse => try self.asmRegisterRegister(
18562 .sse => if (self.hasFeature(.sse2)) try self.asmRegisterRegister(
1721518563 switch (abi_size) {
1721618564 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
1721718565 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
......@@ -17219,7 +18567,20 @@ fn genSetReg(
1721918567 },
1722018568 registerAlias(dst_reg, @max(abi_size, 4)),
1722118569 src_reg.to128(),
17222 ),
18570 ) else {
18571 const frame_index = try self.allocFrameIndex(.init(.{
18572 .size = 4,
18573 .alignment = .@"4",
18574 }));
18575 try self.asmMemoryRegister(.{ ._ss, .mov }, .{
18576 .base = .{ .frame = frame_index },
18577 .mod = .{ .rm = .{ .size = .dword } },
18578 }, src_reg.to128());
18579 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
18580 .base = .{ .frame = frame_index },
18581 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
18582 });
18583 },
1722318584 },
1722418585 .segment => try self.asmRegisterRegister(
1722518586 .{ ._, .mov },
......@@ -17264,17 +18625,17 @@ fn genSetReg(
1726418625 .sse => try self.asmRegisterRegister(
1726518626 @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
1726618627 else => switch (abi_size) {
17267 1...16 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else .{ ._, .movdqa },
17268 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,
18628 1...16 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else .{ ._dqa, .mov },
18629 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null,
1726918630 else => null,
1727018631 },
1727118632 .float => switch (ty.scalarType(zcu).floatBits(self.target.*)) {
1727218633 16, 128 => switch (abi_size) {
1727318634 2...16 => if (self.hasFeature(.avx))
17274 .{ .v_, .movdqa }
18635 .{ .v_dqa, .mov }
1727518636 else
17276 .{ ._, .movdqa },
17277 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,
18637 .{ ._dqa, .mov },
18638 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null,
1727818639 else => null,
1727918640 },
1728018641 32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova },
......@@ -17346,7 +18707,7 @@ fn genSetReg(
1734618707 const pack_lock = self.register_manager.lockReg(pack_reg);
1734718708 defer if (pack_lock) |lock| self.register_manager.unlockReg(lock);
1734818709
17349 var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(), 8));
18710 var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(self.target), 8));
1735018711 switch (src_reg_mask.info.scalar) {
1735118712 else => {},
1735218713 .word => {
......@@ -17355,7 +18716,7 @@ fn genSetReg(
1735518716 if (has_avx) {
1735618717 try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias);
1735718718 } else {
17358 try self.asmRegisterRegister(.{ ._, .movdqa }, pack_alias, src_alias);
18719 try self.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias);
1735918720 try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias);
1736018721 }
1736118722 mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable;
......@@ -17592,7 +18953,10 @@ fn genSetMem(
1759218953 })).write(
1759318954 self,
1759418955 .{ .base = base, .mod = .{ .rm = .{
17595 .size = .fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())),
18956 .size = .fromBitSize(@min(
18957 self.memSize(ty).bitSize(self.target),
18958 src_alias.bitSize(),
18959 )),
1759618960 .disp = disp,
1759718961 } } },
1759818962 src_alias,
......@@ -22460,36 +23824,6 @@ const Temp = struct {
2246023824 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
2246123825 }
2246223826
22463 // i, m, r
22464 fn add(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp {
22465 const res_index = cg.next_temp_index;
22466 var res: Temp = .{ .index = res_index.toIndex() };
22467 try cg.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{
22468 .{ .ops = &.{ .{ .match = 1 }, .r, .i } },
22469 .{ .ops = &.{ .{ .match = 1 }, .m, .i } },
22470 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
22471 .{ .ops = &.{ .{ .match = 1 }, .m, .r } },
22472 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
22473 });
22474 cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
22475 cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg);
22476 return res;
22477 }
22478
22479 fn mul(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp {
22480 const res_index = cg.next_temp_index;
22481 var res: Temp = .{ .index = cg.next_temp_index.toIndex() };
22482 try cg.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{
22483 .{ .ops = &.{ .r, .m, .i } },
22484 .{ .ops = &.{ .r, .r, .i } },
22485 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
22486 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
22487 });
22488 cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
22489 cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg);
22490 return res;
22491 }
22492
2249323827 fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void {
2249423828 if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) {
2249523829 .ref => {
......@@ -22624,1242 +23958,129 @@ fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet)
2262423958 temp_index.tracking(cg).* = .init(
2262523959 .{ .register_pair = try cg.register_manager.allocRegs(2, temp_index.toIndex(), rs) },
2262623960 );
22627 cg.temp_type[@intFromEnum(temp_index)] = ty;
22628 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
22629 return .{ .index = temp_index.toIndex() };
22630}
22631
22632fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp {
22633 const temp_index = cg.next_temp_index;
22634 temp_index.tracking(cg).* = .init(
22635 try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false),
22636 );
22637 cg.temp_type[@intFromEnum(temp_index)] = ty;
22638 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
22639 return .{ .index = temp_index.toIndex() };
22640}
22641
22642fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp {
22643 const temp_index = cg.next_temp_index;
22644 temp_index.tracking(cg).* = .init(value);
22645 cg.temp_type[@intFromEnum(temp_index)] = ty;
22646 try cg.getValue(value, temp_index.toIndex());
22647 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
22648 return .{ .index = temp_index.toIndex() };
22649}
22650
22651fn tempFromOperand(
22652 cg: *CodeGen,
22653 inst: Air.Inst.Index,
22654 op_index: Liveness.OperandInt,
22655 op_ref: Air.Inst.Ref,
22656 ignore_death: bool,
22657) !Temp {
22658 const zcu = cg.pt.zcu;
22659 const ip = &zcu.intern_pool;
22660
22661 if (ignore_death or !cg.liveness.operandDies(inst, op_index)) {
22662 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
22663 const val = op_ref.toInterned().?;
22664 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
22665 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
22666 const const_mcv = try cg.genTypedValue(.fromInterned(val));
22667 switch (const_mcv) {
22668 .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
22669 .elf, .macho => {
22670 if (cg.mod.pic) {
22671 try cg.spillRegisters(&.{ .rdi, .rax });
22672 } else {
22673 try cg.spillRegisters(&.{.rax});
22674 }
22675 const frame_index = try cg.allocFrameIndex(.init(.{
22676 .size = 8,
22677 .alignment = .@"8",
22678 }));
22679 try cg.genSetMem(
22680 .{ .frame = frame_index },
22681 0,
22682 .usize,
22683 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
22684 .{},
22685 );
22686 break :init .{ .load_frame = .{ .index = frame_index } };
22687 },
22688 else => break :init const_mcv,
22689 },
22690 else => break :init const_mcv,
22691 }
22692 });
22693 return cg.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
22694 }
22695
22696 const temp_index = cg.next_temp_index;
22697 const temp: Temp = .{ .index = temp_index.toIndex() };
22698 const op_inst = op_ref.toIndex().?;
22699 const tracking = cg.getResolvedInstValue(op_inst);
22700 temp_index.tracking(cg).* = tracking.*;
22701 if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
22702 cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
22703 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
22704 return temp;
22705}
22706
22707inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
22708 var temps: [op_refs.len]Temp = undefined;
22709 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {
22710 temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| {
22711 if (op_ref == op_refs[prev_op_index]) break true;
22712 } else false);
22713 }
22714 return temps;
22715}
22716
22717const Operand = union(enum) {
22718 none,
22719 reg: Register,
22720 mem: Memory,
22721 imm: Immediate,
22722 inst: Mir.Inst.Index,
22723};
22724
22725const Pattern = struct {
22726 ops: []const Op,
22727 commute: struct { u8, u8 } = .{ 0, 0 },
22728
22729 const Set = struct {
22730 required_features: []const std.Target.x86.Feature = &.{},
22731 scalar: union(enum) {
22732 any,
22733 bool,
22734 float: Memory.Size,
22735 any_int: Memory.Size,
22736 signed_int: Memory.Size,
22737 unsigned_int: Memory.Size,
22738 any_float_or_int: Memory.Size,
22739 } = .any,
22740 clobbers: struct { eflags: bool = false } = .{},
22741 invert_result: bool = false,
22742 loop: enum {
22743 /// only execute the instruction once
22744 once,
22745 /// execute the instruction on all groups of non-overlapping bits in the entire value
22746 bitwise,
22747 /// for each element, execute the instruction on each limb, propagating the carry flag
22748 limbwise_carry,
22749 /// for each element, execute the instruction on each limb, propagating a register
22750 limbwise_reduce,
22751 /// for each element, execute the instruction on pairs of limbs, starting from the
22752 /// least significant, propagating a limb
22753 limbwise_pairs_forward,
22754 /// for each element, execute the instruction on pairs of limbs, starting from the
22755 /// most significant, propagating a limb
22756 limbwise_pairs_reverse,
22757 /// for each element, execute the instruction
22758 elementwise,
22759 } = .once,
22760 mir_tag: Mir.Inst.FixedTag,
22761 final_mir_tag: ?Mir.Inst.FixedTag = null,
22762 patterns: []const Pattern,
22763 };
22764
22765 const Op = union(enum) {
22766 /// reuse another operand
22767 implicit: u8,
22768 /// repeat another operand
22769 explicit: u8,
22770 /// a condition code
22771 cc,
22772 /// any general purpose register
22773 gpr,
22774 /// any 64-bit mmx register
22775 mm,
22776 /// any 128-bit sse register
22777 xmm,
22778 /// any 256-bit sse register
22779 ymm,
22780 /// a 64-bit mmx register mask
22781 mm_mask,
22782 /// a 128-bit sse register mask
22783 xmm_mask,
22784 /// a 256-bit sse register mask
22785 ymm_mask,
22786 /// a 64-bit mmx register sign mask
22787 mm_sign_mask,
22788 /// a 128-bit sse register sign mask
22789 xmm_sign_mask,
22790 /// a 256-bit sse register sign mask
22791 ymm_sign_mask,
22792 /// any memory
22793 mem,
22794 /// a limb stored in a general purpose register
22795 gpr_limb,
22796 /// a limb stored in a 64-bit mmx register
22797 mm_limb,
22798 /// a limb stored in a 128-bit sse register
22799 xmm_limb,
22800 /// a limb stored in a 256-bit sse register
22801 ymm_limb,
22802 /// a limb stored in memory
22803 mem_limb,
22804 /// a mutable limb stored in a general purpose register
22805 mut_gpr_limb,
22806 /// a mutable limb stored in memory
22807 mut_mem_limb,
22808 /// an element stored in a condition code
22809 cc_elem,
22810 /// an element stored in a general purpose register
22811 gpr_elem,
22812 /// an element stored in memory
22813 mem_elem,
22814 /// a limb stored in a 64-bit mmx register mask
22815 mm_mask_limb,
22816 /// a limb stored in a 128-bit sse register masuk
22817 xmm_mask_limb,
22818 /// a limb stored in a 256-bit sse register masuk
22819 ymm_mask_limb,
22820 /// specific immediate
22821 imm: i8,
22822 /// any immediate signed extended from 32 bits
22823 simm32,
22824 /// a temp general purpose register containing all ones
22825 umax_gpr,
22826 /// a temp 64-bit mmx register containing all ones
22827 umax_mm,
22828 /// a temp 128-bit sse register containing all ones
22829 umax_xmm,
22830 /// a temp 256-bit sse register containing all ones
22831 umax_ymm,
22832
22833 fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool {
22834 switch (op) {
22835 .implicit, .explicit, .cc, .cc_elem => unreachable,
22836 else => {},
22837 // temp is undefined
22838 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => return true,
22839 }
22840 const temp_ty = temp.typeOf(cg);
22841 const abi_size = temp_ty.abiSize(cg.pt.zcu);
22842 return switch (op) {
22843 .implicit, .explicit, .cc, .cc_elem, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable,
22844 .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) {
22845 .register => |reg| reg.class() == .general_purpose,
22846 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and
22847 reg_off.off == 0,
22848 else => cg.regClassForType(temp_ty) == .general_purpose,
22849 },
22850 .mm, .mm_mask, .mm_sign_mask => abi_size <= 8 and switch (temp.tracking(cg).short) {
22851 .register => |reg| reg.class() == .mmx,
22852 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
22853 else => cg.regClassForType(temp_ty) == .mmx,
22854 },
22855 .xmm, .xmm_mask, .xmm_sign_mask => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) {
22856 .register => |reg| reg.class() == .sse,
22857 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
22858 else => cg.regClassForType(temp_ty) == .sse,
22859 },
22860 .ymm, .ymm_mask, .ymm_sign_mask => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) {
22861 .register => |reg| reg.class() == .sse,
22862 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
22863 else => cg.regClassForType(temp_ty) == .sse,
22864 },
22865 .mem, .mem_limb, .mut_mem_limb, .mem_elem => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
22866 .gpr_limb, .mut_gpr_limb, .gpr_elem => abi_size > 8 and switch (temp.tracking(cg).short) {
22867 .register, .register_pair, .register_triple, .register_quadruple => true,
22868 else => |mcv| mcv.isMemory(),
22869 },
22870 .mm_limb, .mm_mask_limb => abi_size > 8 and switch (temp.tracking(cg).short) {
22871 inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| {
22872 if (reg.class() != .mmx) break false;
22873 } else true,
22874 else => |mcv| mcv.isMemory() and cg.regClassForType(temp_ty) == .mmx,
22875 },
22876 .xmm_limb, .xmm_mask_limb => abi_size > 16 and switch (temp.tracking(cg).short) {
22877 inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| {
22878 if (reg.class() != .sse) break false;
22879 } else true,
22880 else => |mcv| mcv.isMemory(),
22881 },
22882 .ymm_limb, .ymm_mask_limb => abi_size > 32 and switch (temp.tracking(cg).short) {
22883 inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| {
22884 if (reg.class() != .sse) break false;
22885 } else true,
22886 else => |mcv| mcv.isMemory(),
22887 },
22888 .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(cg).short) {
22889 .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm,
22890 else => false,
22891 },
22892 .simm32 => if (is_mut) unreachable else switch (temp.tracking(cg).short) {
22893 .immediate => |imm| abi_size <= 4 or std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
22894 else => false,
22895 },
22896 };
22897 }
22898 };
22899
22900 const Instruction = struct {
22901 mir_tag: Mir.Inst.FixedTag,
22902 operands: [4]Instruction.Operand,
22903
22904 const Operand = union(enum) {
22905 regb: u8,
22906 regw: u8,
22907 ereg: u8,
22908 rreg: u8,
22909 xmm: u8,
22910 ymm: u8,
22911 };
22912 };
22913};
22914const SelectOptions = struct {
22915 cc: ?Condition = null,
22916 invert_result: bool = false,
22917};
22918fn select(
22919 cg: *CodeGen,
22920 dst_temps: []Temp,
22921 dst_tys: []const Type,
22922 src_temps: []Temp,
22923 pattern_sets: []const Pattern.Set,
22924 opts: SelectOptions,
22925) !void {
22926 var loop: struct {
22927 element_reloc: Mir.Inst.Index,
22928 element_offset: Offset,
22929 element_size: ?u13,
22930 limb_reloc: Mir.Inst.Index,
22931 limb_offset: Offset,
22932 limb_size: ?u8,
22933 shuffle_temp: ?Temp,
22934 mask_limb_temp: ?Temp,
22935 mask_limb_offset: Offset,
22936 mask_limb_offset_lock: ?RegisterLock,
22937 mask_limb_bit_size: ?u7,
22938 mask_store_temp: ?Temp,
22939 mask_store_reg: ?Register,
22940 mask_store_bit_size: ?u7,
22941 remaining_size: ?u64,
22942
22943 const Offset = union(enum) {
22944 unused,
22945 known: u31,
22946 temp: Temp,
22947 };
22948 } = .{
22949 .element_reloc = undefined,
22950 .element_offset = .unused,
22951 .element_size = null,
22952 .limb_reloc = undefined,
22953 .limb_offset = .unused,
22954 .limb_size = null,
22955 .shuffle_temp = null,
22956 .mask_limb_temp = null,
22957 .mask_limb_offset = .unused,
22958 .mask_limb_offset_lock = null,
22959 .mask_limb_bit_size = null,
22960 .mask_store_temp = null,
22961 .mask_store_reg = null,
22962 .mask_store_bit_size = null,
22963 .remaining_size = null,
22964 };
22965 var extra_temps: [4]?Temp = @splat(null);
22966 pattern_sets: for (pattern_sets) |pattern_set| {
22967 for (pattern_set.required_features) |required_feature| if (!cg.hasFeature(required_feature)) continue :pattern_sets;
22968 for (src_temps) |src_temp| switch (pattern_set.scalar) {
22969 .any => {},
22970 .bool => if (src_temp.typeOf(cg).scalarType(cg.pt.zcu).toIntern() != .bool_type) continue :pattern_sets,
22971 .float => |size| {
22972 const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu);
22973 if (!scalar_ty.isRuntimeFloat()) continue :pattern_sets;
22974 if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets;
22975 },
22976 .any_int => |size| {
22977 const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu);
22978 if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets;
22979 if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets;
22980 },
22981 .signed_int => |size| {
22982 const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu);
22983 if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets;
22984 const scalar_info = scalar_ty.intInfo(cg.pt.zcu);
22985 if (scalar_info.signedness != .signed) continue :pattern_sets;
22986 if (scalar_info.bits > size.bitSize()) continue :pattern_sets;
22987 },
22988 .unsigned_int => |size| {
22989 const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu);
22990 if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets;
22991 const scalar_info = scalar_ty.intInfo(cg.pt.zcu);
22992 if (scalar_info.signedness != .unsigned) continue :pattern_sets;
22993 if (scalar_info.bits > size.bitSize()) continue :pattern_sets;
22994 },
22995 .any_float_or_int => |size| {
22996 const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu);
22997 if (scalar_ty.isRuntimeFloat()) {
22998 if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets;
22999 } else if (scalar_ty.isAbiInt(cg.pt.zcu)) {
23000 if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets;
23001 } else continue :pattern_sets;
23002 },
23003 };
23004 patterns: for (pattern_set.patterns) |pattern| {
23005 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
23006 const ref_src_op, const is_mut = switch (src_op) {
23007 .implicit, .explicit => |linked_index| .{ pattern.ops[linked_index], true },
23008 .mut_mem_limb, .mut_gpr_limb => .{ src_op, true },
23009 else => .{ src_op, false },
23010 };
23011 if (!ref_src_op.matches(is_mut, src_temp, cg)) continue :patterns;
23012 }
23013
23014 for (pattern.ops) |op| switch (op) {
23015 else => {},
23016 .cc_elem,
23017 .mm_mask_limb,
23018 .xmm_mask_limb,
23019 .ymm_mask_limb,
23020 => if (loop.mask_limb_offset_lock == null and !cg.hasFeature(.bmi2)) {
23021 try cg.register_manager.getKnownReg(.rcx, null);
23022 loop.mask_limb_offset_lock = cg.register_manager.lockKnownRegAssumeUnused(.rcx);
23023 },
23024 };
23025 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |*src_temp, src_op| {
23026 if (switch (switch (src_op) {
23027 .implicit, .explicit => |linked_index| pattern.ops[linked_index],
23028 else => src_op,
23029 }) {
23030 .implicit, .explicit, .cc, .cc_elem => unreachable,
23031 .gpr => try src_temp.toRegClass(true, .general_purpose, cg),
23032 .mm, .mm_mask, .mm_sign_mask => try src_temp.toRegClass(true, .mmx, cg),
23033 .xmm,
23034 .ymm,
23035 .xmm_mask,
23036 .ymm_mask,
23037 .xmm_sign_mask,
23038 .ymm_sign_mask,
23039 => try src_temp.toRegClass(true, .sse, cg),
23040 .mem => try src_temp.toBase(cg),
23041 .imm, .simm32 => false,
23042 .gpr_limb,
23043 .mm_limb,
23044 .xmm_limb,
23045 .ymm_limb,
23046 .mem_limb,
23047 .mut_gpr_limb,
23048 .mut_mem_limb,
23049 .gpr_elem,
23050 .mem_elem,
23051 => switch (src_temp.tracking(cg).short) {
23052 .register, .register_pair, .register_triple, .register_quadruple => false,
23053 else => try src_temp.toBase(cg),
23054 },
23055 .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => false,
23056 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => false,
23057 }) break;
23058 } else break;
23059
23060 const invert_result = opts.invert_result != pattern_set.invert_result;
23061 var dst_is_linked: std.StaticBitSet(4) = .initEmpty();
23062 var mir_ops_len: usize = 0;
23063 for (pattern.ops[0..dst_temps.len]) |dst_op| switch (dst_op) {
23064 else => mir_ops_len += 1,
23065 .cc, .cc_elem => {},
23066 };
23067 const dst_mir_ops_len = mir_ops_len;
23068 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
23069 defer mir_ops_len += @intFromBool(src_op != .implicit);
23070 const linked_src_op, const extra_temp = op: switch (src_op) {
23071 .implicit, .explicit => |linked_index| {
23072 if (src_temp.isMut(cg)) {
23073 dst_temps[linked_index] = src_temp;
23074 dst_is_linked.set(linked_index);
23075 }
23076 break :op .{ pattern.ops[linked_index], &extra_temps[linked_index] };
23077 },
23078 else => .{ src_op, &extra_temps[mir_ops_len] },
23079 };
23080 const limb_size: u8, const rc = switch (linked_src_op) {
23081 else => continue,
23082 .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ @intCast(@divExact(Memory.Size.bitSize(switch (pattern_set.scalar) {
23083 .any => .qword,
23084 .bool => unreachable,
23085 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
23086 }), 8)), abi.RegisterClass.gp },
23087 .mm_limb, .mm_mask_limb => .{ 8, @panic("TODO") },
23088 .xmm_limb, .xmm_mask_limb => .{ 16, abi.RegisterClass.sse },
23089 .ymm_limb, .ymm_mask_limb => .{ 32, abi.RegisterClass.sse },
23090 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => {
23091 assert(extra_temp.* == null);
23092 extra_temp.* = try cg.tempAllocReg(.noreturn, switch (linked_src_op) {
23093 else => unreachable,
23094 .umax_gpr => abi.RegisterClass.gp,
23095 .umax_mm => @panic("TODO"),
23096 .umax_xmm, .umax_ymm => abi.RegisterClass.sse,
23097 });
23098 continue;
23099 },
23100 };
23101 assert(loop.limb_size == null or loop.limb_size == limb_size);
23102 loop.limb_size = limb_size;
23103 loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu);
23104 const src_mcv = src_temp.tracking(cg).short;
23105 switch (src_mcv) {
23106 .register, .register_pair, .register_triple, .register_quadruple => {
23107 switch (loop.limb_offset) {
23108 .unused, .temp => loop.limb_offset = .{ .known = 0 },
23109 .known => {},
23110 }
23111 if (switch (linked_src_op) {
23112 .mut_gpr_limb => true,
23113 else => !rc.isSet(RegisterManager.indexOfRegIntoTracked(src_mcv.getRegs()[0]).?),
23114 }) {
23115 if (loop.shuffle_temp == null) loop.shuffle_temp = try cg.tempAllocReg(.noreturn, abi.RegisterClass.sse);
23116 assert(extra_temp.* == null);
23117 extra_temp.* = try cg.tempAllocReg(.usize, rc);
23118 }
23119 },
23120 else => {
23121 switch (loop.limb_offset) {
23122 .unused => loop.limb_offset = .{ .temp = undefined },
23123 .known, .temp => {},
23124 }
23125 assert(extra_temp.* == null);
23126 extra_temp.* = try cg.tempAllocReg(.usize, rc);
23127 },
23128 }
23129 }
23130 for (
23131 0..,
23132 dst_temps,
23133 pattern.ops[0..dst_temps.len],
23134 dst_tys,
23135 extra_temps[0..dst_temps.len],
23136 ) |dst_index, *dst_temp, dst_op, dst_ty, *extra_temp| switch (dst_op) {
23137 else => if (!dst_is_linked.isSet(dst_index)) {
23138 dst_temp.* = dst_temp: switch (dst_op) {
23139 .implicit => unreachable,
23140 .explicit => |linked_index| dst_temps[linked_index],
23141 .cc => try cg.tempFromValue(.bool, .{ .eflags = switch (invert_result) {
23142 false => opts.cc.?,
23143 true => opts.cc.?.negate(),
23144 } }),
23145 .gpr => try cg.tempAllocReg(dst_ty, abi.RegisterClass.gp),
23146 .mm, .mm_mask, .mm_sign_mask => @panic("TODO"),
23147 .xmm, .xmm_mask, .xmm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse),
23148 .ymm, .ymm_mask, .ymm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse),
23149 .mem => @panic("TODO"),
23150 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mut_gpr_limb, .gpr_elem => {
23151 if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(.noreturn, switch (dst_op) {
23152 else => unreachable,
23153 .gpr_limb, .mut_gpr_limb, .gpr_elem => abi.RegisterClass.gp,
23154 .mm_limb => @panic("TODO"),
23155 .xmm_limb, .ymm_limb => abi.RegisterClass.sse,
23156 });
23157 break :dst_temp try cg.tempAlloc(dst_ty);
23158 },
23159 .mem_limb, .mut_mem_limb, .mem_elem => try cg.tempAlloc(dst_ty),
23160 .cc_elem, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => unreachable, // already checked
23161 .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination
23162 };
23163 },
23164 .cc_elem, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
23165 const scalar_size = @divExact(Memory.Size.bitSize(switch (pattern_set.scalar) {
23166 .any => .qword,
23167 .bool => unreachable,
23168 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
23169 }), 8);
23170 const mask_bit_size = @divExact(loop.remaining_size.?, scalar_size);
23171 const mask_limb_bit_size: u7 = @intCast(@divExact(loop.limb_size.?, scalar_size));
23172 assert(loop.mask_limb_bit_size == null or loop.mask_limb_bit_size == mask_limb_bit_size);
23173 loop.mask_limb_bit_size = mask_limb_bit_size;
23174 const mask_store_bit_size = mask_store_bit_size: {
23175 // Try to match limb size so that no shifting will be needed.
23176 if (mask_limb_bit_size % 8 == 0) break :mask_store_bit_size mask_limb_bit_size;
23177 // If abi size <= 8 the entire value can be stored at once,
23178 // enabling store forwarding and minimizing store buffer usage.
23179 // Otherwise, we will be performing shifts that need to wrap at
23180 // store size, which for x86 requires 32 or 64, so just pick 64
23181 // for the same reasons as above.
23182 break :mask_store_bit_size @min(mask_bit_size, 64);
23183 };
23184 assert(loop.mask_store_bit_size == null or loop.mask_store_bit_size == mask_store_bit_size);
23185 loop.mask_store_bit_size = mask_store_bit_size;
23186 loop.mask_limb_offset = loop.limb_offset;
23187 if (loop.mask_limb_temp == null) {
23188 loop.mask_limb_temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
23189 if (dst_op == .cc_elem and mask_store_bit_size > 8) {
23190 // setcc only clears 8 bits
23191 const mask_limb_alias = loop.mask_limb_temp.?.tracking(cg).short.register.to32();
23192 try cg.spillEflagsIfOccupied();
23193 try cg.asmRegisterRegister(.{ ._, .xor }, mask_limb_alias, mask_limb_alias);
23194 }
23195 }
23196 if (mask_limb_bit_size < mask_store_bit_size and loop.mask_store_reg == null) {
23197 loop.mask_store_temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
23198 loop.mask_store_reg = loop.mask_store_temp.?.tracking(cg).short.register;
23199 }
23200 dst_temp.* = if (mask_store_bit_size < mask_bit_size)
23201 try cg.tempAllocMem(dst_ty)
23202 else if (loop.mask_store_temp) |mask_store_temp| dst_temp: {
23203 loop.mask_store_temp = null;
23204 break :dst_temp mask_store_temp;
23205 } else try cg.tempAlloc(dst_ty);
23206 },
23207 };
23208 switch (loop.mask_limb_offset) {
23209 .unused, .known => {},
23210 .temp => |*mask_limb_offset| {
23211 mask_limb_offset.* = if (cg.hasFeature(.bmi2))
23212 try cg.tempAllocReg(.usize, abi.RegisterClass.gp)
23213 else if (loop.mask_limb_offset_lock != null)
23214 try cg.tempFromValue(.usize, .{ .register = .rcx })
23215 else
23216 unreachable;
23217 if (loop.mask_store_reg) |mask_store_reg| {
23218 const mask_store_alias =
23219 if (loop.mask_store_bit_size.? > 8) mask_store_reg.to32() else mask_store_reg.to8();
23220 try cg.spillEflagsIfOccupied();
23221 try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias);
23222 }
23223 },
23224 }
23225 if (loop.mask_limb_offset_lock) |lock| cg.register_manager.unlockReg(lock);
23226 loop.mask_limb_offset_lock = null;
23227 switch (loop.element_offset) {
23228 .unused, .known => {},
23229 .temp => |*element_offset| {
23230 element_offset.* = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
23231 const element_offset_reg = element_offset.tracking(cg).short.register;
23232 try cg.spillEflagsIfOccupied();
23233 try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32());
23234 loop.element_reloc = @intCast(cg.mir_instructions.len);
23235 },
23236 }
23237 switch (loop.limb_offset) {
23238 .unused, .known => {},
23239 .temp => |*limb_offset| limb_offset.* = try cg.tempAllocReg(.usize, abi.RegisterClass.gp),
23240 }
23241 while (true) {
23242 switch (loop.mask_limb_offset) {
23243 .unused, .known => {},
23244 .temp => |mask_limb_offset| {
23245 const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32();
23246 try cg.spillEflagsIfOccupied();
23247 try cg.asmRegisterRegister(.{ ._, .xor }, mask_limb_offset_reg, mask_limb_offset_reg);
23248 },
23249 }
23250 switch (loop.limb_offset) {
23251 .unused, .known => {},
23252 .temp => |limb_offset| {
23253 const limb_offset_reg = limb_offset.tracking(cg).short.register.to32();
23254 try cg.spillEflagsIfOccupied();
23255 try cg.asmRegisterRegister(.{ ._, .xor }, limb_offset_reg.to32(), limb_offset_reg.to32());
23256 loop.limb_reloc = @intCast(cg.mir_instructions.len);
23257 },
23258 }
23259 while (true) {
23260 var mir_ops: [4]Operand = @splat(.none);
23261 mir_ops_len = dst_mir_ops_len;
23262 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
23263 defer mir_ops_len += @intFromBool(src_op != .implicit);
23264 const mir_op, const linked_src_op, const extra_temp = switch (src_op) {
23265 .implicit => |linked_index| .{ &mir_ops[linked_index], pattern.ops[linked_index], extra_temps[linked_index] },
23266 .explicit => |linked_index| .{ &mir_ops[mir_ops_len], pattern.ops[linked_index], extra_temps[linked_index] },
23267 else => .{ &mir_ops[mir_ops_len], src_op, extra_temps[mir_ops_len] },
23268 };
23269 const src_mcv = switch (linked_src_op) {
23270 else => src_temp,
23271 // src_temp is undefined
23272 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => extra_temp.?,
23273 }.tracking(cg).short;
23274 switch (linked_src_op) {
23275 else => {},
23276 .gpr_limb,
23277 .mm_limb,
23278 .xmm_limb,
23279 .ymm_limb,
23280 .mut_gpr_limb,
23281 .gpr_elem,
23282 .mm_mask_limb,
23283 .xmm_mask_limb,
23284 .ymm_mask_limb,
23285 => if (extra_temp) |limb_temp| switch (src_mcv) {
23286 .register, .register_pair, .register_triple, .register_quadruple => {
23287 const limb_reg = registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?);
23288 const src_regs = src_mcv.getRegs();
23289 const src_reg_size: u32 = @intCast(switch (src_mcv) {
23290 .register => src_temp.typeOf(cg).abiSize(cg.pt.zcu),
23291 else => @divExact(src_regs[0].bitSize(), 8),
23292 });
23293 const src_reg = src_regs[loop.limb_offset.known / src_reg_size];
23294 assert(src_mcv == .register or src_reg.bitSize() == 8 * src_reg_size);
23295 switch (src_reg.class()) {
23296 else => unreachable,
23297 .general_purpose => try cg.asmRegisterRegister(
23298 .{ ._, .mov },
23299 limb_reg,
23300 registerAlias(src_reg, src_reg_size),
23301 ),
23302 .sse => {
23303 assert(src_reg_size == 16);
23304 const limb_alias_size = @max(loop.limb_size.?, 4);
23305 const limb_alias = registerAlias(limb_reg, limb_alias_size);
23306 const src_reg_offset = loop.limb_offset.known % src_reg_size;
23307 switch (limb_reg_offset: {
23308 extr: {
23309 const limb_size = if (cg.hasFeature(.sse4_1)) loop.limb_size.? else 2;
23310 if (loop.limb_size.? > limb_size) break :extr;
23311 const limb_offset = src_reg_offset / limb_size;
23312 if (limb_offset == 0) break :extr;
23313 try cg.asmRegisterRegisterImmediate(.{ switch (limb_size) {
23314 else => unreachable,
23315 1 => if (cg.hasFeature(.avx)) .vp_b else .p_b,
23316 2 => if (cg.hasFeature(.avx)) .vp_w else .p_w,
23317 4 => if (cg.hasFeature(.avx)) .vp_d else .p_d,
23318 8 => if (cg.hasFeature(.avx)) .vp_q else .p_q,
23319 }, .extr }, limb_alias, src_reg.to128(), .u(limb_offset));
23320 break :limb_reg_offset src_reg_offset % limb_size;
23321 }
23322 try cg.asmRegisterRegister(
23323 .{ switch (limb_alias_size) {
23324 else => unreachable,
23325 4 => ._d,
23326 8 => ._q,
23327 }, .mov },
23328 limb_alias,
23329 if (src_reg_offset < limb_alias_size) src_reg.to128() else shuffle_reg: {
23330 const shuffle_reg = loop.shuffle_temp.?.tracking(cg).short.register.to128();
23331 const mir_fixes: Mir.Inst.Fixes = if (cg.hasFeature(.sse2))
23332 if (src_temp.typeOf(cg).scalarType(cg.pt.zcu).isRuntimeFloat()) switch (limb_alias_size) {
23333 else => unreachable,
23334 4 => if (cg.hasFeature(.avx)) .v_ps else ._ps,
23335 8 => if (cg.hasFeature(.avx)) .v_pd else ._pd,
23336 } else if (cg.hasFeature(.avx)) .vp_d else .p_d
23337 else
23338 ._ps;
23339 try cg.asmRegisterRegisterImmediate(
23340 .{ mir_fixes, .shuf },
23341 shuffle_reg,
23342 src_reg: switch (mir_fixes) {
23343 else => unreachable,
23344 ._ps, ._pd => {
23345 try cg.asmRegisterRegister(.{ mir_fixes, .mova }, shuffle_reg, src_reg.to128());
23346 break :src_reg shuffle_reg;
23347 },
23348 .p_d => src_reg.to128(),
23349 },
23350 .u(switch (mir_fixes) {
23351 else => unreachable,
23352 .v_ps, ._ps, .vp_d, .p_d => switch (limb_alias_size) {
23353 else => unreachable,
23354 4 => switch (src_reg_offset) {
23355 else => unreachable,
23356 4...7 => 0b01_01_01_01,
23357 8...11 => 0b10_10_10_10,
23358 12...15 => 0b11_11_11_11,
23359 },
23360 8 => switch (src_reg_offset) {
23361 else => unreachable,
23362 8...15 => 0b11_10_11_10,
23363 },
23364 },
23365 .v_pd, ._pd => switch (limb_alias_size) {
23366 else => unreachable,
23367 8 => switch (src_reg_offset) {
23368 else => unreachable,
23369 8...15 => 0b1_1,
23370 },
23371 },
23372 }),
23373 );
23374 break :shuffle_reg shuffle_reg;
23375 },
23376 );
23377 break :limb_reg_offset src_reg_offset % limb_alias_size;
23378 }) {
23379 0 => {},
23380 else => |limb_reg_offset| {
23381 try cg.spillEflagsIfOccupied();
23382 try cg.asmRegisterImmediate(.{ ._r, .sh }, limb_alias, .u(limb_reg_offset * 8));
23383 },
23384 }
23385 },
23386 }
23387 },
23388 else => try cg.asmRegisterMemory(
23389 switch (linked_src_op) {
23390 else => unreachable,
23391 .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ ._, .mov },
23392 .mm_limb, .mm_mask_limb => .{ ._q, .mov },
23393 .xmm_limb,
23394 .ymm_limb,
23395 .xmm_mask_limb,
23396 .ymm_mask_limb,
23397 => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
23398 },
23399 registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?),
23400 try src_mcv.mem(cg, switch (loop.limb_offset) {
23401 .unused => unreachable,
23402 .known => |limb_offset| .{
23403 .size = .fromSize(loop.limb_size.?),
23404 .disp = limb_offset,
23405 },
23406 .temp => |limb_offset| .{
23407 .size = .fromSize(loop.limb_size.?),
23408 .index = limb_offset.tracking(cg).short.register.to64(),
23409 },
23410 }),
23411 ),
23412 },
23413 }
23414 mir_op.* = switch (linked_src_op) {
23415 .implicit, .explicit, .cc, .cc_elem => unreachable,
23416 .gpr => .{ .reg = registerAlias(
23417 src_mcv.register,
23418 @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)),
23419 ) },
23420 .umax_gpr => .{ .reg = src_mcv.register.to64() }, // TODO: use other op size?
23421 .mm, .mm_mask, .mm_sign_mask, .umax_mm => .{ .reg = src_mcv.register },
23422 .xmm, .xmm_mask, .xmm_sign_mask, .umax_xmm => .{ .reg = src_mcv.register.to128() },
23423 .ymm, .ymm_mask, .ymm_sign_mask, .umax_ymm => .{ .reg = src_mcv.register.to256() },
23424 .mem => .{ .mem = try src_mcv.mem(cg, .{ .size = cg.memSize(src_temp.typeOf(cg)) }) },
23425 .gpr_limb,
23426 .mm_limb,
23427 .xmm_limb,
23428 .ymm_limb,
23429 .mut_gpr_limb,
23430 .gpr_elem,
23431 .mm_mask_limb,
23432 .xmm_mask_limb,
23433 .ymm_mask_limb,
23434 => .{ .reg = registerAlias(if (extra_temp) |limb_temp|
23435 limb_temp.tracking(cg).short.register
23436 else
23437 src_mcv.getRegs()[@divExact(loop.limb_offset.known, loop.limb_size.?)], loop.limb_size.?) },
23438 .mem_limb, .mut_mem_limb, .mem_elem => .{ .mem = switch (src_mcv) {
23439 .register, .register_pair, .register_triple, .register_quadruple => unreachable,
23440 else => switch (loop.limb_offset) {
23441 .unused => unreachable,
23442 .known => |limb_offset| try src_mcv.mem(cg, .{
23443 .size = .fromSize(loop.limb_size.?),
23444 .disp = limb_offset,
23445 }),
23446 .temp => |limb_offset| try src_mcv.mem(cg, .{
23447 .size = .fromSize(loop.limb_size.?),
23448 .index = limb_offset.tracking(cg).short.register.to64(),
23449 }),
23450 },
23451 } },
23452 .imm => |imm| .{ .imm = .s(imm) },
23453 .simm32 => switch (src_temp.typeOf(cg).abiSize(cg.pt.zcu)) {
23454 else => unreachable,
23455 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small|
23456 .s(small)
23457 else
23458 .u(@as(u8, @intCast(src_mcv.immediate))) },
23459 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small|
23460 .s(small)
23461 else
23462 .u(@as(u16, @intCast(src_mcv.immediate))) },
23463 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small|
23464 .s(small)
23465 else
23466 .u(@as(u32, @intCast(src_mcv.immediate))) },
23467 },
23468 };
23469 switch (src_op) {
23470 else => {},
23471 .explicit => |linked_index| mir_ops[linked_index] = mir_op.*,
23961 cg.temp_type[@intFromEnum(temp_index)] = ty;
23962 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
23963 return .{ .index = temp_index.toIndex() };
23964}
23965
23966fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp {
23967 const temp_index = cg.next_temp_index;
23968 temp_index.tracking(cg).* = .init(
23969 try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false),
23970 );
23971 cg.temp_type[@intFromEnum(temp_index)] = ty;
23972 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
23973 return .{ .index = temp_index.toIndex() };
23974}
23975
23976fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp {
23977 const temp_index = cg.next_temp_index;
23978 temp_index.tracking(cg).* = .init(value);
23979 cg.temp_type[@intFromEnum(temp_index)] = ty;
23980 try cg.getValue(value, temp_index.toIndex());
23981 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
23982 return .{ .index = temp_index.toIndex() };
23983}
23984
23985fn tempFromOperand(
23986 cg: *CodeGen,
23987 inst: Air.Inst.Index,
23988 op_index: Liveness.OperandInt,
23989 op_ref: Air.Inst.Ref,
23990 ignore_death: bool,
23991) !Temp {
23992 const zcu = cg.pt.zcu;
23993 const ip = &zcu.intern_pool;
23994
23995 if (ignore_death or !cg.liveness.operandDies(inst, op_index)) {
23996 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
23997 const val = op_ref.toInterned().?;
23998 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
23999 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
24000 const const_mcv = try cg.genTypedValue(.fromInterned(val));
24001 switch (const_mcv) {
24002 .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
24003 .elf, .macho => {
24004 if (cg.mod.pic) {
24005 try cg.spillRegisters(&.{ .rdi, .rax });
24006 } else {
24007 try cg.spillRegisters(&.{.rax});
2347224008 }
23473 }
23474 for (
23475 mir_ops[0..dst_mir_ops_len],
23476 pattern.ops[0..dst_mir_ops_len],
23477 dst_temps[0..dst_mir_ops_len],
23478 dst_tys[0..dst_mir_ops_len],
23479 extra_temps[0..dst_mir_ops_len],
23480 ) |*mir_op, dst_op, dst_temp, dst_ty, extra_temp| {
23481 if (mir_op.* != .none) continue;
23482 mir_op.* = switch (dst_op) {
23483 .implicit, .cc, .cc_elem => unreachable,
23484 .explicit => |linked_index| mir_ops[linked_index],
23485 .gpr => .{ .reg = registerAlias(
23486 dst_temp.tracking(cg).short.register,
23487 @intCast(dst_ty.abiSize(cg.pt.zcu)),
23488 ) },
23489 .mm, .mm_mask, .mm_sign_mask => @panic("TODO"),
23490 .xmm, .xmm_mask, .xmm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to128() },
23491 .ymm, .ymm_mask, .ymm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to256() },
23492 .mem => @panic("TODO"),
23493 .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ .reg = registerAlias(
23494 extra_temp.?.tracking(cg).short.register,
23495 @intCast(@divExact(Memory.Size.bitSize(switch (pattern_set.scalar) {
23496 .any => .qword,
23497 .bool => unreachable,
23498 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
23499 }), 8)),
23500 ) },
23501 .mm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register },
23502 .xmm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() },
23503 .ymm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() },
23504 .mem_limb, .mut_mem_limb, .mem_elem => .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) {
23505 .unused => unreachable,
23506 .known => |limb_offset| .{
23507 .size = .fromSize(loop.limb_size.?),
23508 .disp = limb_offset,
23509 },
23510 .temp => |limb_offset| .{
23511 .size = .fromSize(loop.limb_size.?),
23512 .index = limb_offset.tracking(cg).short.register.to64(),
23513 },
23514 }) },
23515 .mm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register },
23516 .xmm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() },
23517 .ymm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() },
23518 .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination
23519 };
23520 }
23521 std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]);
23522 if (pattern_set.clobbers.eflags) try cg.spillEflagsIfOccupied();
23523 cg.asmOps((if (loop.remaining_size != null and loop.limb_size != null and
23524 loop.remaining_size.? <= loop.limb_size.?)
23525 pattern_set.final_mir_tag
23526 else
23527 null) orelse pattern_set.mir_tag, mir_ops) catch |err| switch (err) {
23528 error.InvalidInstruction => {
23529 const fixes = @tagName(pattern_set.mir_tag[0]);
23530 const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
23531 return cg.fail(
23532 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
23533 .{
23534 fixes[0..fixes_blank],
23535 @tagName(pattern_set.mir_tag[1]),
23536 fixes[fixes_blank + 1 ..],
23537 @tagName(mir_ops[0]),
23538 @tagName(mir_ops[1]),
23539 @tagName(mir_ops[2]),
23540 @tagName(mir_ops[3]),
23541 },
23542 );
23543 },
23544 else => |e| return e,
23545 };
23546 for (
23547 extra_temps[0..dst_temps.len],
23548 pattern.ops[0..dst_temps.len],
23549 mir_ops[0..dst_temps.len],
23550 dst_temps,
23551 ) |extra_temp, dst_op, mir_op, dst_temp| switch (dst_op) {
23552 else => if (invert_result) {
23553 try cg.spillEflagsIfOccupied();
23554 cg.asmOps(
23555 .{ ._, .not },
23556 .{ mir_op, .none, .none, .none },
23557 ) catch |err| switch (err) {
23558 error.InvalidInstruction => return cg.fail(
23559 "invalid instruction: 'not {s} none none none'",
23560 .{@tagName(mir_op)},
23561 ),
23562 else => |e| return e,
23563 };
23564 },
23565 .mm_mask,
23566 .xmm_mask,
23567 .ymm_mask,
23568 .mm_sign_mask,
23569 .xmm_sign_mask,
23570 .ymm_sign_mask,
23571 => dst_temp.asMask(.{
23572 .kind = switch (dst_op) {
23573 else => unreachable,
23574 .mm_mask, .xmm_mask, .ymm_mask => .all,
23575 .mm_sign_mask, .xmm_sign_mask, .ymm_sign_mask => .sign,
23576 },
23577 .inverted = invert_result,
23578 .scalar = switch (pattern_set.scalar) {
23579 .any, .bool => unreachable,
23580 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
23581 },
23582 }, cg),
23583 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mut_gpr_limb, .gpr_elem => if (extra_temp) |limb_temp| {
23584 const dst_mcv = dst_temp.tracking(cg).short;
23585 switch (dst_mcv) {
23586 .register_pair, .register_triple, .register_quadruple => try cg.asmRegisterRegister(
23587 .{ ._, .mov },
23588 dst_mcv.getRegs()[@divExact(loop.limb_offset.known, loop.limb_size.?)].to64(),
23589 limb_temp.tracking(cg).short.register.to64(),
23590 ),
23591 else => try cg.asmMemoryRegister(
23592 switch (dst_op) {
23593 else => unreachable,
23594 .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ ._, .mov },
23595 .mm_limb => .{ ._q, .mov },
23596 .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
23597 },
23598 try dst_mcv.mem(cg, switch (loop.limb_offset) {
23599 .unused => unreachable,
23600 .known => |limb_offset| .{
23601 .size = .fromSize(loop.limb_size.?),
23602 .disp = limb_offset,
23603 },
23604 .temp => |limb_offset| .{
23605 .size = .fromSize(loop.limb_size.?),
23606 .index = limb_offset.tracking(cg).short.register.to64(),
23607 },
23608 }),
23609 registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?),
23610 ),
23611 }
23612 },
23613 .cc_elem, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
23614 const scalar_size = switch (pattern_set.scalar) {
23615 .any => .qword,
23616 .bool => unreachable,
23617 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
23618 };
23619 const mask_store_size: u4 =
23620 @intCast(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable);
23621 const known_shl_count = if (loop.mask_store_reg) |_| switch (loop.mask_limb_offset) {
23622 .unused => unreachable,
23623 .known => |mask_limb_offset| mask_limb_offset & (loop.mask_store_bit_size.? - 1),
23624 .temp => null,
23625 } else null;
23626 const mask_limb_reg = registerAlias(if (known_shl_count != 0)
23627 loop.mask_limb_temp.?.tracking(cg).short.register
23628 else
23629 loop.mask_store_reg.?, mask_store_size);
23630 switch (dst_op) {
23631 else => unreachable,
23632 .cc_elem => try cg.asmSetccRegister(switch (invert_result) {
23633 false => opts.cc.?,
23634 true => opts.cc.?.negate(),
23635 }, mask_limb_reg.to8()),
23636 .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
23637 if (scalar_size == .word) if (cg.hasFeature(.avx)) try cg.asmRegisterRegisterRegister(
23638 .{ .vp_b, .ackssw },
23639 mir_op.reg,
23640 mir_op.reg,
23641 mir_op.reg,
23642 ) else try cg.asmRegisterRegister(
23643 .{ .p_b, .ackssw },
23644 mir_op.reg,
23645 mir_op.reg,
23646 );
23647 try cg.asmRegisterRegister(switch (scalar_size) {
23648 else => unreachable,
23649 .byte, .word => .{ if (cg.hasFeature(.avx)) .vp_b else .p_b, .movmsk },
23650 .dword => .{ if (cg.hasFeature(.avx)) .v_ps else ._ps, .movmsk },
23651 .qword => .{ if (cg.hasFeature(.avx)) .v_pd else ._pd, .movmsk },
23652 }, mask_limb_reg.to32(), mir_op.reg);
23653 if (invert_result) if (loop.mask_store_reg) |_| {
23654 try cg.spillEflagsIfOccupied();
23655 try cg.asmRegisterImmediate(
23656 .{ ._, .xor },
23657 registerAlias(mask_limb_reg, @min(mask_store_size, 4)),
23658 .u((@as(u32, 1) << @intCast(loop.mask_limb_bit_size.?)) - 1),
23659 );
23660 } else try cg.asmRegister(.{ ._, .not }, mask_limb_reg);
23661 },
23662 }
23663 if (loop.mask_store_reg) |mask_store_reg| {
23664 const mask_store_alias = registerAlias(mask_store_reg, mask_store_size);
23665 switch (loop.mask_limb_offset) {
23666 .unused => unreachable,
23667 .known => if (known_shl_count.? != 0) {
23668 try cg.spillEflagsIfOccupied();
23669 try cg.asmRegisterImmediate(.{ ._l, .sh }, mask_limb_reg, .u(known_shl_count.?));
23670 try cg.spillEflagsIfOccupied();
23671 try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg);
23672 },
23673 .temp => |mask_limb_offset| {
23674 if (cg.hasFeature(.bmi2)) {
23675 const shlx_size = @max(mask_store_size, 4);
23676 const shlx_mask_limb_reg = registerAlias(mask_limb_reg, shlx_size);
23677 try cg.asmRegisterRegisterRegister(
23678 .{ ._lx, .sh },
23679 shlx_mask_limb_reg,
23680 shlx_mask_limb_reg,
23681 registerAlias(mask_limb_offset.tracking(cg).short.register, shlx_size),
23682 );
23683 } else {
23684 try cg.spillEflagsIfOccupied();
23685 try cg.asmRegisterRegister(
23686 .{ ._l, .sh },
23687 mask_limb_reg,
23688 mask_limb_offset.tracking(cg).short.register.to8(),
23689 );
23690 }
23691 try cg.spillEflagsIfOccupied();
23692 try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg);
23693 },
23694 }
23695 }
23696 const dst_mcv = dst_temp.tracking(cg).short;
23697 switch (loop.mask_limb_offset) {
23698 .unused => unreachable,
23699 .known => |*mask_limb_offset| {
23700 mask_limb_offset.* += loop.mask_limb_bit_size.?;
23701 if (mask_limb_offset.* & (loop.mask_store_bit_size.? - 1) == 0) switch (dst_mcv) {
23702 .register => {},
23703 else => {
23704 try cg.asmMemoryRegister(
23705 .{ ._, .mov },
23706 try dst_mcv.mem(cg, .{
23707 .size = .fromSize(mask_store_size),
23708 .disp = @divExact(mask_limb_offset.*, 8) - mask_store_size,
23709 }),
23710 registerAlias(loop.mask_store_reg orelse mask_limb_reg, mask_store_size),
23711 );
23712 if (loop.mask_store_reg) |mask_store_reg| {
23713 const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4));
23714 try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias);
23715 }
23716 },
23717 };
23718 },
23719 .temp => |mask_limb_offset| {
23720 const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32();
23721 if (loop.mask_store_reg) |mask_store_reg| {
23722 try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{
23723 .base = .{ .reg = mask_limb_offset_reg.to64() },
23724 .mod = .{ .rm = .{
23725 .size = .qword,
23726 .disp = loop.mask_limb_bit_size.?,
23727 } },
23728 });
23729 switch (dst_mcv) {
23730 .register => {},
23731 else => {
23732 try cg.spillEflagsIfOccupied();
23733 try cg.asmRegisterImmediate(
23734 .{ ._, .@"test" },
23735 mask_limb_offset_reg,
23736 .u(loop.mask_store_bit_size.? - 1),
23737 );
23738 const skip_store_reloc = try cg.asmJccReloc(.nz, undefined);
23739 const mask_store_offset_reg = mask_limb_reg.to32();
23740 try cg.asmRegisterRegister(.{ ._, .mov }, mask_store_offset_reg, mask_limb_offset_reg);
23741 try cg.asmRegisterImmediate(.{ ._r, .sh }, mask_store_offset_reg, .u(3));
23742 try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{
23743 .size = .fromSize(mask_store_size),
23744 .index = mask_store_offset_reg.to64(),
23745 .disp = -@as(i8, mask_store_size),
23746 }), registerAlias(mask_store_reg, mask_store_size));
23747 const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4));
23748 try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias);
23749 cg.performReloc(skip_store_reloc);
23750 },
23751 }
23752 } else {
23753 switch (dst_mcv) {
23754 .register => {},
23755 else => try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{
23756 .size = .fromSize(mask_store_size),
23757 .index = mask_limb_offset_reg.to64(),
23758 }), mask_limb_reg),
23759 }
23760 try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{
23761 .base = .{ .reg = mask_limb_offset_reg.to64() },
23762 .mod = .{ .rm = .{
23763 .size = .qword,
23764 .disp = mask_store_size,
23765 } },
23766 });
23767 }
23768 },
23769 }
23770 },
23771 };
23772 switch (pattern_set.loop) {
23773 .once => break :pattern_sets,
23774 .bitwise => {},
23775 .limbwise_carry => @panic("TODO"),
23776 .limbwise_reduce => @panic("TODO"),
23777 .limbwise_pairs_forward => @panic("TODO"),
23778 .limbwise_pairs_reverse => @panic("TODO"),
23779 .elementwise => {},
23780 }
23781 switch (loop.limb_offset) {
23782 .unused => break,
23783 .known => |*limb_offset| {
23784 limb_offset.* += loop.limb_size.?;
23785 loop.remaining_size.? -= loop.limb_size.?;
23786 if (loop.remaining_size.? < loop.limb_size.? or
23787 (loop.element_size != null and limb_offset.* >= loop.element_size.?))
23788 {
23789 switch (loop.mask_limb_offset) {
23790 .unused => {},
23791 .known => |*mask_limb_offset| mask_limb_offset.* = 0,
23792 .temp => unreachable,
23793 }
23794 limb_offset.* = 0;
23795 break;
23796 }
23797 },
23798 .temp => |limb_offset| {
23799 const limb_offset_reg = limb_offset.tracking(cg).short.register;
23800 try cg.asmRegisterMemory(.{ ._, .lea }, limb_offset_reg.to32(), .{
23801 .base = .{ .reg = limb_offset_reg.to64() },
23802 .mod = .{ .rm = .{
23803 .size = .qword,
23804 .disp = loop.limb_size.?,
23805 } },
23806 });
23807 try cg.spillEflagsIfOccupied();
23808 try cg.asmRegisterImmediate(
23809 .{ ._, .cmp },
23810 limb_offset_reg.to32(),
23811 .u(loop.element_size orelse loop.remaining_size.?),
23812 );
23813 _ = try cg.asmJccReloc(.b, loop.limb_reloc);
23814 try limb_offset.die(cg);
23815 break;
23816 },
23817 }
23818 }
23819 if (loop.shuffle_temp) |shuffle_temp| try shuffle_temp.die(cg);
23820 if (loop.mask_limb_temp) |mask_limb_temp| try mask_limb_temp.die(cg);
23821 if (loop.mask_store_temp) |mask_store_temp| try mask_store_temp.die(cg);
23822 switch (loop.mask_limb_offset) {
23823 .unused, .known => {},
23824 .temp => |mask_limb_offset| try mask_limb_offset.die(cg),
23825 }
23826 switch (loop.element_offset) {
23827 .unused => break :pattern_sets,
23828 .known => |*element_offset| {
23829 if (loop.remaining_size.? == 0) break :pattern_sets;
23830 element_offset.* += loop.element_size.?;
23831 },
23832 .temp => |element_offset| {
23833 if (true) @panic("TODO");
23834 try element_offset.die(cg);
23835 if (loop.remaining_size.? == 0) break :pattern_sets;
23836 break;
24009 const frame_index = try cg.allocFrameIndex(.init(.{
24010 .size = 8,
24011 .alignment = .@"8",
24012 }));
24013 try cg.genSetMem(
24014 .{ .frame = frame_index },
24015 0,
24016 .usize,
24017 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
24018 .{},
24019 );
24020 break :init .{ .load_frame = .{ .index = frame_index } };
2383724021 },
23838 }
24022 else => break :init const_mcv,
24023 },
24024 else => break :init const_mcv,
2383924025 }
23840 }
23841 } else {
23842 log.err("failed to select {s}:", .{@tagName(pattern_sets[0].mir_tag[1])});
23843 for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(cg)});
23844 return error.SelectFailed;
24026 });
24027 return cg.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
24028 }
24029
24030 const temp_index = cg.next_temp_index;
24031 const temp: Temp = .{ .index = temp_index.toIndex() };
24032 const op_inst = op_ref.toIndex().?;
24033 const tracking = cg.getResolvedInstValue(op_inst);
24034 temp_index.tracking(cg).* = tracking.*;
24035 if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
24036 cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
24037 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
24038 return temp;
24039}
24040
24041inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
24042 var temps: [op_refs.len]Temp = undefined;
24043 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {
24044 temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| {
24045 if (op_ref == op_refs[prev_op_index]) break true;
24046 } else false);
2384524047 }
23846 for (extra_temps) |extra_temp| if (extra_temp) |temp| try temp.die(cg);
24048 return temps;
2384724049}
2384824050
23849const Select2 = struct {
24051const Operand = union(enum) {
24052 none,
24053 reg: Register,
24054 mem: Memory,
24055 imm: Immediate,
24056 inst: Mir.Inst.Index,
24057};
24058
24059const Select = struct {
2385024060 cg: *CodeGen,
23851 case: *const Case,
23852 pattern: *const Select2.Pattern,
23853 extra_temps: [3]Temp,
23854 dst_temps: []const Temp,
23855 src_temps: []const Temp,
23856 commute: struct { u8, u8 },
24061 temps: [@intFromEnum(Select.Operand.Ref.none)]Temp,
24062 labels: [@intFromEnum(Label._)]struct {
24063 backward: ?Mir.Inst.Index,
24064 forward: [1]?Mir.Inst.Index,
24065 },
2385724066 limb: Memory.Mod.Rm,
24067 mask_limb: Memory.Mod.Rm,
24068
24069 fn emitLabel(s: *Select, label_index: Label) void {
24070 if (label_index == ._) return;
24071 const label = &s.labels[@intFromEnum(label_index)];
24072 for (&label.forward) |*reloc| {
24073 if (reloc.*) |r| s.cg.performReloc(r);
24074 reloc.* = null;
24075 }
24076 label.backward = @intCast(s.cg.mir_instructions.len);
24077 }
2385824078
23859 fn emit(s: Select2, inst: Instruction) !void {
23860 const mir_tag: Mir.Inst.FixedTag = .{ inst[0], inst[1] };
24079 fn emit(s: *Select, inst: Instruction) !void {
24080 s.emitLabel(inst[0]);
24081 const mir_tag: Mir.Inst.FixedTag = .{ inst[1], inst[2] };
2386124082 var mir_ops: [4]CodeGen.Operand = undefined;
23862 inline for (&mir_ops, 2..) |*mir_op, inst_index| mir_op.* = try inst[inst_index].lower(s);
24083 inline for (&mir_ops, 3..) |*mir_op, inst_index| mir_op.* = try inst[inst_index].lower(s);
2386324084 s.cg.asmOps(mir_tag, mir_ops) catch |err| switch (err) {
2386424085 error.InvalidInstruction => {
2386524086 const fixes = @tagName(mir_tag[0]);
......@@ -23881,32 +24102,27 @@ const Select2 = struct {
2388124102 };
2388224103 }
2388324104
23884 fn lowerLimb(s: Select2, temp: Temp) !CodeGen.Operand {
23885 return .{ .mem = try temp.tracking(s.cg).short.mem(s.cg, s.limb) };
23886 }
23887
23888 fn srcTemp(s: Select2, index: u8) Temp {
23889 return s.src_temps[
23890 if (index == s.commute[0])
23891 s.commute[1]
23892 else if (index == s.commute[1])
23893 s.commute[0]
23894 else
23895 index
23896 ];
23897 }
23898
2389924105 const Case = struct {
2390024106 required_features: [2]?std.Target.x86.Feature = @splat(null),
23901 constraints: [2]Constraint = @splat(.any),
23902 patterns: []const Select2.Pattern,
24107 dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any),
24108 src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any),
24109 patterns: []const Select.Pattern,
2390324110 clobbers: struct { eflags: bool = false } = .{},
23904 extra_temps: [3]TempSpec = @splat(.unused),
23905 dst_temps: [1]TempSpec.Kind = @splat(.unused),
24111 extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused),
24112 dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused),
2390624113 each: union(enum) {
2390724114 once: []const Instruction,
2390824115 limb: struct {
23909 of: Select2.Operand,
24116 of: Select.Operand.Ref.Sized,
24117 header: []const Instruction = &.{},
24118 first: ?[]const Instruction = null,
24119 body: []const Instruction,
24120 last: ?[]const Instruction = null,
24121 trailer: []const Instruction = &.{},
24122 },
24123 limb_and_mask_limb: struct {
24124 of: Select.Operand.Ref.Sized,
24125 of_mask: Select.Operand.Ref.Sized,
2391024126 header: []const Instruction = &.{},
2391124127 first: ?[]const Instruction = null,
2391224128 body: []const Instruction,
......@@ -23918,6 +24134,7 @@ const Select2 = struct {
2391824134
2391924135 const Constraint = union(enum) {
2392024136 any,
24137 any_bool_vec,
2392124138 any_int,
2392224139 any_float,
2392324140 bool_vec: Memory.Size,
......@@ -23925,37 +24142,35 @@ const Select2 = struct {
2392524142 signed_int: Memory.Size,
2392624143 unsigned_int: Memory.Size,
2392724144
23928 fn accepts(constraint: Constraint, temp: Temp, cg: *CodeGen) bool {
24145 fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool {
2392924146 const zcu = cg.pt.zcu;
2393024147 switch (constraint) {
2393124148 .any => return true,
24149 .any_bool_vec => return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type,
2393224150 .any_int => {
23933 const scalar_ty = temp.typeOf(cg).scalarType(zcu);
24151 const scalar_ty = ty.scalarType(zcu);
2393424152 return scalar_ty.isAbiInt(zcu) or scalar_ty.isPtrAtRuntime(zcu);
2393524153 },
23936 .any_float => return temp.typeOf(cg).scalarType(zcu).isRuntimeFloat(),
23937 .bool_vec => |size| {
23938 const ty = temp.typeOf(cg);
23939 return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and
23940 ty.vectorLen(zcu) <= size.bitSize();
23941 },
24154 .any_float => return ty.scalarType(zcu).isRuntimeFloat(),
24155 .bool_vec => |size| return ty.isVector(zcu) and
24156 ty.scalarType(zcu).toIntern() == .bool_type and ty.vectorLen(zcu) <= size.bitSize(cg.target),
2394224157 .int => |size| {
23943 const scalar_ty = temp.typeOf(cg).scalarType(zcu);
23944 if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize();
23945 return scalar_ty.isAbiInt(zcu) and scalar_ty.intInfo(zcu).bits <= size.bitSize();
24158 const scalar_ty = ty.scalarType(zcu);
24159 if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(cg.target);
24160 return scalar_ty.isAbiInt(zcu) and scalar_ty.intInfo(zcu).bits <= size.bitSize(cg.target);
2394624161 },
2394724162 .signed_int => |size| {
23948 const scalar_ty = temp.typeOf(cg).scalarType(zcu);
24163 const scalar_ty = ty.scalarType(zcu);
2394924164 if (!scalar_ty.isAbiInt(zcu)) return false;
2395024165 const info = scalar_ty.intInfo(zcu);
23951 return info.signedness == .signed and info.bits <= size.bitSize();
24166 return info.signedness == .signed and info.bits <= size.bitSize(cg.target);
2395224167 },
2395324168 .unsigned_int => |size| {
23954 const scalar_ty = temp.typeOf(cg).scalarType(zcu);
23955 if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize();
24169 const scalar_ty = ty.scalarType(zcu);
24170 if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(cg.target);
2395624171 if (!scalar_ty.isAbiInt(zcu)) return false;
2395724172 const info = scalar_ty.intInfo(zcu);
23958 return info.signedness == .unsigned and info.bits <= size.bitSize();
24173 return info.signedness == .unsigned and info.bits <= size.bitSize(cg.target);
2395924174 },
2396024175 }
2396124176 }
......@@ -24083,189 +24298,578 @@ const Select2 = struct {
2408324298 rc: Register.Class,
2408424299 rc_mask: struct { rc: Register.Class, info: MaskInfo },
2408524300 mem,
24086 src: u8,
24087 src_mask: struct { src: u8, info: MaskInfo },
24301 ref: Select.Operand.Ref,
24302 ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo },
2408824303
24089 fn finish(kind: Kind, temp: Temp, s: Select2) void {
24304 fn finish(kind: Kind, temp: Temp, s: *const Select) void {
2409024305 switch (kind) {
2409124306 else => {},
24092 inline .rc_mask, .src_mask => |mask| temp.asMask(mask.info, s.cg),
24307 inline .rc_mask, .ref_mask => |mask| temp.asMask(mask.info, s.cg),
2409324308 }
2409424309 }
2409524310 };
2409624311
24097 fn create(spec: TempSpec, s: Select2) !?Temp {
24312 fn create(spec: TempSpec, s: *Select) !?Temp {
2409824313 return switch (spec.kind) {
2409924314 .unused => null,
2410024315 .any => try s.cg.tempAlloc(spec.type),
2410124316 .cc => |cc| try s.cg.tempFromValue(spec.type, .{ .eflags = cc }),
2410224317 .reg => |reg| try s.cg.tempFromValue(spec.type, .{ .register = reg }),
2410324318 .rc => |rc| try s.cg.tempAllocReg(spec.type, regSetForRegClass(rc)),
24104 .rc_mask => |mask| try s.cg.tempAllocReg(spec.type, regSetForRegClass(mask.rc)),
24319 .rc_mask => |rc_mask| try s.cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)),
2410524320 .mem => try s.cg.tempAllocMem(spec.type),
24106 .src => |src| s.srcTemp(src),
24107 .src_mask => |mask| s.srcTemp(mask.src),
24321 .ref => |ref| ref.deref(s),
24322 .ref_mask => |ref_mask| ref_mask.ref.deref(s),
2410824323 };
2410924324 }
2411024325 };
2411124326
2411224327 const Instruction = struct {
24328 Label,
2411324329 Mir.Inst.Fixes,
2411424330 Mir.Inst.Tag,
24115 Select2.Operand,
24116 Select2.Operand,
24117 Select2.Operand,
24118 Select2.Operand,
24331 Select.Operand,
24332 Select.Operand,
24333 Select.Operand,
24334 Select.Operand,
2411924335 };
24120 const Operand = union(enum) {
24121 none,
24122 extra: struct { Memory.Size, u8 },
24123 dst: struct { Memory.Size, u8 },
24124 src: struct { Memory.Size, u8 },
24125 dst_limb: u8,
24126 src_limb: u8,
24127 simm32: i32,
24128
24129 const tmp0b: Select2.Operand = .{ .extra = .{ .byte, 0 } };
24130 const tmp0w: Select2.Operand = .{ .extra = .{ .word, 0 } };
24131 const etmp0: Select2.Operand = .{ .extra = .{ .dword, 0 } };
24132 const rtmp0: Select2.Operand = .{ .extra = .{ .qword, 0 } };
24133 const xtmp0: Select2.Operand = .{ .extra = .{ .xword, 0 } };
24134 const ytmp0: Select2.Operand = .{ .extra = .{ .yword, 0 } };
24135
24136 const tmp1b: Select2.Operand = .{ .extra = .{ .byte, 1 } };
24137 const tmp1w: Select2.Operand = .{ .extra = .{ .word, 1 } };
24138 const etmp1: Select2.Operand = .{ .extra = .{ .dword, 1 } };
24139 const rtmp1: Select2.Operand = .{ .extra = .{ .qword, 1 } };
24140 const xtmp1: Select2.Operand = .{ .extra = .{ .xword, 1 } };
24141 const ytmp1: Select2.Operand = .{ .extra = .{ .yword, 1 } };
24142
24143 const tmp2b: Select2.Operand = .{ .extra = .{ .byte, 2 } };
24144 const tmp2w: Select2.Operand = .{ .extra = .{ .word, 2 } };
24145 const etmp2: Select2.Operand = .{ .extra = .{ .dword, 2 } };
24146 const rtmp2: Select2.Operand = .{ .extra = .{ .qword, 2 } };
24147 const xtmp2: Select2.Operand = .{ .extra = .{ .xword, 2 } };
24148 const ytmp2: Select2.Operand = .{ .extra = .{ .yword, 2 } };
24149
24150 const dst0b: Select2.Operand = .{ .dst = .{ .byte, 0 } };
24151 const dst0w: Select2.Operand = .{ .dst = .{ .word, 0 } };
24152 const edst0: Select2.Operand = .{ .dst = .{ .dword, 0 } };
24153 const rdst0: Select2.Operand = .{ .dst = .{ .qword, 0 } };
24154 const xdst0: Select2.Operand = .{ .dst = .{ .xword, 0 } };
24155 const ydst0: Select2.Operand = .{ .dst = .{ .yword, 0 } };
24156
24157 const src0b: Select2.Operand = .{ .src = .{ .byte, 0 } };
24158 const src0w: Select2.Operand = .{ .src = .{ .word, 0 } };
24159 const esrc0: Select2.Operand = .{ .src = .{ .dword, 0 } };
24160 const rsrc0: Select2.Operand = .{ .src = .{ .qword, 0 } };
24161 const xsrc0: Select2.Operand = .{ .src = .{ .xword, 0 } };
24162 const ysrc0: Select2.Operand = .{ .src = .{ .yword, 0 } };
24163
24164 const src1b: Select2.Operand = .{ .src = .{ .byte, 1 } };
24165 const src1w: Select2.Operand = .{ .src = .{ .word, 1 } };
24166 const esrc1: Select2.Operand = .{ .src = .{ .dword, 1 } };
24167 const rsrc1: Select2.Operand = .{ .src = .{ .qword, 1 } };
24168 const xsrc1: Select2.Operand = .{ .src = .{ .xword, 1 } };
24169 const ysrc1: Select2.Operand = .{ .src = .{ .yword, 1 } };
24170
24171 fn unwrap(op: Select2.Operand, s: Select2) struct { Memory.Size, Temp } {
24172 return switch (op) {
24173 else => unreachable,
24174 .extra => |extra| .{ extra[0], s.extra_temps[extra[1]] },
24175 .dst => |dst| .{ dst[0], s.dst_temps[dst[1]] },
24176 .src => |src| .{ src[0], s.srcTemp(src[1]) },
24336 const Label = enum { @"0:", @"1:", @"_" };
24337 const Operand = struct {
24338 tag: Tag,
24339 base: Ref.Sized = .none,
24340 index: packed struct(u6) {
24341 ref: Ref,
24342 scale: Memory.Scale,
24343 } = .{ .ref = .none, .scale = .@"1" },
24344 adjust: Adjust = .none,
24345 imm: i32 = 0,
24346
24347 const Tag = enum {
24348 none,
24349 backward_label,
24350 forward_label,
24351 ref,
24352 limb,
24353 mask_limb,
24354 simm,
24355 lea,
24356 mem,
24357 };
24358 const Adjust = enum {
24359 none,
24360 add_access_size,
24361 sub_access_size,
24362 add_size,
24363 sub_size,
24364 add_bit_size,
24365 sub_bit_size,
24366 add_limbs,
24367 sub_limbs,
24368 add_len,
24369 sub_len,
24370 add_elem_size,
24371 sub_elem_size,
24372 add_elem_limbs,
24373 sub_elem_limbs,
24374 };
24375 const Ref = enum(u4) {
24376 tmp0,
24377 tmp1,
24378 tmp2,
24379 tmp3,
24380 tmp4,
24381 tmp5,
24382 dst0,
24383 src0,
24384 src1,
24385 none,
24386
24387 const Sized = packed struct(u8) {
24388 ref: Ref,
24389 size: Memory.Size,
24390
24391 const none: Sized = .{ .ref = .none, .size = .none };
24392
24393 const tmp0: Sized = .{ .ref = .tmp0, .size = .none };
24394 const tmp0b: Sized = .{ .ref = .tmp0, .size = .byte };
24395 const tmp0w: Sized = .{ .ref = .tmp0, .size = .word };
24396 const tmp0d: Sized = .{ .ref = .tmp0, .size = .dword };
24397 const tmp0p: Sized = .{ .ref = .tmp0, .size = .ptr };
24398 const tmp0q: Sized = .{ .ref = .tmp0, .size = .qword };
24399 const tmp0x: Sized = .{ .ref = .tmp0, .size = .xword };
24400 const tmp0y: Sized = .{ .ref = .tmp0, .size = .yword };
24401
24402 const tmp1: Sized = .{ .ref = .tmp1, .size = .none };
24403 const tmp1b: Sized = .{ .ref = .tmp1, .size = .byte };
24404 const tmp1w: Sized = .{ .ref = .tmp1, .size = .word };
24405 const tmp1d: Sized = .{ .ref = .tmp1, .size = .dword };
24406 const tmp1p: Sized = .{ .ref = .tmp1, .size = .ptr };
24407 const tmp1q: Sized = .{ .ref = .tmp1, .size = .qword };
24408 const tmp1x: Sized = .{ .ref = .tmp1, .size = .xword };
24409 const tmp1y: Sized = .{ .ref = .tmp1, .size = .yword };
24410
24411 const tmp2: Sized = .{ .ref = .tmp2, .size = .none };
24412 const tmp2b: Sized = .{ .ref = .tmp2, .size = .byte };
24413 const tmp2w: Sized = .{ .ref = .tmp2, .size = .word };
24414 const tmp2d: Sized = .{ .ref = .tmp2, .size = .dword };
24415 const tmp2p: Sized = .{ .ref = .tmp2, .size = .ptr };
24416 const tmp2q: Sized = .{ .ref = .tmp2, .size = .qword };
24417 const tmp2x: Sized = .{ .ref = .tmp2, .size = .xword };
24418 const tmp2y: Sized = .{ .ref = .tmp2, .size = .yword };
24419
24420 const tmp3: Sized = .{ .ref = .tmp3, .size = .none };
24421 const tmp3b: Sized = .{ .ref = .tmp3, .size = .byte };
24422 const tmp3w: Sized = .{ .ref = .tmp3, .size = .word };
24423 const tmp3d: Sized = .{ .ref = .tmp3, .size = .dword };
24424 const tmp3p: Sized = .{ .ref = .tmp3, .size = .ptr };
24425 const tmp3q: Sized = .{ .ref = .tmp3, .size = .qword };
24426 const tmp3x: Sized = .{ .ref = .tmp3, .size = .xword };
24427 const tmp3y: Sized = .{ .ref = .tmp3, .size = .yword };
24428
24429 const tmp4: Sized = .{ .ref = .tmp4, .size = .none };
24430 const tmp4b: Sized = .{ .ref = .tmp4, .size = .byte };
24431 const tmp4w: Sized = .{ .ref = .tmp4, .size = .word };
24432 const tmp4d: Sized = .{ .ref = .tmp4, .size = .dword };
24433 const tmp4p: Sized = .{ .ref = .tmp4, .size = .ptr };
24434 const tmp4q: Sized = .{ .ref = .tmp4, .size = .qword };
24435 const tmp4x: Sized = .{ .ref = .tmp4, .size = .xword };
24436 const tmp4y: Sized = .{ .ref = .tmp4, .size = .yword };
24437
24438 const dst0: Sized = .{ .ref = .dst0, .size = .none };
24439 const dst0b: Sized = .{ .ref = .dst0, .size = .byte };
24440 const dst0w: Sized = .{ .ref = .dst0, .size = .word };
24441 const dst0d: Sized = .{ .ref = .dst0, .size = .dword };
24442 const dst0p: Sized = .{ .ref = .dst0, .size = .ptr };
24443 const dst0q: Sized = .{ .ref = .dst0, .size = .qword };
24444 const dst0x: Sized = .{ .ref = .dst0, .size = .xword };
24445 const dst0y: Sized = .{ .ref = .dst0, .size = .yword };
24446
24447 const src0: Sized = .{ .ref = .src0, .size = .none };
24448 const src0b: Sized = .{ .ref = .src0, .size = .byte };
24449 const src0w: Sized = .{ .ref = .src0, .size = .word };
24450 const src0d: Sized = .{ .ref = .src0, .size = .dword };
24451 const src0p: Sized = .{ .ref = .src0, .size = .ptr };
24452 const src0q: Sized = .{ .ref = .src0, .size = .qword };
24453 const src0x: Sized = .{ .ref = .src0, .size = .xword };
24454 const src0y: Sized = .{ .ref = .src0, .size = .yword };
24455
24456 const src1: Sized = .{ .ref = .src1, .size = .none };
24457 const src1b: Sized = .{ .ref = .src1, .size = .byte };
24458 const src1w: Sized = .{ .ref = .src1, .size = .word };
24459 const src1d: Sized = .{ .ref = .src1, .size = .dword };
24460 const src1p: Sized = .{ .ref = .src1, .size = .ptr };
24461 const src1q: Sized = .{ .ref = .src1, .size = .qword };
24462 const src1x: Sized = .{ .ref = .src1, .size = .xword };
24463 const src1y: Sized = .{ .ref = .src1, .size = .yword };
2417724464 };
24178 }
2417924465
24180 fn lower(op: Select2.Operand, s: Select2) !CodeGen.Operand {
24181 switch (op) {
24182 .none => return .none,
24183 else => {},
24184 .dst_limb => |dst| return s.lowerLimb(s.dst_temps[dst]),
24185 .src_limb => |src| return s.lowerLimb(s.srcTemp(src)),
24186 .simm32 => |imm| return .{ .imm = .s(imm) },
24466 fn deref(ref: Ref, s: *const Select) Temp {
24467 return s.temps[@intFromEnum(ref)];
2418724468 }
24188 const size, const temp = op.unwrap(s);
24189 return switch (temp.tracking(s.cg).short) {
24190 .immediate => |imm| .{ .imm = switch (size) {
24191 .byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u8, @intCast(imm))),
24192 .word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u16, @intCast(imm))),
24193 .dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u32, @intCast(imm))),
24194 .qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(imm),
24195 else => unreachable,
24469 };
24470
24471 const @"_": Select.Operand = .{ .tag = .none };
24472
24473 const @"0b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp0, .size = .none } };
24474 const @"0f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp0, .size = .none } };
24475 const @"1b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp1, .size = .none } };
24476 const @"1f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp1, .size = .none } };
24477
24478 const tmp0b: Select.Operand = .{ .tag = .ref, .base = .tmp0b };
24479 const tmp0w: Select.Operand = .{ .tag = .ref, .base = .tmp0w };
24480 const tmp0d: Select.Operand = .{ .tag = .ref, .base = .tmp0d };
24481 const tmp0p: Select.Operand = .{ .tag = .ref, .base = .tmp0p };
24482 const tmp0q: Select.Operand = .{ .tag = .ref, .base = .tmp0q };
24483 const tmp0x: Select.Operand = .{ .tag = .ref, .base = .tmp0x };
24484 const tmp0y: Select.Operand = .{ .tag = .ref, .base = .tmp0y };
24485
24486 const tmp1b: Select.Operand = .{ .tag = .ref, .base = .tmp1b };
24487 const tmp1w: Select.Operand = .{ .tag = .ref, .base = .tmp1w };
24488 const tmp1d: Select.Operand = .{ .tag = .ref, .base = .tmp1d };
24489 const tmp1p: Select.Operand = .{ .tag = .ref, .base = .tmp1p };
24490 const tmp1q: Select.Operand = .{ .tag = .ref, .base = .tmp1q };
24491 const tmp1x: Select.Operand = .{ .tag = .ref, .base = .tmp1x };
24492 const tmp1y: Select.Operand = .{ .tag = .ref, .base = .tmp1y };
24493
24494 const tmp2b: Select.Operand = .{ .tag = .ref, .base = .tmp2b };
24495 const tmp2w: Select.Operand = .{ .tag = .ref, .base = .tmp2w };
24496 const tmp2d: Select.Operand = .{ .tag = .ref, .base = .tmp2d };
24497 const tmp2p: Select.Operand = .{ .tag = .ref, .base = .tmp2p };
24498 const tmp2q: Select.Operand = .{ .tag = .ref, .base = .tmp2q };
24499 const tmp2x: Select.Operand = .{ .tag = .ref, .base = .tmp2x };
24500 const tmp2y: Select.Operand = .{ .tag = .ref, .base = .tmp2y };
24501
24502 const tmp3b: Select.Operand = .{ .tag = .ref, .base = .tmp3b };
24503 const tmp3w: Select.Operand = .{ .tag = .ref, .base = .tmp3w };
24504 const tmp3d: Select.Operand = .{ .tag = .ref, .base = .tmp3d };
24505 const tmp3p: Select.Operand = .{ .tag = .ref, .base = .tmp3p };
24506 const tmp3q: Select.Operand = .{ .tag = .ref, .base = .tmp3q };
24507 const tmp3x: Select.Operand = .{ .tag = .ref, .base = .tmp3x };
24508 const tmp3y: Select.Operand = .{ .tag = .ref, .base = .tmp3y };
24509
24510 const tmp4b: Select.Operand = .{ .tag = .ref, .base = .tmp4b };
24511 const tmp4w: Select.Operand = .{ .tag = .ref, .base = .tmp4w };
24512 const tmp4d: Select.Operand = .{ .tag = .ref, .base = .tmp4d };
24513 const tmp4p: Select.Operand = .{ .tag = .ref, .base = .tmp4p };
24514 const tmp4q: Select.Operand = .{ .tag = .ref, .base = .tmp4q };
24515 const tmp4x: Select.Operand = .{ .tag = .ref, .base = .tmp4x };
24516 const tmp4y: Select.Operand = .{ .tag = .ref, .base = .tmp4y };
24517
24518 const dst0b: Select.Operand = .{ .tag = .ref, .base = .dst0b };
24519 const dst0w: Select.Operand = .{ .tag = .ref, .base = .dst0w };
24520 const dst0d: Select.Operand = .{ .tag = .ref, .base = .dst0d };
24521 const dst0p: Select.Operand = .{ .tag = .ref, .base = .dst0p };
24522 const dst0q: Select.Operand = .{ .tag = .ref, .base = .dst0q };
24523 const dst0x: Select.Operand = .{ .tag = .ref, .base = .dst0x };
24524 const dst0y: Select.Operand = .{ .tag = .ref, .base = .dst0y };
24525
24526 const src0b: Select.Operand = .{ .tag = .ref, .base = .src0b };
24527 const src0w: Select.Operand = .{ .tag = .ref, .base = .src0w };
24528 const src0d: Select.Operand = .{ .tag = .ref, .base = .src0d };
24529 const src0p: Select.Operand = .{ .tag = .ref, .base = .src0p };
24530 const src0q: Select.Operand = .{ .tag = .ref, .base = .src0q };
24531 const src0x: Select.Operand = .{ .tag = .ref, .base = .src0x };
24532 const src0y: Select.Operand = .{ .tag = .ref, .base = .src0y };
24533
24534 const src1b: Select.Operand = .{ .tag = .ref, .base = .src1b };
24535 const src1w: Select.Operand = .{ .tag = .ref, .base = .src1w };
24536 const src1d: Select.Operand = .{ .tag = .ref, .base = .src1d };
24537 const src1p: Select.Operand = .{ .tag = .ref, .base = .src1p };
24538 const src1q: Select.Operand = .{ .tag = .ref, .base = .src1q };
24539 const src1x: Select.Operand = .{ .tag = .ref, .base = .src1x };
24540 const src1y: Select.Operand = .{ .tag = .ref, .base = .src1y };
24541
24542 fn limb(ref: Ref.Sized) Select.Operand {
24543 return .{ .tag = .limb, .base = ref };
24544 }
24545 fn maskLimb(ref: Ref.Sized) Select.Operand {
24546 return .{ .tag = .mask_limb, .base = ref };
24547 }
24548
24549 fn i(imm: i32) Select.Operand {
24550 return .{ .tag = .simm, .imm = imm };
24551 }
24552 fn a(base: Ref.Sized, adjust: Adjust) Select.Operand {
24553 return .{ .tag = .simm, .base = base, .adjust = adjust };
24554 }
24555 fn ia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
24556 return .{ .tag = .simm, .base = base, .adjust = adjust, .imm = imm };
24557 }
24558
24559 fn lea(size: Memory.Size, base: Ref) Select.Operand {
24560 return .{
24561 .tag = .lea,
24562 .base = .{ .ref = base, .size = size },
24563 };
24564 }
24565 fn lead(size: Memory.Size, base: Ref, disp: i32) Select.Operand {
24566 return .{
24567 .tag = .lea,
24568 .base = .{ .ref = base, .size = size },
24569 .imm = disp,
24570 };
24571 }
24572 fn leai(size: Memory.Size, base: Ref, index: Ref) Select.Operand {
24573 return .{
24574 .tag = .lea,
24575 .base = .{ .ref = base, .size = size },
24576 .index_ = .{ .ref = index, .scale = .@"1" },
24577 };
24578 }
24579 fn leaid(size: Memory.Size, base: Ref, index: Ref, disp: i32) Select.Operand {
24580 return .{
24581 .tag = .lea,
24582 .base = .{ .ref = base, .size = size },
24583 .index_ = .{ .ref = index, .scale = .@"1" },
24584 .imm = disp,
24585 };
24586 }
24587 fn leasi(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref) Select.Operand {
24588 return .{
24589 .tag = .lea,
24590 .base = .{ .ref = base, .size = size },
24591 .index_ = .{ .ref = index, .scale = scale },
24592 };
24593 }
24594 fn leasid(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
24595 return .{
24596 .tag = .lea,
24597 .base = .{ .ref = base, .size = size },
24598 .index_ = .{ .ref = index, .scale = scale },
24599 .imm = disp,
24600 };
24601 }
24602 fn leasida(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32, adjust: Adjust) Select.Operand {
24603 return .{
24604 .tag = .lea,
24605 .base = .{ .ref = base, .size = size },
24606 .index_ = .{ .ref = index, .scale = scale },
24607 .adjust = adjust,
24608 .imm = disp,
24609 };
24610 }
24611
24612 fn mem(base: Ref.Sized) Select.Operand {
24613 return .{
24614 .tag = .mem,
24615 .base = base,
24616 };
24617 }
24618 fn memd(base: Ref.Sized, disp: i32) Select.Operand {
24619 return .{
24620 .tag = .mem,
24621 .base = base,
24622 .imm = disp,
24623 };
24624 }
24625 fn memi(base: Ref.Sized, index: Ref) Select.Operand {
24626 return .{
24627 .tag = .mem,
24628 .base = base,
24629 .index = .{ .ref = index, .scale = .@"1" },
24630 };
24631 }
24632 fn memia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand {
24633 return .{
24634 .tag = .mem,
24635 .base = base,
24636 .index = .{ .ref = index, .scale = .@"1" },
24637 .adjust = adjust,
24638 };
24639 }
24640 fn memid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand {
24641 return .{
24642 .tag = .mem,
24643 .base = base,
24644 .index = .{ .ref = index, .scale = .@"1" },
24645 .imm = disp,
24646 };
24647 }
24648 fn memsi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand {
24649 return .{
24650 .tag = .mem,
24651 .base = base,
24652 .index = .{ .ref = index, .scale = scale },
24653 };
24654 }
24655 fn memsid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
24656 return .{
24657 .tag = .mem,
24658 .base = base,
24659 .index = .{ .ref = index, .scale = scale },
24660 .imm = disp,
24661 };
24662 }
24663 fn memsida(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32, adjust: Adjust) Select.Operand {
24664 return .{
24665 .tag = .mem,
24666 .base = base,
24667 .index = .{ .ref = index, .scale = scale },
24668 .adjust = adjust,
24669 .imm = disp,
24670 };
24671 }
24672
24673 fn adjustedImm(op: Select.Operand, s: *const Select) i32 {
24674 return switch (op.adjust) {
24675 .none => op.imm,
24676 .add_access_size => op.imm + @as(i32, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))),
24677 .sub_access_size => op.imm - @as(i32, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))),
24678 .add_size => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu))),
24679 .sub_size => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu))),
24680 .add_bit_size => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).bitSize(s.cg.pt.zcu))),
24681 .sub_bit_size => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).bitSize(s.cg.pt.zcu))),
24682 .add_limbs => op.imm + @as(i32, @intCast(@divExact(
24683 op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu),
24684 @divExact(op.base.size.bitSize(s.cg.target), 8),
24685 ))),
24686 .sub_limbs => op.imm + @as(i32, @intCast(@divExact(
24687 op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu),
24688 @divExact(op.base.size.bitSize(s.cg.target), 8),
24689 ))),
24690 .add_len => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu))),
24691 .sub_len => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu))),
24692 .add_elem_size => op.imm + @as(i32, @intCast(
24693 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
24694 )),
24695 .sub_elem_size => op.imm - @as(i32, @intCast(
24696 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
24697 )),
24698 .add_elem_limbs => op.imm + @as(i32, @intCast(@divExact(
24699 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
24700 @divExact(op.base.size.bitSize(s.cg.target), 8),
24701 ))),
24702 .sub_elem_limbs => op.imm - @as(i32, @intCast(@divExact(
24703 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
24704 @divExact(op.base.size.bitSize(s.cg.target), 8),
24705 ))),
24706 };
24707 }
24708
24709 fn lower(op: Select.Operand, s: *Select) !CodeGen.Operand {
24710 return switch (op.tag) {
24711 .none => .none,
24712 .backward_label => .{ .inst = s.labels[@intFromEnum(op.base.ref)].backward.? },
24713 .forward_label => for (&s.labels[@intFromEnum(op.base.ref)].forward) |*label| {
24714 if (label.*) |_| continue;
24715 label.* = @intCast(s.cg.mir_instructions.len);
24716 break .{ .inst = undefined };
24717 } else unreachable,
24718 .ref => switch (op.base.ref.deref(s).tracking(s.cg).short) {
24719 .immediate => |imm| .{ .imm = switch (op.base.size) {
24720 .byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u8, @intCast(imm))),
24721 .word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u16, @intCast(imm))),
24722 .dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u32, @intCast(imm))),
24723 .qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(imm),
24724 else => unreachable,
24725 } },
24726 else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) },
24727 .register => |reg| .{ .reg = registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))) },
24728 },
24729 inline .limb, .mask_limb => |kind| .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, @field(s, @tagName(kind))) },
24730 .simm => .{ .imm = .s(op.adjustedImm(s)) },
24731 .lea => .{ .mem = .{
24732 .base = .{ .reg = registerAlias(op.base.ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)) },
24733 .mod = .{ .rm = .{
24734 .size = op.base.size,
24735 .index = switch (op.index.ref) {
24736 else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
24737 .none => .none,
24738 },
24739 .scale = op.index.scale,
24740 .disp = op.adjustedImm(s),
24741 } },
2419624742 } },
24197 else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = size }) },
24198 .register => |reg| .{ .reg = registerAlias(reg, @intCast(@divExact(size.bitSize(), 8))) },
24743 .mem => .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, .{
24744 .size = op.base.size,
24745 .index = switch (op.index.ref) {
24746 else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
24747 .none => .none,
24748 },
24749 .scale = op.index.scale,
24750 .disp = op.adjustedImm(s),
24751 }) },
2419924752 };
2420024753 }
2420124754 };
2420224755};
24203fn select2(
24756fn select(
2420424757 cg: *CodeGen,
2420524758 dst_temps: []Temp,
2420624759 dst_tys: []const Type,
2420724760 src_temps: []Temp,
24208 cases: []const Select2.Case,
24761 cases: []const Select.Case,
2420924762) !void {
24210 cases: for (cases) |*case| {
24763 cases: for (cases) |case| {
2421124764 for (case.required_features) |required_feature| if (required_feature) |feature| if (!switch (feature) {
24212 .@"64bit" => cg.target.cpu.arch == .x86_64,
24765 .@"64bit" => cg.target.ptrBitWidth() == 64,
2421324766 .mmx => false,
2421424767 else => cg.hasFeature(feature),
2421524768 }) continue :cases;
24216 for (case.constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp, cg)) continue :cases;
24217 patterns: for (case.patterns) |*pattern| {
24769 for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases;
24770 for (case.src_constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp.typeOf(cg), cg)) continue :cases;
24771 patterns: for (case.patterns) |pattern| {
2421824772 for (pattern.src, src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns;
2421924773
24220 var s: Select2 = .{
24774 var s: Select = .{
2422124775 .cg = cg,
24222 .case = case,
24223 .pattern = pattern,
24224 .extra_temps = undefined,
24225 .dst_temps = dst_temps,
24226 .src_temps = src_temps,
24227 .commute = pattern.commute,
24776 .temps = undefined,
24777 .labels = @splat(.{ .forward = @splat(null), .backward = null }),
2422824778 .limb = undefined,
24779 .mask_limb = undefined,
2422924780 };
24230 for (&s.extra_temps, case.extra_temps) |*temp, spec| temp.* = try spec.create(s) orelse continue;
24781 const tmp_slots = s.temps[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)];
24782 const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)];
24783 const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)];
24784
24785 for (tmp_slots, case.extra_temps) |*slot, spec| slot.* = try spec.create(&s) orelse continue;
2423124786
2423224787 while (true) for (pattern.src, src_temps) |src_pattern, *src_temp| {
2423324788 if (try src_pattern.convert(src_temp, cg)) break;
2423424789 } else break;
24790 @memcpy(src_slots[0..src_temps.len], src_temps);
24791 std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
2423524792
2423624793 if (case.clobbers.eflags or case.each != .once) try cg.spillEflagsIfOccupied();
2423724794
2423824795 for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, dst_kind|
24239 dst_temp.* = (try Select2.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, s)).?;
24796 dst_temp.* = (try Select.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, &s)).?;
24797 @memcpy(dst_slots[0..dst_temps.len], dst_temps);
2424024798
2424124799 switch (case.each) {
2424224800 .once => |body| for (body) |inst| try s.emit(inst),
2424324801 .limb => |limb| {
24244 const limb_size, const limb_of_temp = limb.of.unwrap(s);
24245 const limb_of_size: u31 = @intCast(limb_of_temp.typeOf(cg).abiSize(cg.pt.zcu));
24802 const limb_of_size: i32 = @intCast(limb.of.ref.deref(&s).typeOf(cg).abiSize(cg.pt.zcu));
2424624803 s.limb = .{
24247 .size = limb_size,
24248 .index = s.extra_temps[0].tracking(cg).short.register.to64(),
24804 .size = limb.of.size,
24805 .index = (try Select.Operand.tmp0p.lower(&s)).reg,
2424924806 .disp = limb_of_size,
2425024807 };
2425124808 for (limb.header) |inst| try s.emit(inst);
24252 try cg.asmRegisterImmediate(.{ ._, .mov }, s.limb.index, .s(-@as(i32, limb_of_size)));
24253 const limb_loop_reloc: u32 = @intCast(cg.mir_instructions.len);
24809 try s.emit(.{ ._, ._, .mov, .tmp0p, .i(-limb_of_size), ._, ._ });
24810 assert(s.labels[0].backward == null);
24811 s.labels[0].backward = @intCast(cg.mir_instructions.len);
2425424812 for (limb.body) |inst| try s.emit(inst);
24255 try cg.asmRegisterImmediate(
24256 .{ ._, .add },
24257 s.limb.index,
24258 .s(@intCast(@divExact(limb_size.bitSize(), 8))),
24259 );
24260 _ = try cg.asmJccReloc(.nc, limb_loop_reloc);
24813 try s.emit(.{ ._, ._, .add, .tmp0p, .i(@intCast(@divExact(limb.of.size.bitSize(cg.target), 8))), ._, ._ });
24814 try s.emit(.{ ._, ._nc, .j, .@"0b", ._, ._, ._ });
2426124815 for (limb.trailer) |inst| try s.emit(inst);
2426224816 },
24817 .limb_and_mask_limb => |limb| {
24818 const limb_of_size: i32 = @intCast(limb.of.ref.deref(&s).typeOf(cg).abiSize(cg.pt.zcu));
24819 s.limb = .{
24820 .size = limb.of.size,
24821 .index = (try Select.Operand.tmp0p.lower(&s)).reg,
24822 .disp = limb_of_size,
24823 };
24824 const mask_limb_bit_size: u31 = @intCast(@divExact(
24825 limb.of.size.bitSize(cg.target),
24826 limb.of_mask.size.bitSize(cg.target),
24827 ));
24828 if (mask_limb_bit_size >= 8) {
24829 s.mask_limb = .{
24830 .size = .fromBitSize(mask_limb_bit_size),
24831 .index = (try Select.Operand.tmp1p.lower(&s)).reg,
24832 };
24833 for (limb.header) |inst| try s.emit(inst);
24834 try s.emit(.{ ._, ._, .mov, .tmp0p, .i(-limb_of_size), ._, ._ });
24835 try s.emit(.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ });
24836 assert(s.labels[0].backward == null);
24837 s.labels[0].backward = @intCast(cg.mir_instructions.len);
24838 for (limb.body) |inst| try s.emit(inst);
24839 try s.emit(.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, @divExact(mask_limb_bit_size, 8)), ._, ._ });
24840 try s.emit(.{ ._, ._, .add, .tmp0p, .i(@intCast(@divExact(limb.of.size.bitSize(cg.target), 8))), ._, ._ });
24841 try s.emit(.{ ._, ._nc, .j, .@"0b", ._, ._, ._ });
24842 for (limb.trailer) |inst| try s.emit(inst);
24843 } else {
24844 for (limb.header) |inst| try s.emit(inst);
24845 try s.emit(.{ ._, ._, .mov, .tmp0p, .i(-limb_of_size), ._, ._ });
24846 try s.emit(.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ });
24847 try s.emit(.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ });
24848 assert(s.labels[0].backward == null);
24849 s.labels[0].backward = @intCast(cg.mir_instructions.len);
24850 for (limb.body) |inst| try s.emit(inst);
24851 try s.emit(.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, mask_limb_bit_size), ._, ._ });
24852 try s.emit(.{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ });
24853 try s.emit(.{ ._, ._nz, .j, .@"1f", ._, ._, ._ });
24854 try s.emit(.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ });
24855 try s.emit(.{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ });
24856 try s.emit(.{ ._, ._, .mov, .memid(.{ .ref = limb.of_mask.ref, .size = .byte }, .tmp3, -1), .tmp2b, ._, ._ });
24857 try s.emit(.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ });
24858 try s.emit(.{ .@"1:", ._, .add, .tmp0p, .i(@intCast(@divExact(limb.of.size.bitSize(cg.target), 8))), ._, ._ });
24859 try s.emit(.{ ._, ._nc, .j, .@"0b", ._, ._, ._ });
24860 try s.emit(.{ ._, ._, .lea, .tmp3d, .lead(.none, .tmp1, -1), ._, ._ });
24861 try s.emit(.{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ });
24862 try s.emit(.{ ._, ._, .mov, .memi(.{ .ref = limb.of_mask.ref, .size = .byte }, .tmp3), .tmp2b, ._, ._ });
24863 for (limb.trailer) |inst| try s.emit(inst);
24864 }
24865 },
2426324866 }
24867 s.emitLabel(.@"0:");
2426424868
24265 for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, s);
24266 for (case.extra_temps, s.extra_temps) |spec, temp| if (spec.kind != .unused) try temp.die(cg);
24869 for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, &s);
24870 for (case.extra_temps, tmp_slots) |spec, temp| if (spec.kind != .unused) try temp.die(cg);
2426724871 return;
2426824872 }
2426924873 }
24270 return error.Select2Failed;
24874 return error.SelectFailed;
2427124875}
src/arch/x86_64/Encoding.zig+20-9
......@@ -30,9 +30,10 @@ pub fn findByMnemonic(
3030 prefix: Instruction.Prefix,
3131 mnemonic: Mnemonic,
3232 ops: []const Instruction.Operand,
33 target: *const std.Target,
3334) !?Encoding {
34 var input_ops = [1]Op{.none} ** 4;
35 for (input_ops[0..ops.len], ops) |*input_op, op| input_op.* = Op.fromOperand(op);
35 var input_ops: [4]Op = @splat(.none);
36 for (input_ops[0..ops.len], ops) |*input_op, op| input_op.* = Op.fromOperand(op, target);
3637
3738 const rex_required = for (ops) |op| switch (op) {
3839 .reg => |r| switch (r) {
......@@ -57,6 +58,16 @@ pub fn findByMnemonic(
5758 var shortest_enc: ?Encoding = null;
5859 var shortest_len: ?usize = null;
5960 next: for (mnemonic_to_encodings_map[@intFromEnum(mnemonic)]) |data| {
61 if (!switch (data.feature) {
62 .none => true,
63 inline else => |tag| has_features: {
64 comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' ');
65 comptime var features: []const std.Target.x86.Feature = &.{};
66 inline while (comptime feature_it.next()) |feature| features = features ++ .{@field(std.Target.x86.Feature, feature)};
67 break :has_features std.Target.x86.featureSetHasAll(target.cpu.features, features[0..features.len].*);
68 },
69 }) continue;
70
6071 switch (data.mode) {
6172 .none, .short => if (rex_required) continue,
6273 .rex, .rex_short => if (!rex_required) continue,
......@@ -64,7 +75,7 @@ pub fn findByMnemonic(
6475 }
6576 for (input_ops, data.ops) |input_op, data_op| if (!input_op.isSubset(data_op)) continue :next;
6677
67 const enc = Encoding{ .mnemonic = mnemonic, .data = data };
78 const enc: Encoding = .{ .mnemonic = mnemonic, .data = data };
6879 if (shortest_enc) |previous_shortest_enc| {
6980 const len = estimateInstructionLength(prefix, enc, ops);
7081 const previous_shortest_len = shortest_len orelse
......@@ -474,7 +485,7 @@ pub const Op = enum {
474485 ymm, ymm_m256,
475486 // zig fmt: on
476487
477 pub fn fromOperand(operand: Instruction.Operand) Op {
488 pub fn fromOperand(operand: Instruction.Operand, target: *const std.Target) Op {
478489 return switch (operand) {
479490 .none => .none,
480491
......@@ -516,7 +527,7 @@ pub const Op = enum {
516527
517528 .mem => |mem| switch (mem) {
518529 .moffs => .moffs,
519 .sib, .rip => switch (mem.bitSize()) {
530 .sib, .rip => switch (mem.bitSize(target)) {
520531 0 => .m,
521532 8 => .m8,
522533 16 => .m16,
......@@ -835,7 +846,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
835846 var inst = Instruction{
836847 .prefix = prefix,
837848 .encoding = encoding,
838 .ops = [1]Operand{.none} ** 4,
849 .ops = @splat(.none),
839850 };
840851 @memcpy(inst.ops[0..ops.len], ops);
841852
......@@ -850,7 +861,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
850861const mnemonic_to_encodings_map = init: {
851862 @setEvalBranchQuota(5_000);
852863 const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len;
853 var mnemonic_map: [mnemonic_count][]Data = .{&.{}} ** mnemonic_count;
864 var mnemonic_map: [mnemonic_count][]Data = @splat(&.{});
854865 const encodings = @import("encodings.zig");
855866 for (encodings.table) |entry| mnemonic_map[@intFromEnum(entry[0])].len += 1;
856867 var data_storage: [encodings.table.len]Data = undefined;
......@@ -859,7 +870,7 @@ const mnemonic_to_encodings_map = init: {
859870 value.ptr = data_storage[storage_i..].ptr;
860871 storage_i += value.len;
861872 }
862 var mnemonic_i: [mnemonic_count]usize = .{0} ** mnemonic_count;
873 var mnemonic_i: [mnemonic_count]usize = @splat(0);
863874 const ops_len = @typeInfo(std.meta.FieldType(Data, .ops)).array.len;
864875 const opc_len = @typeInfo(std.meta.FieldType(Data, .opc)).array.len;
865876 for (encodings.table) |entry| {
......@@ -876,7 +887,7 @@ const mnemonic_to_encodings_map = init: {
876887 i.* += 1;
877888 }
878889 const final_storage = data_storage;
879 var final_map: [mnemonic_count][]const Data = .{&.{}} ** mnemonic_count;
890 var final_map: [mnemonic_count][]const Data = @splat(&.{});
880891 storage_i = 0;
881892 for (&final_map, mnemonic_map) |*final_value, value| {
882893 final_value.* = final_storage[storage_i..][0..value.len];
src/arch/x86_64/Lower.zig+17-16
......@@ -1,6 +1,7 @@
11//! This file contains the functionality for lowering x86_64 MIR to Instructions
22
33bin_file: *link.File,
4target: *const std.Target,
45output_mode: std.builtin.OutputMode,
56link_mode: std.builtin.LinkMode,
67pic: bool,
......@@ -193,7 +194,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
193194 .pseudo_probe_align_ri_s => {
194195 try lower.emit(.none, .@"test", &.{
195196 .{ .reg = inst.data.ri.r1 },
196 .{ .imm = Immediate.s(@bitCast(inst.data.ri.i)) },
197 .{ .imm = .s(@bitCast(inst.data.ri.i)) },
197198 });
198199 try lower.emit(.none, .jz, &.{
199200 .{ .imm = lower.reloc(.{ .inst = index + 1 }, 0) },
......@@ -229,14 +230,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
229230 }
230231 try lower.emit(.none, .sub, &.{
231232 .{ .reg = inst.data.ri.r1 },
232 .{ .imm = Immediate.s(@bitCast(inst.data.ri.i)) },
233 .{ .imm = .s(@bitCast(inst.data.ri.i)) },
233234 });
234235 assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts);
235236 },
236237 .pseudo_probe_adjust_setup_rri_s => {
237238 try lower.emit(.none, .mov, &.{
238239 .{ .reg = inst.data.rri.r2.to32() },
239 .{ .imm = Immediate.s(@bitCast(inst.data.rri.i)) },
240 .{ .imm = .s(@bitCast(inst.data.rri.i)) },
240241 });
241242 try lower.emit(.none, .sub, &.{
242243 .{ .reg = inst.data.rri.r1 },
......@@ -255,7 +256,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
255256 });
256257 try lower.emit(.none, .sub, &.{
257258 .{ .reg = inst.data.rr.r2 },
258 .{ .imm = Immediate.s(page_size) },
259 .{ .imm = .s(page_size) },
259260 });
260261 try lower.emit(.none, .jae, &.{
261262 .{ .imm = lower.reloc(.{ .inst = index }, 0) },
......@@ -355,7 +356,7 @@ pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
355356 .mi_s,
356357 .rmi_s,
357358 .pseudo_dbg_local_ai_s,
358 => Immediate.s(@bitCast(i)),
359 => .s(@bitCast(i)),
359360
360361 .rrri,
361362 .rri_u,
......@@ -368,11 +369,11 @@ pub fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
368369 .rrm,
369370 .rrmi,
370371 .pseudo_dbg_local_ai_u,
371 => Immediate.u(i),
372 => .u(i),
372373
373374 .ri_64,
374375 .pseudo_dbg_local_ai_64,
375 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
376 => .u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
376377
377378 else => unreachable,
378379 };
......@@ -389,7 +390,7 @@ fn reloc(lower: *Lower, target: Reloc.Target, off: i32) Immediate {
389390 .off = off,
390391 };
391392 lower.result_relocs_len += 1;
392 return Immediate.s(0);
393 return .s(0);
393394}
394395
395396fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
......@@ -421,15 +422,15 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
421422 try Instruction.new(.none, .lea, &[_]Operand{
422423 .{ .reg = .rdi },
423424 .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) },
424 });
425 }, lower.target);
425426 lower.result_insts_len += 1;
426427 _ = lower.reloc(.{
427428 .linker_extern_fn = try elf_file.getGlobalSymbol("__tls_get_addr", null),
428429 }, 0);
429430 lower.result_insts[lower.result_insts_len] =
430431 try Instruction.new(.none, .call, &[_]Operand{
431 .{ .imm = Immediate.s(0) },
432 });
432 .{ .imm = .s(0) },
433 }, lower.target);
433434 lower.result_insts_len += 1;
434435 _ = lower.reloc(.{ .linker_dtpoff = sym_index }, 0);
435436 emit_mnemonic = .lea;
......@@ -443,7 +444,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
443444 try Instruction.new(.none, .mov, &[_]Operand{
444445 .{ .reg = .rax },
445446 .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .fs } }) },
446 });
447 }, lower.target);
447448 lower.result_insts_len += 1;
448449 _ = lower.reloc(.{ .linker_reloc = sym_index }, 0);
449450 emit_mnemonic = .lea;
......@@ -467,7 +468,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
467468 try Instruction.new(.none, .mov, &[_]Operand{
468469 .{ .reg = reg.to64() },
469470 .{ .mem = Memory.initRip(.qword, 0) },
470 });
471 }, lower.target);
471472 lower.result_insts_len += 1;
472473 break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{ .base = .{
473474 .reg = reg.to64(),
......@@ -482,7 +483,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
482483 }) },
483484 .lea => {
484485 emit_mnemonic = .mov;
485 break :op .{ .imm = Immediate.s(0) };
486 break :op .{ .imm = .s(0) };
486487 },
487488 .mov => break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{
488489 .base = .{ .reg = .ds },
......@@ -541,7 +542,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
541542 };
542543 }
543544 lower.result_insts[lower.result_insts_len] =
544 try Instruction.new(emit_prefix, emit_mnemonic, emit_ops);
545 try Instruction.new(emit_prefix, emit_mnemonic, emit_ops, lower.target);
545546 lower.result_insts_len += 1;
546547}
547548
......@@ -743,7 +744,7 @@ fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Er
743744 while (it.next()) |i| {
744745 try lower.emit(.directive, .@".cfi_rel_offset", &.{
745746 .{ .reg = callee_preserved_regs[i] },
746 .{ .imm = Immediate.s(off) },
747 .{ .imm = .s(off) },
747748 });
748749 off += 8;
749750 }
src/arch/x86_64/Mir.zig+10-4
......@@ -214,6 +214,10 @@ pub const Inst = struct {
214214 p_q,
215215 /// Packed ___ Double Quadword
216216 p_dq,
217 /// ___ Aligned Packed Integer Values
218 _dqa,
219 /// ___ Unaligned Packed Integer Values
220 _dqu,
217221
218222 /// ___ Scalar Single-Precision Values
219223 _ss,
......@@ -234,6 +238,10 @@ pub const Inst = struct {
234238 v_d,
235239 /// VEX-Encoded ___ QuadWord
236240 v_q,
241 /// VEX-Encoded ___ Aligned Packed Integer Values
242 v_dqa,
243 /// VEX-Encoded ___ Unaligned Packed Integer Values
244 v_dqu,
237245 /// VEX-Encoded ___ Integer Data
238246 v_i128,
239247 /// VEX-Encoded Packed ___
......@@ -362,6 +370,8 @@ pub const Inst = struct {
362370 /// Move scalar double-precision floating-point value
363371 /// Move doubleword
364372 /// Move quadword
373 /// Move aligned packed integer values
374 /// Move unaligned packed integer values
365375 mov,
366376 /// Move data after swapping bytes
367377 movbe,
......@@ -609,10 +619,6 @@ pub const Inst = struct {
609619 cvttps2dq,
610620 /// Convert with truncation scalar double-precision floating-point value to doubleword integer
611621 cvttsd2si,
612 /// Move aligned packed integer values
613 movdqa,
614 /// Move unaligned packed integer values
615 movdqu,
616622 /// Packed interleave shuffle of quadruplets of single-precision floating-point values
617623 /// Packed interleave shuffle of pairs of double-precision floating-point values
618624 /// Shuffle packed doublewords
src/arch/x86_64/bits.zig+11-6
......@@ -479,8 +479,8 @@ pub const RegisterOffset = struct { reg: Register, off: i32 = 0 };
479479pub const SymbolOffset = struct { sym_index: u32, off: i32 = 0 };
480480
481481pub const Memory = struct {
482 base: Base,
483 mod: Mod,
482 base: Base = .none,
483 mod: Mod = .{ .rm = .{} },
484484
485485 pub const Base = union(enum(u2)) {
486486 none,
......@@ -503,7 +503,7 @@ pub const Memory = struct {
503503 off: u64,
504504
505505 pub const Rm = struct {
506 size: Size,
506 size: Size = .none,
507507 index: Register = .none,
508508 scale: Scale = .@"1",
509509 disp: i32 = 0,
......@@ -512,6 +512,7 @@ pub const Memory = struct {
512512
513513 pub const Size = enum(u4) {
514514 none,
515 ptr,
515516 byte,
516517 word,
517518 dword,
......@@ -548,9 +549,10 @@ pub const Memory = struct {
548549 };
549550 }
550551
551 pub fn bitSize(s: Size) u64 {
552 pub fn bitSize(s: Size, target: *const std.Target) u64 {
552553 return switch (s) {
553554 .none => 0,
555 .ptr => target.ptrBitWidth(),
554556 .byte => 8,
555557 .word => 16,
556558 .dword => 32,
......@@ -569,8 +571,11 @@ pub const Memory = struct {
569571 writer: anytype,
570572 ) @TypeOf(writer).Error!void {
571573 if (s == .none) return;
572 try writer.writeAll(@tagName(s));
573 try writer.writeAll(" ptr");
574 if (s != .ptr) {
575 try writer.writeAll(@tagName(s));
576 try writer.writeByte(' ');
577 }
578 try writer.writeAll("ptr");
574579 }
575580 };
576581
src/arch/x86_64/encoder.zig+19-14
......@@ -167,11 +167,11 @@ pub const Instruction = struct {
167167 };
168168 }
169169
170 pub fn bitSize(mem: Memory) u64 {
170 pub fn bitSize(mem: Memory, target: *const std.Target) u64 {
171171 return switch (mem) {
172 .rip => |r| r.ptr_size.bitSize(),
173 .sib => |s| s.ptr_size.bitSize(),
174 .moffs => 64,
172 .rip => |r| r.ptr_size.bitSize(target),
173 .sib => |s| s.ptr_size.bitSize(target),
174 .moffs => target.ptrBitWidth(),
175175 };
176176 }
177177 };
......@@ -314,16 +314,21 @@ pub const Instruction = struct {
314314 }
315315 };
316316
317 pub fn new(prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) !Instruction {
317 pub fn new(
318 prefix: Prefix,
319 mnemonic: Mnemonic,
320 ops: []const Operand,
321 target: *const std.Target,
322 ) !Instruction {
318323 const encoding: Encoding = switch (prefix) {
319 else => (try Encoding.findByMnemonic(prefix, mnemonic, ops)) orelse {
324 else => (try Encoding.findByMnemonic(prefix, mnemonic, ops, target)) orelse {
320325 log.err("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
321326 @tagName(prefix),
322327 @tagName(mnemonic),
323 @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0]) else .none),
324 @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1]) else .none),
325 @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2]) else .none),
326 @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3]) else .none),
328 @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none),
329 @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none),
330 @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none),
331 @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none),
327332 });
328333 return error.InvalidInstruction;
329334 },
......@@ -332,10 +337,10 @@ pub const Instruction = struct {
332337 .data = .{
333338 .op_en = .zo,
334339 .ops = .{
335 if (ops.len > 0) Encoding.Op.fromOperand(ops[0]) else .none,
336 if (ops.len > 1) Encoding.Op.fromOperand(ops[1]) else .none,
337 if (ops.len > 2) Encoding.Op.fromOperand(ops[2]) else .none,
338 if (ops.len > 3) Encoding.Op.fromOperand(ops[3]) else .none,
340 if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none,
341 if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none,
342 if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none,
343 if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none,
339344 },
340345 .opc_len = 0,
341346 .opc = undefined,
src/link/Elf/Atom.zig+19-17
......@@ -976,6 +976,7 @@ const x86_64 = struct {
976976 it: *RelocsIterator,
977977 ) !void {
978978 dev.check(.x86_64_backend);
979 const t = &elf_file.base.comp.root_mod.resolved_target.result;
979980 const is_static = elf_file.base.isStatic();
980981 const is_dyn_lib = elf_file.isEffectivelyDynLib();
981982
......@@ -1046,7 +1047,7 @@ const x86_64 = struct {
10461047 .GOTTPOFF => {
10471048 const should_relax = blk: {
10481049 if (is_dyn_lib or symbol.flags.import) break :blk false;
1049 if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..])) break :blk false;
1050 if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..], t)) break :blk false;
10501051 break :blk true;
10511052 };
10521053 if (!should_relax) {
......@@ -1090,6 +1091,7 @@ const x86_64 = struct {
10901091 stream: anytype,
10911092 ) (error{ InvalidInstruction, CannotEncode } || RelocError)!void {
10921093 dev.check(.x86_64_backend);
1094 const t = &elf_file.base.comp.root_mod.resolved_target.result;
10931095 const diags = &elf_file.base.comp.link_diags;
10941096 const r_type: elf.R_X86_64 = @enumFromInt(rel.r_type());
10951097 const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow;
......@@ -1120,7 +1122,7 @@ const x86_64 = struct {
11201122
11211123 .GOTPCRELX => {
11221124 if (!target.flags.import and !target.isIFunc(elf_file) and !target.isAbs(elf_file)) blk: {
1123 x86_64.relaxGotpcrelx(code[r_offset - 2 ..]) catch break :blk;
1125 x86_64.relaxGotpcrelx(code[r_offset - 2 ..], t) catch break :blk;
11241126 try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little);
11251127 return;
11261128 }
......@@ -1129,7 +1131,7 @@ const x86_64 = struct {
11291131
11301132 .REX_GOTPCRELX => {
11311133 if (!target.flags.import and !target.isIFunc(elf_file) and !target.isAbs(elf_file)) blk: {
1132 x86_64.relaxRexGotpcrelx(code[r_offset - 3 ..]) catch break :blk;
1134 x86_64.relaxRexGotpcrelx(code[r_offset - 3 ..], t) catch break :blk;
11331135 try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little);
11341136 return;
11351137 }
......@@ -1184,7 +1186,7 @@ const x86_64 = struct {
11841186 const S_ = target.tlsDescAddress(elf_file);
11851187 try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
11861188 } else {
1187 x86_64.relaxGotPcTlsDesc(code[r_offset - 3 ..]) catch {
1189 x86_64.relaxGotPcTlsDesc(code[r_offset - 3 ..], t) catch {
11881190 var err = try diags.addErrorWithNotes(1);
11891191 try err.addMsg("could not relax {s}", .{@tagName(r_type)});
11901192 err.addNote("in {}:{s} at offset 0x{x}", .{
......@@ -1208,7 +1210,7 @@ const x86_64 = struct {
12081210 const S_ = target.gotTpAddress(elf_file);
12091211 try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little);
12101212 } else {
1211 x86_64.relaxGotTpOff(code[r_offset - 3 ..]);
1213 x86_64.relaxGotTpOff(code[r_offset - 3 ..], t);
12121214 try cwriter.writeInt(i32, @as(i32, @intCast(S - TP)), .little);
12131215 }
12141216 },
......@@ -1269,31 +1271,31 @@ const x86_64 = struct {
12691271 }
12701272 }
12711273
1272 fn relaxGotpcrelx(code: []u8) !void {
1274 fn relaxGotpcrelx(code: []u8, t: *const std.Target) !void {
12731275 dev.check(.x86_64_backend);
12741276 const old_inst = disassemble(code) orelse return error.RelaxFailure;
12751277 const inst = switch (old_inst.encoding.mnemonic) {
12761278 .call => try Instruction.new(old_inst.prefix, .call, &.{
12771279 // TODO: hack to force imm32s in the assembler
12781280 .{ .imm = Immediate.s(-129) },
1279 }),
1281 }, t),
12801282 .jmp => try Instruction.new(old_inst.prefix, .jmp, &.{
12811283 // TODO: hack to force imm32s in the assembler
12821284 .{ .imm = Immediate.s(-129) },
1283 }),
1285 }, t),
12841286 else => return error.RelaxFailure,
12851287 };
12861288 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
1287 const nop = try Instruction.new(.none, .nop, &.{});
1289 const nop = try Instruction.new(.none, .nop, &.{}, t);
12881290 try encode(&.{ nop, inst }, code);
12891291 }
12901292
1291 fn relaxRexGotpcrelx(code: []u8) !void {
1293 fn relaxRexGotpcrelx(code: []u8, t: *const std.Target) !void {
12921294 dev.check(.x86_64_backend);
12931295 const old_inst = disassemble(code) orelse return error.RelaxFailure;
12941296 switch (old_inst.encoding.mnemonic) {
12951297 .mov => {
1296 const inst = try Instruction.new(old_inst.prefix, .lea, &old_inst.ops);
1298 const inst = try Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t);
12971299 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
12981300 try encode(&.{inst}, code);
12991301 },
......@@ -1398,7 +1400,7 @@ const x86_64 = struct {
13981400 }
13991401 }
14001402
1401 fn canRelaxGotTpOff(code: []const u8) bool {
1403 fn canRelaxGotTpOff(code: []const u8, t: *const std.Target) bool {
14021404 dev.check(.x86_64_backend);
14031405 const old_inst = disassemble(code) orelse return false;
14041406 switch (old_inst.encoding.mnemonic) {
......@@ -1406,7 +1408,7 @@ const x86_64 = struct {
14061408 old_inst.ops[0],
14071409 // TODO: hack to force imm32s in the assembler
14081410 .{ .imm = Immediate.s(-129) },
1409 })) |inst| {
1411 }, t)) |inst| {
14101412 inst.encode(std.io.null_writer, .{}) catch return false;
14111413 return true;
14121414 } else |_| return false,
......@@ -1414,7 +1416,7 @@ const x86_64 = struct {
14141416 }
14151417 }
14161418
1417 fn relaxGotTpOff(code: []u8) void {
1419 fn relaxGotTpOff(code: []u8, t: *const std.Target) void {
14181420 dev.check(.x86_64_backend);
14191421 const old_inst = disassemble(code) orelse unreachable;
14201422 switch (old_inst.encoding.mnemonic) {
......@@ -1423,7 +1425,7 @@ const x86_64 = struct {
14231425 old_inst.ops[0],
14241426 // TODO: hack to force imm32s in the assembler
14251427 .{ .imm = Immediate.s(-129) },
1426 }) catch unreachable;
1428 }, t) catch unreachable;
14271429 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
14281430 encode(&.{inst}, code) catch unreachable;
14291431 },
......@@ -1431,7 +1433,7 @@ const x86_64 = struct {
14311433 }
14321434 }
14331435
1434 fn relaxGotPcTlsDesc(code: []u8) !void {
1436 fn relaxGotPcTlsDesc(code: []u8, target: *const std.Target) !void {
14351437 dev.check(.x86_64_backend);
14361438 const old_inst = disassemble(code) orelse return error.RelaxFailure;
14371439 switch (old_inst.encoding.mnemonic) {
......@@ -1440,7 +1442,7 @@ const x86_64 = struct {
14401442 old_inst.ops[0],
14411443 // TODO: hack to force imm32s in the assembler
14421444 .{ .imm = Immediate.s(-129) },
1443 });
1445 }, target);
14441446 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
14451447 try encode(&.{inst}, code);
14461448 },
test/behavior/x86_64/math.zig+38-1
......@@ -264,11 +264,48 @@ fn testBinary(comptime op: anytype) !void {
264264 0xed533d18f8657f3f, 0x1ddd7cd7f6bab957,
265265 });
266266
267 if (false) try testType(@Vector(1, u128), .{
267 try testType(@Vector(1, u128), .{
268268 0x5f11e16b0ca3392f907a857881455d2e,
269269 }, .{
270270 0xf9142d73b408fd6955922f9fc147f7d7,
271271 });
272 try testType(@Vector(2, u128), .{
273 0xee0fb41fabd805923fb21b5c658e3a87,
274 0x2352e74aad6c58b3255ff0bba5aa6552,
275 }, .{
276 0x8d822f9fdd9cb9a5b43513b14419b224,
277 0x1aef2a02704379e38ead4d53d69e4cc4,
278 });
279 try testType(@Vector(4, u128), .{
280 0xc74437a4ea3bbbb193dbf0ea2f0c5281,
281 0x039e4b1640868248780db1834a0027eb,
282 0xb9e8bb34155b2b238da20331d08ff85b,
283 0x863802d34a54c2e6aa71dd0f067c4904,
284 }, .{
285 0x7471bae24ff7b84ab107f86ba2b7d1e7,
286 0x8f34c449d0576e682c20bda74aa6b6c9,
287 0x1f34c3efa167b61c48c9d5ec01a1a93f,
288 0x71c8318fcf3ddc7be058c73a52dce9e3,
289 });
290 try testType(@Vector(8, u128), .{
291 0xbf2db71463037f55ee338431f902a906,
292 0xb7ad317626655f38ab25ae30d8a1aa67,
293 0x7d3c5a3ffaa607b5560d69ae3fcf7863,
294 0x009a39a8badf8b628c686dc176aa1273,
295 0x49dba3744c91304cc7bbbdab61b6c969,
296 0x6ec664b624f7acf79ce69d80ed7bc85c,
297 0xe02d7a303c0f00c39010f3b815547f1c,
298 0xb13e1ee914616f58cffe6acd33d9b5c8,
299 }, .{
300 0x2f2d355a071942a7384f82ba72a945b8,
301 0x61f151b3afec8cb7664f813cecf581d1,
302 0x5bfbf5484f3a07f0eacc4739ff48af80,
303 0x59c0abbf8d829cf525a87d5c9c41a38a,
304 0xdad8b18eb680f0520ca49ebfb5842e22,
305 0xa05adcaedd9057480b3ba0413d003cec,
306 0x8b0b4a27fc94a0e90652d19bc755b63d,
307 0xa858bce5ad0e48c13588a4e170e8667c,
308 });
272309}
273310
274311inline fn bitAnd(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs & rhs) {