authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-12-28 22:51:41-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:42:08-05:00
logac1a975f9b5a7d939663fa90556a2f038250c531
tree9df8cf0677d30645b13249b6357240bdf533a67e
parenta7efc56d8680bb51cc2488bbc0680b2fc080174f

x86_64: implement clz and not

19 files changed, 6965 insertions(+), 1949 deletions(-)

lib/std/Target/Query.zig+4-4
......@@ -6,13 +6,13 @@
66/// `null` means native.
77cpu_arch: ?Target.Cpu.Arch = null,
88
9cpu_model: CpuModel = CpuModel.determined_by_arch_os,
9cpu_model: CpuModel = .determined_by_arch_os,
1010
1111/// Sparse set of CPU features to add to the set from `cpu_model`.
12cpu_features_add: Target.Cpu.Feature.Set = Target.Cpu.Feature.Set.empty,
12cpu_features_add: Target.Cpu.Feature.Set = .empty,
1313
1414/// Sparse set of CPU features to remove from the set from `cpu_model`.
15cpu_features_sub: Target.Cpu.Feature.Set = Target.Cpu.Feature.Set.empty,
15cpu_features_sub: Target.Cpu.Feature.Set = .empty,
1616
1717/// `null` means native.
1818os_tag: ?Target.Os.Tag = null,
......@@ -38,7 +38,7 @@ abi: ?Target.Abi = null,
3838
3939/// When `os_tag` is `null`, then `null` means native. Otherwise it means the standard path
4040/// based on the `os_tag`.
41dynamic_linker: Target.DynamicLinker = Target.DynamicLinker.none,
41dynamic_linker: Target.DynamicLinker = .none,
4242
4343/// `null` means default for the cpu/arch/os combo.
4444ofmt: ?Target.ObjectFormat = null,
lib/std/Target/x86.zig+65
......@@ -47,6 +47,7 @@ pub const Feature = enum {
4747 bmi2,
4848 branch_hint,
4949 branchfusion,
50 bsf_bsr_0_clobbers_result,
5051 ccmp,
5152 cf,
5253 cldemote,
......@@ -167,6 +168,8 @@ pub const Feature = enum {
167168 slow_unaligned_mem_32,
168169 sm3,
169170 sm4,
171 smap,
172 smep,
170173 soft_float,
171174 sse,
172175 sse2,
......@@ -497,6 +500,11 @@ pub const all_features = blk: {
497500 .description = "CMP/TEST can be fused with conditional branches",
498501 .dependencies = featureSet(&[_]Feature{}),
499502 };
503 result[@intFromEnum(Feature.bsf_bsr_0_clobbers_result)] = .{
504 .llvm_name = null,
505 .description = "BSF/BSR may clobber the lower 32-bits of the result register when the source is zero",
506 .dependencies = featureSet(&[_]Feature{}),
507 };
500508 result[@intFromEnum(Feature.ccmp)] = .{
501509 .llvm_name = "ccmp",
502510 .description = "Support conditional cmp & test instructions",
......@@ -1127,6 +1135,16 @@ pub const all_features = blk: {
11271135 .avx2,
11281136 }),
11291137 };
1138 result[@intFromEnum(Feature.smap)] = .{
1139 .llvm_name = null,
1140 .description = "Enable Supervisor Mode Access Prevention",
1141 .dependencies = featureSet(&[_]Feature{}),
1142 };
1143 result[@intFromEnum(Feature.smep)] = .{
1144 .llvm_name = null,
1145 .description = "Enable Supervisor Mode Execution Prevention",
1146 .dependencies = featureSet(&[_]Feature{}),
1147 };
11301148 result[@intFromEnum(Feature.soft_float)] = .{
11311149 .llvm_name = "soft-float",
11321150 .description = "Use software floating point features",
......@@ -1371,6 +1389,8 @@ pub const cpu = struct {
13711389 .sha,
13721390 .shstk,
13731391 .slow_3ops_lea,
1392 .smap,
1393 .smep,
13741394 .tuning_fast_imm_vector_shift,
13751395 .vaes,
13761396 .vpclmulqdq,
......@@ -1467,6 +1487,8 @@ pub const cpu = struct {
14671487 .sha,
14681488 .shstk,
14691489 .slow_3ops_lea,
1490 .smap,
1491 .smep,
14701492 .tuning_fast_imm_vector_shift,
14711493 .uintr,
14721494 .vaes,
......@@ -1545,6 +1567,8 @@ pub const cpu = struct {
15451567 .slow_3ops_lea,
15461568 .sm3,
15471569 .sm4,
1570 .smap,
1571 .smep,
15481572 .tuning_fast_imm_vector_shift,
15491573 .uintr,
15501574 .vaes,
......@@ -1783,6 +1807,8 @@ pub const cpu = struct {
17831807 .sahf,
17841808 .sbb_dep_breaking,
17851809 .slow_shld,
1810 .smap,
1811 .smep,
17861812 .sse4a,
17871813 .vzeroupper,
17881814 .x87,
......@@ -1995,6 +2021,8 @@ pub const cpu = struct {
19952021 .rdseed,
19962022 .sahf,
19972023 .slow_3ops_lea,
2024 .smap,
2025 .smep,
19982026 .vzeroupper,
19992027 .x87,
20002028 .xsaveopt,
......@@ -2136,6 +2164,8 @@ pub const cpu = struct {
21362164 .sahf,
21372165 .sha,
21382166 .slow_3ops_lea,
2167 .smap,
2168 .smep,
21392169 .tuning_fast_imm_vector_shift,
21402170 .vzeroupper,
21412171 .x87,
......@@ -2195,6 +2225,8 @@ pub const cpu = struct {
21952225 .rdseed,
21962226 .sahf,
21972227 .slow_3ops_lea,
2228 .smap,
2229 .smep,
21982230 .tuning_fast_imm_vector_shift,
21992231 .vzeroupper,
22002232 .x87,
......@@ -2450,6 +2482,8 @@ pub const cpu = struct {
24502482 .serialize,
24512483 .sha,
24522484 .shstk,
2485 .smap,
2486 .smep,
24532487 .tsxldtrk,
24542488 .tuning_fast_imm_vector_shift,
24552489 .uintr,
......@@ -2519,6 +2553,8 @@ pub const cpu = struct {
25192553 .slow_incdec,
25202554 .slow_lea,
25212555 .slow_two_mem_ops,
2556 .smap,
2557 .smep,
25222558 .sse4_2,
25232559 .use_glm_div_sqrt_costs,
25242560 .vzeroupper,
......@@ -2898,6 +2934,7 @@ pub const cpu = struct {
28982934 .rdrnd,
28992935 .sahf,
29002936 .slow_3ops_lea,
2937 .smep,
29012938 .vzeroupper,
29022939 .x87,
29032940 .xsaveopt,
......@@ -2907,6 +2944,7 @@ pub const cpu = struct {
29072944 .name = "i386",
29082945 .llvm_name = "i386",
29092946 .features = featureSet(&[_]Feature{
2947 .bsf_bsr_0_clobbers_result,
29102948 .slow_unaligned_mem_16,
29112949 .vzeroupper,
29122950 .x87,
......@@ -2916,6 +2954,7 @@ pub const cpu = struct {
29162954 .name = "i486",
29172955 .llvm_name = "i486",
29182956 .features = featureSet(&[_]Feature{
2957 .bsf_bsr_0_clobbers_result,
29192958 .slow_unaligned_mem_16,
29202959 .vzeroupper,
29212960 .x87,
......@@ -3096,6 +3135,7 @@ pub const cpu = struct {
30963135 .sahf,
30973136 .slow_3ops_lea,
30983137 .slow_unaligned_mem_32,
3138 .smep,
30993139 .vzeroupper,
31003140 .x87,
31013141 .xsaveopt,
......@@ -3403,6 +3443,8 @@ pub const cpu = struct {
34033443 .sha,
34043444 .shstk,
34053445 .slow_3ops_lea,
3446 .smap,
3447 .smep,
34063448 .tuning_fast_imm_vector_shift,
34073449 .vaes,
34083450 .vpclmulqdq,
......@@ -3766,6 +3808,8 @@ pub const cpu = struct {
37663808 .sha,
37673809 .shstk,
37683810 .slow_3ops_lea,
3811 .smap,
3812 .smep,
37693813 .tuning_fast_imm_vector_shift,
37703814 .vaes,
37713815 .vpclmulqdq,
......@@ -3831,6 +3875,8 @@ pub const cpu = struct {
38313875 .rdseed,
38323876 .sahf,
38333877 .sha,
3878 .smap,
3879 .smep,
38343880 .tuning_fast_imm_vector_shift,
38353881 .vaes,
38363882 .vpclmulqdq,
......@@ -3939,6 +3985,8 @@ pub const cpu = struct {
39393985 .serialize,
39403986 .sha,
39413987 .shstk,
3988 .smap,
3989 .smep,
39423990 .tsxldtrk,
39433991 .tuning_fast_imm_vector_shift,
39443992 .uintr,
......@@ -4042,6 +4090,7 @@ pub const cpu = struct {
40424090 .slow_lea,
40434091 .slow_pmulld,
40444092 .slow_two_mem_ops,
4093 .smep,
40454094 .sse4_2,
40464095 .use_slm_arith_costs,
40474096 .vzeroupper,
......@@ -4098,6 +4147,8 @@ pub const cpu = struct {
40984147 .rdseed,
40994148 .sahf,
41004149 .slow_3ops_lea,
4150 .smap,
4151 .smep,
41014152 .tuning_fast_imm_vector_shift,
41024153 .vzeroupper,
41034154 .x87,
......@@ -4150,6 +4201,8 @@ pub const cpu = struct {
41504201 .rdseed,
41514202 .sahf,
41524203 .slow_3ops_lea,
4204 .smap,
4205 .smep,
41534206 .vzeroupper,
41544207 .x87,
41554208 .xsavec,
......@@ -4305,6 +4358,8 @@ pub const cpu = struct {
43054358 .sahf,
43064359 .sha,
43074360 .shstk,
4361 .smap,
4362 .smep,
43084363 .tuning_fast_imm_vector_shift,
43094364 .vaes,
43104365 .vpclmulqdq,
......@@ -4574,6 +4629,8 @@ pub const cpu = struct {
45744629 .sbb_dep_breaking,
45754630 .sha,
45764631 .slow_shld,
4632 .smap,
4633 .smep,
45774634 .sse4a,
45784635 .vzeroupper,
45794636 .x87,
......@@ -4629,6 +4686,8 @@ pub const cpu = struct {
46294686 .sbb_dep_breaking,
46304687 .sha,
46314688 .slow_shld,
4689 .smap,
4690 .smep,
46324691 .sse4a,
46334692 .vzeroupper,
46344693 .wbnoinvd,
......@@ -4686,6 +4745,8 @@ pub const cpu = struct {
46864745 .sbb_dep_breaking,
46874746 .sha,
46884747 .slow_shld,
4748 .smap,
4749 .smep,
46894750 .sse4a,
46904751 .vaes,
46914752 .vpclmulqdq,
......@@ -4757,6 +4818,8 @@ pub const cpu = struct {
47574818 .sha,
47584819 .shstk,
47594820 .slow_shld,
4821 .smap,
4822 .smep,
47604823 .sse4a,
47614824 .vaes,
47624825 .vpclmulqdq,
......@@ -4833,6 +4896,8 @@ pub const cpu = struct {
48334896 .sha,
48344897 .shstk,
48354898 .slow_shld,
4899 .smap,
4900 .smep,
48364901 .sse4a,
48374902 .vaes,
48384903 .vpclmulqdq,
lib/std/math/big/int.zig+8-5
......@@ -2520,12 +2520,13 @@ pub const Const = struct {
25202520 return order(a, b) == .eq;
25212521 }
25222522
2523 /// Returns the number of leading zeros in twos-complement form.
25232524 pub fn clz(a: Const, bits: Limb) Limb {
2524 // Limbs are stored in little-endian order but we need
2525 // to iterate big-endian.
2525 // Limbs are stored in little-endian order but we need to iterate big-endian.
2526 if (!a.positive) return 0;
25262527 var total_limb_lz: Limb = 0;
25272528 var i: usize = a.limbs.len;
2528 const bits_per_limb = @sizeOf(Limb) * 8;
2529 const bits_per_limb = @bitSizeOf(Limb);
25292530 while (i != 0) {
25302531 i -= 1;
25312532 const limb = a.limbs[i];
......@@ -2537,13 +2538,15 @@ pub const Const = struct {
25372538 return total_limb_lz + bits - total_limb_bits;
25382539 }
25392540
2541 /// Returns the number of trailing zeros in twos-complement form.
25402542 pub fn ctz(a: Const, bits: Limb) Limb {
2541 // Limbs are stored in little-endian order.
2543 // Limbs are stored in little-endian order. Converting a negative number to twos-complement
2544 // flips all bits above the lowest set bit, which does not affect the trailing zero count.
25422545 var result: Limb = 0;
25432546 for (a.limbs) |limb| {
25442547 const limb_tz = @ctz(limb);
25452548 result += limb_tz;
2546 if (limb_tz != @sizeOf(Limb) * 8) break;
2549 if (limb_tz != @bitSizeOf(Limb)) break;
25472550 }
25482551 return @min(result, bits);
25492552 }
lib/std/zig/system/x86.zig+2
......@@ -369,6 +369,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
369369 setFeature(cpu, .bmi, bit(leaf.ebx, 3));
370370 // AVX2 is only supported if we have the OS save support from AVX.
371371 setFeature(cpu, .avx2, bit(leaf.ebx, 5) and has_avx_save);
372 setFeature(cpu, .smep, bit(leaf.ebx, 7));
372373 setFeature(cpu, .bmi2, bit(leaf.ebx, 8));
373374 setFeature(cpu, .invpcid, bit(leaf.ebx, 10));
374375 setFeature(cpu, .rtm, bit(leaf.ebx, 11));
......@@ -377,6 +378,7 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
377378 setFeature(cpu, .avx512dq, bit(leaf.ebx, 17) and has_avx512_save);
378379 setFeature(cpu, .rdseed, bit(leaf.ebx, 18));
379380 setFeature(cpu, .adx, bit(leaf.ebx, 19));
381 setFeature(cpu, .smap, bit(leaf.ebx, 20));
380382 setFeature(cpu, .avx512ifma, bit(leaf.ebx, 21) and has_avx512_save);
381383 setFeature(cpu, .clflushopt, bit(leaf.ebx, 23));
382384 setFeature(cpu, .clwb, bit(leaf.ebx, 24));
src/arch/x86_64/CodeGen.zig+5635-1781
......@@ -1390,7 +1390,7 @@ fn asmOps(self: *CodeGen, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
13901390
13911391/// A `cc` of `.z_and_np` clobbers `reg2`!
13921392fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2: Register) !void {
1393 _ = try self.addInst(.{
1393 if (self.hasFeature(.cmov)) _ = try self.addInst(.{
13941394 .tag = switch (cc) {
13951395 else => .cmov,
13961396 .z_and_np, .nz_or_p => .pseudo,
......@@ -1408,12 +1408,16 @@ fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2
14081408 .r1 = reg1,
14091409 .r2 = reg2,
14101410 } },
1411 });
1411 }) else {
1412 const reloc = try self.asmJccReloc(cc.negate(), undefined);
1413 try self.asmRegisterRegister(.{ ._, .mov }, reg1, reg2);
1414 self.performReloc(reloc);
1415 }
14121416}
14131417
14141418/// A `cc` of `.z_and_np` is not supported by this encoding!
14151419fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memory) !void {
1416 _ = try self.addInst(.{
1420 if (self.hasFeature(.cmov)) _ = try self.addInst(.{
14171421 .tag = switch (cc) {
14181422 else => .cmov,
14191423 .z_and_np => unreachable,
......@@ -1433,7 +1437,11 @@ fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memo
14331437 .r1 = reg,
14341438 .payload = try self.addExtra(Mir.Memory.encode(m)),
14351439 } },
1436 });
1440 }) else {
1441 const reloc = try self.asmJccReloc(cc.negate(), undefined);
1442 try self.asmRegisterMemory(.{ ._, .mov }, reg, m);
1443 self.performReloc(reloc);
1444 }
14371445}
14381446
14391447fn asmSetccRegister(self: *CodeGen, cc: Condition, reg: Register) !void {
......@@ -2319,6 +2327,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
23192327}
23202328
23212329fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2330 @setEvalBranchQuota(1_600);
23222331 const pt = cg.pt;
23232332 const zcu = pt.zcu;
23242333 const ip = &zcu.intern_pool;
......@@ -2354,9 +2363,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
23542363 try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1);
23552364 switch (air_tags[@intFromEnum(inst)]) {
23562365 // zig fmt: off
2357 .not,
2358 => |air_tag| try cg.airUnOp(inst, air_tag),
2359
23602366 .add,
23612367 .add_wrap,
23622368 .sub,
......@@ -2434,7 +2440,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24342440 .memset_safe => try cg.airMemset(inst, true),
24352441 .set_union_tag => try cg.airSetUnionTag(inst),
24362442 .get_union_tag => try cg.airGetUnionTag(inst),
2437 .clz => try cg.airClz(inst),
24382443 .ctz => try cg.airCtz(inst),
24392444 .popcount => try cg.airPopCount(inst),
24402445 .byte_swap => try cg.airByteSwap(inst),
......@@ -2525,146 +2530,156 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25252530 })) {
25262531 else => unreachable,
25272532 inline .@"and", .@"or", .xor => |mir_tag| comptime &.{ .{
2528 .required_features = .{ .avx2, null },
2533 .required_features = .{ .avx2, null, null, null },
2534 .src_constraints = .{ .{ .int_or_vec = .yword }, .{ .int_or_vec = .yword } },
25292535 .patterns = &.{
2530 .{ .src = .{ .ymm, .mem } },
2531 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
2532 .{ .src = .{ .ymm, .ymm } },
2536 .{ .src = .{ .to_ymm, .mem } },
2537 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
2538 .{ .src = .{ .to_ymm, .to_ymm } },
25332539 },
25342540 .dst_temps = .{.{ .rc = .sse }},
25352541 .each = .{ .once = &.{
25362542 .{ ._, .vp_, mir_tag, .dst0y, .src0y, .src1y, ._ },
25372543 } },
25382544 }, .{
2539 .required_features = .{ .avx, null },
2545 .required_features = .{ .avx, null, null, null },
2546 .src_constraints = .{ .{ .int_or_vec = .yword }, .{ .int_or_vec = .yword } },
25402547 .patterns = &.{
2541 .{ .src = .{ .ymm, .mem } },
2542 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
2543 .{ .src = .{ .ymm, .ymm } },
2548 .{ .src = .{ .to_ymm, .mem } },
2549 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
2550 .{ .src = .{ .to_ymm, .to_ymm } },
25442551 },
25452552 .dst_temps = .{.{ .rc = .sse }},
25462553 .each = .{ .once = &.{
25472554 .{ ._, .v_pd, mir_tag, .dst0y, .src0y, .src1y, ._ },
25482555 } },
25492556 }, .{
2550 .required_features = .{ .avx, null },
2557 .required_features = .{ .avx, null, null, null },
2558 .src_constraints = .{ .{ .int_or_vec = .xword }, .{ .int_or_vec = .xword } },
25512559 .patterns = &.{
2552 .{ .src = .{ .xmm, .mem } },
2553 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
2554 .{ .src = .{ .xmm, .xmm } },
2560 .{ .src = .{ .to_xmm, .mem } },
2561 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
2562 .{ .src = .{ .to_xmm, .to_xmm } },
25552563 },
25562564 .dst_temps = .{.{ .rc = .sse }},
25572565 .each = .{ .once = &.{
25582566 .{ ._, .vp_, mir_tag, .dst0x, .src0x, .src1x, ._ },
25592567 } },
25602568 }, .{
2561 .required_features = .{ .sse2, null },
2569 .required_features = .{ .sse2, null, null, null },
2570 .src_constraints = .{ .{ .int_or_vec = .xword }, .{ .int_or_vec = .xword } },
25622571 .patterns = &.{
2563 .{ .src = .{ .mut_xmm, .mem } },
2564 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
2565 .{ .src = .{ .mut_xmm, .xmm } },
2572 .{ .src = .{ .to_mut_xmm, .mem } },
2573 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
2574 .{ .src = .{ .to_mut_xmm, .to_xmm } },
25662575 },
25672576 .dst_temps = .{.{ .ref = .src0 }},
25682577 .each = .{ .once = &.{
25692578 .{ ._, .p_, mir_tag, .dst0x, .src1x, ._, ._ },
25702579 } },
25712580 }, .{
2572 .required_features = .{ .sse, null },
2581 .required_features = .{ .sse, null, null, null },
2582 .src_constraints = .{ .{ .int_or_vec = .xword }, .{ .int_or_vec = .xword } },
25732583 .patterns = &.{
2574 .{ .src = .{ .mut_xmm, .mem } },
2575 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
2576 .{ .src = .{ .mut_xmm, .xmm } },
2584 .{ .src = .{ .to_mut_xmm, .mem } },
2585 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
2586 .{ .src = .{ .to_mut_xmm, .to_xmm } },
25772587 },
25782588 .dst_temps = .{.{ .ref = .src0 }},
25792589 .each = .{ .once = &.{
25802590 .{ ._, ._ps, mir_tag, .dst0x, .src1x, ._, ._ },
25812591 } },
25822592 }, .{
2583 .required_features = .{ .mmx, null },
2593 .required_features = .{ .mmx, null, null, null },
2594 .src_constraints = .{ .{ .int_or_vec = .qword }, .{ .int_or_vec = .qword } },
25842595 .patterns = &.{
2585 .{ .src = .{ .mut_mm, .mem } },
2586 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
2587 .{ .src = .{ .mut_mm, .mm } },
2596 .{ .src = .{ .to_mut_mm, .mem } },
2597 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
2598 .{ .src = .{ .to_mut_mm, .to_mm } },
25882599 },
25892600 .dst_temps = .{.{ .ref = .src0 }},
25902601 .each = .{ .once = &.{
25912602 .{ ._, .p_, mir_tag, .dst0q, .src1q, ._, ._ },
25922603 } },
25932604 }, .{
2594 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
2605 .src_constraints = .{ .{ .int_or_vec = .byte }, .{ .int_or_vec = .byte } },
25952606 .patterns = &.{
25962607 .{ .src = .{ .mut_mem, .imm8 } },
25972608 .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
2598 .{ .src = .{ .mut_gpr, .imm8 } },
2599 .{ .src = .{ .imm8, .mut_gpr }, .commute = .{ 0, 1 } },
2600 .{ .src = .{ .mut_mem, .gpr } },
2601 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
2602 .{ .src = .{ .mut_gpr, .mem } },
2603 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
2604 .{ .src = .{ .mut_gpr, .gpr } },
2609 .{ .src = .{ .to_mut_gpr, .imm8 } },
2610 .{ .src = .{ .imm8, .to_mut_gpr }, .commute = .{ 0, 1 } },
2611 .{ .src = .{ .mut_mem, .to_gpr } },
2612 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2613 .{ .src = .{ .to_mut_gpr, .mem } },
2614 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2615 .{ .src = .{ .to_mut_gpr, .to_gpr } },
26052616 },
2606 .clobbers = .{ .eflags = true },
26072617 .dst_temps = .{.{ .ref = .src0 }},
2618 .clobbers = .{ .eflags = true },
26082619 .each = .{ .once = &.{
26092620 .{ ._, ._, mir_tag, .dst0b, .src1b, ._, ._ },
26102621 } },
26112622 }, .{
2612 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
2623 .src_constraints = .{ .{ .int_or_vec = .word }, .{ .int_or_vec = .word } },
26132624 .patterns = &.{
26142625 .{ .src = .{ .mut_mem, .imm16 } },
26152626 .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
2616 .{ .src = .{ .mut_gpr, .imm16 } },
2617 .{ .src = .{ .imm16, .mut_gpr }, .commute = .{ 0, 1 } },
2618 .{ .src = .{ .mut_mem, .gpr } },
2619 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
2620 .{ .src = .{ .mut_gpr, .mem } },
2621 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
2622 .{ .src = .{ .mut_gpr, .gpr } },
2627 .{ .src = .{ .to_mut_gpr, .imm16 } },
2628 .{ .src = .{ .imm16, .to_mut_gpr }, .commute = .{ 0, 1 } },
2629 .{ .src = .{ .mut_mem, .to_gpr } },
2630 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2631 .{ .src = .{ .to_mut_gpr, .mem } },
2632 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2633 .{ .src = .{ .to_mut_gpr, .to_gpr } },
26232634 },
2624 .clobbers = .{ .eflags = true },
26252635 .dst_temps = .{.{ .ref = .src0 }},
2636 .clobbers = .{ .eflags = true },
26262637 .each = .{ .once = &.{
26272638 .{ ._, ._, mir_tag, .dst0w, .src1w, ._, ._ },
26282639 } },
26292640 }, .{
2630 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
2641 .src_constraints = .{ .{ .int_or_vec = .dword }, .{ .int_or_vec = .dword } },
26312642 .patterns = &.{
26322643 .{ .src = .{ .mut_mem, .imm32 } },
26332644 .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
2634 .{ .src = .{ .mut_gpr, .imm32 } },
2635 .{ .src = .{ .imm32, .mut_gpr }, .commute = .{ 0, 1 } },
2636 .{ .src = .{ .mut_mem, .gpr } },
2637 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
2638 .{ .src = .{ .mut_gpr, .mem } },
2639 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
2640 .{ .src = .{ .mut_gpr, .gpr } },
2645 .{ .src = .{ .to_mut_gpr, .imm32 } },
2646 .{ .src = .{ .imm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
2647 .{ .src = .{ .mut_mem, .to_gpr } },
2648 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2649 .{ .src = .{ .to_mut_gpr, .mem } },
2650 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2651 .{ .src = .{ .to_mut_gpr, .to_gpr } },
26412652 },
2642 .clobbers = .{ .eflags = true },
26432653 .dst_temps = .{.{ .ref = .src0 }},
2654 .clobbers = .{ .eflags = true },
26442655 .each = .{ .once = &.{
26452656 .{ ._, ._, mir_tag, .dst0d, .src1d, ._, ._ },
26462657 } },
26472658 }, .{
2648 .required_features = .{ .@"64bit", null },
2649 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
2659 .required_features = .{ .@"64bit", null, null, null },
2660 .src_constraints = .{ .{ .int_or_vec = .qword }, .{ .int_or_vec = .qword } },
26502661 .patterns = &.{
26512662 .{ .src = .{ .mut_mem, .simm32 } },
26522663 .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
2653 .{ .src = .{ .mut_gpr, .simm32 } },
2654 .{ .src = .{ .simm32, .mut_gpr }, .commute = .{ 0, 1 } },
2655 .{ .src = .{ .mut_mem, .gpr } },
2656 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
2657 .{ .src = .{ .mut_gpr, .mem } },
2658 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
2659 .{ .src = .{ .mut_gpr, .gpr } },
2664 .{ .src = .{ .to_mut_gpr, .simm32 } },
2665 .{ .src = .{ .simm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
2666 .{ .src = .{ .mut_mem, .to_gpr } },
2667 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
2668 .{ .src = .{ .to_mut_gpr, .mem } },
2669 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
2670 .{ .src = .{ .to_mut_gpr, .to_gpr } },
26602671 },
2661 .clobbers = .{ .eflags = true },
26622672 .dst_temps = .{.{ .ref = .src0 }},
2673 .clobbers = .{ .eflags = true },
26632674 .each = .{ .once = &.{
26642675 .{ ._, ._, mir_tag, .dst0q, .src1q, ._, ._ },
26652676 } },
26662677 }, .{
2667 .required_features = .{ .avx2, null },
2678 .required_features = .{ .avx2, null, null, null },
2679 .src_constraints = .{
2680 .{ .exact_remainder_int_or_vec = .{ .of = .yword, .is = .yword } },
2681 .{ .exact_remainder_int_or_vec = .{ .of = .yword, .is = .yword } },
2682 },
26682683 .patterns = &.{
26692684 .{ .src = .{ .to_mem, .to_mem } },
26702685 },
......@@ -2677,16 +2692,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26772692 .unused,
26782693 },
26792694 .dst_temps = .{.mem},
2695 .clobbers = .{ .eflags = true },
26802696 .each = .{ .once = &.{
2681 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
2697 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26822698 .{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
26832699 .{ ._, .vp_, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
26842700 .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ },
2685 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
2701 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
26862702 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26872703 } },
26882704 }, .{
2689 .required_features = .{ .avx, null },
2705 .required_features = .{ .avx, null, null, null },
2706 .src_constraints = .{
2707 .{ .exact_remainder_int_or_vec = .{ .of = .yword, .is = .yword } },
2708 .{ .exact_remainder_int_or_vec = .{ .of = .yword, .is = .yword } },
2709 },
26902710 .patterns = &.{
26912711 .{ .src = .{ .to_mem, .to_mem } },
26922712 },
......@@ -2699,16 +2719,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26992719 .unused,
27002720 },
27012721 .dst_temps = .{.mem},
2722 .clobbers = .{ .eflags = true },
27022723 .each = .{ .once = &.{
2703 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
2724 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27042725 .{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
27052726 .{ ._, .v_pd, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
27062727 .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ },
2707 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
2728 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
27082729 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27092730 } },
27102731 }, .{
2711 .required_features = .{ .avx, null },
2732 .required_features = .{ .avx, null, null, null },
2733 .src_constraints = .{
2734 .{ .exact_remainder_int_or_vec = .{ .of = .xword, .is = .xword } },
2735 .{ .exact_remainder_int_or_vec = .{ .of = .xword, .is = .xword } },
2736 },
27122737 .patterns = &.{
27132738 .{ .src = .{ .to_mem, .to_mem } },
27142739 },
......@@ -2721,16 +2746,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27212746 .unused,
27222747 },
27232748 .dst_temps = .{.mem},
2749 .clobbers = .{ .eflags = true },
27242750 .each = .{ .once = &.{
2725 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
2751 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27262752 .{ .@"0:", .v_dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
27272753 .{ ._, .vp_, mir_tag, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ },
27282754 .{ ._, .v_dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
2729 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
2755 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27302756 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27312757 } },
27322758 }, .{
2733 .required_features = .{ .sse2, null },
2759 .required_features = .{ .sse2, null, null, null },
2760 .src_constraints = .{
2761 .{ .exact_remainder_int_or_vec = .{ .of = .xword, .is = .xword } },
2762 .{ .exact_remainder_int_or_vec = .{ .of = .xword, .is = .xword } },
2763 },
27342764 .patterns = &.{
27352765 .{ .src = .{ .to_mem, .to_mem } },
27362766 },
......@@ -2743,16 +2773,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27432773 .unused,
27442774 },
27452775 .dst_temps = .{.mem},
2776 .clobbers = .{ .eflags = true },
27462777 .each = .{ .once = &.{
2747 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
2778 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27482779 .{ .@"0:", ._dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
27492780 .{ ._, .p_, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
27502781 .{ ._, ._dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
2751 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
2782 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27522783 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27532784 } },
27542785 }, .{
2755 .required_features = .{ .sse, null },
2786 .required_features = .{ .sse, null, null, null },
2787 .src_constraints = .{
2788 .{ .exact_remainder_int_or_vec = .{ .of = .xword, .is = .xword } },
2789 .{ .exact_remainder_int_or_vec = .{ .of = .xword, .is = .xword } },
2790 },
27562791 .patterns = &.{
27572792 .{ .src = .{ .to_mem, .to_mem } },
27582793 },
......@@ -2765,16 +2800,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27652800 .unused,
27662801 },
27672802 .dst_temps = .{.mem},
2803 .clobbers = .{ .eflags = true },
27682804 .each = .{ .once = &.{
2769 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
2805 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27702806 .{ .@"0:", ._ps, .movu, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
27712807 .{ ._, ._ps, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
27722808 .{ ._, ._ps, .movu, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
2773 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
2809 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27742810 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27752811 } },
27762812 }, .{
2777 .required_features = .{ .mmx, null },
2813 .required_features = .{ .mmx, null, null, null },
2814 .src_constraints = .{
2815 .{ .exact_remainder_int_or_vec = .{ .of = .qword, .is = .qword } },
2816 .{ .exact_remainder_int_or_vec = .{ .of = .qword, .is = .qword } },
2817 },
27782818 .patterns = &.{
27792819 .{ .src = .{ .to_mem, .to_mem } },
27802820 },
......@@ -2787,15 +2827,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27872827 .unused,
27882828 },
27892829 .dst_temps = .{.mem},
2830 .clobbers = .{ .eflags = true },
27902831 .each = .{ .once = &.{
2791 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
2832 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
27922833 .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
27932834 .{ ._, .p_, mir_tag, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
27942835 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ },
2795 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
2836 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27962837 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27972838 } },
27982839 }, .{
2840 .src_constraints = .{
2841 .{ .exact_remainder_int_or_vec = .{ .of = .qword, .is = .qword } },
2842 .{ .exact_remainder_int_or_vec = .{ .of = .qword, .is = .qword } },
2843 },
27992844 .patterns = &.{
28002845 .{ .src = .{ .to_mem, .to_mem } },
28012846 },
......@@ -2808,16 +2853,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
28082853 .unused,
28092854 },
28102855 .dst_temps = .{.mem},
2856 .clobbers = .{ .eflags = true },
28112857 .each = .{ .once = &.{
2812 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
2858 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
28132859 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
28142860 .{ ._, ._, mir_tag, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
28152861 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
2816 .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ },
2862 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
28172863 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
28182864 } },
28192865 } },
2820 }) catch |err2| switch (err2) {
2866 }) catch |err| switch (err) {
28212867 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
28222868 @tagName(air_tag),
28232869 cg.typeOf(bin_op.lhs).fmt(pt),
......@@ -2830,6 +2876,1058 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
28302876 if (ops[1].index != res[0].index) try ops[1].die(cg);
28312877 try res[0].moveTo(inst, cg);
28322878 },
2879 .not => |air_tag| if (use_old) try cg.airUnOp(inst, air_tag) else {
2880 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
2881 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
2882 var res: [1]Temp = undefined;
2883 cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, comptime &.{ .{
2884 .src_constraints = .{ .{ .signed_or_exact_int = .byte }, .any },
2885 .patterns = &.{
2886 .{ .src = .{ .mut_mem, .none } },
2887 .{ .src = .{ .to_mut_gpr, .none } },
2888 },
2889 .dst_temps = .{.{ .ref = .src0 }},
2890 .each = .{ .once = &.{
2891 .{ ._, ._, .not, .dst0b, ._, ._, ._ },
2892 } },
2893 }, .{
2894 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
2895 .patterns = &.{
2896 .{ .src = .{ .mut_mem, .none } },
2897 .{ .src = .{ .to_mut_gpr, .none } },
2898 },
2899 .dst_temps = .{.{ .ref = .src0 }},
2900 .clobbers = .{ .eflags = true },
2901 .each = .{ .once = &.{
2902 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_umax), ._, ._ },
2903 } },
2904 }, .{
2905 .src_constraints = .{ .{ .signed_or_exact_int = .word }, .any },
2906 .patterns = &.{
2907 .{ .src = .{ .mut_mem, .none } },
2908 .{ .src = .{ .to_mut_gpr, .none } },
2909 },
2910 .dst_temps = .{.{ .ref = .src0 }},
2911 .each = .{ .once = &.{
2912 .{ ._, ._, .not, .dst0w, ._, ._, ._ },
2913 } },
2914 }, .{
2915 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
2916 .patterns = &.{
2917 .{ .src = .{ .mut_mem, .none } },
2918 .{ .src = .{ .to_mut_gpr, .none } },
2919 },
2920 .dst_temps = .{.{ .ref = .src0 }},
2921 .clobbers = .{ .eflags = true },
2922 .each = .{ .once = &.{
2923 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_umax), ._, ._ },
2924 } },
2925 }, .{
2926 .src_constraints = .{ .{ .signed_or_exact_int = .dword }, .any },
2927 .patterns = &.{
2928 .{ .src = .{ .mut_mem, .none } },
2929 .{ .src = .{ .to_mut_gpr, .none } },
2930 },
2931 .dst_temps = .{.{ .ref = .src0 }},
2932 .each = .{ .once = &.{
2933 .{ ._, ._, .not, .dst0d, ._, ._, ._ },
2934 } },
2935 }, .{
2936 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
2937 .patterns = &.{
2938 .{ .src = .{ .mut_mem, .none } },
2939 .{ .src = .{ .to_mut_gpr, .none } },
2940 },
2941 .dst_temps = .{.{ .ref = .src0 }},
2942 .clobbers = .{ .eflags = true },
2943 .each = .{ .once = &.{
2944 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_umax), ._, ._ },
2945 } },
2946 }, .{
2947 .required_features = .{ .@"64bit", null, null, null },
2948 .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any },
2949 .patterns = &.{
2950 .{ .src = .{ .mut_mem, .none } },
2951 .{ .src = .{ .to_mut_gpr, .none } },
2952 },
2953 .dst_temps = .{.{ .ref = .src0 }},
2954 .each = .{ .once = &.{
2955 .{ ._, ._, .not, .dst0q, ._, ._, ._ },
2956 } },
2957 }, .{
2958 .required_features = .{ .@"64bit", null, null, null },
2959 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
2960 .patterns = &.{
2961 .{ .src = .{ .mem, .none } },
2962 .{ .src = .{ .to_gpr, .none } },
2963 },
2964 .dst_temps = .{.{ .rc = .general_purpose }},
2965 .each = .{ .once = &.{
2966 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
2967 .{ ._, ._, .xor, .dst0q, .src0q, ._, ._ },
2968 } },
2969 }, .{
2970 .required_features = .{ .mmx, null, null, null },
2971 .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any },
2972 .patterns = &.{
2973 .{ .src = .{ .mem, .none } },
2974 .{ .src = .{ .to_mm, .none } },
2975 },
2976 .dst_temps = .{.{ .rc = .mmx }},
2977 .each = .{ .once = &.{
2978 .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ },
2979 .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ },
2980 } },
2981 }, .{
2982 .required_features = .{ .mmx, null, null, null },
2983 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
2984 .patterns = &.{
2985 .{ .src = .{ .to_mut_mm, .none } },
2986 },
2987 .extra_temps = .{
2988 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
2989 .{ .kind = .{ .umax_mem = .src0 } },
2990 .unused,
2991 .unused,
2992 .unused,
2993 .unused,
2994 },
2995 .dst_temps = .{.{ .ref = .src0 }},
2996 .each = .{ .once = &.{
2997 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
2998 .{ ._, .p_, .xor, .dst0q, .lea(.qword, .tmp0), ._, ._ },
2999 } },
3000 }, .{
3001 .required_features = .{ .avx, null, null, null },
3002 .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any },
3003 .patterns = &.{
3004 .{ .src = .{ .mem, .none } },
3005 .{ .src = .{ .to_xmm, .none } },
3006 },
3007 .dst_temps = .{.{ .rc = .sse }},
3008 .each = .{ .once = &.{
3009 .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ },
3010 .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ },
3011 } },
3012 }, .{
3013 .required_features = .{ .avx, null, null, null },
3014 .src_constraints = .{ .{ .unsigned_int = .xword }, .any },
3015 .patterns = &.{
3016 .{ .src = .{ .to_xmm, .none } },
3017 },
3018 .extra_temps = .{
3019 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3020 .{ .kind = .{ .umax_mem = .src0 } },
3021 .unused,
3022 .unused,
3023 .unused,
3024 .unused,
3025 },
3026 .dst_temps = .{.{ .rc = .sse }},
3027 .each = .{ .once = &.{
3028 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3029 .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
3030 } },
3031 }, .{
3032 .required_features = .{ .sse2, null, null, null },
3033 .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any },
3034 .patterns = &.{
3035 .{ .src = .{ .mem, .none } },
3036 .{ .src = .{ .to_xmm, .none } },
3037 },
3038 .dst_temps = .{.{ .rc = .sse }},
3039 .each = .{ .once = &.{
3040 .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ },
3041 .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ },
3042 } },
3043 }, .{
3044 .required_features = .{ .sse2, null, null, null },
3045 .src_constraints = .{ .{ .unsigned_int = .xword }, .any },
3046 .patterns = &.{
3047 .{ .src = .{ .to_mut_xmm, .none } },
3048 },
3049 .extra_temps = .{
3050 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3051 .{ .kind = .{ .umax_mem = .src0 } },
3052 .unused,
3053 .unused,
3054 .unused,
3055 .unused,
3056 },
3057 .dst_temps = .{.{ .ref = .src0 }},
3058 .each = .{ .once = &.{
3059 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3060 .{ ._, .p_, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
3061 } },
3062 }, .{
3063 .required_features = .{ .sse, null, null, null },
3064 .src_constraints = .{ .{ .int = .xword }, .any },
3065 .patterns = &.{
3066 .{ .src = .{ .to_mut_xmm, .none } },
3067 },
3068 .extra_temps = .{
3069 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3070 .{ .kind = .{ .umax_mem = .src0 } },
3071 .unused,
3072 .unused,
3073 .unused,
3074 .unused,
3075 },
3076 .dst_temps = .{.{ .ref = .src0 }},
3077 .each = .{ .once = &.{
3078 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3079 .{ ._, ._ps, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
3080 } },
3081 }, .{
3082 .required_features = .{ .avx2, null, null, null },
3083 .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any },
3084 .patterns = &.{
3085 .{ .src = .{ .mem, .none } },
3086 .{ .src = .{ .to_ymm, .none } },
3087 },
3088 .dst_temps = .{.{ .rc = .sse }},
3089 .each = .{ .once = &.{
3090 .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ },
3091 .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ },
3092 } },
3093 }, .{
3094 .required_features = .{ .avx2, null, null, null },
3095 .src_constraints = .{ .{ .unsigned_int = .yword }, .any },
3096 .patterns = &.{
3097 .{ .src = .{ .to_ymm, .none } },
3098 },
3099 .extra_temps = .{
3100 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3101 .{ .kind = .{ .umax_mem = .src0 } },
3102 .unused,
3103 .unused,
3104 .unused,
3105 .unused,
3106 },
3107 .dst_temps = .{.{ .rc = .sse }},
3108 .each = .{ .once = &.{
3109 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3110 .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
3111 } },
3112 }, .{
3113 .required_features = .{ .avx, null, null, null },
3114 .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any },
3115 .patterns = &.{
3116 .{ .src = .{ .mem, .none } },
3117 .{ .src = .{ .to_ymm, .none } },
3118 },
3119 .dst_temps = .{.{ .rc = .sse }},
3120 .each = .{ .once = &.{
3121 .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) },
3122 .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ },
3123 } },
3124 }, .{
3125 .required_features = .{ .avx, null, null, null },
3126 .src_constraints = .{ .{ .unsigned_int = .yword }, .any },
3127 .patterns = &.{
3128 .{ .src = .{ .to_ymm, .none } },
3129 },
3130 .extra_temps = .{
3131 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3132 .{ .kind = .{ .umax_mem = .src0 } },
3133 .unused,
3134 .unused,
3135 .unused,
3136 .unused,
3137 },
3138 .dst_temps = .{.{ .rc = .sse }},
3139 .each = .{ .once = &.{
3140 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3141 .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
3142 } },
3143 }, .{
3144 .required_features = .{ .avx2, null, null, null },
3145 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any },
3146 .patterns = &.{
3147 .{ .src = .{ .to_mem, .none } },
3148 },
3149 .extra_temps = .{
3150 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3151 .{ .kind = .{ .rc = .sse } },
3152 .{ .kind = .{ .rc = .sse } },
3153 .unused,
3154 .unused,
3155 .unused,
3156 },
3157 .dst_temps = .{.mem},
3158 .clobbers = .{ .eflags = true },
3159 .each = .{ .once = &.{
3160 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3161 .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
3162 .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
3163 .{ ._, .v_dqu, .mov, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
3164 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3165 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3166 .{ .@"0:", .vp_, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
3167 .{ ._, .v_dqa, .mov, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
3168 } },
3169 }, .{
3170 .required_features = .{ .avx2, null, null, null },
3171 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any },
3172 .patterns = &.{
3173 .{ .src = .{ .to_mem, .none } },
3174 },
3175 .extra_temps = .{
3176 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3177 .{ .kind = .{ .rc = .sse } },
3178 .{ .kind = .{ .rc = .sse } },
3179 .unused,
3180 .unused,
3181 .unused,
3182 },
3183 .dst_temps = .{.mem},
3184 .clobbers = .{ .eflags = true },
3185 .each = .{ .once = &.{
3186 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3187 .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
3188 .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
3189 .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
3190 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3191 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3192 } },
3193 }, .{
3194 .required_features = .{ .avx, null, null, null },
3195 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any },
3196 .patterns = &.{
3197 .{ .src = .{ .to_mem, .none } },
3198 },
3199 .extra_temps = .{
3200 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3201 .{ .kind = .{ .rc = .sse } },
3202 .{ .kind = .{ .rc = .sse } },
3203 .unused,
3204 .unused,
3205 .unused,
3206 },
3207 .dst_temps = .{.mem},
3208 .clobbers = .{ .eflags = true },
3209 .each = .{ .once = &.{
3210 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3211 .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) },
3212 .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
3213 .{ ._, .v_pd, .movu, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
3214 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3215 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3216 .{ .@"0:", .v_pd, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
3217 .{ ._, .v_pd, .mova, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
3218 } },
3219 }, .{
3220 .required_features = .{ .avx, null, null, null },
3221 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any },
3222 .patterns = &.{
3223 .{ .src = .{ .to_mem, .none } },
3224 },
3225 .extra_temps = .{
3226 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3227 .{ .kind = .{ .rc = .sse } },
3228 .{ .kind = .{ .rc = .sse } },
3229 .unused,
3230 .unused,
3231 .unused,
3232 },
3233 .dst_temps = .{.mem},
3234 .clobbers = .{ .eflags = true },
3235 .each = .{ .once = &.{
3236 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3237 .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) },
3238 .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
3239 .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
3240 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
3241 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3242 } },
3243 }, .{
3244 .required_features = .{ .avx, null, null, null },
3245 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3246 .patterns = &.{
3247 .{ .src = .{ .to_mem, .none } },
3248 },
3249 .extra_temps = .{
3250 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3251 .{ .kind = .{ .rc = .sse } },
3252 .{ .kind = .{ .rc = .sse } },
3253 .unused,
3254 .unused,
3255 .unused,
3256 },
3257 .dst_temps = .{.mem},
3258 .clobbers = .{ .eflags = true },
3259 .each = .{ .once = &.{
3260 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3261 .{ ._, .vp_q, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ },
3262 .{ .@"0:", .v_, .xor, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._ },
3263 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
3264 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3265 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3266 } },
3267 }, .{
3268 .required_features = .{ .sse2, null, null, null },
3269 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3270 .patterns = &.{
3271 .{ .src = .{ .to_mem, .none } },
3272 },
3273 .extra_temps = .{
3274 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3275 .{ .kind = .{ .rc = .sse } },
3276 .{ .kind = .{ .rc = .sse } },
3277 .unused,
3278 .unused,
3279 .unused,
3280 },
3281 .dst_temps = .{.mem},
3282 .clobbers = .{ .eflags = true },
3283 .each = .{ .once = &.{
3284 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3285 .{ ._, .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ },
3286 .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3287 .{ ._, .p_, .xor, .tmp2x, .tmp1x, ._, ._ },
3288 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
3289 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3290 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3291 } },
3292 }, .{
3293 .required_features = .{ .@"64bit", null, null, null },
3294 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3295 .patterns = &.{
3296 .{ .src = .{ .mut_mem, .none } },
3297 },
3298 .extra_temps = .{
3299 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3300 .unused,
3301 .unused,
3302 .unused,
3303 .unused,
3304 .unused,
3305 },
3306 .dst_temps = .{.{ .ref = .src0 }},
3307 .clobbers = .{ .eflags = true },
3308 .each = .{ .once = &.{
3309 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3310 .{ .@"0:", ._, .not, .memia(.dst0q, .tmp0, .add_size), ._, ._, ._ },
3311 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, 8), ._, ._, ._ },
3312 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3313 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3314 } },
3315 }, .{
3316 .required_features = .{ .@"64bit", null, null, null },
3317 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3318 .patterns = &.{
3319 .{ .src = .{ .to_mem, .none } },
3320 },
3321 .extra_temps = .{
3322 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3323 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3324 .unused,
3325 .unused,
3326 .unused,
3327 .unused,
3328 },
3329 .dst_temps = .{.mem},
3330 .clobbers = .{ .eflags = true },
3331 .each = .{ .once = &.{
3332 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
3333 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
3334 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3335 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ },
3336 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3337 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3338 } },
3339 }, .{
3340 .required_features = .{ .@"64bit", null, null, null },
3341 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3342 .patterns = &.{
3343 .{ .src = .{ .mut_mem, .none } },
3344 },
3345 .extra_temps = .{
3346 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3347 .unused,
3348 .unused,
3349 .unused,
3350 .unused,
3351 .unused,
3352 },
3353 .dst_temps = .{.{ .ref = .src0 }},
3354 .clobbers = .{ .eflags = true },
3355 .each = .{ .once = &.{
3356 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3357 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3358 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3359 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3360 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3361 .{ ._, ._, .not, .memad(.dst0d, .add_size, -16), ._, ._, ._ },
3362 } },
3363 }, .{
3364 .required_features = .{ .@"64bit", null, null, null },
3365 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3366 .patterns = &.{
3367 .{ .src = .{ .to_mem, .none } },
3368 },
3369 .extra_temps = .{
3370 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3371 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3372 .unused,
3373 .unused,
3374 .unused,
3375 .unused,
3376 },
3377 .dst_temps = .{.mem},
3378 .clobbers = .{ .eflags = true },
3379 .each = .{ .once = &.{
3380 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3381 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
3382 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3383 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
3384 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3385 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3386 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ },
3387 .{ ._, ._, .not, .tmp0d, ._, ._, ._ },
3388 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ },
3389 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ },
3390 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ },
3391 } },
3392 }, .{
3393 .required_features = .{ .@"64bit", null, null, null },
3394 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
3395 .patterns = &.{
3396 .{ .src = .{ .mut_mem, .none } },
3397 },
3398 .extra_temps = .{
3399 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3400 .unused,
3401 .unused,
3402 .unused,
3403 .unused,
3404 .unused,
3405 },
3406 .dst_temps = .{.{ .ref = .src0 }},
3407 .clobbers = .{ .eflags = true },
3408 .each = .{ .once = &.{
3409 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3410 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3411 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3412 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3413 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3414 .{ ._, ._, .not, .memad(.dst0q, .add_size, -16), ._, ._, ._ },
3415 } },
3416 }, .{
3417 .required_features = .{ .@"64bit", null, null, null },
3418 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = .qword } }, .any },
3419 .patterns = &.{
3420 .{ .src = .{ .to_mem, .none } },
3421 },
3422 .extra_temps = .{
3423 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3424 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3425 .unused,
3426 .unused,
3427 .unused,
3428 .unused,
3429 },
3430 .dst_temps = .{.mem},
3431 .clobbers = .{ .eflags = true },
3432 .each = .{ .once = &.{
3433 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3434 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3435 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3436 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3437 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3438 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3439 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ },
3440 } },
3441 }, .{
3442 .required_features = .{ .@"64bit", null, null, null },
3443 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = .dword } }, .any },
3444 .patterns = &.{
3445 .{ .src = .{ .mut_mem, .none } },
3446 },
3447 .extra_temps = .{
3448 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3449 .unused,
3450 .unused,
3451 .unused,
3452 .unused,
3453 .unused,
3454 },
3455 .dst_temps = .{.{ .ref = .src0 }},
3456 .clobbers = .{ .eflags = true },
3457 .each = .{ .once = &.{
3458 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3459 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
3460 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3461 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3462 .{ ._, ._, .not, .memad(.dst0d, .add_size, -8), ._, ._, ._ },
3463 } },
3464 }, .{
3465 .required_features = .{ .@"64bit", null, null, null },
3466 .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = .dword } }, .any },
3467 .patterns = &.{
3468 .{ .src = .{ .to_mem, .none } },
3469 },
3470 .extra_temps = .{
3471 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3472 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3473 .unused,
3474 .unused,
3475 .unused,
3476 .unused,
3477 },
3478 .dst_temps = .{.mem},
3479 .clobbers = .{ .eflags = true },
3480 .each = .{ .once = &.{
3481 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3482 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3483 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3484 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3485 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3486 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3487 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ },
3488 .{ ._, ._, .not, .tmp0d, ._, ._, ._ },
3489 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ },
3490 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ },
3491 } },
3492 }, .{
3493 .required_features = .{ .@"64bit", null, null, null },
3494 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3495 .patterns = &.{
3496 .{ .src = .{ .mut_mem, .none } },
3497 },
3498 .extra_temps = .{
3499 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3500 .unused,
3501 .unused,
3502 .unused,
3503 .unused,
3504 .unused,
3505 },
3506 .dst_temps = .{.{ .ref = .src0 }},
3507 .clobbers = .{ .eflags = true },
3508 .each = .{ .once = &.{
3509 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3510 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3511 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3512 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3513 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3514 .{ ._, ._, .xor, .memad(.dst0d, .add_size, -16), .sa(.src0, .add_umax), ._, ._ },
3515 } },
3516 }, .{
3517 .required_features = .{ .@"64bit", null, null, null },
3518 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any },
3519 .patterns = &.{
3520 .{ .src = .{ .to_mem, .none } },
3521 },
3522 .extra_temps = .{
3523 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3524 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3525 .unused,
3526 .unused,
3527 .unused,
3528 .unused,
3529 },
3530 .dst_temps = .{.mem},
3531 .clobbers = .{ .eflags = true },
3532 .each = .{ .once = &.{
3533 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3534 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
3535 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3536 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
3537 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3538 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3539 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ },
3540 .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ },
3541 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ },
3542 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ },
3543 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ },
3544 } },
3545 }, .{
3546 .required_features = .{ .@"64bit", null, null, null },
3547 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any },
3548 .patterns = &.{
3549 .{ .src = .{ .mut_mem, .none } },
3550 },
3551 .extra_temps = .{
3552 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3553 .unused,
3554 .unused,
3555 .unused,
3556 .unused,
3557 .unused,
3558 },
3559 .dst_temps = .{.{ .ref = .src0 }},
3560 .clobbers = .{ .eflags = true },
3561 .each = .{ .once = &.{
3562 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3563 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
3564 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3565 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3566 .{ ._, ._, .xor, .memad(.dst0d, .add_size, -8), .sa(.src0, .add_umax), ._, ._ },
3567 } },
3568 }, .{
3569 .required_features = .{ .@"64bit", null, null, null },
3570 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any },
3571 .patterns = &.{
3572 .{ .src = .{ .to_mem, .none } },
3573 },
3574 .extra_temps = .{
3575 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3576 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3577 .unused,
3578 .unused,
3579 .unused,
3580 .unused,
3581 },
3582 .dst_temps = .{.mem},
3583 .clobbers = .{ .eflags = true },
3584 .each = .{ .once = &.{
3585 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3586 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3587 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3588 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3589 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3590 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3591 .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ },
3592 .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ },
3593 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ },
3594 .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ },
3595 } },
3596 }, .{
3597 .required_features = .{ .@"64bit", null, null, null },
3598 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
3599 .patterns = &.{
3600 .{ .src = .{ .mut_mem, .none } },
3601 },
3602 .extra_temps = .{
3603 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3604 .unused,
3605 .unused,
3606 .unused,
3607 .unused,
3608 .unused,
3609 },
3610 .dst_temps = .{.{ .ref = .src0 }},
3611 .clobbers = .{ .eflags = true },
3612 .each = .{ .once = &.{
3613 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3614 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ },
3615 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ },
3616 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
3617 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3618 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3619 .{ ._, ._, .xor, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ },
3620 } },
3621 }, .{
3622 .required_features = .{ .@"64bit", null, null, null },
3623 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
3624 .patterns = &.{
3625 .{ .src = .{ .to_mem, .none } },
3626 },
3627 .extra_temps = .{
3628 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3629 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3630 .unused,
3631 .unused,
3632 .unused,
3633 .unused,
3634 },
3635 .dst_temps = .{.mem},
3636 .clobbers = .{ .eflags = true },
3637 .each = .{ .once = &.{
3638 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
3639 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ },
3640 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3641 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ },
3642 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3643 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3644 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3645 .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ },
3646 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ },
3647 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ },
3648 } },
3649 }, .{
3650 .required_features = .{ .@"64bit", null, null, null },
3651 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3652 .patterns = &.{
3653 .{ .src = .{ .mut_mem, .none } },
3654 },
3655 .extra_temps = .{
3656 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3657 .unused,
3658 .unused,
3659 .unused,
3660 .unused,
3661 .unused,
3662 },
3663 .dst_temps = .{.{ .ref = .src0 }},
3664 .clobbers = .{ .eflags = true },
3665 .each = .{ .once = &.{
3666 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3667 .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ },
3668 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3669 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3670 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3671 .{ ._, ._, .xor, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ },
3672 } },
3673 }, .{
3674 .required_features = .{ .@"64bit", null, null, null },
3675 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
3676 .patterns = &.{
3677 .{ .src = .{ .to_mem, .none } },
3678 },
3679 .extra_temps = .{
3680 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3681 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3682 .unused,
3683 .unused,
3684 .unused,
3685 .unused,
3686 },
3687 .dst_temps = .{.mem},
3688 .clobbers = .{ .eflags = true },
3689 .each = .{ .once = &.{
3690 .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ },
3691 .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ },
3692 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
3693 .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ },
3694 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3695 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3696 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
3697 .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ },
3698 .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ },
3699 } },
3700 }, .{
3701 .required_features = .{ .mmx, null, null, null },
3702 .src_constraints = .{ .{ .signed_int_or_full_vec = .qword }, .any },
3703 .patterns = &.{
3704 .{ .src = .{ .mem, .none } },
3705 .{ .src = .{ .to_mm, .none } },
3706 },
3707 .dst_temps = .{.{ .rc = .mmx }},
3708 .each = .{ .once = &.{
3709 .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ },
3710 .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ },
3711 } },
3712 }, .{
3713 .required_features = .{ .mmx, null, null, null },
3714 .src_constraints = .{ .{ .unsigned_int_vec = .qword }, .any },
3715 .patterns = &.{
3716 .{ .src = .{ .to_mut_mm, .none } },
3717 },
3718 .extra_temps = .{
3719 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3720 .{ .kind = .{ .umax_mem = .src0 } },
3721 .unused,
3722 .unused,
3723 .unused,
3724 .unused,
3725 },
3726 .dst_temps = .{.{ .ref = .src0 }},
3727 .each = .{ .once = &.{
3728 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3729 .{ ._, .p_, .xor, .dst0q, .lea(.qword, .tmp0), ._, ._ },
3730 } },
3731 }, .{
3732 .required_features = .{ .avx, null, null, null },
3733 .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any },
3734 .patterns = &.{
3735 .{ .src = .{ .mem, .none } },
3736 .{ .src = .{ .to_xmm, .none } },
3737 },
3738 .dst_temps = .{.{ .rc = .sse }},
3739 .each = .{ .once = &.{
3740 .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ },
3741 .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ },
3742 } },
3743 }, .{
3744 .required_features = .{ .avx, null, null, null },
3745 .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any },
3746 .patterns = &.{
3747 .{ .src = .{ .to_xmm, .none } },
3748 },
3749 .extra_temps = .{
3750 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3751 .{ .kind = .{ .umax_mem = .src0 } },
3752 .unused,
3753 .unused,
3754 .unused,
3755 .unused,
3756 },
3757 .dst_temps = .{.{ .rc = .sse }},
3758 .each = .{ .once = &.{
3759 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3760 .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
3761 } },
3762 }, .{
3763 .required_features = .{ .sse2, null, null, null },
3764 .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any },
3765 .patterns = &.{
3766 .{ .src = .{ .mem, .none } },
3767 .{ .src = .{ .to_xmm, .none } },
3768 },
3769 .dst_temps = .{.{ .rc = .sse }},
3770 .each = .{ .once = &.{
3771 .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ },
3772 .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ },
3773 } },
3774 }, .{
3775 .required_features = .{ .sse2, null, null, null },
3776 .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any },
3777 .patterns = &.{
3778 .{ .src = .{ .to_mut_xmm, .none } },
3779 },
3780 .extra_temps = .{
3781 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3782 .{ .kind = .{ .umax_mem = .src0 } },
3783 .unused,
3784 .unused,
3785 .unused,
3786 .unused,
3787 },
3788 .dst_temps = .{.{ .ref = .src0 }},
3789 .each = .{ .once = &.{
3790 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3791 .{ ._, .p_, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
3792 } },
3793 }, .{
3794 .required_features = .{ .sse, null, null, null },
3795 .src_constraints = .{ .{ .vec = .xword }, .any },
3796 .patterns = &.{
3797 .{ .src = .{ .to_mut_xmm, .none } },
3798 },
3799 .extra_temps = .{
3800 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3801 .{ .kind = .{ .umax_mem = .src0 } },
3802 .unused,
3803 .unused,
3804 .unused,
3805 .unused,
3806 },
3807 .dst_temps = .{.{ .ref = .src0 }},
3808 .each = .{ .once = &.{
3809 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3810 .{ ._, ._ps, .xor, .dst0x, .lea(.xword, .tmp0), ._, ._ },
3811 } },
3812 }, .{
3813 .required_features = .{ .avx2, null, null, null },
3814 .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any },
3815 .patterns = &.{
3816 .{ .src = .{ .mem, .none } },
3817 .{ .src = .{ .to_ymm, .none } },
3818 },
3819 .dst_temps = .{.{ .rc = .sse }},
3820 .each = .{ .once = &.{
3821 .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ },
3822 .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ },
3823 } },
3824 }, .{
3825 .required_features = .{ .avx2, null, null, null },
3826 .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any },
3827 .patterns = &.{
3828 .{ .src = .{ .to_ymm, .none } },
3829 },
3830 .extra_temps = .{
3831 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3832 .{ .kind = .{ .umax_mem = .src0 } },
3833 .unused,
3834 .unused,
3835 .unused,
3836 .unused,
3837 },
3838 .dst_temps = .{.{ .rc = .sse }},
3839 .each = .{ .once = &.{
3840 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3841 .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
3842 } },
3843 }, .{
3844 .required_features = .{ .avx, null, null, null },
3845 .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any },
3846 .patterns = &.{
3847 .{ .src = .{ .mem, .none } },
3848 .{ .src = .{ .to_ymm, .none } },
3849 },
3850 .dst_temps = .{.{ .rc = .sse }},
3851 .each = .{ .once = &.{
3852 .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) },
3853 .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ },
3854 } },
3855 }, .{
3856 .required_features = .{ .avx, null, null, null },
3857 .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any },
3858 .patterns = &.{
3859 .{ .src = .{ .to_ymm, .none } },
3860 },
3861 .extra_temps = .{
3862 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3863 .{ .kind = .{ .umax_mem = .src0 } },
3864 .unused,
3865 .unused,
3866 .unused,
3867 .unused,
3868 },
3869 .dst_temps = .{.{ .rc = .sse }},
3870 .each = .{ .once = &.{
3871 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
3872 .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
3873 } },
3874 }, .{
3875 .required_features = .{ .@"64bit", null, null, null },
3876 .patterns = &.{
3877 .{ .src = .{ .to_mem, .none } },
3878 },
3879 .extra_temps = .{
3880 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3881 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3882 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
3883 .{ .kind = .{ .umax_mem = .src0 } },
3884 .unused,
3885 .unused,
3886 },
3887 .dst_temps = .{.mem},
3888 .each = .{ .once = &.{
3889 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_src0_size), ._, ._ },
3890 .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
3891 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_src0_size), ._, ._ },
3892 .{ ._, ._, .xor, .tmp2q, .leaia(.qword, .tmp1, .tmp0, .add_src0_size), ._, ._ },
3893 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_src0_size), .tmp2q, ._, ._ },
3894 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
3895 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3896 } },
3897 }, .{
3898 .patterns = &.{
3899 .{ .src = .{ .to_mem, .none } },
3900 },
3901 .extra_temps = .{
3902 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3903 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3904 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3905 .{ .kind = .{ .umax_mem = .src0 } },
3906 .unused,
3907 .unused,
3908 },
3909 .dst_temps = .{.mem},
3910 .each = .{ .once = &.{
3911 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_src0_size), ._, ._ },
3912 .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
3913 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_src0_size), ._, ._ },
3914 .{ ._, ._, .xor, .tmp2d, .leaia(.dword, .tmp1, .tmp0, .add_src0_size), ._, ._ },
3915 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_src0_size), .tmp2d, ._, ._ },
3916 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
3917 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3918 } },
3919 } }) catch |err| switch (err) {
3920 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
3921 @tagName(air_tag),
3922 cg.typeOf(ty_op.operand).fmt(pt),
3923 ops[0].tracking(cg),
3924 }),
3925 else => |e| return e,
3926 };
3927 if (ops[0].index != res[0].index) try ops[0].die(cg);
3928 try res[0].moveTo(inst, cg);
3929 },
3930
28333931 .block => if (use_old) try cg.airBlock(inst) else {
28343932 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
28353933 const extra = cg.air.extraData(Air.Block, ty_pl.payload);
......@@ -2880,91 +3978,2760 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
28803978 .call_never_tail => try cg.airCall(inst, .never_tail),
28813979 .call_never_inline => try cg.airCall(inst, .never_inline),
28823980
2883 .cmp_vector, .cmp_vector_optimized => |air_tag| if (use_old) try cg.airCmpVector(inst) else fallback: {
2884 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
2885 const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
2886 switch (extra.compareOperator()) {
2887 .eq, .neq => {},
2888 else => break :fallback try cg.airCmpVector(inst),
2889 }
2890 var ops = try cg.tempsFromOperands(inst, .{ extra.lhs, extra.rhs });
3981 .clz => |air_tag| if (use_old) try cg.airClz(inst) else {
3982 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
3983 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
28913984 var res: [1]Temp = undefined;
2892 switch (extra.compareOperator()) {
2893 .lt => unreachable,
2894 .lte => unreachable,
2895 .eq, .neq => |cmp_op| cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (cmp_op) {
2896 else => unreachable,
2897 .eq => .e,
2898 .neq => .ne,
2899 })) {
2900 else => unreachable,
2901 inline .e, .ne => |cc| comptime &.{ .{
2902 .required_features = .{ .avx2, null },
2903 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
2904 .patterns = &.{
2905 .{ .src = .{ .ymm, .mem } },
2906 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
2907 .{ .src = .{ .ymm, .ymm } },
2908 },
2909 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
2910 .kind = .all,
2911 .inverted = switch (cc) {
2912 else => unreachable,
2913 .e => false,
2914 .ne => true,
2915 },
2916 .scalar = .byte,
2917 } } }},
2918 .each = .{ .once = &.{
2919 .{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ },
2920 } },
2921 }, .{
2922 .required_features = .{ .avx2, null },
2923 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
2924 .patterns = &.{
2925 .{ .src = .{ .ymm, .mem } },
2926 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
2927 .{ .src = .{ .ymm, .ymm } },
2928 },
2929 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
2930 .kind = .all,
2931 .inverted = switch (cc) {
2932 else => unreachable,
2933 .e => false,
2934 .ne => true,
2935 },
2936 .scalar = .word,
2937 } } }},
2938 .each = .{ .once = &.{
2939 .{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ },
2940 } },
2941 }, .{
2942 .required_features = .{ .avx2, null },
2943 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
2944 .patterns = &.{
2945 .{ .src = .{ .ymm, .mem } },
2946 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
2947 .{ .src = .{ .ymm, .ymm } },
2948 },
2949 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
2950 .kind = .all,
2951 .inverted = switch (cc) {
2952 else => unreachable,
2953 .e => false,
2954 .ne => true,
2955 },
2956 .scalar = .dword,
2957 } } }},
2958 .each = .{ .once = &.{
2959 .{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ },
2960 } },
2961 }, .{
2962 .required_features = .{ .avx2, null },
2963 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
2964 .patterns = &.{
2965 .{ .src = .{ .ymm, .mem } },
2966 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
2967 .{ .src = .{ .ymm, .ymm } },
3985 cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, comptime &.{ .{
3986 .required_features = .{ .slow_incdec, null, null, null },
3987 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any },
3988 .patterns = &.{
3989 .{ .src = .{ .mut_mem, .none } },
3990 .{ .src = .{ .to_mut_gpr, .none } },
3991 },
3992 .dst_temps = .{.{ .ref = .src0 }},
3993 .clobbers = .{ .eflags = true },
3994 .each = .{ .once = &.{
3995 .{ ._, ._, .add, .dst0b, .si(1), ._, ._ },
3996 } },
3997 }, .{
3998 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any },
3999 .patterns = &.{
4000 .{ .src = .{ .mut_mem, .none } },
4001 .{ .src = .{ .to_mut_gpr, .none } },
4002 },
4003 .dst_temps = .{.{ .ref = .src0 }},
4004 .clobbers = .{ .eflags = true },
4005 .each = .{ .once = &.{
4006 .{ ._, ._, .inc, .dst0b, ._, ._, ._ },
4007 } },
4008 }, .{
4009 .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any },
4010 .patterns = &.{
4011 .{ .src = .{ .mut_mem, .none } },
4012 .{ .src = .{ .to_mut_gpr, .none } },
4013 },
4014 .dst_temps = .{.{ .ref = .src0 }},
4015 .clobbers = .{ .eflags = true },
4016 .each = .{ .once = &.{
4017 .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ },
4018 } },
4019 }, .{
4020 .required_features = .{ .lzcnt, null, null, null },
4021 .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any },
4022 .patterns = &.{
4023 .{ .src = .{ .mem, .none } },
4024 .{ .src = .{ .to_gpr, .none } },
4025 },
4026 .dst_temps = .{.{ .rc = .general_purpose }},
4027 .clobbers = .{ .eflags = true },
4028 .each = .{ .once = &.{
4029 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4030 .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ },
4031 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4032 } },
4033 }, .{
4034 .required_features = .{ .lzcnt, null, null, null },
4035 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4036 .patterns = &.{
4037 .{ .src = .{ .mem, .none } },
4038 .{ .src = .{ .to_gpr, .none } },
4039 },
4040 .dst_temps = .{.{ .rc = .general_purpose }},
4041 .clobbers = .{ .eflags = true },
4042 .each = .{ .once = &.{
4043 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4044 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4045 .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ },
4046 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4047 } },
4048 }, .{
4049 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4050 .src_constraints = .{ .{ .exact_int = 16 }, .any },
4051 .patterns = &.{
4052 .{ .src = .{ .to_mut_gpr, .none } },
4053 },
4054 .dst_temps = .{.{ .ref = .src0 }},
4055 .clobbers = .{ .eflags = true },
4056 .each = .{ .once = &.{
4057 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4058 } },
4059 }, .{
4060 .required_features = .{ .lzcnt, null, null, null },
4061 .src_constraints = .{ .{ .exact_int = 16 }, .any },
4062 .patterns = &.{
4063 .{ .src = .{ .mem, .none } },
4064 .{ .src = .{ .to_gpr, .none } },
4065 },
4066 .dst_temps = .{.{ .rc = .general_purpose }},
4067 .clobbers = .{ .eflags = true },
4068 .each = .{ .once = &.{
4069 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4070 } },
4071 }, .{
4072 .required_features = .{ .lzcnt, null, null, null },
4073 .src_constraints = .{ .{ .signed_int = .word }, .any },
4074 .patterns = &.{
4075 .{ .src = .{ .to_mut_gpr, .none } },
4076 },
4077 .dst_temps = .{.{ .ref = .src0 }},
4078 .clobbers = .{ .eflags = true },
4079 .each = .{ .once = &.{
4080 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4081 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4082 .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
4083 } },
4084 }, .{
4085 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4086 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4087 .patterns = &.{
4088 .{ .src = .{ .to_mut_gpr, .none } },
4089 },
4090 .dst_temps = .{.{ .ref = .src0 }},
4091 .clobbers = .{ .eflags = true },
4092 .each = .{ .once = &.{
4093 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4094 .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
4095 } },
4096 }, .{
4097 .required_features = .{ .lzcnt, null, null, null },
4098 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4099 .patterns = &.{
4100 .{ .src = .{ .mem, .none } },
4101 .{ .src = .{ .to_gpr, .none } },
4102 },
4103 .dst_temps = .{.{ .rc = .general_purpose }},
4104 .clobbers = .{ .eflags = true },
4105 .each = .{ .once = &.{
4106 .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ },
4107 .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ },
4108 } },
4109 }, .{
4110 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4111 .src_constraints = .{ .{ .exact_int = 32 }, .any },
4112 .patterns = &.{
4113 .{ .src = .{ .to_mut_gpr, .none } },
4114 },
4115 .dst_temps = .{.{ .ref = .src0 }},
4116 .clobbers = .{ .eflags = true },
4117 .each = .{ .once = &.{
4118 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4119 } },
4120 }, .{
4121 .required_features = .{ .lzcnt, null, null, null },
4122 .src_constraints = .{ .{ .exact_int = 32 }, .any },
4123 .patterns = &.{
4124 .{ .src = .{ .mem, .none } },
4125 .{ .src = .{ .to_gpr, .none } },
4126 },
4127 .dst_temps = .{.{ .rc = .general_purpose }},
4128 .clobbers = .{ .eflags = true },
4129 .each = .{ .once = &.{
4130 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4131 } },
4132 }, .{
4133 .required_features = .{ .lzcnt, null, null, null },
4134 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4135 .patterns = &.{
4136 .{ .src = .{ .to_mut_gpr, .none } },
4137 },
4138 .dst_temps = .{.{ .ref = .src0 }},
4139 .clobbers = .{ .eflags = true },
4140 .each = .{ .once = &.{
4141 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
4142 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4143 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4144 } },
4145 }, .{
4146 .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null },
4147 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4148 .patterns = &.{
4149 .{ .src = .{ .to_mut_gpr, .none } },
4150 },
4151 .dst_temps = .{.{ .ref = .src0 }},
4152 .clobbers = .{ .eflags = true },
4153 .each = .{ .once = &.{
4154 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4155 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4156 } },
4157 }, .{
4158 .required_features = .{ .lzcnt, null, null, null },
4159 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4160 .patterns = &.{
4161 .{ .src = .{ .mem, .none } },
4162 .{ .src = .{ .to_gpr, .none } },
4163 },
4164 .dst_temps = .{.{ .rc = .general_purpose }},
4165 .clobbers = .{ .eflags = true },
4166 .each = .{ .once = &.{
4167 .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ },
4168 .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ },
4169 } },
4170 }, .{
4171 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
4172 .src_constraints = .{ .{ .exact_int = 64 }, .any },
4173 .patterns = &.{
4174 .{ .src = .{ .to_mut_gpr, .none } },
4175 },
4176 .dst_temps = .{.{ .ref = .src0 }},
4177 .clobbers = .{ .eflags = true },
4178 .each = .{ .once = &.{
4179 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4180 } },
4181 }, .{
4182 .required_features = .{ .@"64bit", .lzcnt, null, null },
4183 .src_constraints = .{ .{ .exact_int = 64 }, .any },
4184 .patterns = &.{
4185 .{ .src = .{ .mem, .none } },
4186 .{ .src = .{ .to_gpr, .none } },
4187 },
4188 .dst_temps = .{.{ .rc = .general_purpose }},
4189 .clobbers = .{ .eflags = true },
4190 .each = .{ .once = &.{
4191 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4192 } },
4193 }, .{
4194 .required_features = .{ .@"64bit", .lzcnt, null, null },
4195 .src_constraints = .{ .{ .signed_int = .qword }, .any },
4196 .patterns = &.{
4197 .{ .src = .{ .mem, .none } },
4198 .{ .src = .{ .to_gpr, .none } },
4199 },
4200 .dst_temps = .{.{ .rc = .general_purpose }},
4201 .clobbers = .{ .eflags = true },
4202 .each = .{ .once = &.{
4203 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
4204 .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ },
4205 .{ ._, ._, .lzcnt, .dst0q, .dst0q, ._, ._ },
4206 .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
4207 } },
4208 }, .{
4209 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
4210 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
4211 .patterns = &.{
4212 .{ .src = .{ .to_mut_gpr, .none } },
4213 },
4214 .dst_temps = .{.{ .ref = .src0 }},
4215 .clobbers = .{ .eflags = true },
4216 .each = .{ .once = &.{
4217 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4218 .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
4219 } },
4220 }, .{
4221 .required_features = .{ .@"64bit", .lzcnt, null, null },
4222 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
4223 .patterns = &.{
4224 .{ .src = .{ .mem, .none } },
4225 .{ .src = .{ .to_gpr, .none } },
4226 },
4227 .dst_temps = .{.{ .rc = .general_purpose }},
4228 .clobbers = .{ .eflags = true },
4229 .each = .{ .once = &.{
4230 .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ },
4231 .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ },
4232 } },
4233 }, .{
4234 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4235 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
4236 .patterns = &.{
4237 .{ .src = .{ .mem, .none } },
4238 .{ .src = .{ .to_gpr, .none } },
4239 },
4240 .extra_temps = .{
4241 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4242 .unused,
4243 .unused,
4244 .unused,
4245 .unused,
4246 .unused,
4247 },
4248 .dst_temps = .{.{ .rc = .general_purpose }},
4249 .clobbers = .{ .eflags = true },
4250 .each = .{ .once = &.{
4251 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4252 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4253 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4254 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
4255 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4256 } },
4257 }, .{
4258 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4259 .src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
4260 .patterns = &.{
4261 .{ .src = .{ .mem, .none } },
4262 .{ .src = .{ .to_gpr, .none } },
4263 },
4264 .extra_temps = .{
4265 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4266 .unused,
4267 .unused,
4268 .unused,
4269 .unused,
4270 .unused,
4271 },
4272 .dst_temps = .{.{ .rc = .general_purpose }},
4273 .clobbers = .{ .eflags = true },
4274 .each = .{ .once = &.{
4275 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4276 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4277 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4278 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4279 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
4280 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4281 } },
4282 }, .{
4283 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4284 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4285 .patterns = &.{
4286 .{ .src = .{ .mem, .none } },
4287 .{ .src = .{ .to_gpr, .none } },
4288 },
4289 .extra_temps = .{
4290 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4291 .unused,
4292 .unused,
4293 .unused,
4294 .unused,
4295 .unused,
4296 },
4297 .dst_temps = .{.{ .rc = .general_purpose }},
4298 .clobbers = .{ .eflags = true },
4299 .each = .{ .once = &.{
4300 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4301 .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
4302 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4303 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4304 .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
4305 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4306 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4307 } },
4308 }, .{
4309 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4310 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
4311 .patterns = &.{
4312 .{ .src = .{ .mem, .none } },
4313 .{ .src = .{ .to_gpr, .none } },
4314 },
4315 .extra_temps = .{
4316 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4317 .unused,
4318 .unused,
4319 .unused,
4320 .unused,
4321 .unused,
4322 },
4323 .dst_temps = .{.{ .rc = .general_purpose }},
4324 .clobbers = .{ .eflags = true },
4325 .each = .{ .once = &.{
4326 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4327 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4328 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4329 .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
4330 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4331 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4332 } },
4333 }, .{
4334 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4335 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
4336 .patterns = &.{
4337 .{ .src = .{ .mem, .none } },
4338 .{ .src = .{ .to_gpr, .none } },
4339 },
4340 .dst_temps = .{.{ .rc = .general_purpose }},
4341 .clobbers = .{ .eflags = true },
4342 .each = .{ .once = &.{
4343 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4344 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4345 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4346 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4347 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4348 } },
4349 }, .{
4350 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4351 .src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
4352 .patterns = &.{
4353 .{ .src = .{ .mem, .none } },
4354 .{ .src = .{ .to_gpr, .none } },
4355 },
4356 .dst_temps = .{.{ .rc = .general_purpose }},
4357 .clobbers = .{ .eflags = true },
4358 .each = .{ .once = &.{
4359 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4360 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4361 .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ },
4362 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4363 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4364 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4365 } },
4366 }, .{
4367 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4368 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4369 .patterns = &.{
4370 .{ .src = .{ .mem, .none } },
4371 .{ .src = .{ .to_gpr, .none } },
4372 },
4373 .extra_temps = .{
4374 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4375 .unused,
4376 .unused,
4377 .unused,
4378 .unused,
4379 .unused,
4380 },
4381 .dst_temps = .{.{ .rc = .general_purpose }},
4382 .clobbers = .{ .eflags = true },
4383 .each = .{ .once = &.{
4384 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4385 .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
4386 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4387 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4388 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4389 .{ ._, ._c, .st, ._, ._, ._, ._ },
4390 .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
4391 } },
4392 }, .{
4393 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4394 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
4395 .patterns = &.{
4396 .{ .src = .{ .mem, .none } },
4397 .{ .src = .{ .to_gpr, .none } },
4398 },
4399 .extra_temps = .{
4400 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4401 .unused,
4402 .unused,
4403 .unused,
4404 .unused,
4405 .unused,
4406 },
4407 .dst_temps = .{.{ .rc = .general_purpose }},
4408 .clobbers = .{ .eflags = true },
4409 .each = .{ .once = &.{
4410 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4411 .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ },
4412 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4413 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4414 .{ ._, ._c, .st, ._, ._, ._, ._ },
4415 .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
4416 } },
4417 }, .{
4418 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any },
4419 .patterns = &.{
4420 .{ .src = .{ .mem, .none } },
4421 .{ .src = .{ .to_gpr, .none } },
4422 },
4423 .extra_temps = .{
4424 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4425 .unused,
4426 .unused,
4427 .unused,
4428 .unused,
4429 .unused,
4430 },
4431 .dst_temps = .{.{ .rc = .general_purpose }},
4432 .clobbers = .{ .eflags = true },
4433 .each = .{ .once = &.{
4434 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4435 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4436 .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ },
4437 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4438 } },
4439 }, .{
4440 .src_constraints = .{ .{ .signed_po2_int = .byte }, .any },
4441 .patterns = &.{
4442 .{ .src = .{ .mem, .none } },
4443 .{ .src = .{ .to_gpr, .none } },
4444 },
4445 .extra_temps = .{
4446 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4447 .unused,
4448 .unused,
4449 .unused,
4450 .unused,
4451 .unused,
4452 },
4453 .dst_temps = .{.{ .rc = .general_purpose }},
4454 .clobbers = .{ .eflags = true },
4455 .each = .{ .once = &.{
4456 .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ },
4457 .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ },
4458 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4459 .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ },
4460 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4461 } },
4462 }, .{
4463 .src_constraints = .{ .{ .signed_int = .byte }, .any },
4464 .patterns = &.{
4465 .{ .src = .{ .mem, .none } },
4466 .{ .src = .{ .to_gpr, .none } },
4467 },
4468 .extra_temps = .{
4469 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4470 .unused,
4471 .unused,
4472 .unused,
4473 .unused,
4474 .unused,
4475 },
4476 .dst_temps = .{.{ .rc = .general_purpose }},
4477 .clobbers = .{ .eflags = true },
4478 .each = .{ .once = &.{
4479 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4480 .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ },
4481 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4482 .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ },
4483 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4484 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4485 } },
4486 }, .{
4487 .src_constraints = .{ .{ .unsigned_int = .byte }, .any },
4488 .patterns = &.{
4489 .{ .src = .{ .mem, .none } },
4490 .{ .src = .{ .to_gpr, .none } },
4491 },
4492 .extra_temps = .{
4493 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4494 .unused,
4495 .unused,
4496 .unused,
4497 .unused,
4498 .unused,
4499 },
4500 .dst_temps = .{.{ .rc = .general_purpose }},
4501 .clobbers = .{ .eflags = true },
4502 .each = .{ .once = &.{
4503 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
4504 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4505 .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ },
4506 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4507 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4508 } },
4509 }, .{
4510 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4511 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
4512 .patterns = &.{
4513 .{ .src = .{ .to_mut_gpr, .none } },
4514 },
4515 .dst_temps = .{.{ .rc = .general_purpose }},
4516 .clobbers = .{ .eflags = true },
4517 .each = .{ .once = &.{
4518 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4519 .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4520 .{ ._, ._nz, .cmov, .dst0w, .src0w, ._, ._ },
4521 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4522 } },
4523 }, .{
4524 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4525 .src_constraints = .{ .{ .signed_int = .word }, .any },
4526 .patterns = &.{
4527 .{ .src = .{ .to_mut_gpr, .none } },
4528 },
4529 .dst_temps = .{.{ .rc = .general_purpose }},
4530 .clobbers = .{ .eflags = true },
4531 .each = .{ .once = &.{
4532 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4533 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4534 .{ ._, ._, .mov, .dst0w, .si(0xff), ._, ._ },
4535 .{ ._, ._z, .cmov, .src0w, .dst0w, ._, ._ },
4536 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4537 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4538 } },
4539 }, .{
4540 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4541 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4542 .patterns = &.{
4543 .{ .src = .{ .to_mut_gpr, .none } },
4544 },
4545 .dst_temps = .{.{ .rc = .general_purpose }},
4546 .clobbers = .{ .eflags = true },
4547 .each = .{ .once = &.{
4548 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4549 .{ ._, ._, .mov, .dst0w, .si(0xff), ._, ._ },
4550 .{ ._, ._z, .cmov, .src0w, .dst0w, ._, ._ },
4551 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4552 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4553 } },
4554 }, .{
4555 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4556 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
4557 .patterns = &.{
4558 .{ .src = .{ .to_mut_gpr, .none } },
4559 },
4560 .dst_temps = .{.{ .ref = .src0 }},
4561 .clobbers = .{ .eflags = true },
4562 .each = .{ .once = &.{
4563 .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ },
4564 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4565 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4566 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4567 } },
4568 }, .{
4569 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4570 .src_constraints = .{ .{ .signed_int = .word }, .any },
4571 .patterns = &.{
4572 .{ .src = .{ .to_mut_gpr, .none } },
4573 },
4574 .dst_temps = .{.{ .rc = .general_purpose }},
4575 .clobbers = .{ .eflags = true },
4576 .each = .{ .once = &.{
4577 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4578 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4579 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4580 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4581 .{ ._, ._c, .st, ._, ._, ._, ._ },
4582 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4583 } },
4584 }, .{
4585 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4586 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4587 .patterns = &.{
4588 .{ .src = .{ .to_mut_gpr, .none } },
4589 },
4590 .dst_temps = .{.{ .rc = .general_purpose }},
4591 .clobbers = .{ .eflags = true },
4592 .each = .{ .once = &.{
4593 .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ },
4594 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4595 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4596 .{ ._, ._c, .st, ._, ._, ._, ._ },
4597 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4598 } },
4599 }, .{
4600 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any },
4601 .patterns = &.{
4602 .{ .src = .{ .mem, .none } },
4603 .{ .src = .{ .to_gpr, .none } },
4604 },
4605 .dst_temps = .{.{ .rc = .general_purpose }},
4606 .clobbers = .{ .eflags = true },
4607 .each = .{ .once = &.{
4608 .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4609 .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ },
4610 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4611 } },
4612 }, .{
4613 .src_constraints = .{ .{ .signed_int = .word }, .any },
4614 .patterns = &.{
4615 .{ .src = .{ .to_mut_gpr, .none } },
4616 },
4617 .extra_temps = .{
4618 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4619 .unused,
4620 .unused,
4621 .unused,
4622 .unused,
4623 .unused,
4624 },
4625 .dst_temps = .{.{ .rc = .general_purpose }},
4626 .clobbers = .{ .eflags = true },
4627 .each = .{ .once = &.{
4628 .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ },
4629 .{ ._, ._, .mov, .tmp0w, .si(0xff), ._, ._ },
4630 .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ },
4631 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4632 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4633 } },
4634 }, .{
4635 .src_constraints = .{ .{ .unsigned_int = .word }, .any },
4636 .patterns = &.{
4637 .{ .src = .{ .mem, .none } },
4638 .{ .src = .{ .to_gpr, .none } },
4639 },
4640 .extra_temps = .{
4641 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4642 .unused,
4643 .unused,
4644 .unused,
4645 .unused,
4646 .unused,
4647 },
4648 .dst_temps = .{.{ .rc = .general_purpose }},
4649 .clobbers = .{ .eflags = true },
4650 .each = .{ .once = &.{
4651 .{ ._, ._, .mov, .tmp0w, .si(0xff), ._, ._ },
4652 .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ },
4653 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4654 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4655 } },
4656 }, .{
4657 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4658 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
4659 .patterns = &.{
4660 .{ .src = .{ .to_mut_gpr, .none } },
4661 },
4662 .dst_temps = .{.{ .rc = .general_purpose }},
4663 .clobbers = .{ .eflags = true },
4664 .each = .{ .once = &.{
4665 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4666 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4667 .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ },
4668 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4669 } },
4670 }, .{
4671 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4672 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4673 .patterns = &.{
4674 .{ .src = .{ .to_mut_gpr, .none } },
4675 },
4676 .dst_temps = .{.{ .rc = .general_purpose }},
4677 .clobbers = .{ .eflags = true },
4678 .each = .{ .once = &.{
4679 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
4680 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4681 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4682 .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
4683 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4684 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4685 } },
4686 }, .{
4687 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
4688 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4689 .patterns = &.{
4690 .{ .src = .{ .to_mut_gpr, .none } },
4691 },
4692 .dst_temps = .{.{ .rc = .general_purpose }},
4693 .clobbers = .{ .eflags = true },
4694 .each = .{ .once = &.{
4695 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4696 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4697 .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
4698 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4699 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4700 } },
4701 }, .{
4702 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4703 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
4704 .patterns = &.{
4705 .{ .src = .{ .to_mut_gpr, .none } },
4706 },
4707 .dst_temps = .{.{ .ref = .src0 }},
4708 .clobbers = .{ .eflags = true },
4709 .each = .{ .once = &.{
4710 .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ },
4711 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4712 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4713 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4714 } },
4715 }, .{
4716 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4717 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4718 .patterns = &.{
4719 .{ .src = .{ .to_mut_gpr, .none } },
4720 },
4721 .dst_temps = .{.{ .rc = .general_purpose }},
4722 .clobbers = .{ .eflags = true },
4723 .each = .{ .once = &.{
4724 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
4725 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4726 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4727 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4728 .{ ._, ._c, .st, ._, ._, ._, ._ },
4729 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4730 } },
4731 }, .{
4732 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
4733 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4734 .patterns = &.{
4735 .{ .src = .{ .to_mut_gpr, .none } },
4736 },
4737 .dst_temps = .{.{ .rc = .general_purpose }},
4738 .clobbers = .{ .eflags = true },
4739 .each = .{ .once = &.{
4740 .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ },
4741 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4742 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4743 .{ ._, ._c, .st, ._, ._, ._, ._ },
4744 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4745 } },
4746 }, .{
4747 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any },
4748 .patterns = &.{
4749 .{ .src = .{ .mem, .none } },
4750 .{ .src = .{ .to_gpr, .none } },
4751 },
4752 .dst_temps = .{.{ .rc = .general_purpose }},
4753 .clobbers = .{ .eflags = true },
4754 .each = .{ .once = &.{
4755 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4756 .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ },
4757 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4758 } },
4759 }, .{
4760 .src_constraints = .{ .{ .signed_int = .dword }, .any },
4761 .patterns = &.{
4762 .{ .src = .{ .to_mut_gpr, .none } },
4763 },
4764 .extra_temps = .{
4765 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4766 .unused,
4767 .unused,
4768 .unused,
4769 .unused,
4770 .unused,
4771 },
4772 .dst_temps = .{.{ .rc = .general_purpose }},
4773 .clobbers = .{ .eflags = true },
4774 .each = .{ .once = &.{
4775 .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ },
4776 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4777 .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ },
4778 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4779 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4780 } },
4781 }, .{
4782 .src_constraints = .{ .{ .unsigned_int = .dword }, .any },
4783 .patterns = &.{
4784 .{ .src = .{ .mem, .none } },
4785 .{ .src = .{ .to_gpr, .none } },
4786 },
4787 .extra_temps = .{
4788 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4789 .unused,
4790 .unused,
4791 .unused,
4792 .unused,
4793 .unused,
4794 },
4795 .dst_temps = .{.{ .rc = .general_purpose }},
4796 .clobbers = .{ .eflags = true },
4797 .each = .{ .once = &.{
4798 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4799 .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ },
4800 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4801 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4802 } },
4803 }, .{
4804 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
4805 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
4806 .patterns = &.{
4807 .{ .src = .{ .to_mut_gpr, .none } },
4808 },
4809 .dst_temps = .{.{ .rc = .general_purpose }},
4810 .clobbers = .{ .eflags = true },
4811 .each = .{ .once = &.{
4812 .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
4813 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4814 .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ },
4815 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4816 } },
4817 }, .{
4818 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
4819 .src_constraints = .{ .{ .signed_int = .qword }, .any },
4820 .patterns = &.{
4821 .{ .src = .{ .mem, .none } },
4822 .{ .src = .{ .to_gpr, .none } },
4823 },
4824 .extra_temps = .{
4825 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4826 .unused,
4827 .unused,
4828 .unused,
4829 .unused,
4830 .unused,
4831 },
4832 .dst_temps = .{.{ .rc = .general_purpose }},
4833 .clobbers = .{ .eflags = true },
4834 .each = .{ .once = &.{
4835 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
4836 .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ },
4837 .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ },
4838 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4839 .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ },
4840 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4841 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4842 } },
4843 }, .{
4844 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
4845 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
4846 .patterns = &.{
4847 .{ .src = .{ .to_mut_gpr, .none } },
4848 },
4849 .dst_temps = .{.{ .rc = .general_purpose }},
4850 .clobbers = .{ .eflags = true },
4851 .each = .{ .once = &.{
4852 .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
4853 .{ ._, ._, .mov, .dst0d, .si(0xff), ._, ._ },
4854 .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ },
4855 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4856 .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ },
4857 } },
4858 }, .{
4859 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
4860 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
4861 .patterns = &.{
4862 .{ .src = .{ .to_mut_gpr, .none } },
4863 },
4864 .dst_temps = .{.{ .ref = .src0 }},
4865 .clobbers = .{ .eflags = true },
4866 .each = .{ .once = &.{
4867 .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ },
4868 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
4869 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4870 .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4871 } },
4872 }, .{
4873 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
4874 .src_constraints = .{ .{ .signed_int = .qword }, .any },
4875 .patterns = &.{
4876 .{ .src = .{ .mem, .none } },
4877 .{ .src = .{ .to_gpr, .none } },
4878 },
4879 .extra_temps = .{
4880 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4881 .unused,
4882 .unused,
4883 .unused,
4884 .unused,
4885 .unused,
4886 },
4887 .dst_temps = .{.{ .rc = .general_purpose }},
4888 .clobbers = .{ .eflags = true },
4889 .each = .{ .once = &.{
4890 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ },
4891 .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ },
4892 .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ },
4893 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4894 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4895 .{ ._, ._c, .st, ._, ._, ._, ._ },
4896 .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ },
4897 } },
4898 }, .{
4899 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
4900 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
4901 .patterns = &.{
4902 .{ .src = .{ .to_mut_gpr, .none } },
4903 },
4904 .dst_temps = .{.{ .rc = .general_purpose }},
4905 .clobbers = .{ .eflags = true },
4906 .each = .{ .once = &.{
4907 .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ },
4908 .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ },
4909 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4910 .{ ._, ._c, .st, ._, ._, ._, ._ },
4911 .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ },
4912 } },
4913 }, .{
4914 .required_features = .{ .@"64bit", null, null, null },
4915 .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any },
4916 .patterns = &.{
4917 .{ .src = .{ .mem, .none } },
4918 .{ .src = .{ .to_gpr, .none } },
4919 },
4920 .dst_temps = .{.{ .rc = .general_purpose }},
4921 .clobbers = .{ .eflags = true },
4922 .each = .{ .once = &.{
4923 .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ },
4924 .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ },
4925 .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4926 } },
4927 }, .{
4928 .required_features = .{ .@"64bit", null, null, null },
4929 .src_constraints = .{ .{ .signed_int = .qword }, .any },
4930 .patterns = &.{
4931 .{ .src = .{ .mem, .none } },
4932 .{ .src = .{ .to_gpr, .none } },
4933 },
4934 .extra_temps = .{
4935 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4936 .unused,
4937 .unused,
4938 .unused,
4939 .unused,
4940 .unused,
4941 },
4942 .dst_temps = .{.{ .rc = .general_purpose }},
4943 .clobbers = .{ .eflags = true },
4944 .each = .{ .once = &.{
4945 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
4946 .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ },
4947 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4948 .{ ._, ._r, .bs, .tmp0q, .dst0q, ._, ._ },
4949 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4950 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4951 } },
4952 }, .{
4953 .required_features = .{ .@"64bit", null, null, null },
4954 .src_constraints = .{ .{ .unsigned_int = .qword }, .any },
4955 .patterns = &.{
4956 .{ .src = .{ .mem, .none } },
4957 .{ .src = .{ .to_gpr, .none } },
4958 },
4959 .extra_temps = .{
4960 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4961 .unused,
4962 .unused,
4963 .unused,
4964 .unused,
4965 .unused,
4966 },
4967 .dst_temps = .{.{ .rc = .general_purpose }},
4968 .clobbers = .{ .eflags = true },
4969 .each = .{ .once = &.{
4970 .{ ._, ._, .mov, .tmp0d, .si(0xff), ._, ._ },
4971 .{ ._, ._r, .bs, .tmp0q, .src0q, ._, ._ },
4972 .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
4973 .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ },
4974 } },
4975 }, .{
4976 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
4977 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
4978 .patterns = &.{
4979 .{ .src = .{ .to_mem, .none } },
4980 },
4981 .extra_temps = .{
4982 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4983 .unused,
4984 .unused,
4985 .unused,
4986 .unused,
4987 .unused,
4988 },
4989 .dst_temps = .{.{ .rc = .general_purpose }},
4990 .each = .{ .once = &.{
4991 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
4992 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
4993 .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
4994 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
4995 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
4996 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4997 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4998 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
4999 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5000 } },
5001 }, .{
5002 .required_features = .{ .@"64bit", .lzcnt, null, null },
5003 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5004 .patterns = &.{
5005 .{ .src = .{ .to_mem, .none } },
5006 },
5007 .extra_temps = .{
5008 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5009 .unused,
5010 .unused,
5011 .unused,
5012 .unused,
5013 .unused,
5014 },
5015 .dst_temps = .{.{ .rc = .general_purpose }},
5016 .each = .{ .once = &.{
5017 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5018 .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5019 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5020 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5021 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5022 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5023 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5024 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5025 } },
5026 }, .{
5027 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
5028 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5029 .patterns = &.{
5030 .{ .src = .{ .to_mem, .none } },
5031 },
5032 .extra_temps = .{
5033 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5034 .unused,
5035 .unused,
5036 .unused,
5037 .unused,
5038 .unused,
5039 },
5040 .dst_temps = .{.{ .rc = .general_purpose }},
5041 .each = .{ .once = &.{
5042 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5043 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5044 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5045 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5046 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5047 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5048 .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ },
5049 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5050 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5051 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5052 } },
5053 }, .{
5054 .required_features = .{ .@"64bit", null, null, null },
5055 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5056 .patterns = &.{
5057 .{ .src = .{ .to_mem, .none } },
5058 },
5059 .extra_temps = .{
5060 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5061 .unused,
5062 .unused,
5063 .unused,
5064 .unused,
5065 .unused,
5066 },
5067 .dst_temps = .{.{ .rc = .general_purpose }},
5068 .each = .{ .once = &.{
5069 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5070 .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ },
5071 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5072 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5073 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5074 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5075 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5076 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5077 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5078 } },
5079 }, .{
5080 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
5081 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5082 .patterns = &.{
5083 .{ .src = .{ .to_mem, .none } },
5084 },
5085 .extra_temps = .{
5086 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5087 .unused,
5088 .unused,
5089 .unused,
5090 .unused,
5091 .unused,
5092 },
5093 .dst_temps = .{.{ .rc = .general_purpose }},
5094 .each = .{ .once = &.{
5095 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5096 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5097 .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5098 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5099 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5100 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5101 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5102 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5103 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5104 } },
5105 }, .{
5106 .required_features = .{ .@"64bit", .lzcnt, null, null },
5107 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5108 .patterns = &.{
5109 .{ .src = .{ .to_mem, .none } },
5110 },
5111 .extra_temps = .{
5112 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5113 .unused,
5114 .unused,
5115 .unused,
5116 .unused,
5117 .unused,
5118 },
5119 .dst_temps = .{.{ .rc = .general_purpose }},
5120 .each = .{ .once = &.{
5121 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5122 .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5123 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5124 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5125 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5126 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5127 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5128 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5129 } },
5130 }, .{
5131 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
5132 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5133 .patterns = &.{
5134 .{ .src = .{ .to_mem, .none } },
5135 },
5136 .extra_temps = .{
5137 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5138 .unused,
5139 .unused,
5140 .unused,
5141 .unused,
5142 .unused,
5143 },
5144 .dst_temps = .{.{ .rc = .general_purpose }},
5145 .each = .{ .once = &.{
5146 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5147 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5148 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5149 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5150 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5151 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5152 .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ },
5153 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5154 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5155 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5156 } },
5157 }, .{
5158 .required_features = .{ .@"64bit", null, null, null },
5159 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5160 .patterns = &.{
5161 .{ .src = .{ .to_mem, .none } },
5162 },
5163 .extra_temps = .{
5164 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5165 .unused,
5166 .unused,
5167 .unused,
5168 .unused,
5169 .unused,
5170 },
5171 .dst_temps = .{.{ .rc = .general_purpose }},
5172 .each = .{ .once = &.{
5173 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5174 .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ },
5175 .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5176 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5177 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5178 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5179 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5180 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5181 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5182 } },
5183 }, .{
5184 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
5185 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5186 .patterns = &.{
5187 .{ .src = .{ .to_mem, .none } },
5188 },
5189 .extra_temps = .{
5190 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5191 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5192 .unused,
5193 .unused,
5194 .unused,
5195 .unused,
5196 },
5197 .dst_temps = .{.{ .rc = .general_purpose }},
5198 .each = .{ .once = &.{
5199 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5200 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5201 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5202 .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5203 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5204 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5205 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5206 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5207 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5208 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5209 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5210 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5211 } },
5212 }, .{
5213 .required_features = .{ .@"64bit", .lzcnt, null, null },
5214 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5215 .patterns = &.{
5216 .{ .src = .{ .to_mem, .none } },
5217 },
5218 .extra_temps = .{
5219 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5220 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5221 .unused,
5222 .unused,
5223 .unused,
5224 .unused,
5225 },
5226 .dst_temps = .{.{ .rc = .general_purpose }},
5227 .each = .{ .once = &.{
5228 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5229 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5230 .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5231 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5232 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5233 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5234 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5235 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5236 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5237 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5238 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5239 } },
5240 }, .{
5241 .required_features = .{ .@"64bit", null, null, null },
5242 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any },
5243 .patterns = &.{
5244 .{ .src = .{ .to_mem, .none } },
5245 },
5246 .extra_temps = .{
5247 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5248 .unused,
5249 .unused,
5250 .unused,
5251 .unused,
5252 .unused,
5253 },
5254 .dst_temps = .{.{ .rc = .general_purpose }},
5255 .each = .{ .once = &.{
5256 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
5257 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
5258 .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5259 .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ },
5260 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5261 .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ },
5262 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5263 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5264 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5265 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5266 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5267 } },
5268 }, .{
5269 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
5270 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5271 .patterns = &.{
5272 .{ .src = .{ .to_mem, .none } },
5273 },
5274 .extra_temps = .{
5275 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5276 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5277 .unused,
5278 .unused,
5279 .unused,
5280 .unused,
5281 },
5282 .dst_temps = .{.{ .rc = .general_purpose }},
5283 .each = .{ .once = &.{
5284 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5285 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5286 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
5287 .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5288 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5289 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5290 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5291 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5292 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5293 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5294 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5295 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5296 } },
5297 }, .{
5298 .required_features = .{ .@"64bit", .lzcnt, null, null },
5299 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5300 .patterns = &.{
5301 .{ .src = .{ .to_mem, .none } },
5302 },
5303 .extra_temps = .{
5304 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5305 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5306 .unused,
5307 .unused,
5308 .unused,
5309 .unused,
5310 },
5311 .dst_temps = .{.{ .rc = .general_purpose }},
5312 .each = .{ .once = &.{
5313 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5314 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5315 .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
5316 .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ },
5317 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
5318 .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ },
5319 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5320 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5321 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5322 .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ },
5323 .{ ._, ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ },
5324 } },
5325 }, .{
5326 .required_features = .{ .@"64bit", null, null, null },
5327 .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any },
5328 .patterns = &.{
5329 .{ .src = .{ .to_mem, .none } },
5330 },
5331 .extra_temps = .{
5332 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5333 .unused,
5334 .unused,
5335 .unused,
5336 .unused,
5337 .unused,
5338 },
5339 .dst_temps = .{.{ .rc = .general_purpose }},
5340 .each = .{ .once = &.{
5341 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
5342 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
5343 .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ },
5344 .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ },
5345 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
5346 .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ },
5347 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
5348 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5349 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
5350 .{ .@"0:", ._, .lea, .dst0d, .leasiad(.none, .dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ },
5351 .{ ._, ._, .neg, .dst0d, ._, ._, ._ },
5352 } },
5353 }, .{
5354 .required_features = .{ .lzcnt, .slow_incdec, null, null },
5355 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5356 .patterns = &.{
5357 .{ .src = .{ .to_mem, .none } },
5358 },
5359 .extra_temps = .{
5360 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5361 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5362 .unused,
5363 .unused,
5364 .unused,
5365 .unused,
5366 },
5367 .dst_temps = .{.mem},
5368 .each = .{ .once = &.{
5369 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5370 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5371 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5372 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5373 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5374 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5375 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5376 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5377 } },
5378 }, .{
5379 .required_features = .{ .lzcnt, null, null, null },
5380 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5381 .patterns = &.{
5382 .{ .src = .{ .to_mem, .none } },
5383 },
5384 .extra_temps = .{
5385 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5386 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5387 .unused,
5388 .unused,
5389 .unused,
5390 .unused,
5391 },
5392 .dst_temps = .{.mem},
5393 .each = .{ .once = &.{
5394 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5395 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5396 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5397 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5398 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5399 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5400 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5401 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5402 } },
5403 }, .{
5404 .required_features = .{ .lzcnt, .slow_incdec, null, null },
5405 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5406 .patterns = &.{
5407 .{ .src = .{ .to_mem, .none } },
5408 },
5409 .extra_temps = .{
5410 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5411 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5412 .unused,
5413 .unused,
5414 .unused,
5415 .unused,
5416 },
5417 .dst_temps = .{.mem},
5418 .each = .{ .once = &.{
5419 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5420 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5421 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5422 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5423 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5424 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5425 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5426 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5427 } },
5428 }, .{
5429 .required_features = .{ .lzcnt, null, null, null },
5430 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5431 .patterns = &.{
5432 .{ .src = .{ .to_mem, .none } },
5433 },
5434 .extra_temps = .{
5435 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5436 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5437 .unused,
5438 .unused,
5439 .unused,
5440 .unused,
5441 },
5442 .dst_temps = .{.mem},
5443 .each = .{ .once = &.{
5444 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5445 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5446 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5447 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5448 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5449 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5450 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5451 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5452 } },
5453 }, .{
5454 .required_features = .{ .lzcnt, .slow_incdec, null, null },
5455 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5456 .patterns = &.{
5457 .{ .src = .{ .to_mem, .none } },
5458 },
5459 .extra_temps = .{
5460 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5461 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5462 .unused,
5463 .unused,
5464 .unused,
5465 .unused,
5466 },
5467 .dst_temps = .{.mem},
5468 .each = .{ .once = &.{
5469 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5470 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5471 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5472 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5473 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5474 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5475 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5476 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5477 } },
5478 }, .{
5479 .required_features = .{ .lzcnt, null, null, null },
5480 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5481 .patterns = &.{
5482 .{ .src = .{ .to_mem, .none } },
5483 },
5484 .extra_temps = .{
5485 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5486 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5487 .unused,
5488 .unused,
5489 .unused,
5490 .unused,
5491 },
5492 .dst_temps = .{.mem},
5493 .each = .{ .once = &.{
5494 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5495 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5496 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5497 .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ },
5498 .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ },
5499 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5500 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5501 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5502 } },
5503 }, .{
5504 .required_features = .{ .@"64bit", .lzcnt, .slow_incdec, null },
5505 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
5506 .patterns = &.{
5507 .{ .src = .{ .to_mem, .none } },
5508 },
5509 .extra_temps = .{
5510 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5511 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5512 .unused,
5513 .unused,
5514 .unused,
5515 .unused,
5516 },
5517 .dst_temps = .{.mem},
5518 .each = .{ .once = &.{
5519 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5520 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5521 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
5522 .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ },
5523 .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ },
5524 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5525 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5526 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5527 } },
5528 }, .{
5529 .required_features = .{ .@"64bit", .lzcnt, null, null },
5530 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
5531 .patterns = &.{
5532 .{ .src = .{ .to_mem, .none } },
5533 },
5534 .extra_temps = .{
5535 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5536 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5537 .unused,
5538 .unused,
5539 .unused,
5540 .unused,
5541 },
5542 .dst_temps = .{.mem},
5543 .each = .{ .once = &.{
5544 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5545 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
5546 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
5547 .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ },
5548 .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ },
5549 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5550 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5551 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5552 } },
5553 }, .{
5554 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
5555 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5556 .patterns = &.{
5557 .{ .src = .{ .to_mem, .none } },
5558 },
5559 .extra_temps = .{
5560 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5561 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5562 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5563 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5564 .unused,
5565 .unused,
5566 },
5567 .dst_temps = .{.mem},
5568 .each = .{ .once = &.{
5569 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5570 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5571 .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5572 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5573 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5574 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5575 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5576 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5577 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5578 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5579 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5580 } },
5581 }, .{
5582 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
5583 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5584 .patterns = &.{
5585 .{ .src = .{ .to_mem, .none } },
5586 },
5587 .extra_temps = .{
5588 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5589 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5590 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5591 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5592 .unused,
5593 .unused,
5594 },
5595 .dst_temps = .{.mem},
5596 .each = .{ .once = &.{
5597 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5598 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5599 .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5600 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5601 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5602 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5603 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5604 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5605 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5606 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5607 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5608 } },
5609 }, .{
5610 .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
5611 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5612 .patterns = &.{
5613 .{ .src = .{ .to_mem, .none } },
5614 },
5615 .extra_temps = .{
5616 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5617 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5618 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5619 .unused,
5620 .unused,
5621 .unused,
5622 },
5623 .dst_temps = .{.mem},
5624 .each = .{ .once = &.{
5625 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5626 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5627 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5628 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5629 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5630 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5631 .{ ._, ._c, .st, ._, ._, ._, ._ },
5632 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5633 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5634 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5635 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5636 } },
5637 }, .{
5638 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
5639 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5640 .patterns = &.{
5641 .{ .src = .{ .to_mem, .none } },
5642 },
5643 .extra_temps = .{
5644 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5645 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5646 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5647 .unused,
5648 .unused,
5649 .unused,
5650 },
5651 .dst_temps = .{.mem},
5652 .each = .{ .once = &.{
5653 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5654 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5655 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5656 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5657 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5658 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5659 .{ ._, ._c, .st, ._, ._, ._, ._ },
5660 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5661 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5662 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5663 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5664 } },
5665 }, .{
5666 .required_features = .{ .slow_incdec, null, null, null },
5667 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5668 .patterns = &.{
5669 .{ .src = .{ .to_mem, .none } },
5670 },
5671 .extra_temps = .{
5672 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5673 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5674 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5675 .unused,
5676 .unused,
5677 .unused,
5678 },
5679 .dst_temps = .{.mem},
5680 .each = .{ .once = &.{
5681 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5682 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5683 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5684 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
5685 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
5686 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5687 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
5688 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5689 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5690 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5691 } },
5692 }, .{
5693 .src_constraints = .{ .{ .scalar_int = .byte }, .any },
5694 .patterns = &.{
5695 .{ .src = .{ .to_mem, .none } },
5696 },
5697 .extra_temps = .{
5698 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5699 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5700 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5701 .unused,
5702 .unused,
5703 .unused,
5704 },
5705 .dst_temps = .{.mem},
5706 .each = .{ .once = &.{
5707 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5708 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ },
5709 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5710 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
5711 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
5712 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5713 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
5714 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5715 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5716 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5717 } },
5718 }, .{
5719 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
5720 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5721 .patterns = &.{
5722 .{ .src = .{ .to_mem, .none } },
5723 },
5724 .extra_temps = .{
5725 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5726 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5727 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5728 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5729 .unused,
5730 .unused,
5731 },
5732 .dst_temps = .{.mem},
5733 .each = .{ .once = &.{
5734 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5735 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5736 .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5737 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5738 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5739 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5740 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5741 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5742 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5743 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5744 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5745 } },
5746 }, .{
5747 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
5748 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5749 .patterns = &.{
5750 .{ .src = .{ .to_mem, .none } },
5751 },
5752 .extra_temps = .{
5753 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5754 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5755 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5756 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5757 .unused,
5758 .unused,
5759 },
5760 .dst_temps = .{.mem},
5761 .each = .{ .once = &.{
5762 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5763 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5764 .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5765 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5766 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5767 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5768 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5769 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5770 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5771 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5772 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5773 } },
5774 }, .{
5775 .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
5776 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5777 .patterns = &.{
5778 .{ .src = .{ .to_mem, .none } },
5779 },
5780 .extra_temps = .{
5781 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5782 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5783 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5784 .unused,
5785 .unused,
5786 .unused,
5787 },
5788 .dst_temps = .{.mem},
5789 .each = .{ .once = &.{
5790 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5791 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5792 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5793 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5794 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5795 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5796 .{ ._, ._c, .st, ._, ._, ._, ._ },
5797 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5798 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5799 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5800 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5801 } },
5802 }, .{
5803 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
5804 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5805 .patterns = &.{
5806 .{ .src = .{ .to_mem, .none } },
5807 },
5808 .extra_temps = .{
5809 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5810 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5811 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5812 .unused,
5813 .unused,
5814 .unused,
5815 },
5816 .dst_temps = .{.mem},
5817 .each = .{ .once = &.{
5818 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5819 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5820 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5821 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5822 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5823 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5824 .{ ._, ._c, .st, ._, ._, ._, ._ },
5825 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5826 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5827 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5828 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5829 } },
5830 }, .{
5831 .required_features = .{ .slow_incdec, null, null, null },
5832 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5833 .patterns = &.{
5834 .{ .src = .{ .to_mem, .none } },
5835 },
5836 .extra_temps = .{
5837 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5838 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5839 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5840 .unused,
5841 .unused,
5842 .unused,
5843 },
5844 .dst_temps = .{.mem},
5845 .each = .{ .once = &.{
5846 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5847 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5848 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5849 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
5850 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
5851 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5852 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
5853 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5854 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5855 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5856 } },
5857 }, .{
5858 .src_constraints = .{ .{ .scalar_int = .word }, .any },
5859 .patterns = &.{
5860 .{ .src = .{ .to_mem, .none } },
5861 },
5862 .extra_temps = .{
5863 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5864 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5865 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5866 .unused,
5867 .unused,
5868 .unused,
5869 },
5870 .dst_temps = .{.mem},
5871 .each = .{ .once = &.{
5872 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5873 .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ },
5874 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5875 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
5876 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
5877 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5878 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
5879 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
5880 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5881 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5882 } },
5883 }, .{
5884 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null },
5885 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5886 .patterns = &.{
5887 .{ .src = .{ .to_mem, .none } },
5888 },
5889 .extra_temps = .{
5890 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5891 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5892 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5893 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5894 .unused,
5895 .unused,
5896 },
5897 .dst_temps = .{.mem},
5898 .each = .{ .once = &.{
5899 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5900 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5901 .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5902 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5903 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5904 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5905 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5906 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5907 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5908 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5909 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5910 } },
5911 }, .{
5912 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
5913 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5914 .patterns = &.{
5915 .{ .src = .{ .to_mem, .none } },
5916 },
5917 .extra_temps = .{
5918 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5919 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5920 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5921 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5922 .unused,
5923 .unused,
5924 },
5925 .dst_temps = .{.mem},
5926 .each = .{ .once = &.{
5927 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5928 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
5929 .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5930 .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ },
5931 .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ },
5932 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
5933 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
5934 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
5935 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
5936 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5937 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5938 } },
5939 }, .{
5940 .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null },
5941 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5942 .patterns = &.{
5943 .{ .src = .{ .to_mem, .none } },
5944 },
5945 .extra_temps = .{
5946 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5947 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5948 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5949 .unused,
5950 .unused,
5951 .unused,
5952 },
5953 .dst_temps = .{.mem},
5954 .each = .{ .once = &.{
5955 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5956 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5957 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5958 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5959 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5960 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5961 .{ ._, ._c, .st, ._, ._, ._, ._ },
5962 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5963 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5964 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
5965 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5966 } },
5967 }, .{
5968 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
5969 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5970 .patterns = &.{
5971 .{ .src = .{ .to_mem, .none } },
5972 },
5973 .extra_temps = .{
5974 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5975 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5976 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5977 .unused,
5978 .unused,
5979 .unused,
5980 },
5981 .dst_temps = .{.mem},
5982 .each = .{ .once = &.{
5983 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
5984 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
5985 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
5986 .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ },
5987 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
5988 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
5989 .{ ._, ._c, .st, ._, ._, ._, ._ },
5990 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
5991 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
5992 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
5993 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5994 } },
5995 }, .{
5996 .required_features = .{ .slow_incdec, null, null, null },
5997 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
5998 .patterns = &.{
5999 .{ .src = .{ .to_mem, .none } },
6000 },
6001 .extra_temps = .{
6002 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6003 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6004 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6005 .unused,
6006 .unused,
6007 .unused,
6008 },
6009 .dst_temps = .{.mem},
6010 .each = .{ .once = &.{
6011 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6012 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6013 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6014 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6015 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
6016 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6017 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6018 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6019 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6020 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6021 } },
6022 }, .{
6023 .src_constraints = .{ .{ .scalar_int = .dword }, .any },
6024 .patterns = &.{
6025 .{ .src = .{ .to_mem, .none } },
6026 },
6027 .extra_temps = .{
6028 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6029 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6030 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6031 .unused,
6032 .unused,
6033 .unused,
6034 },
6035 .dst_temps = .{.mem},
6036 .each = .{ .once = &.{
6037 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6038 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ },
6039 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ },
6040 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6041 .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ },
6042 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6043 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6044 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6045 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6046 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6047 } },
6048 }, .{
6049 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec },
6050 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6051 .patterns = &.{
6052 .{ .src = .{ .to_mem, .none } },
6053 },
6054 .extra_temps = .{
6055 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6056 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6057 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6058 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6059 .unused,
6060 .unused,
6061 },
6062 .dst_temps = .{.mem},
6063 .each = .{ .once = &.{
6064 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6065 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
6066 .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ },
6067 .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6068 .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ },
6069 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
6070 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6071 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6072 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6073 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6074 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6075 } },
6076 }, .{
6077 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
6078 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6079 .patterns = &.{
6080 .{ .src = .{ .to_mem, .none } },
6081 },
6082 .extra_temps = .{
6083 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6084 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6085 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6086 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6087 .unused,
6088 .unused,
6089 },
6090 .dst_temps = .{.mem},
6091 .each = .{ .once = &.{
6092 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6093 .{ ._, ._, .mov, .tmp1d, .si(0xff), ._, ._ },
6094 .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ },
6095 .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6096 .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ },
6097 .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ },
6098 .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6099 .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ },
6100 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6101 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6102 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6103 } },
6104 }, .{
6105 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, .slow_incdec, null },
6106 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6107 .patterns = &.{
6108 .{ .src = .{ .to_mem, .none } },
6109 },
6110 .extra_temps = .{
6111 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6112 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6113 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6114 .unused,
6115 .unused,
6116 .unused,
6117 },
6118 .dst_temps = .{.mem},
6119 .each = .{ .once = &.{
6120 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6121 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6122 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6123 .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ },
6124 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6125 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6126 .{ ._, ._c, .st, ._, ._, ._, ._ },
6127 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6128 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6129 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6130 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6131 } },
6132 }, .{
6133 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
6134 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6135 .patterns = &.{
6136 .{ .src = .{ .to_mem, .none } },
6137 },
6138 .extra_temps = .{
6139 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6140 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6141 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6142 .unused,
6143 .unused,
6144 .unused,
6145 },
6146 .dst_temps = .{.mem},
6147 .each = .{ .once = &.{
6148 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6149 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6150 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6151 .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ },
6152 .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ },
6153 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
6154 .{ ._, ._c, .st, ._, ._, ._, ._ },
6155 .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ },
6156 .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ },
6157 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6158 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6159 } },
6160 }, .{
6161 .required_features = .{ .@"64bit", .slow_incdec, null, null },
6162 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6163 .patterns = &.{
6164 .{ .src = .{ .to_mem, .none } },
6165 },
6166 .extra_temps = .{
6167 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6168 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6169 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6170 .unused,
6171 .unused,
6172 .unused,
6173 },
6174 .dst_temps = .{.mem},
6175 .each = .{ .once = &.{
6176 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6177 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6178 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6179 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6180 .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ },
6181 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6182 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6183 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6184 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6185 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6186 } },
6187 }, .{
6188 .required_features = .{ .@"64bit", null, null, null },
6189 .src_constraints = .{ .{ .scalar_int = .qword }, .any },
6190 .patterns = &.{
6191 .{ .src = .{ .to_mem, .none } },
6192 },
6193 .extra_temps = .{
6194 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6195 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6196 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6197 .unused,
6198 .unused,
6199 .unused,
6200 },
6201 .dst_temps = .{.mem},
6202 .each = .{ .once = &.{
6203 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6204 .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ },
6205 .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ },
6206 .{ ._, ._, .mov, .tmp2d, .si(0xff), ._, ._ },
6207 .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ },
6208 .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ },
6209 .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ },
6210 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ },
6211 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
6212 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
6213 } },
6214 }, .{
6215 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6216 .dst_constraints = .{.{ .scalar_int = .byte }},
6217 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6218 .patterns = &.{
6219 .{ .src = .{ .to_mem, .none } },
6220 },
6221 .extra_temps = .{
6222 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6223 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6224 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6225 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6226 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6227 .unused,
6228 },
6229 .dst_temps = .{.mem},
6230 .each = .{ .once = &.{
6231 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6232 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6233 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6234 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6235 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6236 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6237 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6238 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6239 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6240 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6241 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6242 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6243 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6244 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6245 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6246 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6247 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6248 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6249 } },
6250 }, .{
6251 .required_features = .{ .@"64bit", .lzcnt, null, null },
6252 .dst_constraints = .{.{ .scalar_int = .byte }},
6253 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6254 .patterns = &.{
6255 .{ .src = .{ .to_mem, .none } },
6256 },
6257 .extra_temps = .{
6258 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6259 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6260 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6261 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6262 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6263 .unused,
6264 },
6265 .dst_temps = .{.mem},
6266 .each = .{ .once = &.{
6267 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6268 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6269 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6270 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6271 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6272 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6273 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6274 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6275 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6276 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6277 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6278 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6279 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6280 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6281 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6282 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6283 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6284 } },
6285 }, .{
6286 .required_features = .{ .@"64bit", null, null, null },
6287 .dst_constraints = .{.{ .scalar_int = .byte }},
6288 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6289 .patterns = &.{
6290 .{ .src = .{ .to_mem, .none } },
6291 },
6292 .extra_temps = .{
6293 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6294 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6295 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6296 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6297 .unused,
6298 .unused,
6299 },
6300 .dst_temps = .{.mem},
6301 .each = .{ .once = &.{
6302 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6303 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6304 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6305 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6306 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6307 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6308 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6309 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6310 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6311 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6312 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6313 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6314 .{ ._, ._, .neg, .tmp3b, ._, ._, ._ },
6315 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6316 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6317 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6318 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6319 } },
6320 }, .{
6321 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6322 .dst_constraints = .{.{ .scalar_int = .byte }},
6323 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6324 .patterns = &.{
6325 .{ .src = .{ .to_mem, .none } },
6326 },
6327 .extra_temps = .{
6328 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6329 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6330 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6331 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6332 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6333 .unused,
6334 },
6335 .dst_temps = .{.mem},
6336 .each = .{ .once = &.{
6337 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6338 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6339 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6340 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6341 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6342 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6343 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6344 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6345 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6346 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6347 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6348 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6349 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6350 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6351 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6352 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6353 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6354 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6355 } },
6356 }, .{
6357 .required_features = .{ .@"64bit", .lzcnt, null, null },
6358 .dst_constraints = .{.{ .scalar_int = .byte }},
6359 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6360 .patterns = &.{
6361 .{ .src = .{ .to_mem, .none } },
6362 },
6363 .extra_temps = .{
6364 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6365 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6366 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6367 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6368 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6369 .unused,
6370 },
6371 .dst_temps = .{.mem},
6372 .each = .{ .once = &.{
6373 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6374 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6375 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6376 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6377 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6378 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6379 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6380 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6381 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6382 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6383 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6384 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6385 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6386 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6387 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6388 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6389 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6390 } },
6391 }, .{
6392 .required_features = .{ .@"64bit", null, null, null },
6393 .dst_constraints = .{.{ .scalar_int = .byte }},
6394 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6395 .patterns = &.{
6396 .{ .src = .{ .to_mem, .none } },
6397 },
6398 .extra_temps = .{
6399 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6400 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6401 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6402 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6403 .unused,
6404 .unused,
6405 },
6406 .dst_temps = .{.mem},
6407 .each = .{ .once = &.{
6408 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6409 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6410 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6411 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6412 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6413 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6414 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6415 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6416 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6417 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6418 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6419 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6420 .{ ._, ._, .neg, .tmp3b, ._, ._, ._ },
6421 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ },
6422 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6423 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6424 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6425 } },
6426 }, .{
6427 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6428 .dst_constraints = .{.{ .scalar_int = .word }},
6429 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6430 .patterns = &.{
6431 .{ .src = .{ .to_mem, .none } },
6432 },
6433 .extra_temps = .{
6434 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6435 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6436 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6437 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6438 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6439 .unused,
6440 },
6441 .dst_temps = .{.mem},
6442 .each = .{ .once = &.{
6443 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6444 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6445 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6446 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6447 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6448 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6449 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6450 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6451 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6452 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6453 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6454 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6455 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6456 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6457 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6458 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6459 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6460 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6461 } },
6462 }, .{
6463 .required_features = .{ .@"64bit", .lzcnt, null, null },
6464 .dst_constraints = .{.{ .scalar_int = .word }},
6465 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6466 .patterns = &.{
6467 .{ .src = .{ .to_mem, .none } },
6468 },
6469 .extra_temps = .{
6470 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6471 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6472 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6473 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6474 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6475 .unused,
6476 },
6477 .dst_temps = .{.mem},
6478 .each = .{ .once = &.{
6479 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6480 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6481 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6482 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6483 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6484 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6485 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6486 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6487 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6488 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6489 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6490 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6491 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6492 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6493 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6494 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6495 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6496 } },
6497 }, .{
6498 .required_features = .{ .@"64bit", null, null, null },
6499 .dst_constraints = .{.{ .scalar_int = .word }},
6500 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any },
6501 .patterns = &.{
6502 .{ .src = .{ .to_mem, .none } },
6503 },
6504 .extra_temps = .{
6505 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6506 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6507 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6508 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6509 .unused,
6510 .unused,
6511 },
6512 .dst_temps = .{.mem},
6513 .each = .{ .once = &.{
6514 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6515 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6516 .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ },
6517 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6518 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6519 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6520 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6521 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6522 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6523 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6524 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6525 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6526 .{ ._, ._, .neg, .tmp3d, ._, ._, ._ },
6527 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6528 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6529 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6530 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6531 } },
6532 }, .{
6533 .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null },
6534 .dst_constraints = .{.{ .scalar_int = .word }},
6535 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6536 .patterns = &.{
6537 .{ .src = .{ .to_mem, .none } },
6538 },
6539 .extra_temps = .{
6540 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6541 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6542 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6543 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6544 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6545 .unused,
6546 },
6547 .dst_temps = .{.mem},
6548 .each = .{ .once = &.{
6549 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6550 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6551 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6552 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6553 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6554 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6555 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6556 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6557 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6558 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6559 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6560 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6561 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6562 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6563 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6564 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6565 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6566 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6567 } },
6568 }, .{
6569 .required_features = .{ .@"64bit", .lzcnt, null, null },
6570 .dst_constraints = .{.{ .scalar_int = .word }},
6571 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6572 .patterns = &.{
6573 .{ .src = .{ .to_mem, .none } },
6574 },
6575 .extra_temps = .{
6576 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6577 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6578 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6579 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6580 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6581 .unused,
6582 },
6583 .dst_temps = .{.mem},
6584 .each = .{ .once = &.{
6585 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6586 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6587 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6588 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6589 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6590 .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ },
6591 .{ ._, ._nc, .j, .@"1f", ._, ._, ._ },
6592 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6593 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6594 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6595 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6596 .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ },
6597 .{ ._, ._, .lea, .tmp3d, .leasiad(.none, .tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ },
6598 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6599 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6600 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6601 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6602 } },
6603 }, .{
6604 .required_features = .{ .@"64bit", null, null, null },
6605 .dst_constraints = .{.{ .scalar_int = .word }},
6606 .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any },
6607 .patterns = &.{
6608 .{ .src = .{ .to_mem, .none } },
6609 },
6610 .extra_temps = .{
6611 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6612 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6613 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
6614 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6615 .unused,
6616 .unused,
6617 },
6618 .dst_temps = .{.mem},
6619 .each = .{ .once = &.{
6620 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ },
6621 .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ },
6622 .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ },
6623 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ },
6624 .{ .@"1:", ._, .@"and", .tmp3q, .leai(.qword, .tmp1, .tmp2), ._, ._ },
6625 .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ },
6626 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
6627 .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ },
6628 .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ },
6629 .{ ._, ._nc, .j, .@"1b", ._, ._, ._ },
6630 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6631 .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.none, .tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ },
6632 .{ ._, ._, .neg, .tmp3d, ._, ._, ._ },
6633 .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ },
6634 .{ ._, ._, .lea, .tmp1q, .leaa(.none, .tmp1, .add_src0_elem_size), ._, ._ },
6635 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
6636 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6637 } },
6638 } }) catch |err| switch (err) {
6639 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
6640 @tagName(air_tag),
6641 cg.typeOf(ty_op.operand).fmt(pt),
6642 ops[0].tracking(cg),
6643 }),
6644 else => |e| return e,
6645 };
6646 if (ops[0].index != res[0].index) try ops[0].die(cg);
6647 try res[0].moveTo(inst, cg);
6648 },
6649
6650 .cmp_vector, .cmp_vector_optimized => |air_tag| if (use_old) try cg.airCmpVector(inst) else fallback: {
6651 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
6652 const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
6653 switch (extra.compareOperator()) {
6654 .eq, .neq => {},
6655 else => break :fallback try cg.airCmpVector(inst),
6656 }
6657 var ops = try cg.tempsFromOperands(inst, .{ extra.lhs, extra.rhs });
6658 var res: [1]Temp = undefined;
6659 switch (extra.compareOperator()) {
6660 .lt => unreachable,
6661 .lte => unreachable,
6662 .eq, .neq => |cmp_op| cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (cmp_op) {
6663 else => unreachable,
6664 .eq => .e,
6665 .neq => .ne,
6666 })) {
6667 else => unreachable,
6668 inline .e, .ne => |cc| comptime &.{ .{
6669 .required_features = .{ .avx2, null, null, null },
6670 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
6671 .patterns = &.{
6672 .{ .src = .{ .to_ymm, .mem } },
6673 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
6674 .{ .src = .{ .to_ymm, .to_ymm } },
6675 },
6676 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
6677 .kind = .all,
6678 .inverted = switch (cc) {
6679 else => unreachable,
6680 .e => false,
6681 .ne => true,
6682 },
6683 .scalar = .byte,
6684 } } }},
6685 .each = .{ .once = &.{
6686 .{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ },
6687 } },
6688 }, .{
6689 .required_features = .{ .avx2, null, null, null },
6690 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
6691 .patterns = &.{
6692 .{ .src = .{ .to_ymm, .mem } },
6693 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
6694 .{ .src = .{ .to_ymm, .to_ymm } },
6695 },
6696 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
6697 .kind = .all,
6698 .inverted = switch (cc) {
6699 else => unreachable,
6700 .e => false,
6701 .ne => true,
6702 },
6703 .scalar = .word,
6704 } } }},
6705 .each = .{ .once = &.{
6706 .{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ },
6707 } },
6708 }, .{
6709 .required_features = .{ .avx2, null, null, null },
6710 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
6711 .patterns = &.{
6712 .{ .src = .{ .to_ymm, .mem } },
6713 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
6714 .{ .src = .{ .to_ymm, .to_ymm } },
6715 },
6716 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
6717 .kind = .all,
6718 .inverted = switch (cc) {
6719 else => unreachable,
6720 .e => false,
6721 .ne => true,
6722 },
6723 .scalar = .dword,
6724 } } }},
6725 .each = .{ .once = &.{
6726 .{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ },
6727 } },
6728 }, .{
6729 .required_features = .{ .avx2, null, null, null },
6730 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
6731 .patterns = &.{
6732 .{ .src = .{ .to_ymm, .mem } },
6733 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
6734 .{ .src = .{ .to_ymm, .to_ymm } },
29686735 },
29696736 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
29706737 .kind = .all,
......@@ -2979,12 +6746,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
29796746 .{ ._, .vp_q, .cmpeq, .dst0y, .src0y, .src1y, ._ },
29806747 } },
29816748 }, .{
2982 .required_features = .{ .avx, null },
2983 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
6749 .required_features = .{ .avx, null, null, null },
6750 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
29846751 .patterns = &.{
2985 .{ .src = .{ .xmm, .mem } },
2986 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
2987 .{ .src = .{ .xmm, .xmm } },
6752 .{ .src = .{ .to_xmm, .mem } },
6753 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
6754 .{ .src = .{ .to_xmm, .to_xmm } },
29886755 },
29896756 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
29906757 .kind = .all,
......@@ -2999,12 +6766,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
29996766 .{ ._, .vp_b, .cmpeq, .dst0x, .src0x, .src1x, ._ },
30006767 } },
30016768 }, .{
3002 .required_features = .{ .avx, null },
3003 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
6769 .required_features = .{ .avx, null, null, null },
6770 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
30046771 .patterns = &.{
3005 .{ .src = .{ .xmm, .mem } },
3006 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
3007 .{ .src = .{ .xmm, .xmm } },
6772 .{ .src = .{ .to_xmm, .mem } },
6773 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
6774 .{ .src = .{ .to_xmm, .to_xmm } },
30086775 },
30096776 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
30106777 .kind = .all,
......@@ -3019,12 +6786,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30196786 .{ ._, .vp_w, .cmpeq, .dst0x, .src0x, .src1x, ._ },
30206787 } },
30216788 }, .{
3022 .required_features = .{ .avx, null },
3023 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
6789 .required_features = .{ .avx, null, null, null },
6790 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
30246791 .patterns = &.{
3025 .{ .src = .{ .xmm, .mem } },
3026 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
3027 .{ .src = .{ .xmm, .xmm } },
6792 .{ .src = .{ .to_xmm, .mem } },
6793 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
6794 .{ .src = .{ .to_xmm, .to_xmm } },
30286795 },
30296796 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
30306797 .kind = .all,
......@@ -3039,12 +6806,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30396806 .{ ._, .vp_d, .cmpeq, .dst0x, .src0x, .src1x, ._ },
30406807 } },
30416808 }, .{
3042 .required_features = .{ .avx, null },
3043 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
6809 .required_features = .{ .avx, null, null, null },
6810 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
30446811 .patterns = &.{
3045 .{ .src = .{ .xmm, .mem } },
3046 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
3047 .{ .src = .{ .xmm, .xmm } },
6812 .{ .src = .{ .to_xmm, .mem } },
6813 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
6814 .{ .src = .{ .to_xmm, .to_xmm } },
30486815 },
30496816 .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{
30506817 .kind = .all,
......@@ -3059,12 +6826,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30596826 .{ ._, .vp_q, .cmpeq, .dst0x, .src0x, .src1x, ._ },
30606827 } },
30616828 }, .{
3062 .required_features = .{ .sse2, null },
3063 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
6829 .required_features = .{ .sse2, null, null, null },
6830 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
30646831 .patterns = &.{
3065 .{ .src = .{ .mut_xmm, .mem } },
3066 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
3067 .{ .src = .{ .mut_xmm, .xmm } },
6832 .{ .src = .{ .to_mut_xmm, .mem } },
6833 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
6834 .{ .src = .{ .to_mut_xmm, .to_xmm } },
30686835 },
30696836 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
30706837 .kind = .all,
......@@ -3079,12 +6846,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30796846 .{ ._, .p_b, .cmpeq, .dst0x, .src1x, ._, ._ },
30806847 } },
30816848 }, .{
3082 .required_features = .{ .sse2, null },
3083 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
6849 .required_features = .{ .sse2, null, null, null },
6850 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
30846851 .patterns = &.{
3085 .{ .src = .{ .mut_xmm, .mem } },
3086 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
3087 .{ .src = .{ .mut_xmm, .xmm } },
6852 .{ .src = .{ .to_mut_xmm, .mem } },
6853 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
6854 .{ .src = .{ .to_mut_xmm, .to_xmm } },
30886855 },
30896856 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
30906857 .kind = .all,
......@@ -3099,12 +6866,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30996866 .{ ._, .p_w, .cmpeq, .dst0x, .src1x, ._, ._ },
31006867 } },
31016868 }, .{
3102 .required_features = .{ .sse2, null },
3103 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
6869 .required_features = .{ .sse2, null, null, null },
6870 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
31046871 .patterns = &.{
3105 .{ .src = .{ .mut_xmm, .mem } },
3106 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
3107 .{ .src = .{ .mut_xmm, .xmm } },
6872 .{ .src = .{ .to_mut_xmm, .mem } },
6873 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
6874 .{ .src = .{ .to_mut_xmm, .to_xmm } },
31086875 },
31096876 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31106877 .kind = .all,
......@@ -3119,12 +6886,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31196886 .{ ._, .p_d, .cmpeq, .dst0x, .src1x, ._, ._ },
31206887 } },
31216888 }, .{
3122 .required_features = .{ .sse4_1, null },
3123 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
6889 .required_features = .{ .sse4_1, null, null, null },
6890 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
31246891 .patterns = &.{
3125 .{ .src = .{ .mut_xmm, .mem } },
3126 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
3127 .{ .src = .{ .mut_xmm, .xmm } },
6892 .{ .src = .{ .to_mut_xmm, .mem } },
6893 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
6894 .{ .src = .{ .to_mut_xmm, .to_xmm } },
31286895 },
31296896 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31306897 .kind = .all,
......@@ -3139,12 +6906,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31396906 .{ ._, .p_q, .cmpeq, .dst0x, .src1x, ._, ._ },
31406907 } },
31416908 }, .{
3142 .required_features = .{ .mmx, null },
3143 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
6909 .required_features = .{ .mmx, null, null, null },
6910 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
31446911 .patterns = &.{
3145 .{ .src = .{ .mut_mm, .mem } },
3146 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
3147 .{ .src = .{ .mut_mm, .mm } },
6912 .{ .src = .{ .to_mut_mm, .mem } },
6913 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
6914 .{ .src = .{ .to_mut_mm, .to_mm } },
31486915 },
31496916 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31506917 .kind = .all,
......@@ -3159,12 +6926,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31596926 .{ ._, .p_b, .cmpeq, .dst0q, .src1q, ._, ._ },
31606927 } },
31616928 }, .{
3162 .required_features = .{ .mmx, null },
3163 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
6929 .required_features = .{ .mmx, null, null, null },
6930 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
31646931 .patterns = &.{
3165 .{ .src = .{ .mut_mm, .mem } },
3166 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
3167 .{ .src = .{ .mut_mm, .mm } },
6932 .{ .src = .{ .to_mut_mm, .mem } },
6933 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
6934 .{ .src = .{ .to_mut_mm, .to_mm } },
31686935 },
31696936 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31706937 .kind = .all,
......@@ -3179,12 +6946,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
31796946 .{ ._, .p_w, .cmpeq, .dst0q, .src1q, ._, ._ },
31806947 } },
31816948 }, .{
3182 .required_features = .{ .mmx, null },
3183 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
6949 .required_features = .{ .mmx, null, null, null },
6950 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
31846951 .patterns = &.{
3185 .{ .src = .{ .mut_mm, .mem } },
3186 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
3187 .{ .src = .{ .mut_mm, .mm } },
6952 .{ .src = .{ .to_mut_mm, .mem } },
6953 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
6954 .{ .src = .{ .to_mut_mm, .to_mm } },
31886955 },
31896956 .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{
31906957 .kind = .all,
......@@ -3203,550 +6970,182 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32036970 .patterns = &.{
32046971 .{ .src = .{ .mut_mem, .imm8 } },
32056972 .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } },
3206 .{ .src = .{ .mut_gpr, .imm8 } },
3207 .{ .src = .{ .imm8, .mut_gpr }, .commute = .{ 0, 1 } },
3208 .{ .src = .{ .mut_mem, .gpr } },
3209 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
3210 .{ .src = .{ .mut_gpr, .mem } },
3211 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
3212 .{ .src = .{ .mut_gpr, .gpr } },
3213 },
3214 .clobbers = .{ .eflags = true },
3215 .dst_temps = .{.{ .ref = .src0 }},
3216 .each = .{ .once = switch (cc) {
3217 else => unreachable,
3218 .e => &.{
3219 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
3220 .{ ._, ._, .not, .dst0b, ._, ._, ._ },
3221 },
3222 .ne => &.{
3223 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
3224 },
3225 } },
3226 }, .{
3227 .src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } },
3228 .patterns = &.{
3229 .{ .src = .{ .mut_mem, .imm16 } },
3230 .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
3231 .{ .src = .{ .mut_gpr, .imm16 } },
3232 .{ .src = .{ .imm16, .mut_gpr }, .commute = .{ 0, 1 } },
3233 .{ .src = .{ .mut_mem, .gpr } },
3234 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
3235 .{ .src = .{ .mut_gpr, .mem } },
3236 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
3237 .{ .src = .{ .mut_gpr, .gpr } },
3238 },
3239 .clobbers = .{ .eflags = true },
3240 .dst_temps = .{.{ .ref = .src0 }},
3241 .each = .{ .once = switch (cc) {
3242 else => unreachable,
3243 .e => &.{
3244 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
3245 .{ ._, ._, .not, .dst0w, ._, ._, ._ },
3246 },
3247 .ne => &.{
3248 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
3249 },
3250 } },
3251 }, .{
3252 .src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } },
3253 .patterns = &.{
3254 .{ .src = .{ .mut_mem, .imm32 } },
3255 .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
3256 .{ .src = .{ .mut_gpr, .imm32 } },
3257 .{ .src = .{ .imm32, .mut_gpr }, .commute = .{ 0, 1 } },
3258 .{ .src = .{ .mut_mem, .gpr } },
3259 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
3260 .{ .src = .{ .mut_gpr, .mem } },
3261 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
3262 .{ .src = .{ .mut_gpr, .gpr } },
3263 },
3264 .clobbers = .{ .eflags = true },
3265 .dst_temps = .{.{ .ref = .src0 }},
3266 .each = .{ .once = switch (cc) {
3267 else => unreachable,
3268 .e => &.{
3269 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
3270 .{ ._, ._, .not, .dst0d, ._, ._, ._ },
3271 },
3272 .ne => &.{
3273 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
3274 },
3275 } },
3276 }, .{
3277 .required_features = .{ .@"64bit", null },
3278 .src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } },
3279 .patterns = &.{
3280 .{ .src = .{ .mut_mem, .simm32 } },
3281 .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
3282 .{ .src = .{ .mut_gpr, .simm32 } },
3283 .{ .src = .{ .simm32, .mut_gpr }, .commute = .{ 0, 1 } },
3284 .{ .src = .{ .mut_mem, .gpr } },
3285 .{ .src = .{ .gpr, .mut_mem }, .commute = .{ 0, 1 } },
3286 .{ .src = .{ .mut_gpr, .mem } },
3287 .{ .src = .{ .mem, .mut_gpr }, .commute = .{ 0, 1 } },
3288 .{ .src = .{ .mut_gpr, .gpr } },
6973 .{ .src = .{ .to_mut_gpr, .imm8 } },
6974 .{ .src = .{ .imm8, .to_mut_gpr }, .commute = .{ 0, 1 } },
6975 .{ .src = .{ .mut_mem, .to_gpr } },
6976 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
6977 .{ .src = .{ .to_mut_gpr, .mem } },
6978 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
6979 .{ .src = .{ .to_mut_gpr, .to_gpr } },
32896980 },
3290 .clobbers = .{ .eflags = true },
32916981 .dst_temps = .{.{ .ref = .src0 }},
3292 .each = .{ .once = switch (cc) {
3293 else => unreachable,
3294 .e => &.{
3295 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
3296 .{ ._, ._, .not, .dst0q, ._, ._, ._ },
3297 },
3298 .ne => &.{
3299 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
3300 },
3301 } },
3302 }, .{
3303 .src_constraints = .{ .any_bool_vec, .any_bool_vec },
3304 .patterns = &.{
3305 .{ .src = .{ .to_mem, .to_mem } },
3306 },
33076982 .clobbers = .{ .eflags = true },
3308 .extra_temps = .{
3309 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3310 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3311 .unused,
3312 .unused,
3313 .unused,
3314 .unused,
3315 },
3316 .dst_temps = .{.mem},
3317 .each = .{ .once = switch (cc) {
3318 else => unreachable,
3319 .e => &.{
3320 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3321 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
3322 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
3323 .{ ._, ._, .not, .tmp1p, ._, ._, ._ },
3324 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
3325 .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ },
3326 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3327 },
3328 .ne => &.{
3329 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3330 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
3331 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
3332 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
3333 .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ },
3334 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3335 },
3336 } },
3337 }, .{
3338 .required_features = .{ .avx2, null },
3339 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
3340 .patterns = &.{
3341 .{ .src = .{ .to_mem, .to_mem } },
3342 },
3343 .extra_temps = .{
3344 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3345 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3346 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3347 .{ .kind = .{ .rc = .sse } },
3348 .unused,
3349 .unused,
3350 },
3351 .dst_temps = .{.mem},
3352 .each = .{ .once = switch (cc) {
3353 else => unreachable,
3354 .e => &.{
3355 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3356 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3357 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3358 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
3359 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3360 .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
3361 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3362 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
3363 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3364 },
3365 .ne => &.{
3366 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3367 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3368 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3369 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
3370 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3371 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
3372 .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
3373 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3374 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
3375 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3376 },
3377 } },
3378 }, .{
3379 .required_features = .{ .avx2, null },
3380 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
3381 .patterns = &.{
3382 .{ .src = .{ .to_mem, .to_mem } },
3383 },
3384 .extra_temps = .{
3385 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3386 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3387 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3388 .{ .kind = .{ .rc = .sse } },
3389 .unused,
3390 .unused,
3391 },
3392 .dst_temps = .{.mem},
3393 .each = .{ .once = switch (cc) {
3394 else => unreachable,
3395 .e => &.{
3396 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3397 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3398 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3399 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
3400 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
3401 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3402 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
3403 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3404 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
3405 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3406 },
3407 .ne => &.{
3408 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3409 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3410 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3411 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
3412 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
3413 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3414 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
3415 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
3416 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3417 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
3418 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3419 },
3420 } },
3421 }, .{
3422 .required_features = .{ .avx2, null },
3423 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
3424 .patterns = &.{
3425 .{ .src = .{ .to_mem, .to_mem } },
3426 },
3427 .extra_temps = .{
3428 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3429 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3430 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3431 .{ .kind = .{ .rc = .sse } },
3432 .unused,
3433 .unused,
3434 },
3435 .dst_temps = .{.mem},
3436 .each = .{ .once = switch (cc) {
3437 else => unreachable,
3438 .e => &.{
3439 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3440 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3441 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3442 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
3443 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3444 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
3445 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
3446 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
3447 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3448 },
3449 .ne => &.{
3450 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3451 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3452 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3453 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
3454 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
3455 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
3456 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
3457 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
3458 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
3459 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6983 .each = .{ .once = switch (cc) {
6984 else => unreachable,
6985 .e => &.{
6986 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
6987 .{ ._, ._, .not, .dst0b, ._, ._, ._ },
6988 },
6989 .ne => &.{
6990 .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ },
34606991 },
34616992 } },
34626993 }, .{
3463 .required_features = .{ .avx2, null },
3464 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
6994 .src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } },
34656995 .patterns = &.{
3466 .{ .src = .{ .to_mem, .to_mem } },
3467 },
3468 .extra_temps = .{
3469 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3470 .{ .type = .u32, .kind = .{ .reg = .rcx } },
3471 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3472 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3473 .{ .kind = .{ .rc = .sse } },
3474 .unused,
6996 .{ .src = .{ .mut_mem, .imm16 } },
6997 .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } },
6998 .{ .src = .{ .to_mut_gpr, .imm16 } },
6999 .{ .src = .{ .imm16, .to_mut_gpr }, .commute = .{ 0, 1 } },
7000 .{ .src = .{ .mut_mem, .to_gpr } },
7001 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
7002 .{ .src = .{ .to_mut_gpr, .mem } },
7003 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
7004 .{ .src = .{ .to_mut_gpr, .to_gpr } },
34757005 },
3476 .dst_temps = .{.mem},
7006 .dst_temps = .{.{ .ref = .src0 }},
7007 .clobbers = .{ .eflags = true },
34777008 .each = .{ .once = switch (cc) {
34787009 else => unreachable,
34797010 .e => &.{
3480 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3481 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3482 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3483 .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3484 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
3485 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
3486 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3487 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3488 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3489 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3490 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3491 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3492 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3493 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3494 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3495 .{ .@"1:", ._, .add, .tmp0p, .i(32), ._, ._ },
3496 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3497 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3498 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3499 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3500 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3501 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7011 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
7012 .{ ._, ._, .not, .dst0w, ._, ._, ._ },
35027013 },
35037014 .ne => &.{
3504 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3505 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3506 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3507 .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
3508 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
3509 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
3510 .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ },
3511 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3512 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3513 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3514 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3515 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3516 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3517 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3518 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3519 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3520 .{ .@"1:", ._, .add, .tmp0p, .i(32), ._, ._ },
3521 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3522 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3523 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3524 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3525 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3526 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7015 .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ },
35277016 },
35287017 } },
35297018 }, .{
3530 .required_features = .{ .avx, null },
3531 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
7019 .src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } },
35327020 .patterns = &.{
3533 .{ .src = .{ .to_mem, .to_mem } },
3534 },
3535 .extra_temps = .{
3536 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3537 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3538 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
3539 .{ .kind = .{ .rc = .sse } },
3540 .unused,
3541 .unused,
7021 .{ .src = .{ .mut_mem, .imm32 } },
7022 .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } },
7023 .{ .src = .{ .to_mut_gpr, .imm32 } },
7024 .{ .src = .{ .imm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
7025 .{ .src = .{ .mut_mem, .to_gpr } },
7026 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
7027 .{ .src = .{ .to_mut_gpr, .mem } },
7028 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
7029 .{ .src = .{ .to_mut_gpr, .to_gpr } },
35427030 },
3543 .dst_temps = .{.mem},
7031 .dst_temps = .{.{ .ref = .src0 }},
7032 .clobbers = .{ .eflags = true },
35447033 .each = .{ .once = switch (cc) {
35457034 else => unreachable,
35467035 .e => &.{
3547 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3548 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3549 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3550 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
3551 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3552 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
3553 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3554 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
3555 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7036 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
7037 .{ ._, ._, .not, .dst0d, ._, ._, ._ },
35567038 },
35577039 .ne => &.{
3558 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3559 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3560 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3561 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
3562 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3563 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
3564 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
3565 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3566 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
3567 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7040 .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ },
35687041 },
35697042 } },
35707043 }, .{
3571 .required_features = .{ .avx, null },
3572 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
7044 .required_features = .{ .@"64bit", null, null, null },
7045 .src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } },
35737046 .patterns = &.{
3574 .{ .src = .{ .to_mem, .to_mem } },
3575 },
3576 .extra_temps = .{
3577 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3578 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3579 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3580 .{ .kind = .{ .rc = .sse } },
3581 .unused,
3582 .unused,
7047 .{ .src = .{ .mut_mem, .simm32 } },
7048 .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } },
7049 .{ .src = .{ .to_mut_gpr, .simm32 } },
7050 .{ .src = .{ .simm32, .to_mut_gpr }, .commute = .{ 0, 1 } },
7051 .{ .src = .{ .mut_mem, .to_gpr } },
7052 .{ .src = .{ .to_gpr, .mut_mem }, .commute = .{ 0, 1 } },
7053 .{ .src = .{ .to_mut_gpr, .mem } },
7054 .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } },
7055 .{ .src = .{ .to_mut_gpr, .to_gpr } },
35837056 },
3584 .dst_temps = .{.mem},
7057 .dst_temps = .{.{ .ref = .src0 }},
7058 .clobbers = .{ .eflags = true },
35857059 .each = .{ .once = switch (cc) {
35867060 else => unreachable,
35877061 .e => &.{
3588 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3589 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3590 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3591 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
3592 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
3593 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3594 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
3595 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
3596 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
3597 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7062 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
7063 .{ ._, ._, .not, .dst0q, ._, ._, ._ },
35987064 },
35997065 .ne => &.{
3600 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3601 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3602 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3603 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
3604 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
3605 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
3606 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
3607 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
3608 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
3609 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
3610 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7066 .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ },
36117067 },
36127068 } },
36137069 }, .{
3614 .required_features = .{ .avx, null },
3615 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
7070 .src_constraints = .{ .any_bool_vec, .any_bool_vec },
36167071 .patterns = &.{
36177072 .{ .src = .{ .to_mem, .to_mem } },
36187073 },
36197074 .extra_temps = .{
36207075 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3621 .{ .type = .u32, .kind = .{ .reg = .rcx } },
3622 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3623 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3624 .{ .kind = .{ .rc = .sse } },
7076 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7077 .unused,
7078 .unused,
7079 .unused,
36257080 .unused,
36267081 },
36277082 .dst_temps = .{.mem},
7083 .clobbers = .{ .eflags = true },
36287084 .each = .{ .once = switch (cc) {
36297085 else => unreachable,
36307086 .e => &.{
3631 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3632 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3633 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3634 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3635 .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
3636 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3637 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3638 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3639 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3640 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3641 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3642 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3643 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3644 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3645 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3646 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7087 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7088 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
7089 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
7090 .{ ._, ._, .not, .tmp1p, ._, ._, ._ },
7091 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
7092 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
36477093 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3648 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3649 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3650 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3651 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3652 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
36537094 },
36547095 .ne => &.{
3655 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
3656 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3657 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3658 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3659 .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
3660 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3661 .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ },
3662 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3663 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3664 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3665 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3666 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3667 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3668 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3669 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3670 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3671 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7096 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7097 .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
7098 .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
7099 .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ },
7100 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
36727101 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3673 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3674 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3675 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3676 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3677 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
36787102 },
36797103 } },
36807104 }, .{
3681 .required_features = .{ .avx, null },
3682 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
7105 .required_features = .{ .avx2, null, null, null },
7106 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
36837107 .patterns = &.{
36847108 .{ .src = .{ .to_mem, .to_mem } },
36857109 },
36867110 .extra_temps = .{
36877111 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3688 .{ .type = .u32, .kind = .{ .reg = .rcx } },
3689 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3690 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7112 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7113 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
36917114 .{ .kind = .{ .rc = .sse } },
36927115 .unused,
7116 .unused,
36937117 },
36947118 .dst_temps = .{.mem},
7119 .clobbers = .{ .eflags = true },
36957120 .each = .{ .once = switch (cc) {
36967121 else => unreachable,
36977122 .e => &.{
3698 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7123 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
36997124 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3700 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3701 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3702 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
3703 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3704 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3705 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3706 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3707 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3708 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3709 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3710 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3711 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3712 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3713 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7125 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7126 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7127 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7128 .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
7129 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7130 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
37147131 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3715 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3716 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3717 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3718 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3719 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
37207132 },
37217133 .ne => &.{
3722 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7134 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
37237135 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3724 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3725 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3726 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
3727 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3728 .{ ._, ._, .xor, .tmp3b, .i(0b11), ._, ._ },
3729 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3730 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
3731 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3732 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3733 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3734 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3735 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3736 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3737 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3738 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7136 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7137 .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7138 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
7139 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
7140 .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ },
7141 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7142 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
37397143 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3740 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3741 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3742 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3743 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3744 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
37457144 },
37467145 } },
37477146 }, .{
3748 .required_features = .{ .sse2, null },
3749 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
7147 .required_features = .{ .avx2, null, null, null },
7148 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
37507149 .patterns = &.{
37517150 .{ .src = .{ .to_mem, .to_mem } },
37527151 },
......@@ -3759,35 +7158,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
37597158 .unused,
37607159 },
37617160 .dst_temps = .{.mem},
7161 .clobbers = .{ .eflags = true },
37627162 .each = .{ .once = switch (cc) {
37637163 else => unreachable,
37647164 .e => &.{
3765 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7165 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
37667166 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3767 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3768 .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3769 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7167 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7168 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7169 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
7170 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
37707171 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
37717172 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3772 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
7173 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
37737174 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
37747175 },
37757176 .ne => &.{
3776 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7177 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
37777178 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3778 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3779 .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3780 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7179 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7180 .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7181 .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ },
7182 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ },
37817183 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
37827184 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
37837185 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3784 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
7186 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
37857187 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
37867188 },
37877189 } },
37887190 }, .{
3789 .required_features = .{ .sse2, null },
3790 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
7191 .required_features = .{ .avx2, null, null, null },
7192 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
37917193 .patterns = &.{
37927194 .{ .src = .{ .to_mem, .to_mem } },
37937195 },
......@@ -3800,37 +7202,36 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
38007202 .unused,
38017203 },
38027204 .dst_temps = .{.mem},
7205 .clobbers = .{ .eflags = true },
38037206 .each = .{ .once = switch (cc) {
38047207 else => unreachable,
38057208 .e => &.{
3806 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7209 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
38077210 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3808 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3809 .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3810 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
3811 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7211 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7212 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7213 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
38127214 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
38137215 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
3814 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
7216 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
38157217 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
38167218 },
38177219 .ne => &.{
3818 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7220 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
38197221 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3820 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3821 .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3822 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
3823 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7222 .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7223 .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ },
7224 .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ },
38247225 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
38257226 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
38267227 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
3827 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
7228 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
38287229 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
38297230 },
38307231 } },
38317232 }, .{
3832 .required_features = .{ .sse2, null },
3833 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
7233 .required_features = .{ .avx2, null, null, null },
7234 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
38347235 .patterns = &.{
38357236 .{ .src = .{ .to_mem, .to_mem } },
38367237 },
......@@ -3843,128 +7244,104 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
38437244 .unused,
38447245 },
38457246 .dst_temps = .{.mem},
7247 .clobbers = .{ .eflags = true },
38467248 .each = .{ .once = switch (cc) {
38477249 else => unreachable,
38487250 .e => &.{
3849 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7251 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
38507252 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
38517253 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3852 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3853 .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3854 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7254 .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7255 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
7256 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
38557257 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
38567258 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
38577259 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3858 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7260 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
38597261 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
38607262 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3861 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
7263 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
38627264 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
38637265 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3864 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7266 .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
38657267 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3866 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7268 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
38677269 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
38687270 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3869 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
7271 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
38707272 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
38717273 },
38727274 .ne => &.{
3873 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7275 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
38747276 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
38757277 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3876 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3877 .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3878 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3879 .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ },
7278 .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
7279 .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ },
7280 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ },
7281 .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
38807282 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
38817283 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
38827284 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
3883 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7285 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
38847286 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
38857287 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3886 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
7288 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
38877289 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
38887290 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3889 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7291 .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
38907292 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3891 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7293 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
38927294 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
38937295 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3894 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
7296 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
38957297 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
38967298 },
38977299 } },
38987300 }, .{
3899 .required_features = .{ .sse4_1, null },
3900 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
7301 .required_features = .{ .avx, null, null, null },
7302 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
39017303 .patterns = &.{
39027304 .{ .src = .{ .to_mem, .to_mem } },
39037305 },
39047306 .extra_temps = .{
39057307 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
3906 .{ .type = .u32, .kind = .{ .reg = .rcx } },
3907 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3908 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7308 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7309 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
39097310 .{ .kind = .{ .rc = .sse } },
39107311 .unused,
7312 .unused,
39117313 },
39127314 .dst_temps = .{.mem},
7315 .clobbers = .{ .eflags = true },
39137316 .each = .{ .once = switch (cc) {
39147317 else => unreachable,
39157318 .e => &.{
3916 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7319 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
39177320 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3918 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3919 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3920 .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3921 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3922 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3923 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7321 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7322 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7323 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7324 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
39247325 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3925 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3926 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3927 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3928 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3929 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3930 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3931 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7326 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
39327327 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3933 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3934 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3935 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3936 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3937 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
39387328 },
39397329 .ne => &.{
3940 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7330 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
39417331 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3942 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3943 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
3944 .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
3945 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
3946 .{ ._, ._, .xor, .tmp3b, .i(0b11), ._, ._ },
3947 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
3948 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7332 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7333 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7334 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7335 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
7336 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
39497337 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
3950 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3951 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
3952 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3953 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3954 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
3955 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
3956 .{ .@"1:", ._, .add, .tmp0p, .i(16), ._, ._ },
7338 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
39577339 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3958 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
3959 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
3960 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
3961 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
3962 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
39637340 },
39647341 } },
39657342 }, .{
3966 .required_features = .{ .sse, .mmx },
3967 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
7343 .required_features = .{ .avx, null, null, null },
7344 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
39687345 .patterns = &.{
39697346 .{ .src = .{ .to_mem, .to_mem } },
39707347 },
......@@ -3972,111 +7349,111 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
39727349 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
39737350 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
39747351 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
3975 .{ .kind = .{ .rc = .mmx } },
7352 .{ .kind = .{ .rc = .sse } },
39767353 .unused,
39777354 .unused,
39787355 },
39797356 .dst_temps = .{.mem},
7357 .clobbers = .{ .eflags = true },
39807358 .each = .{ .once = switch (cc) {
39817359 else => unreachable,
39827360 .e => &.{
3983 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7361 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
39847362 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3985 .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
3986 .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
3987 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
7363 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7364 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7365 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
7366 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
39887367 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
39897368 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
3990 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
7369 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
39917370 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
39927371 },
39937372 .ne => &.{
3994 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7373 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
39957374 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
3996 .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
3997 .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
3998 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
7375 .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7376 .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ },
7377 .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ },
7378 .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
39997379 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
40007380 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
40017381 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
4002 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
7382 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
40037383 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
40047384 },
40057385 } },
40067386 }, .{
4007 .required_features = .{ .sse, .mmx },
4008 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
7387 .required_features = .{ .avx, null, null, null },
7388 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
40097389 .patterns = &.{
40107390 .{ .src = .{ .to_mem, .to_mem } },
40117391 },
40127392 .extra_temps = .{
40137393 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4014 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7394 .{ .type = .u32, .kind = .{ .reg = .rcx } },
40157395 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4016 .{ .kind = .{ .rc = .mmx } },
40177396 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4018 .{ .kind = .{ .rc = .mmx } },
7397 .{ .kind = .{ .rc = .sse } },
7398 .unused,
40197399 },
40207400 .dst_temps = .{.mem},
7401 .clobbers = .{ .eflags = true },
40217402 .each = .{ .once = switch (cc) {
40227403 else => unreachable,
40237404 .e => &.{
4024 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7405 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
40257406 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
40267407 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4027 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
4028 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4029 .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4030 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
4031 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
4032 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
4033 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7408 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7409 .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7410 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7411 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7412 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
40347413 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
4035 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7414 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
40367415 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
4037 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4038 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4039 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
7416 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7417 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7418 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
40407419 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4041 .{ .@"1:", ._, .add, .tmp0p, .i(8), ._, ._ },
7420 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
40427421 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4043 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7422 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
40447423 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4045 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4046 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4047 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
7424 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7425 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7426 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
40487427 },
40497428 .ne => &.{
4050 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7429 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
40517430 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
40527431 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4053 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
4054 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4055 .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4056 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
4057 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
4058 .{ ._, ._, .xor, .tmp4b, .i(0b1111), ._, ._ },
4059 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
4060 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7432 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7433 .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7434 .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7435 .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
7436 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7437 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
40617438 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
4062 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7439 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
40637440 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
4064 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4065 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4066 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
7441 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7442 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7443 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
40677444 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4068 .{ .@"1:", ._, .add, .tmp0p, .i(8), ._, ._ },
7445 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
40697446 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4070 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7447 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
40717448 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4072 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4073 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4074 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
7449 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7450 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7451 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
40757452 },
40767453 } },
40777454 }, .{
4078 .required_features = .{ .sse, .mmx },
4079 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
7455 .required_features = .{ .avx, null, null, null },
7456 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
40807457 .patterns = &.{
40817458 .{ .src = .{ .to_mem, .to_mem } },
40827459 },
......@@ -4084,578 +7461,535 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40847461 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
40857462 .{ .type = .u32, .kind = .{ .reg = .rcx } },
40867463 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4087 .{ .kind = .{ .rc = .mmx } },
40887464 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4089 .{ .kind = .{ .rc = .mmx } },
7465 .{ .kind = .{ .rc = .sse } },
7466 .unused,
40907467 },
40917468 .dst_temps = .{.mem},
7469 .clobbers = .{ .eflags = true },
40927470 .each = .{ .once = switch (cc) {
40937471 else => unreachable,
40947472 .e => &.{
4095 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7473 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
40967474 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
40977475 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4098 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
4099 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4100 .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4101 .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
4102 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
4103 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
4104 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
4105 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7476 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7477 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7478 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7479 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7480 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
41067481 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
4107 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7482 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
41087483 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
4109 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4110 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4111 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
7484 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7485 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7486 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
41127487 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4113 .{ .@"1:", ._, .add, .tmp0p, .i(8), ._, ._ },
7488 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
41147489 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4115 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7490 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
41167491 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4117 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4118 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4119 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
7492 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7493 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7494 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
41207495 },
41217496 .ne => &.{
4122 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7497 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
41237498 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
41247499 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4125 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
4126 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4127 .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4128 .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
4129 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
4130 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
4131 .{ ._, ._, .xor, .tmp4b, .i(0b11), ._, ._ },
4132 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
4133 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7500 .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7501 .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ },
7502 .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7503 .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ },
7504 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7505 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
41347506 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
4135 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
7507 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
41367508 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
4137 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4138 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4139 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
4140 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
4141 .{ .@"1:", ._, .add, .tmp0p, .i(8), ._, ._ },
4142 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4143 .{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ },
4144 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
4145 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
4146 .{ ._, ._r, .sh, .tmp4d, .i(3), ._, ._ },
4147 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
4148 },
4149 } },
4150 }, .{
4151 .required_features = .{ .slow_incdec, null },
4152 .dst_constraints = .{.{ .bool_vec = .byte }},
4153 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4154 .patterns = &.{
4155 .{ .src = .{ .to_mem, .to_mem } },
4156 },
4157 .extra_temps = .{
4158 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4159 .{ .type = .u8, .kind = .{ .reg = .cl } },
4160 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4161 .unused,
4162 .unused,
4163 .unused,
4164 },
4165 .dst_temps = .{.{ .rc = .general_purpose }},
4166 .each = .{ .once = &.{
4167 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4168 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4169 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4170 .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4171 .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4172 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4173 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4174 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4175 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4176 .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ },
4177 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4178 } },
4179 }, .{
4180 .dst_constraints = .{.{ .bool_vec = .byte }},
4181 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
4182 .patterns = &.{
4183 .{ .src = .{ .to_mem, .to_mem } },
4184 },
4185 .extra_temps = .{
4186 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4187 .{ .type = .u8, .kind = .{ .reg = .cl } },
4188 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4189 .unused,
4190 .unused,
4191 .unused,
4192 },
4193 .dst_temps = .{.{ .rc = .general_purpose }},
4194 .each = .{ .once = &.{
4195 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4196 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4197 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4198 .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4199 .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4200 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4201 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4202 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4203 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4204 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
4205 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
4206 } },
4207 }, .{
4208 .required_features = .{ .slow_incdec, null },
4209 .dst_constraints = .{.{ .bool_vec = .byte }},
4210 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
4211 .patterns = &.{
4212 .{ .src = .{ .to_mem, .to_mem } },
4213 },
4214 .extra_temps = .{
4215 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4216 .{ .type = .u8, .kind = .{ .reg = .cl } },
4217 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4218 .unused,
4219 .unused,
4220 .unused,
4221 },
4222 .dst_temps = .{.{ .rc = .general_purpose }},
4223 .each = .{ .once = &.{
4224 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4225 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4226 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4227 .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4228 .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4229 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4230 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4231 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4232 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4233 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4234 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4235 } },
4236 }, .{
4237 .dst_constraints = .{.{ .bool_vec = .byte }},
4238 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
4239 .patterns = &.{
4240 .{ .src = .{ .to_mem, .to_mem } },
4241 },
4242 .extra_temps = .{
4243 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4244 .{ .type = .u8, .kind = .{ .reg = .cl } },
4245 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4246 .unused,
4247 .unused,
4248 .unused,
4249 },
4250 .dst_temps = .{.{ .rc = .general_purpose }},
4251 .each = .{ .once = &.{
4252 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4253 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4254 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4255 .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4256 .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4257 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4258 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4259 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4260 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4261 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4262 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7509 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7510 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7511 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7512 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7513 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7514 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7515 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7516 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7517 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7518 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7519 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7520 },
42637521 } },
42647522 }, .{
4265 .required_features = .{ .slow_incdec, null },
4266 .dst_constraints = .{.{ .bool_vec = .byte }},
4267 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
7523 .required_features = .{ .sse2, null, null, null },
7524 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
42687525 .patterns = &.{
42697526 .{ .src = .{ .to_mem, .to_mem } },
42707527 },
42717528 .extra_temps = .{
42727529 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4273 .{ .type = .u8, .kind = .{ .reg = .cl } },
42747530 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4275 .unused,
7531 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
7532 .{ .kind = .{ .rc = .sse } },
42767533 .unused,
42777534 .unused,
42787535 },
4279 .dst_temps = .{.{ .rc = .general_purpose }},
4280 .each = .{ .once = &.{
4281 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4282 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4283 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4284 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4285 .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4286 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4287 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4288 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4289 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4290 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4291 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7536 .dst_temps = .{.mem},
7537 .clobbers = .{ .eflags = true },
7538 .each = .{ .once = switch (cc) {
7539 else => unreachable,
7540 .e => &.{
7541 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7542 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7543 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7544 .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7545 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7546 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7547 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7548 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7549 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7550 },
7551 .ne => &.{
7552 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7553 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7554 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7555 .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7556 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7557 .{ ._, ._, .not, .tmp2d, ._, ._, ._ },
7558 .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ },
7559 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7560 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7561 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7562 },
42927563 } },
42937564 }, .{
4294 .dst_constraints = .{.{ .bool_vec = .byte }},
4295 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
7565 .required_features = .{ .sse2, null, null, null },
7566 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
42967567 .patterns = &.{
42977568 .{ .src = .{ .to_mem, .to_mem } },
42987569 },
42997570 .extra_temps = .{
43007571 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4301 .{ .type = .u8, .kind = .{ .reg = .cl } },
43027572 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4303 .unused,
7573 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7574 .{ .kind = .{ .rc = .sse } },
43047575 .unused,
43057576 .unused,
43067577 },
4307 .dst_temps = .{.{ .rc = .general_purpose }},
4308 .each = .{ .once = &.{
4309 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4310 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4311 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4312 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4313 .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4314 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4315 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4316 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4317 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4318 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4319 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7578 .dst_temps = .{.mem},
7579 .clobbers = .{ .eflags = true },
7580 .each = .{ .once = switch (cc) {
7581 else => unreachable,
7582 .e => &.{
7583 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7584 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7585 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7586 .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7587 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
7588 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7589 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7590 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7591 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7592 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7593 },
7594 .ne => &.{
7595 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7596 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7597 .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7598 .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7599 .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ },
7600 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ },
7601 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
7602 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7603 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7604 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7605 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7606 },
43207607 } },
43217608 }, .{
4322 .required_features = .{ .@"64bit", .slow_incdec },
4323 .dst_constraints = .{.{ .bool_vec = .byte }},
4324 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
7609 .required_features = .{ .sse2, null, null, null },
7610 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
43257611 .patterns = &.{
43267612 .{ .src = .{ .to_mem, .to_mem } },
43277613 },
43287614 .extra_temps = .{
43297615 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4330 .{ .type = .u8, .kind = .{ .reg = .cl } },
4331 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4332 .unused,
4333 .unused,
7616 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7617 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7618 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7619 .{ .kind = .{ .rc = .sse } },
43347620 .unused,
43357621 },
4336 .dst_temps = .{.{ .rc = .general_purpose }},
4337 .each = .{ .once = &.{
4338 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4339 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4340 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4341 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4342 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4343 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4344 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4345 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4346 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4347 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4348 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7622 .dst_temps = .{.mem},
7623 .clobbers = .{ .eflags = true },
7624 .each = .{ .once = switch (cc) {
7625 else => unreachable,
7626 .e => &.{
7627 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7628 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7629 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7630 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7631 .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7632 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7633 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7634 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7635 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7636 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7637 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7638 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7639 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7640 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7641 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7642 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7643 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7644 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7645 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7646 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7647 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7648 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7649 },
7650 .ne => &.{
7651 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7652 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7653 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7654 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7655 .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7656 .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7657 .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ },
7658 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7659 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7660 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7661 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7662 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7663 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7664 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7665 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7666 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7667 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7668 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7669 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7670 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7671 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7672 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7673 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7674 },
43497675 } },
43507676 }, .{
4351 .required_features = .{ .@"64bit", null },
4352 .dst_constraints = .{.{ .bool_vec = .byte }},
4353 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
7677 .required_features = .{ .sse4_1, null, null, null },
7678 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
43547679 .patterns = &.{
43557680 .{ .src = .{ .to_mem, .to_mem } },
43567681 },
43577682 .extra_temps = .{
43587683 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4359 .{ .type = .u8, .kind = .{ .reg = .cl } },
4360 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4361 .unused,
4362 .unused,
4363 .unused,
4364 },
4365 .dst_temps = .{.{ .rc = .general_purpose }},
4366 .each = .{ .once = &.{
4367 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4368 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4369 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4370 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4371 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
4372 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4373 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4374 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4375 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4376 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4377 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4378 } },
4379 }, .{
4380 .required_features = .{ .slow_incdec, null },
4381 .dst_constraints = .{.{ .bool_vec = .byte }},
4382 .patterns = &.{
4383 .{ .src = .{ .to_mem, .to_mem } },
4384 },
4385 .extra_temps = .{
4386 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4387 .{ .type = .u8, .kind = .{ .reg = .cl } },
4388 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4389 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4390 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4391 .unused,
4392 },
4393 .dst_temps = .{.{ .rc = .general_purpose }},
4394 .each = .{ .once = &.{
4395 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4396 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4397 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4398 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4399 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4400 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4401 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4402 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4403 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4404 .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ },
4405 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
4406 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4407 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4408 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4409 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4410 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4411 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4412 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
4413 } },
4414 }, .{
4415 .dst_constraints = .{.{ .bool_vec = .byte }},
4416 .patterns = &.{
4417 .{ .src = .{ .to_mem, .to_mem } },
4418 },
4419 .extra_temps = .{
4420 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4421 .{ .type = .u8, .kind = .{ .reg = .cl } },
4422 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4423 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4424 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7684 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7685 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7686 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7687 .{ .kind = .{ .rc = .sse } },
44257688 .unused,
44267689 },
4427 .dst_temps = .{.{ .rc = .general_purpose }},
4428 .each = .{ .once = &.{
4429 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
4430 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
4431 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4432 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4433 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4434 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4435 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4436 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4437 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4438 .{ ._, ._, .dec, .tmp2d, ._, ._, ._ },
4439 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
4440 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
4441 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4442 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
4443 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
4444 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4445 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4446 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
7690 .dst_temps = .{.mem},
7691 .clobbers = .{ .eflags = true },
7692 .each = .{ .once = switch (cc) {
7693 else => unreachable,
7694 .e => &.{
7695 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7696 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7697 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7698 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7699 .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7700 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7701 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7702 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7703 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7704 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7705 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7706 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7707 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7708 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7709 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7710 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7711 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7712 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7713 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7714 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7715 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7716 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7717 },
7718 .ne => &.{
7719 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7720 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7721 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7722 .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
7723 .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
7724 .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ },
7725 .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ },
7726 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
7727 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
7728 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7729 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7730 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7731 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7732 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7733 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
7734 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7735 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
7736 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7737 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7738 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7739 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
7740 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
7741 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
7742 },
44477743 } },
44487744 }, .{
4449 .required_features = .{ .slow_incdec, null },
4450 .dst_constraints = .{.{ .bool_vec = .dword }},
4451 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
7745 .required_features = .{ .sse, .mmx, null, null },
7746 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
44527747 .patterns = &.{
44537748 .{ .src = .{ .to_mem, .to_mem } },
44547749 },
44557750 .extra_temps = .{
44567751 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4457 .{ .type = .u8, .kind = .{ .reg = .cl } },
44587752 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
44597753 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7754 .{ .kind = .{ .rc = .mmx } },
44607755 .unused,
44617756 .unused,
44627757 },
4463 .dst_temps = .{.{ .rc = .general_purpose }},
4464 .each = .{ .once = &.{
4465 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4466 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4467 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4468 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4469 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4470 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4471 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4472 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4473 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4474 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4475 .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ },
4476 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7758 .dst_temps = .{.mem},
7759 .clobbers = .{ .eflags = true },
7760 .each = .{ .once = switch (cc) {
7761 else => unreachable,
7762 .e => &.{
7763 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7764 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7765 .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
7766 .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
7767 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
7768 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7769 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7770 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7771 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7772 },
7773 .ne => &.{
7774 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7775 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7776 .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
7777 .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
7778 .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ },
7779 .{ ._, ._, .not, .tmp2b, ._, ._, ._ },
7780 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
7781 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
7782 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7783 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7784 },
44777785 } },
44787786 }, .{
4479 .dst_constraints = .{.{ .bool_vec = .dword }},
4480 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
7787 .required_features = .{ .sse, .mmx, null, null },
7788 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
44817789 .patterns = &.{
44827790 .{ .src = .{ .to_mem, .to_mem } },
44837791 },
44847792 .extra_temps = .{
44857793 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4486 .{ .type = .u8, .kind = .{ .reg = .cl } },
44877794 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
44887795 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
4489 .unused,
4490 .unused,
7796 .{ .kind = .{ .rc = .mmx } },
7797 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7798 .{ .kind = .{ .rc = .mmx } },
44917799 },
4492 .dst_temps = .{.{ .rc = .general_purpose }},
4493 .each = .{ .once = &.{
4494 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4495 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4496 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4497 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4498 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4499 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
4500 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4501 .{ ._, ._l, .sh, .tmp2d, .tmp1d, ._, ._ },
4502 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4503 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4504 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
4505 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
7800 .dst_temps = .{.mem},
7801 .clobbers = .{ .eflags = true },
7802 .each = .{ .once = switch (cc) {
7803 else => unreachable,
7804 .e => &.{
7805 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7806 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7807 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7808 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
7809 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
7810 .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
7811 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
7812 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
7813 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
7814 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7815 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7816 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7817 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7818 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7819 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7820 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
7821 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7822 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
7823 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7824 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7825 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7826 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7827 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7828 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
7829 },
7830 .ne => &.{
7831 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7832 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7833 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7834 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
7835 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
7836 .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
7837 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
7838 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
7839 .{ ._, ._, .xor, .tmp4b, .si(0b1111), ._, ._ },
7840 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
7841 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7842 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ },
7843 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7844 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7845 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7846 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7847 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
7848 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7849 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
7850 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7851 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7852 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7853 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7854 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7855 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
7856 },
45067857 } },
45077858 }, .{
4508 .required_features = .{ .slow_incdec, null },
4509 .dst_constraints = .{.{ .bool_vec = .dword }},
4510 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
7859 .required_features = .{ .sse, .mmx, null, null },
7860 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
45117861 .patterns = &.{
45127862 .{ .src = .{ .to_mem, .to_mem } },
45137863 },
45147864 .extra_temps = .{
45157865 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4516 .{ .type = .u8, .kind = .{ .reg = .cl } },
4517 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4518 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4519 .unused,
4520 .unused,
7866 .{ .type = .u32, .kind = .{ .reg = .rcx } },
7867 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7868 .{ .kind = .{ .rc = .mmx } },
7869 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7870 .{ .kind = .{ .rc = .mmx } },
45217871 },
4522 .dst_temps = .{.{ .rc = .general_purpose }},
4523 .each = .{ .once = &.{
4524 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4525 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4526 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4527 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4528 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4529 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4530 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4531 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4532 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4533 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4534 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4535 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7872 .dst_temps = .{.mem},
7873 .clobbers = .{ .eflags = true },
7874 .each = .{ .once = switch (cc) {
7875 else => unreachable,
7876 .e => &.{
7877 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7878 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7879 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7880 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
7881 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
7882 .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
7883 .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
7884 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
7885 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
7886 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
7887 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7888 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7889 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7890 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7891 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7892 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7893 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
7894 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7895 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
7896 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7897 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7898 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7899 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7900 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7901 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
7902 },
7903 .ne => &.{
7904 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7905 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
7906 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7907 .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ },
7908 .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
7909 .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
7910 .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ },
7911 .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ },
7912 .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ },
7913 .{ ._, ._, .xor, .tmp4b, .si(0b11), ._, ._ },
7914 .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ },
7915 .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ },
7916 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ },
7917 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7918 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
7919 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7920 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7921 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
7922 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
7923 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
7924 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7925 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
7926 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
7927 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
7928 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
7929 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
7930 },
45367931 } },
45377932 }, .{
4538 .dst_constraints = .{.{ .bool_vec = .dword }},
4539 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
7933 .dst_constraints = .{.{ .bool_vec = .byte }},
7934 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
45407935 .patterns = &.{
45417936 .{ .src = .{ .to_mem, .to_mem } },
45427937 },
45437938 .extra_temps = .{
45447939 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
45457940 .{ .type = .u8, .kind = .{ .reg = .cl } },
4546 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4547 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
4548 .unused,
7941 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
45497942 .unused,
4550 },
4551 .dst_temps = .{.{ .rc = .general_purpose }},
4552 .each = .{ .once = &.{
4553 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4554 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4555 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4556 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4557 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4558 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
4559 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4560 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4561 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4562 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4563 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
4564 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4565 } },
4566 }, .{
4567 .required_features = .{ .slow_incdec, null },
4568 .dst_constraints = .{.{ .bool_vec = .dword }},
4569 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
4570 .patterns = &.{
4571 .{ .src = .{ .to_mem, .to_mem } },
4572 },
4573 .extra_temps = .{
4574 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4575 .{ .type = .u8, .kind = .{ .reg = .cl } },
4576 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4577 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
45787943 .unused,
45797944 .unused,
45807945 },
45817946 .dst_temps = .{.{ .rc = .general_purpose }},
7947 .clobbers = .{ .eflags = true },
45827948 .each = .{ .once = &.{
4583 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4584 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7949 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
7950 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
45857951 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4586 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4587 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4588 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
7952 .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
7953 .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
45897954 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4590 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4591 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4592 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4593 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
7955 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
7956 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
7957 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
7958 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
45947959 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
45957960 } },
45967961 }, .{
4597 .dst_constraints = .{.{ .bool_vec = .dword }},
4598 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
7962 .dst_constraints = .{.{ .bool_vec = .byte }},
7963 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
45997964 .patterns = &.{
46007965 .{ .src = .{ .to_mem, .to_mem } },
46017966 },
46027967 .extra_temps = .{
46037968 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
46047969 .{ .type = .u8, .kind = .{ .reg = .cl } },
4605 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4606 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4607 .unused,
7970 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
46087971 .unused,
4609 },
4610 .dst_temps = .{.{ .rc = .general_purpose }},
4611 .each = .{ .once = &.{
4612 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4613 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4614 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4615 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4616 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4617 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
4618 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4619 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4620 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4621 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4622 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4623 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4624 } },
4625 }, .{
4626 .required_features = .{ .@"64bit", .slow_incdec },
4627 .dst_constraints = .{.{ .bool_vec = .dword }},
4628 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
4629 .patterns = &.{
4630 .{ .src = .{ .to_mem, .to_mem } },
4631 },
4632 .extra_temps = .{
4633 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4634 .{ .type = .u8, .kind = .{ .reg = .cl } },
4635 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4636 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
46377972 .unused,
46387973 .unused,
46397974 },
46407975 .dst_temps = .{.{ .rc = .general_purpose }},
7976 .clobbers = .{ .eflags = true },
46417977 .each = .{ .once = &.{
4642 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4643 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
7978 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
7979 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
46447980 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4645 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4646 .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4647 .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
7981 .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
7982 .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
46487983 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
46497984 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
46507985 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4651 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4652 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
7986 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
7987 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
46537988 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
46547989 } },
46557990 }, .{
4656 .required_features = .{ .@"64bit", null },
4657 .dst_constraints = .{.{ .bool_vec = .dword }},
4658 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
7991 .dst_constraints = .{.{ .bool_vec = .byte }},
7992 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
46597993 .patterns = &.{
46607994 .{ .src = .{ .to_mem, .to_mem } },
46617995 },
......@@ -4663,249 +7997,251 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
46637997 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
46647998 .{ .type = .u8, .kind = .{ .reg = .cl } },
46657999 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4666 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8000 .unused,
46678001 .unused,
46688002 .unused,
46698003 },
46708004 .dst_temps = .{.{ .rc = .general_purpose }},
8005 .clobbers = .{ .eflags = true },
46718006 .each = .{ .once = &.{
4672 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4673 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8007 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
8008 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
46748009 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4675 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4676 .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4677 .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8010 .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
8011 .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
46788012 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4679 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4680 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4681 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4682 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
8013 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
8014 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
8015 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8016 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
46838017 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
46848018 } },
46858019 }, .{
4686 .required_features = .{ .slow_incdec, null },
4687 .dst_constraints = .{.{ .bool_vec = .dword }},
8020 .required_features = .{ .@"64bit", null, null, null },
8021 .dst_constraints = .{.{ .bool_vec = .byte }},
8022 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
46888023 .patterns = &.{
46898024 .{ .src = .{ .to_mem, .to_mem } },
46908025 },
46918026 .extra_temps = .{
4692 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8027 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
46938028 .{ .type = .u8, .kind = .{ .reg = .cl } },
4694 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4695 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4696 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8029 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8030 .unused,
8031 .unused,
46978032 .unused,
46988033 },
46998034 .dst_temps = .{.{ .rc = .general_purpose }},
8035 .clobbers = .{ .eflags = true },
47008036 .each = .{ .once = &.{
4701 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4702 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
8037 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
8038 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
47038039 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4704 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
4705 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
4706 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
4707 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
4708 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4709 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4710 .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ },
4711 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
4712 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4713 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
8040 .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8041 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
47148042 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4715 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4716 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
4717 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4718 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
4719 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
8043 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
8044 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
8045 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8046 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
8047 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
47208048 } },
47218049 }, .{
4722 .dst_constraints = .{.{ .bool_vec = .dword }},
8050 .dst_constraints = .{.{ .bool_vec = .byte }},
47238051 .patterns = &.{
47248052 .{ .src = .{ .to_mem, .to_mem } },
47258053 },
47268054 .extra_temps = .{
47278055 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
47288056 .{ .type = .u8, .kind = .{ .reg = .cl } },
4729 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8057 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
47308058 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
47318059 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
47328060 .unused,
47338061 },
47348062 .dst_temps = .{.{ .rc = .general_purpose }},
8063 .clobbers = .{ .eflags = true },
47358064 .each = .{ .once = &.{
4736 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
8065 .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ },
47378066 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
47388067 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4739 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
8068 .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
47408069 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
47418070 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
47428071 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
47438072 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
4744 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
4745 .{ ._, ._, .dec, .tmp2d, ._, ._, ._ },
4746 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
4747 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8073 .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
8074 .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
8075 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
47488076 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
47498077 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4750 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
4751 .{ ._, ._, .@"or", .dst0d, .tmp2b, ._, ._ },
4752 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4753 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
8078 .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ },
8079 .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ },
8080 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8081 .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
47548082 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
47558083 } },
47568084 }, .{
4757 .required_features = .{ .@"64bit", .slow_incdec },
4758 .dst_constraints = .{.{ .bool_vec = .qword }},
4759 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
8085 .dst_constraints = .{.{ .bool_vec = .dword }},
8086 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
47608087 .patterns = &.{
47618088 .{ .src = .{ .to_mem, .to_mem } },
47628089 },
47638090 .extra_temps = .{
47648091 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
47658092 .{ .type = .u8, .kind = .{ .reg = .cl } },
4766 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8093 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
47678094 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
47688095 .unused,
47698096 .unused,
47708097 },
47718098 .dst_temps = .{.{ .rc = .general_purpose }},
8099 .clobbers = .{ .eflags = true },
47728100 .each = .{ .once = &.{
47738101 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4774 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8102 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
47758103 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
47768104 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
47778105 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
47788106 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
47798107 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4780 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4781 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4782 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4783 .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ },
8108 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8109 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8110 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8111 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
47848112 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
47858113 } },
47868114 }, .{
4787 .required_features = .{ .@"64bit", null },
4788 .dst_constraints = .{.{ .bool_vec = .qword }},
4789 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
8115 .dst_constraints = .{.{ .bool_vec = .dword }},
8116 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
47908117 .patterns = &.{
47918118 .{ .src = .{ .to_mem, .to_mem } },
47928119 },
47938120 .extra_temps = .{
47948121 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
47958122 .{ .type = .u8, .kind = .{ .reg = .cl } },
4796 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4797 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
8123 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8124 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
47988125 .unused,
47998126 .unused,
48008127 },
48018128 .dst_temps = .{.{ .rc = .general_purpose }},
8129 .clobbers = .{ .eflags = true },
48028130 .each = .{ .once = &.{
48038131 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4804 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8132 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
48058133 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
48068134 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4807 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
4808 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
8135 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
8136 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
48098137 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4810 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4811 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4812 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4813 .{ ._, ._, .inc, .tmp0p, ._, ._, ._ },
4814 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
8138 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8139 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8140 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8141 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
8142 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
48158143 } },
48168144 }, .{
4817 .required_features = .{ .@"64bit", .slow_incdec },
4818 .dst_constraints = .{.{ .bool_vec = .qword }},
4819 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
8145 .dst_constraints = .{.{ .bool_vec = .dword }},
8146 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
48208147 .patterns = &.{
48218148 .{ .src = .{ .to_mem, .to_mem } },
48228149 },
48238150 .extra_temps = .{
48248151 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
48258152 .{ .type = .u8, .kind = .{ .reg = .cl } },
4826 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4827 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
8153 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8154 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
48288155 .unused,
48298156 .unused,
48308157 },
48318158 .dst_temps = .{.{ .rc = .general_purpose }},
8159 .clobbers = .{ .eflags = true },
48328160 .each = .{ .once = &.{
48338161 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4834 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8162 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
8163 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
48358164 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4836 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4837 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
8165 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
8166 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
48388167 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4839 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4840 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4841 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4842 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
8168 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8169 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8170 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8171 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
48438172 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
48448173 } },
48458174 }, .{
4846 .required_features = .{ .@"64bit", null },
4847 .dst_constraints = .{.{ .bool_vec = .qword }},
4848 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
8175 .required_features = .{ .@"64bit", null, null, null },
8176 .dst_constraints = .{.{ .bool_vec = .dword }},
8177 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
48498178 .patterns = &.{
48508179 .{ .src = .{ .to_mem, .to_mem } },
48518180 },
48528181 .extra_temps = .{
48538182 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
48548183 .{ .type = .u8, .kind = .{ .reg = .cl } },
8184 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
48558185 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4856 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
48578186 .unused,
48588187 .unused,
48598188 },
48608189 .dst_temps = .{.{ .rc = .general_purpose }},
8190 .clobbers = .{ .eflags = true },
48618191 .each = .{ .once = &.{
48628192 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4863 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8193 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
48648194 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
48658195 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4866 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
4867 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
8196 .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8197 .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
48688198 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4869 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4870 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4871 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4872 .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ },
8199 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8200 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8201 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8202 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
48738203 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
48748204 } },
48758205 }, .{
4876 .required_features = .{ .@"64bit", .slow_incdec },
4877 .dst_constraints = .{.{ .bool_vec = .qword }},
4878 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
8206 .dst_constraints = .{.{ .bool_vec = .dword }},
48798207 .patterns = &.{
48808208 .{ .src = .{ .to_mem, .to_mem } },
48818209 },
48828210 .extra_temps = .{
4883 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8211 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
48848212 .{ .type = .u8, .kind = .{ .reg = .cl } },
4885 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
48868213 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4887 .unused,
8214 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8215 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
48888216 .unused,
48898217 },
48908218 .dst_temps = .{.{ .rc = .general_purpose }},
8219 .clobbers = .{ .eflags = true },
48918220 .each = .{ .once = &.{
48928221 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4893 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8222 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
48948223 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
4895 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4896 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4897 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
8224 .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
8225 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
8226 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
8227 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
8228 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
8229 .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
8230 .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
8231 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
8232 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8233 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
48988234 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
4899 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
4900 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4901 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4902 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
4903 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8235 .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ },
8236 .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ },
8237 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8238 .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
8239 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
49048240 } },
49058241 }, .{
4906 .required_features = .{ .@"64bit", null },
8242 .required_features = .{ .@"64bit", null, null, null },
49078243 .dst_constraints = .{.{ .bool_vec = .qword }},
4908 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
8244 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
49098245 .patterns = &.{
49108246 .{ .src = .{ .to_mem, .to_mem } },
49118247 },
......@@ -4913,29 +8249,30 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
49138249 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
49148250 .{ .type = .u8, .kind = .{ .reg = .cl } },
49158251 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4916 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8252 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
49178253 .unused,
49188254 .unused,
49198255 },
49208256 .dst_temps = .{.{ .rc = .general_purpose }},
8257 .clobbers = .{ .eflags = true },
49218258 .each = .{ .once = &.{
49228259 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4923 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8260 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
49248261 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
49258262 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4926 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
4927 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
8263 .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
8264 .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
49288265 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
49298266 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
49308267 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4931 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4932 .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ },
8268 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8269 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
49338270 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
49348271 } },
49358272 }, .{
4936 .required_features = .{ .@"64bit", .slow_incdec },
8273 .required_features = .{ .@"64bit", null, null, null },
49378274 .dst_constraints = .{.{ .bool_vec = .qword }},
4938 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
8275 .src_constraints = .{ .{ .scalar_int = .word }, .{ .scalar_int = .word } },
49398276 .patterns = &.{
49408277 .{ .src = .{ .to_mem, .to_mem } },
49418278 },
......@@ -4943,29 +8280,29 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
49438280 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
49448281 .{ .type = .u8, .kind = .{ .reg = .cl } },
49458282 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4946 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8283 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
49478284 .unused,
49488285 .unused,
49498286 },
49508287 .dst_temps = .{.{ .rc = .general_purpose }},
8288 .clobbers = .{ .eflags = true },
49518289 .each = .{ .once = &.{
49528290 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4953 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
4954 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
8291 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
49558292 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4956 .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4957 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8293 .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ },
8294 .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ },
49588295 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
49598296 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
49608297 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4961 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
4962 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
8298 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8299 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
49638300 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
49648301 } },
49658302 }, .{
4966 .required_features = .{ .@"64bit", null },
8303 .required_features = .{ .@"64bit", null, null, null },
49678304 .dst_constraints = .{.{ .bool_vec = .qword }},
4968 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
8305 .src_constraints = .{ .{ .scalar_int = .dword }, .{ .scalar_int = .dword } },
49698306 .patterns = &.{
49708307 .{ .src = .{ .to_mem, .to_mem } },
49718308 },
......@@ -4973,63 +8310,59 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
49738310 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
49748311 .{ .type = .u8, .kind = .{ .reg = .cl } },
49758312 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4976 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8313 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
49778314 .unused,
49788315 .unused,
49798316 },
49808317 .dst_temps = .{.{ .rc = .general_purpose }},
8318 .clobbers = .{ .eflags = true },
49818319 .each = .{ .once = &.{
49828320 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
4983 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8321 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
49848322 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
49858323 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
4986 .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
4987 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
8324 .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
8325 .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
49888326 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
49898327 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
49908328 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
4991 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
4992 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
8329 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8330 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
49938331 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
49948332 } },
49958333 }, .{
4996 .required_features = .{ .@"64bit", .slow_incdec },
8334 .required_features = .{ .@"64bit", null, null, null },
49978335 .dst_constraints = .{.{ .bool_vec = .qword }},
8336 .src_constraints = .{ .{ .scalar_int = .qword }, .{ .scalar_int = .qword } },
49988337 .patterns = &.{
49998338 .{ .src = .{ .to_mem, .to_mem } },
50008339 },
50018340 .extra_temps = .{
5002 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8341 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
50038342 .{ .type = .u8, .kind = .{ .reg = .cl } },
5004 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5005 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5006 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
8343 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8344 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
8345 .unused,
50078346 .unused,
50088347 },
50098348 .dst_temps = .{.{ .rc = .general_purpose }},
8349 .clobbers = .{ .eflags = true },
50108350 .each = .{ .once = &.{
50118351 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
5012 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
8352 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
50138353 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
5014 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
5015 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
5016 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
5017 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
5018 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
5019 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
5020 .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ },
5021 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
5022 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
5023 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
8354 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
8355 .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
8356 .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
50248357 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
50258358 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
50268359 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
5027 .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ },
5028 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
5029 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
8360 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8361 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
8362 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
50308363 } },
50318364 }, .{
5032 .required_features = .{ .@"64bit", null },
8365 .required_features = .{ .@"64bit", null, null, null },
50338366 .dst_constraints = .{.{ .bool_vec = .qword }},
50348367 .patterns = &.{
50358368 .{ .src = .{ .to_mem, .to_mem } },
......@@ -5043,30 +8376,30 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
50438376 .unused,
50448377 },
50458378 .dst_temps = .{.{ .rc = .general_purpose }},
8379 .clobbers = .{ .eflags = true },
50468380 .each = .{ .once = &.{
50478381 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
50488382 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
50498383 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
5050 .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ },
8384 .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ },
50518385 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
50528386 .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ },
50538387 .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ },
50548388 .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ },
5055 .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ },
5056 .{ ._, ._, .dec, .tmp2d, ._, ._, ._ },
5057 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
8389 .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ },
8390 .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ },
8391 .{ ._, ._b, .j, .@"1b", ._, ._, ._ },
50588392 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
50598393 .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ },
50608394 .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ },
50618395 .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ },
50628396 .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ },
5063 .{ ._, ._, .inc, .tmp1b, ._, ._, ._ },
5064 .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ },
8397 .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ },
8398 .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ },
50658399 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
50668400 } },
50678401 }, .{
5068 .required_features = .{ .slow_incdec, null },
5069 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
8402 .src_constraints = .{ .{ .scalar_int = .byte }, .{ .scalar_int = .byte } },
50708403 .patterns = &.{
50718404 .{ .src = .{ .to_mem, .to_mem } },
50728405 },
......@@ -5079,8 +8412,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
50798412 .unused,
50808413 },
50818414 .dst_temps = .{.mem},
8415 .clobbers = .{ .eflags = true },
50828416 .each = .{ .once = &.{
5083 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8417 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
50848418 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
50858419 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
50868420 .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
......@@ -5089,62 +8423,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
50898423 .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ },
50908424 .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ },
50918425 .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ },
5092 .{ ._, ._, .add, .tmp1d, .i(1), ._, ._ },
5093 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .none, .add_ptr_bit_size), ._, ._ },
8426 .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ },
8427 .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
50948428 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
50958429 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
5096 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
8430 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
50978431 .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ },
50988432 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
5099 .{ .@"1:", ._, .add, .tmp0p, .i(1), ._, ._ },
8433 .{ .@"1:", ._, .add, .tmp0p, .si(1), ._, ._ },
51008434 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5101 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .none, .add_ptr_bit_size), ._, ._ },
5102 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
5103 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
5104 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
5105 .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ },
5106 } },
5107 }, .{
5108 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
5109 .patterns = &.{
5110 .{ .src = .{ .to_mem, .to_mem } },
5111 },
5112 .extra_temps = .{
5113 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5114 .{ .type = .u32, .kind = .{ .reg = .ecx } },
5115 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5116 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5117 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5118 .unused,
5119 },
5120 .dst_temps = .{.mem},
5121 .each = .{ .once = &.{
5122 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
5123 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
5124 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
5125 .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
5126 .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ },
5127 .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ },
5128 .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ },
5129 .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ },
5130 .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ },
5131 .{ ._, ._, .inc, .tmp1d, ._, ._, ._ },
5132 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .none, .add_ptr_bit_size), ._, ._ },
5133 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
5134 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
5135 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
5136 .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ },
5137 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
5138 .{ .@"1:", ._, .inc, .tmp0p, ._, ._, ._ },
5139 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
5140 .{ ._, ._, .@"test", .tmp1d, .ia(-1, .none, .add_ptr_bit_size), ._, ._ },
8435 .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
51418436 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
51428437 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
5143 .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ },
8438 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
51448439 .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ },
51458440 } },
51468441 } },
5147 }) catch |err2| switch (err2) {
8442 }) catch |err| switch (err) {
51488443 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
51498444 @tagName(air_tag),
51508445 cg.typeOf(extra.lhs).fmt(pt),
......@@ -5175,9 +8470,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
51758470 .cmp_lte, .cmp_lte_optimized => .lte,
51768471 .cmp_gte, .cmp_gte_optimized => .gte,
51778472 .cmp_gt, .cmp_gt_optimized => .gt,
5178 }) else {
8473 }) else fallback: {
51798474 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
51808475 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
8476 if (scalar_ty.isRuntimeFloat()) break :fallback try cg.airCmp(inst, switch (air_tag) {
8477 else => unreachable,
8478 .cmp_lt, .cmp_lt_optimized => .lt,
8479 .cmp_lte, .cmp_lte_optimized => .lte,
8480 .cmp_gte, .cmp_gte_optimized => .gte,
8481 .cmp_gt, .cmp_gt_optimized => .gt,
8482 });
51818483 const signedness = if (scalar_ty.isAbiInt(zcu))
51828484 scalar_ty.intInfo(zcu).signedness
51838485 else
......@@ -5205,11 +8507,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52058507 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
52068508 .patterns = &.{
52078509 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
5208 .{ .src = .{ .imm8, .gpr }, .commute = .{ 0, 1 } },
5209 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
8510 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
8511 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
52108512 },
5211 .clobbers = .{ .eflags = true },
52128513 .dst_temps = .{.{ .cc = cc.commute() }},
8514 .clobbers = .{ .eflags = true },
52138515 .each = .{ .once = &.{
52148516 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
52158517 } },
......@@ -5217,12 +8519,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52178519 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
52188520 .patterns = &.{
52198521 .{ .src = .{ .mem, .imm8 } },
5220 .{ .src = .{ .gpr, .imm8 } },
5221 .{ .src = .{ .gpr, .mem } },
5222 .{ .src = .{ .gpr, .gpr } },
8522 .{ .src = .{ .to_gpr, .imm8 } },
8523 .{ .src = .{ .to_gpr, .mem } },
8524 .{ .src = .{ .to_gpr, .to_gpr } },
52238525 },
5224 .clobbers = .{ .eflags = true },
52258526 .dst_temps = .{.{ .cc = cc }},
8527 .clobbers = .{ .eflags = true },
52268528 .each = .{ .once = &.{
52278529 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
52288530 } },
......@@ -5230,11 +8532,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52308532 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
52318533 .patterns = &.{
52328534 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
5233 .{ .src = .{ .imm16, .gpr }, .commute = .{ 0, 1 } },
5234 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
8535 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
8536 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
52358537 },
5236 .clobbers = .{ .eflags = true },
52378538 .dst_temps = .{.{ .cc = cc.commute() }},
8539 .clobbers = .{ .eflags = true },
52388540 .each = .{ .once = &.{
52398541 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
52408542 } },
......@@ -5242,12 +8544,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52428544 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
52438545 .patterns = &.{
52448546 .{ .src = .{ .mem, .imm16 } },
5245 .{ .src = .{ .gpr, .imm16 } },
5246 .{ .src = .{ .gpr, .mem } },
5247 .{ .src = .{ .gpr, .gpr } },
8547 .{ .src = .{ .to_gpr, .imm16 } },
8548 .{ .src = .{ .to_gpr, .mem } },
8549 .{ .src = .{ .to_gpr, .to_gpr } },
52488550 },
5249 .clobbers = .{ .eflags = true },
52508551 .dst_temps = .{.{ .cc = cc }},
8552 .clobbers = .{ .eflags = true },
52518553 .each = .{ .once = &.{
52528554 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
52538555 } },
......@@ -5255,11 +8557,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52558557 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
52568558 .patterns = &.{
52578559 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
5258 .{ .src = .{ .imm32, .gpr }, .commute = .{ 0, 1 } },
5259 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
8560 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
8561 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
52608562 },
5261 .clobbers = .{ .eflags = true },
52628563 .dst_temps = .{.{ .cc = cc.commute() }},
8564 .clobbers = .{ .eflags = true },
52638565 .each = .{ .once = &.{
52648566 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
52658567 } },
......@@ -5267,45 +8569,50 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52678569 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
52688570 .patterns = &.{
52698571 .{ .src = .{ .mem, .imm32 } },
5270 .{ .src = .{ .gpr, .imm32 } },
5271 .{ .src = .{ .gpr, .mem } },
5272 .{ .src = .{ .gpr, .gpr } },
8572 .{ .src = .{ .to_gpr, .imm32 } },
8573 .{ .src = .{ .to_gpr, .mem } },
8574 .{ .src = .{ .to_gpr, .to_gpr } },
52738575 },
5274 .clobbers = .{ .eflags = true },
52758576 .dst_temps = .{.{ .cc = cc }},
8577 .clobbers = .{ .eflags = true },
52768578 .each = .{ .once = &.{
52778579 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
52788580 } },
52798581 }, .{
5280 .required_features = .{ .@"64bit", null },
8582 .required_features = .{ .@"64bit", null, null, null },
52818583 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
52828584 .patterns = &.{
52838585 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
5284 .{ .src = .{ .simm32, .gpr }, .commute = .{ 0, 1 } },
5285 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
8586 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
8587 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
52868588 },
5287 .clobbers = .{ .eflags = true },
52888589 .dst_temps = .{.{ .cc = cc.commute() }},
8590 .clobbers = .{ .eflags = true },
52898591 .each = .{ .once = &.{
52908592 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
52918593 } },
52928594 }, .{
5293 .required_features = .{ .@"64bit", null },
8595 .required_features = .{ .@"64bit", null, null, null },
52948596 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
52958597 .patterns = &.{
52968598 .{ .src = .{ .mem, .simm32 } },
5297 .{ .src = .{ .gpr, .simm32 } },
5298 .{ .src = .{ .gpr, .mem } },
5299 .{ .src = .{ .gpr, .gpr } },
8599 .{ .src = .{ .to_gpr, .simm32 } },
8600 .{ .src = .{ .to_gpr, .mem } },
8601 .{ .src = .{ .to_gpr, .to_gpr } },
53008602 },
5301 .clobbers = .{ .eflags = true },
53028603 .dst_temps = .{.{ .cc = cc }},
8604 .clobbers = .{ .eflags = true },
53038605 .each = .{ .once = &.{
53048606 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
53058607 } },
53068608 }, .{
8609 .src_constraints = .{ .any_int, .any_int },
53078610 .patterns = &.{
5308 .{ .src = .{ .to_mem, .to_mem } },
8611 .{ .src = .{ .to_mem, .to_mem }, .commute = switch (cc) {
8612 else => unreachable,
8613 .l, .ge, .b, .ae => .{ 0, 0 },
8614 .le, .g, .be, .a => .{ 0, 1 },
8615 } },
53098616 },
53108617 .extra_temps = .{
53118618 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
......@@ -5315,17 +8622,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
53158622 .unused,
53168623 .unused,
53178624 },
5318 .clobbers = .{ .eflags = true },
53198625 .dst_temps = .{.{ .rc = .general_purpose }},
8626 .clobbers = .{ .eflags = true },
53208627 .each = .{ .once = &.{
5321 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8628 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
53228629 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
5323 .{ .@"0:", ._r, .sh, .tmp1b, .i(1), ._, ._ },
8630 .{ .@"0:", ._r, .sh, .tmp1b, .si(1), ._, ._ },
53248631 .{ ._, ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
53258632 .{ ._, ._, .sbb, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
53268633 .{ ._, ._c, .set, .tmp1b, ._, ._, ._ },
5327 .{ ._, .fromCondition(cc), .set, .dst0b, ._, ._, ._ },
5328 .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ },
8634 .{ ._, .fromCondition(switch (cc) {
8635 else => unreachable,
8636 .l, .ge, .b, .ae => cc,
8637 .le, .g, .be, .a => cc.commute(),
8638 }), .set, .dst0b, ._, ._, ._ },
8639 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
53298640 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
53308641 } },
53318642 } },
......@@ -5342,13 +8653,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
53428653 if (ops[1].index != res[0].index) try ops[1].die(cg);
53438654 try res[0].moveTo(inst, cg);
53448655 },
5345 .cmp_eq, .cmp_eq_optimized, .cmp_neq, .cmp_neq_optimized => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
8656 .cmp_eq,
8657 .cmp_eq_optimized,
8658 .cmp_neq,
8659 .cmp_neq_optimized,
8660 => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
53468661 else => unreachable,
53478662 .cmp_eq, .cmp_eq_optimized => .eq,
53488663 .cmp_neq, .cmp_neq_optimized => .neq,
53498664 }) else fallback: {
53508665 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
5351 if (ip.isOptionalType(cg.typeOf(bin_op.lhs).toIntern())) break :fallback try cg.airCmp(inst, switch (air_tag) {
8666 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
8667 if (scalar_ty.isRuntimeFloat() or ip.isOptionalType(scalar_ty.toIntern())) break :fallback try cg.airCmp(inst, switch (air_tag) {
53528668 else => unreachable,
53538669 .cmp_eq, .cmp_eq_optimized => .eq,
53548670 .cmp_neq, .cmp_neq_optimized => .neq,
......@@ -5362,14 +8678,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
53628678 })) {
53638679 else => unreachable,
53648680 inline .e, .ne => |cc| comptime &.{ .{
5365 .required_features = .{ .avx2, null },
5366 .src_constraints = .{ .any_int, .any_int },
8681 .required_features = .{ .avx2, null, null, null },
8682 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
53678683 .patterns = &.{
5368 .{ .src = .{ .ymm, .mem } },
5369 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
5370 .{ .src = .{ .ymm, .ymm } },
8684 .{ .src = .{ .to_ymm, .mem } },
8685 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
8686 .{ .src = .{ .to_ymm, .to_ymm } },
53718687 },
5372 .clobbers = .{ .eflags = true },
53738688 .extra_temps = .{
53748689 .{ .kind = .{ .rc = .sse } },
53758690 .unused,
......@@ -5379,19 +8694,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
53798694 .unused,
53808695 },
53818696 .dst_temps = .{.{ .cc = cc }},
8697 .clobbers = .{ .eflags = true },
53828698 .each = .{ .once = &.{
53838699 .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ },
53848700 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
53858701 } },
53868702 }, .{
5387 .required_features = .{ .avx, null },
5388 .src_constraints = .{ .any_int, .any_int },
8703 .required_features = .{ .avx, null, null, null },
8704 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
53898705 .patterns = &.{
5390 .{ .src = .{ .ymm, .mem } },
5391 .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } },
5392 .{ .src = .{ .ymm, .ymm } },
8706 .{ .src = .{ .to_ymm, .mem } },
8707 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
8708 .{ .src = .{ .to_ymm, .to_ymm } },
53938709 },
5394 .clobbers = .{ .eflags = true },
53958710 .extra_temps = .{
53968711 .{ .kind = .{ .rc = .sse } },
53978712 .unused,
......@@ -5401,19 +8716,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
54018716 .unused,
54028717 },
54038718 .dst_temps = .{.{ .cc = cc }},
8719 .clobbers = .{ .eflags = true },
54048720 .each = .{ .once = &.{
54058721 .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ },
54068722 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
54078723 } },
54088724 }, .{
5409 .required_features = .{ .avx, null },
5410 .src_constraints = .{ .any_int, .any_int },
8725 .required_features = .{ .avx, null, null, null },
8726 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
54118727 .patterns = &.{
5412 .{ .src = .{ .xmm, .mem } },
5413 .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } },
5414 .{ .src = .{ .xmm, .xmm } },
8728 .{ .src = .{ .to_xmm, .mem } },
8729 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
8730 .{ .src = .{ .to_xmm, .to_xmm } },
54158731 },
5416 .clobbers = .{ .eflags = true },
54178732 .extra_temps = .{
54188733 .{ .kind = .{ .rc = .sse } },
54198734 .unused,
......@@ -5423,33 +8738,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
54238738 .unused,
54248739 },
54258740 .dst_temps = .{.{ .cc = cc }},
8741 .clobbers = .{ .eflags = true },
54268742 .each = .{ .once = &.{
54278743 .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ },
54288744 .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ },
54298745 } },
54308746 }, .{
5431 .required_features = .{ .sse4_1, null },
5432 .src_constraints = .{ .any_int, .any_int },
8747 .required_features = .{ .sse4_1, null, null, null },
8748 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
54338749 .patterns = &.{
5434 .{ .src = .{ .mut_xmm, .mem } },
5435 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
5436 .{ .src = .{ .mut_xmm, .xmm } },
8750 .{ .src = .{ .to_mut_xmm, .mem } },
8751 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
8752 .{ .src = .{ .to_mut_xmm, .to_xmm } },
54378753 },
5438 .clobbers = .{ .eflags = true },
54398754 .dst_temps = .{.{ .cc = cc }},
8755 .clobbers = .{ .eflags = true },
54408756 .each = .{ .once = &.{
54418757 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
54428758 .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ },
54438759 } },
54448760 }, .{
5445 .required_features = .{ .sse2, null },
5446 .src_constraints = .{ .any_int, .any_int },
8761 .required_features = .{ .sse2, null, null, null },
8762 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
54478763 .patterns = &.{
5448 .{ .src = .{ .mut_xmm, .mem } },
5449 .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } },
5450 .{ .src = .{ .mut_xmm, .xmm } },
8764 .{ .src = .{ .to_mut_xmm, .mem } },
8765 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
8766 .{ .src = .{ .to_mut_xmm, .to_xmm } },
54518767 },
5452 .clobbers = .{ .eflags = true },
54538768 .extra_temps = .{
54548769 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
54558770 .{ .kind = .{ .rc = .sse } },
......@@ -5459,22 +8774,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
54598774 .unused,
54608775 },
54618776 .dst_temps = .{.{ .cc = cc }},
8777 .clobbers = .{ .eflags = true },
54628778 .each = .{ .once = &.{
54638779 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
54648780 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
54658781 .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ },
54668782 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
5467 .{ ._, ._, .xor, .tmp0d, .i(0xffff), ._, ._ },
8783 .{ ._, ._, .xor, .tmp0d, .si(0xffff), ._, ._ },
54688784 } },
54698785 }, .{
5470 .required_features = .{ .sse2, .mmx },
5471 .src_constraints = .{ .any_int, .any_int },
8786 .required_features = .{ .sse, .mmx, null, null },
8787 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
54728788 .patterns = &.{
5473 .{ .src = .{ .mut_mm, .mem } },
5474 .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } },
5475 .{ .src = .{ .mut_mm, .mm } },
8789 .{ .src = .{ .to_mut_mm, .mem } },
8790 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
8791 .{ .src = .{ .to_mut_mm, .to_mm } },
54768792 },
5477 .clobbers = .{ .eflags = true },
54788793 .extra_temps = .{
54798794 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
54808795 .{ .kind = .{ .rc = .mmx } },
......@@ -5484,26 +8799,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
54848799 .unused,
54858800 },
54868801 .dst_temps = .{.{ .cc = cc }},
8802 .clobbers = .{ .eflags = true },
54878803 .each = .{ .once = &.{
54888804 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
54898805 .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ },
54908806 .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ },
54918807 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
5492 .{ ._, ._, .xor, .tmp0d, .i(0xff), ._, ._ },
8808 .{ ._, ._, .xor, .tmp0d, .si(0xff), ._, ._ },
54938809 } },
54948810 }, .{
54958811 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
54968812 .patterns = &.{
54978813 .{ .src = .{ .mem, .imm8 } },
54988814 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
5499 .{ .src = .{ .gpr, .imm8 } },
5500 .{ .src = .{ .imm8, .gpr }, .commute = .{ 0, 1 } },
5501 .{ .src = .{ .gpr, .mem } },
5502 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
5503 .{ .src = .{ .gpr, .gpr } },
8815 .{ .src = .{ .to_gpr, .imm8 } },
8816 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
8817 .{ .src = .{ .to_gpr, .mem } },
8818 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8819 .{ .src = .{ .to_gpr, .to_gpr } },
55048820 },
5505 .clobbers = .{ .eflags = true },
55068821 .dst_temps = .{.{ .cc = cc }},
8822 .clobbers = .{ .eflags = true },
55078823 .each = .{ .once = &.{
55088824 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
55098825 } },
......@@ -5512,14 +8828,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
55128828 .patterns = &.{
55138829 .{ .src = .{ .mem, .imm16 } },
55148830 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
5515 .{ .src = .{ .gpr, .imm16 } },
5516 .{ .src = .{ .imm16, .gpr }, .commute = .{ 0, 1 } },
5517 .{ .src = .{ .gpr, .mem } },
5518 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
5519 .{ .src = .{ .gpr, .gpr } },
8831 .{ .src = .{ .to_gpr, .imm16 } },
8832 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
8833 .{ .src = .{ .to_gpr, .mem } },
8834 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8835 .{ .src = .{ .to_gpr, .to_gpr } },
55208836 },
5521 .clobbers = .{ .eflags = true },
55228837 .dst_temps = .{.{ .cc = cc }},
8838 .clobbers = .{ .eflags = true },
55238839 .each = .{ .once = &.{
55248840 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
55258841 } },
......@@ -5528,36 +8844,68 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
55288844 .patterns = &.{
55298845 .{ .src = .{ .mem, .imm32 } },
55308846 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
5531 .{ .src = .{ .gpr, .imm32 } },
5532 .{ .src = .{ .imm32, .gpr }, .commute = .{ 0, 1 } },
5533 .{ .src = .{ .gpr, .mem } },
5534 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
5535 .{ .src = .{ .gpr, .gpr } },
8847 .{ .src = .{ .to_gpr, .imm32 } },
8848 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
8849 .{ .src = .{ .to_gpr, .mem } },
8850 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8851 .{ .src = .{ .to_gpr, .to_gpr } },
55368852 },
5537 .clobbers = .{ .eflags = true },
55388853 .dst_temps = .{.{ .cc = cc }},
8854 .clobbers = .{ .eflags = true },
55398855 .each = .{ .once = &.{
55408856 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
55418857 } },
55428858 }, .{
5543 .required_features = .{ .@"64bit", null },
8859 .required_features = .{ .@"64bit", null, null, null },
55448860 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
55458861 .patterns = &.{
55468862 .{ .src = .{ .mem, .simm32 } },
55478863 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
5548 .{ .src = .{ .gpr, .simm32 } },
5549 .{ .src = .{ .simm32, .gpr }, .commute = .{ 0, 1 } },
5550 .{ .src = .{ .gpr, .mem } },
5551 .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } },
5552 .{ .src = .{ .gpr, .gpr } },
8864 .{ .src = .{ .to_gpr, .simm32 } },
8865 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
8866 .{ .src = .{ .to_gpr, .mem } },
8867 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
8868 .{ .src = .{ .to_gpr, .to_gpr } },
55538869 },
5554 .clobbers = .{ .eflags = true },
55558870 .dst_temps = .{.{ .cc = cc }},
8871 .clobbers = .{ .eflags = true },
55568872 .each = .{ .once = &.{
55578873 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
55588874 } },
55598875 }, .{
5560 .required_features = .{ .avx2, null },
8876 .required_features = .{ .avx2, null, null, null },
8877 .src_constraints = .{
8878 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
8879 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
8880 },
8881 .patterns = &.{
8882 .{ .src = .{ .to_mem, .to_mem } },
8883 },
8884 .extra_temps = .{
8885 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8886 .{ .kind = .{ .rc = .sse } },
8887 .{ .kind = .{ .rc = .sse } },
8888 .unused,
8889 .unused,
8890 .unused,
8891 },
8892 .dst_temps = .{.{ .cc = cc }},
8893 .clobbers = .{ .eflags = true },
8894 .each = .{ .once = &.{
8895 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
8896 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
8897 .{ .@"0:", .v_dqu, .mov, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
8898 .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
8899 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
8900 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
8901 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8902 .{ ._, .v_dqa, .mov, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
8903 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
8904 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
8905 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
8906 } },
8907 }, .{
8908 .required_features = .{ .avx2, null, null, null },
55618909 .patterns = &.{
55628910 .{ .src = .{ .to_mem, .to_mem } },
55638911 },
......@@ -5570,18 +8918,51 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
55708918 .unused,
55718919 },
55728920 .dst_temps = .{.{ .cc = cc }},
8921 .clobbers = .{ .eflags = true },
55738922 .each = .{ .once = &.{
5574 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8923 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
55758924 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
55768925 .{ .@"0:", .v_dqu, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
55778926 .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
55788927 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
5579 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
8928 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
8929 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8930 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
8931 } },
8932 }, .{
8933 .required_features = .{ .avx, null, null, null },
8934 .src_constraints = .{
8935 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
8936 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
8937 },
8938 .patterns = &.{
8939 .{ .src = .{ .to_mem, .to_mem } },
8940 },
8941 .extra_temps = .{
8942 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
8943 .{ .kind = .{ .rc = .sse } },
8944 .{ .kind = .{ .rc = .sse } },
8945 .unused,
8946 .unused,
8947 .unused,
8948 },
8949 .dst_temps = .{.{ .cc = cc }},
8950 .clobbers = .{ .eflags = true },
8951 .each = .{ .once = &.{
8952 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
8953 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
8954 .{ .@"0:", .v_pd, .movu, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
8955 .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
8956 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
8957 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
55808958 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
8959 .{ ._, .v_pd, .mova, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
8960 .{ ._, .v_pd, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
8961 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
55818962 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
55828963 } },
55838964 }, .{
5584 .required_features = .{ .avx, null },
8965 .required_features = .{ .avx, null, null, null },
55858966 .patterns = &.{
55868967 .{ .src = .{ .to_mem, .to_mem } },
55878968 },
......@@ -5594,18 +8975,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
55948975 .unused,
55958976 },
55968977 .dst_temps = .{.{ .cc = cc }},
8978 .clobbers = .{ .eflags = true },
55978979 .each = .{ .once = &.{
5598 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
8980 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
55998981 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
56008982 .{ .@"0:", .v_pd, .movu, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
56018983 .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
56028984 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
5603 .{ ._, ._, .add, .tmp0p, .i(32), ._, ._ },
8985 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
56048986 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
56058987 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
56068988 } },
56078989 }, .{
5608 .required_features = .{ .avx, null },
8990 .required_features = .{ .avx, null, null, null },
56098991 .patterns = &.{
56108992 .{ .src = .{ .to_mem, .to_mem } },
56118993 },
......@@ -5618,18 +9000,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56189000 .unused,
56199001 },
56209002 .dst_temps = .{.{ .cc = cc }},
9003 .clobbers = .{ .eflags = true },
56219004 .each = .{ .once = &.{
5622 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
9005 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
56239006 .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
56249007 .{ .@"0:", .v_dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
56259008 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._ },
56269009 .{ ._, .vp_, .@"or", .tmp1x, .tmp1x, .tmp2x, ._ },
5627 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
9010 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
56289011 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
56299012 .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ },
56309013 } },
56319014 }, .{
5632 .required_features = .{ .sse4_1, null },
9015 .required_features = .{ .sse4_1, null, null, null },
56339016 .patterns = &.{
56349017 .{ .src = .{ .to_mem, .to_mem } },
56359018 },
......@@ -5642,18 +9025,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56429025 .unused,
56439026 },
56449027 .dst_temps = .{.{ .cc = cc }},
9028 .clobbers = .{ .eflags = true },
56459029 .each = .{ .once = &.{
5646 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
9030 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
56479031 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
56489032 .{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
56499033 .{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
56509034 .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
5651 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
9035 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
56529036 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
56539037 .{ ._, .p_, .@"test", .tmp1x, .tmp1x, ._, ._ },
56549038 } },
56559039 }, .{
5656 .required_features = .{ .sse2, null },
9040 .required_features = .{ .sse2, null, null, null },
56579041 .patterns = &.{
56589042 .{ .src = .{ .to_mem, .to_mem } },
56599043 },
......@@ -5666,21 +9050,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56669050 .unused,
56679051 },
56689052 .dst_temps = .{.{ .cc = cc }},
9053 .clobbers = .{ .eflags = true },
56699054 .each = .{ .once = &.{
5670 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
9055 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
56719056 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
56729057 .{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
56739058 .{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
56749059 .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
5675 .{ ._, ._, .add, .tmp0p, .i(16), ._, ._ },
9060 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
56769061 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
56779062 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
56789063 .{ ._, .p_b, .cmpeq, .tmp1x, .tmp2x, ._, ._ },
56799064 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
5680 .{ ._, ._, .cmp, .tmp0d, .i(0xffff), ._, ._ },
9065 .{ ._, ._, .cmp, .tmp0d, .si(0xffff), ._, ._ },
56819066 } },
56829067 }, .{
5683 .required_features = .{ .sse, .mmx },
9068 .required_features = .{ .sse, .mmx, null, null },
56849069 .patterns = &.{
56859070 .{ .src = .{ .to_mem, .to_mem } },
56869071 },
......@@ -5693,18 +9078,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56939078 .unused,
56949079 },
56959080 .dst_temps = .{.{ .cc = cc }},
9081 .clobbers = .{ .eflags = true },
56969082 .each = .{ .once = &.{
5697 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
9083 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
56989084 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
56999085 .{ .@"0:", ._q, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
57009086 .{ ._, .p_, .xor, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
57019087 .{ ._, .p_, .@"or", .tmp1q, .tmp2q, ._, ._ },
5702 .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ },
9088 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
57039089 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
57049090 .{ ._, .p_, .xor, .tmp2q, .tmp2q, ._, ._ },
57059091 .{ ._, .p_b, .cmpeq, .tmp1q, .tmp2q, ._, ._ },
57069092 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
5707 .{ ._, ._, .cmp, .tmp0d, .i(0xff), ._, ._ },
9093 .{ ._, ._, .cmp, .tmp0d, .si(0xff), ._, ._ },
57089094 } },
57099095 }, .{
57109096 .patterns = &.{
......@@ -5719,13 +9105,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
57199105 .unused,
57209106 },
57219107 .dst_temps = .{.{ .cc = cc }},
9108 .clobbers = .{ .eflags = true },
57229109 .each = .{ .once = &.{
5723 .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ },
9110 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
57249111 .{ ._, ._, .xor, .tmp1p, .tmp1p, ._, ._ },
57259112 .{ .@"0:", ._, .mov, .tmp2p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
57269113 .{ ._, ._, .xor, .tmp2p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
57279114 .{ ._, ._, .@"or", .tmp1p, .tmp2p, ._, ._ },
5728 .{ ._, ._, .add, .tmp0p, .a(.tmp2, .add_size), ._, ._ },
9115 .{ ._, ._, .add, .tmp0p, .sa(.tmp2, .add_size), ._, ._ },
57299116 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
57309117 .{ ._, ._, .@"test", .tmp1p, .tmp1p, ._, ._ },
57319118 } },
......@@ -6453,11 +9840,8 @@ fn regClassForType(self: *CodeGen, ty: Type) Register.Class {
64539840 else => .sse,
64549841 },
64559842 .vector => switch (ty.childType(zcu).toIntern()) {
6456 .bool_type, .u1_type => .general_purpose,
6457 else => if (ty.isAbiInt(zcu) and ty.intInfo(zcu).bits == 1)
6458 .general_purpose
6459 else
6460 .sse,
9843 .bool_type => .general_purpose,
9844 else => .sse,
64619845 },
64629846 else => .general_purpose,
64639847 };
......@@ -8383,32 +11767,54 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
838311767
838411768 const lhs_mcv = try self.resolveInst(bin_op.lhs);
838511769 const rhs_mcv = try self.resolveInst(bin_op.rhs);
8386 const mat_lhs_mcv = switch (lhs_mcv) {
8387 .load_symbol => mat_lhs_mcv: {
11770 const mat_lhs_mcv = mat_lhs_mcv: switch (lhs_mcv) {
11771 .register => |lhs_reg| switch (lhs_reg.class()) {
11772 else => lhs_mcv,
11773 .sse => {
11774 const mat_lhs_mcv: MCValue = .{
11775 .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
11776 };
11777 try self.genCopy(dst_ty, mat_lhs_mcv, lhs_mcv, .{});
11778 break :mat_lhs_mcv mat_lhs_mcv;
11779 },
11780 },
11781 .load_symbol => {
838811782 // TODO clean this up!
838911783 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
839011784 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
839111785 },
839211786 else => lhs_mcv,
839311787 };
8394 const mat_lhs_lock = switch (mat_lhs_mcv) {
8395 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
8396 else => null,
11788 const mat_lhs_locks: [2]?RegisterLock = switch (mat_lhs_mcv) {
11789 .register_pair => |mat_lhs_regs| self.register_manager.lockRegs(2, mat_lhs_regs),
11790 .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null },
11791 else => @splat(null),
839711792 };
8398 defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
8399 const mat_rhs_mcv = switch (rhs_mcv) {
8400 .load_symbol => mat_rhs_mcv: {
11793 defer for (mat_lhs_locks) |mat_lhs_lock| if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
11794 const mat_rhs_mcv = mat_rhs_mcv: switch (rhs_mcv) {
11795 .register => |rhs_reg| switch (rhs_reg.class()) {
11796 else => rhs_mcv,
11797 .sse => {
11798 const mat_rhs_mcv: MCValue = .{
11799 .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
11800 };
11801 try self.genCopy(dst_ty, mat_rhs_mcv, rhs_mcv, .{});
11802 break :mat_rhs_mcv mat_rhs_mcv;
11803 },
11804 },
11805 .load_symbol => {
840111806 // TODO clean this up!
840211807 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
840311808 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
840411809 },
840511810 else => rhs_mcv,
840611811 };
8407 const mat_rhs_lock = switch (mat_rhs_mcv) {
8408 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
8409 else => null,
11812 const mat_rhs_locks: [2]?RegisterLock = switch (mat_rhs_mcv) {
11813 .register_pair => |mat_rhs_regs| self.register_manager.lockRegs(2, mat_rhs_regs),
11814 .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null },
11815 else => @splat(null),
841011816 };
8411 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
11817 defer for (mat_rhs_locks) |mat_rhs_lock| if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
841211818
841311819 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
841411820 .{ ._, .mov },
......@@ -10003,7 +13409,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
1000313409 } },
1000413410 }, .u(0));
1000513411 _ = try self.asmJccReloc(.e, loop);
10006 try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{
13412 try self.asmRegisterMemory(.{ ._r, .bs }, dst_reg.to64(), .{
1000713413 .base = .{ .frame = src_frame_addr.index },
1000813414 .mod = .{ .rm = .{
1000913415 .size = .qword,
......@@ -10080,8 +13486,8 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
1008013486 defer self.register_manager.unlockReg(wide_lock);
1008113487
1008213488 try self.truncateRegister(src_ty, wide_reg);
10083 try self.genBinOpMir(.{ ._, .bsr }, .u16, dst_mcv, .{ .register = wide_reg });
10084 } else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
13489 try self.genBinOpMir(.{ ._r, .bs }, .u16, dst_mcv, .{ .register = wide_reg });
13490 } else try self.genBinOpMir(.{ ._r, .bs }, src_ty, dst_mcv, mat_src_mcv);
1008513491
1008613492 try self.asmCmovccRegisterRegister(
1008713493 .z,
......@@ -10103,7 +13509,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
1010313509
1010413510 try self.truncateRegister(src_ty, wide_reg);
1010513511 try self.genBinOpMir(
10106 .{ ._, .bsr },
13512 .{ ._r, .bs },
1010713513 if (src_bits <= 8) .u16 else src_ty,
1010813514 dst_mcv,
1010913515 .{ .register = wide_reg },
......@@ -10200,7 +13606,7 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
1020013606 } },
1020113607 }, .u(0));
1020213608 _ = try self.asmJccReloc(.e, loop);
10203 try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{
13609 try self.asmRegisterMemory(.{ ._f, .bs }, dst_reg.to64(), .{
1020413610 .base = .{ .frame = src_frame_addr.index },
1020513611 .mod = .{ .rm = .{
1020613612 .size = .qword,
......@@ -10280,8 +13686,8 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
1028013686 defer self.register_manager.unlockReg(wide_lock);
1028113687
1028213688 try self.truncateRegister(src_ty, wide_reg);
10283 try self.genBinOpMir(.{ ._, .bsf }, wide_ty, dst_mcv, .{ .register = wide_reg });
10284 } else try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
13689 try self.genBinOpMir(.{ ._f, .bs }, wide_ty, dst_mcv, .{ .register = wide_reg });
13690 } else try self.genBinOpMir(.{ ._f, .bs }, src_ty, dst_mcv, mat_src_mcv);
1028513691
1028613692 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
1028713693 try self.asmCmovccRegisterRegister(
......@@ -12975,7 +16381,18 @@ fn genShiftBinOp(
1297516381 const rcx_lock = self.register_manager.lockReg(.rcx);
1297616382 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
1297716383
12978 const lhs_lock = switch (lhs_mcv) {
16384 const mat_lhs_mcv: MCValue, const can_reuse_lhs = switch (lhs_mcv) {
16385 .register => |lhs_reg| switch (lhs_reg.class()) {
16386 .general_purpose => .{ lhs_mcv, true },
16387 else => lhs: {
16388 const mat_lhs_mcv = try self.allocTempRegOrMem(lhs_ty, true);
16389 try self.genCopy(lhs_ty, mat_lhs_mcv, lhs_mcv, .{});
16390 break :lhs .{ mat_lhs_mcv, false };
16391 },
16392 },
16393 else => .{ lhs_mcv, true },
16394 };
16395 const lhs_lock = switch (mat_lhs_mcv) {
1297916396 .register => |reg| self.register_manager.lockReg(reg),
1298016397 else => null,
1298116398 };
......@@ -12988,12 +16405,12 @@ fn genShiftBinOp(
1298816405 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1298916406
1299016407 const dst_mcv: MCValue = dst: {
12991 if (maybe_inst) |inst| {
16408 if (can_reuse_lhs) if (maybe_inst) |inst| {
1299216409 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
12993 if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
12994 }
16410 if (self.reuseOperand(inst, bin_op.lhs, 0, mat_lhs_mcv)) break :dst mat_lhs_mcv;
16411 };
1299516412 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
12996 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{});
16413 try self.genCopy(lhs_ty, dst_mcv, mat_lhs_mcv, .{});
1299716414 break :dst dst_mcv;
1299816415 };
1299916416
......@@ -18337,12 +21754,28 @@ const MoveStrategy = union(enum) {
1833721754 try self.asmRegister(.{ .f_, .ld }, src_reg);
1833821755 try self.asmMemory(.{ .f_p, .st }, dst_mem);
1833921756 },
18340 .insert_extract, .vex_insert_extract => |ie| try self.asmMemoryRegisterImmediate(
18341 ie.extract,
18342 dst_mem,
18343 src_reg,
18344 .u(0),
18345 ),
21757 .insert_extract, .vex_insert_extract => |ie| if (ie.extract[0] != .p_w or self.hasFeature(.sse4_1))
21758 try self.asmMemoryRegisterImmediate(ie.extract, dst_mem, src_reg, .u(0))
21759 else if (self.hasFeature(.sse2)) {
21760 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
21761 try self.asmRegisterRegisterImmediate(ie.extract, tmp_reg.to32(), src_reg.to128(), .u(0));
21762 try self.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16());
21763 } else {
21764 const tmp_frame_index = try self.allocFrameIndex(.init(.{
21765 .size = 16,
21766 .alignment = .@"16",
21767 }));
21768 try self.asmMemoryRegister(.{ ._ps, .mova }, .{
21769 .base = .{ .frame = tmp_frame_index },
21770 .mod = .{ .rm = .{ .size = .xword } },
21771 }, src_reg.to128());
21772 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
21773 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg.to16(), .{
21774 .base = .{ .frame = tmp_frame_index },
21775 .mod = .{ .rm = .{ .size = .word } },
21776 });
21777 try self.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16());
21778 },
1834621779 }
1834721780 }
1834821781};
......@@ -18400,8 +21833,10 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1840021833 .{ ._ss, .mov } },
1840121834 5...8 => return .{ .move = if (self.hasFeature(.avx))
1840221835 .{ .v_sd, .mov }
21836 else if (self.hasFeature(.sse2))
21837 .{ ._sd, .mov }
1840321838 else
18404 .{ ._sd, .mov } },
21839 .{ ._ps, .movl } },
1840521840 9...16 => return .{ .move = if (self.hasFeature(.avx))
1840621841 .{ .v_pd, if (aligned) .mova else .movu }
1840721842 else if (self.hasFeature(.sse2))
......@@ -18427,8 +21862,10 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool)
1842721862 .{ ._ss, .mov } },
1842821863 64 => return .{ .move = if (self.hasFeature(.avx))
1842921864 .{ .v_sd, .mov }
21865 else if (self.hasFeature(.sse2))
21866 .{ ._sd, .mov }
1843021867 else
18431 .{ ._sd, .mov } },
21868 .{ ._ps, .movl } },
1843221869 128 => return .{ .move = if (self.hasFeature(.avx))
1843321870 .{ if (aligned) .v_dqa else .v_dqu, .mov }
1843421871 else if (self.hasFeature(.sse2))
......@@ -18623,6 +22060,30 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
1862322060 }, opts),
1862422061 inline .register_pair, .register_triple, .register_quadruple => |dst_regs| {
1862522062 const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) {
22063 .register => |src_reg| switch (dst_regs[0].class()) {
22064 .general_purpose => switch (src_reg.class()) {
22065 else => unreachable,
22066 .sse => if (ty.abiSize(pt.zcu) <= 16) {
22067 if (self.hasFeature(.avx)) {
22068 try self.asmRegisterRegister(.{ .v_q, .mov }, dst_regs[0].to64(), src_reg.to128());
22069 try self.asmRegisterRegisterImmediate(.{ .vp_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1));
22070 } else if (self.hasFeature(.sse4_1)) {
22071 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
22072 try self.asmRegisterRegisterImmediate(.{ .p_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1));
22073 } else {
22074 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
22075 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
22076 defer self.register_manager.unlockReg(tmp_lock);
22077
22078 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128());
22079 try self.asmRegisterRegister(.{ ._ps, .movhl }, tmp_reg.to128(), src_reg.to128());
22080 try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[1].to64(), src_reg.to128());
22081 }
22082 return;
22083 } else unreachable,
22084 },
22085 else => unreachable,
22086 },
1862622087 .register_pair, .memory, .indirect, .load_frame => null,
1862722088 .load_symbol, .load_direct, .load_got, .load_tlv => src: {
1862822089 const src_addr_reg =
......@@ -18863,7 +22324,39 @@ fn genSetReg(
1886322324 inline .register_pair,
1886422325 .register_triple,
1886522326 .register_quadruple,
18866 => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts),
22327 => |src_regs| switch (dst_reg.class()) {
22328 .general_purpose => switch (src_regs[0].class()) {
22329 .general_purpose => try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts),
22330 else => unreachable,
22331 },
22332 .sse => switch (src_regs[0].class()) {
22333 .general_purpose => if (abi_size <= 16) {
22334 if (self.hasFeature(.avx)) {
22335 try self.asmRegisterRegister(.{ .v_q, .mov }, dst_reg.to128(), src_regs[0].to64());
22336 try self.asmRegisterRegisterRegisterImmediate(
22337 .{ .vp_q, .insr },
22338 dst_reg.to128(),
22339 dst_reg.to128(),
22340 src_regs[1].to64(),
22341 .u(1),
22342 );
22343 } else if (self.hasFeature(.sse4_1)) {
22344 try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
22345 try self.asmRegisterRegisterImmediate(.{ .p_q, .insr }, dst_reg.to128(), src_regs[1].to64(), .u(1));
22346 } else {
22347 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
22348 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
22349 defer self.register_manager.unlockReg(tmp_lock);
22350
22351 try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64());
22352 try self.asmRegisterRegister(.{ ._q, .mov }, tmp_reg.to128(), src_regs[1].to64());
22353 try self.asmRegisterRegister(.{ ._ps, .movlh }, dst_reg.to128(), tmp_reg.to128());
22354 }
22355 } else unreachable,
22356 else => unreachable,
22357 },
22358 else => unreachable,
22359 },
1886722360 .register_offset,
1886822361 .indirect,
1886922362 .load_frame,
......@@ -23517,8 +27010,6 @@ fn promoteVarArg(self: *CodeGen, ty: Type) Type {
2351727010 }
2351827011}
2351927012
23520// ====================================== rewrite starts here ======================================
23521
2352227013const Temp = struct {
2352327014 index: Air.Inst.Index,
2352427015
......@@ -24311,13 +27802,13 @@ const Select = struct {
2431127802 }
2431227803
2431327804 const Case = struct {
24314 required_features: [2]?std.Target.x86.Feature = @splat(null),
27805 required_features: [4]?std.Target.x86.Feature = @splat(null),
2431527806 dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any),
2431627807 src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any),
2431727808 patterns: []const Select.Pattern,
24318 clobbers: struct { eflags: bool = false } = .{},
2431927809 extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused),
2432027810 dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused),
27811 clobbers: struct { eflags: bool = false } = .{},
2432127812 each: union(enum) {
2432227813 once: []const Instruction,
2432327814 },
......@@ -24327,9 +27818,32 @@ const Select = struct {
2432727818 any,
2432827819 any_bool_vec,
2432927820 any_int,
27821 any_signed_int,
2433027822 any_float,
2433127823 bool_vec: Memory.Size,
27824 vec: Memory.Size,
27825 signed_int_vec: Memory.Size,
27826 signed_int_or_full_vec: Memory.Size,
27827 unsigned_int_vec: Memory.Size,
27828 int_or_vec: Memory.Size,
27829 exact_remainder_int_or_vec: struct { of: Memory.Size, is: Memory.Size },
2433227830 int: Memory.Size,
27831 scalar_int: Memory.Size,
27832 scalar_signed_int: Memory.Size,
27833 scalar_unsigned_int: Memory.Size,
27834 scalar_remainder_int: struct { of: Memory.Size, is: Memory.Size },
27835 exact_int: u16,
27836 exact_signed_int: u16,
27837 exact_unsigned_int: u16,
27838 signed_or_exact_int: Memory.Size,
27839 unsigned_or_exact_int: Memory.Size,
27840 po2_int: Memory.Size,
27841 signed_po2_int: Memory.Size,
27842 unsigned_po2_or_exact_int: Memory.Size,
27843 remainder_int: struct { of: Memory.Size, is: Memory.Size },
27844 exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
27845 signed_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
27846 unsigned_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
2433327847 signed_int: Memory.Size,
2433427848 unsigned_int: Memory.Size,
2433527849
......@@ -24338,30 +27852,183 @@ const Select = struct {
2433827852 switch (constraint) {
2433927853 .any => return true,
2434027854 .any_bool_vec => return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type,
24341 .any_int => {
27855 .any_int => return ty.toIntern() == .bool_type or ty.isPtrAtRuntime(zcu) or ty.isAbiInt(zcu),
27856 .any_signed_int => return ty.isAbiInt(zcu) and ty.intInfo(zcu).signedness == .signed,
27857 .any_float => return ty.scalarType(zcu).isRuntimeFloat(),
27858 .bool_vec => |size| return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and
27859 size.bitSize(cg.target) >= ty.vectorLen(zcu),
27860 .vec => |size| return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() != .bool_type and
27861 size.bitSize(cg.target) >= ty.abiSize(zcu),
27862 .signed_int_vec => |size| {
27863 if (!ty.isVector(zcu) or size.bitSize(cg.target) < 8 * ty.abiSize(zcu)) return false;
2434227864 const scalar_ty = ty.scalarType(zcu);
24343 return scalar_ty.isAbiInt(zcu) or scalar_ty.isPtrAtRuntime(zcu);
27865 return scalar_ty.isAbiInt(zcu) and scalar_ty.intInfo(zcu).signedness == .signed;
27866 },
27867 .signed_int_or_full_vec => |size| {
27868 if (!ty.isVector(zcu) or size.bitSize(cg.target) < 8 * ty.abiSize(zcu)) return false;
27869 const scalar_ty = ty.scalarType(zcu);
27870 if (scalar_ty.isPtrAtRuntime(zcu)) return true;
27871 if (!scalar_ty.isAbiInt(zcu)) return false;
27872 const scalar_int_info = scalar_ty.intInfo(zcu);
27873 return switch (scalar_int_info.signedness) {
27874 .signed => true,
27875 .unsigned => scalar_int_info.bits >= 8 and std.math.isPowerOfTwo(scalar_int_info.bits),
27876 };
27877 },
27878 .unsigned_int_vec => |size| {
27879 if (!ty.isVector(zcu) or size.bitSize(cg.target) < ty.bitSize(zcu)) return false;
27880 const scalar_ty = ty.scalarType(zcu);
27881 if (scalar_ty.isPtrAtRuntime(zcu)) return true;
27882 return scalar_ty.isAbiInt(zcu) and scalar_ty.intInfo(zcu).signedness == .unsigned;
27883 },
27884 .int_or_vec => |size| {
27885 if (ty.isVector(zcu)) return ty.scalarType(zcu).toIntern() != .bool_type and
27886 size.bitSize(cg.target) >= 8 * ty.abiSize(zcu);
27887 if (ty.toIntern() == .bool_type) return true;
27888 if (ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
27889 return ty.isAbiInt(zcu) and size.bitSize(cg.target) >= ty.intInfo(zcu).bits;
27890 },
27891 .exact_remainder_int_or_vec => |of_is| {
27892 if (ty.isVector(zcu)) return ty.scalarType(zcu).toIntern() != .bool_type and
27893 of_is.is.bitSize(cg.target) == (8 * ty.abiSize(zcu) - 1) % of_is.of.bitSize(cg.target) + 1;
27894 if (ty.isPtrAtRuntime(zcu))
27895 return of_is.is.bitSize(cg.target) == (cg.target.ptrBitWidth() - 1) % of_is.of.bitSize(cg.target) + 1;
27896 if (!ty.isAbiInt(zcu)) return false;
27897 return of_is.is.bitSize(cg.target) == (ty.intInfo(zcu).bits - 1) % of_is.of.bitSize(cg.target) + 1;
2434427898 },
24345 .any_float => return ty.scalarType(zcu).isRuntimeFloat(),
24346 .bool_vec => |size| return ty.isVector(zcu) and
24347 ty.scalarType(zcu).toIntern() == .bool_type and ty.vectorLen(zcu) <= size.bitSize(cg.target),
2434827899 .int => |size| {
27900 if (ty.toIntern() == .bool_type) return true;
27901 if (ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
27902 return ty.isAbiInt(zcu) and size.bitSize(cg.target) >= ty.intInfo(zcu).bits;
27903 },
27904 .scalar_int => |size| {
2434927905 const scalar_ty = ty.scalarType(zcu);
24350 if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(cg.target);
24351 return scalar_ty.isAbiInt(zcu) and scalar_ty.intInfo(zcu).bits <= size.bitSize(cg.target);
27906 if (scalar_ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
27907 return scalar_ty.isAbiInt(zcu) and size.bitSize(cg.target) >= scalar_ty.intInfo(zcu).bits;
2435227908 },
24353 .signed_int => |size| {
27909 .scalar_signed_int => |size| {
2435427910 const scalar_ty = ty.scalarType(zcu);
2435527911 if (!scalar_ty.isAbiInt(zcu)) return false;
24356 const info = scalar_ty.intInfo(zcu);
24357 return info.signedness == .signed and info.bits <= size.bitSize(cg.target);
27912 const scalar_int_info = scalar_ty.intInfo(zcu);
27913 return scalar_int_info.signedness == .signed and size.bitSize(cg.target) >= scalar_int_info.bits;
2435827914 },
24359 .unsigned_int => |size| {
27915 .scalar_unsigned_int => |size| {
27916 const scalar_ty = ty.scalarType(zcu);
27917 if (scalar_ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
27918 if (!scalar_ty.isAbiInt(zcu)) return false;
27919 const scalar_int_info = scalar_ty.intInfo(zcu);
27920 return scalar_int_info.signedness == .unsigned and size.bitSize(cg.target) >= scalar_int_info.bits;
27921 },
27922 .scalar_remainder_int => |of_is| {
2436027923 const scalar_ty = ty.scalarType(zcu);
24361 if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(cg.target);
27924 if (scalar_ty.isPtrAtRuntime(zcu))
27925 return of_is.is.bitSize(cg.target) >= (cg.target.ptrBitWidth() - 1) % of_is.of.bitSize(cg.target) + 1;
2436227926 if (!scalar_ty.isAbiInt(zcu)) return false;
24363 const info = scalar_ty.intInfo(zcu);
24364 return info.signedness == .unsigned and info.bits <= size.bitSize(cg.target);
27927 return of_is.is.bitSize(cg.target) >= (scalar_ty.intInfo(zcu).bits - 1) % of_is.of.bitSize(cg.target) + 1;
27928 },
27929 .exact_int => |bit_size| {
27930 if (ty.toIntern() == .bool_type) return bit_size == 1;
27931 if (ty.isPtrAtRuntime(zcu)) return bit_size == cg.target.ptrBitWidth();
27932 return ty.isAbiInt(zcu) and bit_size == ty.intInfo(zcu).bits;
27933 },
27934 .exact_signed_int => |bit_size| {
27935 if (!ty.isAbiInt(zcu)) return false;
27936 const int_info = ty.intInfo(zcu);
27937 return int_info.signedness == .signed and bit_size == int_info.bits;
27938 },
27939 .exact_unsigned_int => |bit_size| {
27940 if (ty.toIntern() == .bool_type) return bit_size == 1;
27941 if (ty.isPtrAtRuntime(zcu)) return bit_size == cg.target.ptrBitWidth();
27942 if (!ty.isAbiInt(zcu)) return false;
27943 const int_info = ty.intInfo(zcu);
27944 return int_info.signedness == .unsigned and bit_size == int_info.bits;
27945 },
27946 .signed_or_exact_int => |size| {
27947 if (ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) == cg.target.ptrBitWidth();
27948 if (!ty.isAbiInt(zcu)) return false;
27949 const int_info = ty.intInfo(zcu);
27950 return switch (int_info.signedness) {
27951 .signed => size.bitSize(cg.target) >= int_info.bits,
27952 .unsigned => size.bitSize(cg.target) == int_info.bits,
27953 };
27954 },
27955 .unsigned_or_exact_int => |size| {
27956 if (ty.toIntern() == .bool_type or ty.isPtrAtRuntime(zcu)) return true;
27957 if (!ty.isAbiInt(zcu)) return false;
27958 const int_info = ty.intInfo(zcu);
27959 return switch (int_info.signedness) {
27960 .signed => size.bitSize(cg.target) == int_info.bits,
27961 .unsigned => size.bitSize(cg.target) >= int_info.bits,
27962 };
27963 },
27964 .po2_int => |size| {
27965 if (ty.toIntern() == .bool_type) return true;
27966 if (ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
27967 if (!ty.isAbiInt(zcu)) return false;
27968 const bit_size = ty.intInfo(zcu).bits;
27969 return std.math.isPowerOfTwo(bit_size) and size.bitSize(cg.target) >= bit_size;
27970 },
27971 .signed_po2_int => |size| {
27972 if (!ty.isAbiInt(zcu)) return false;
27973 const int_info = ty.intInfo(zcu);
27974 return int_info.signedness == .signed and std.math.isPowerOfTwo(int_info.bits) and
27975 size.bitSize(cg.target) >= int_info.bits;
27976 },
27977 .unsigned_po2_or_exact_int => |size| {
27978 if (ty.toIntern() == .bool_type) return true;
27979 if (ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
27980 if (!ty.isAbiInt(zcu)) return false;
27981 const int_info = ty.intInfo(zcu);
27982 return switch (int_info.signedness) {
27983 .signed => size.bitSize(cg.target) == int_info.bits,
27984 .unsigned => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits,
27985 };
27986 },
27987 .remainder_int => |of_is| {
27988 if (ty.toIntern() == .bool_type) return true;
27989 if (ty.isPtrAtRuntime(zcu))
27990 return of_is.is.bitSize(cg.target) >= (cg.target.ptrBitWidth() - 1) % of_is.of.bitSize(cg.target) + 1;
27991 if (!ty.isAbiInt(zcu)) return false;
27992 return of_is.is.bitSize(cg.target) >= (ty.intInfo(zcu).bits - 1) % of_is.of.bitSize(cg.target) + 1;
27993 },
27994 .exact_remainder_int => |of_is| {
27995 if (ty.isPtrAtRuntime(zcu))
27996 return of_is.is.bitSize(cg.target) == (cg.target.ptrBitWidth() - 1) % of_is.of.bitSize(cg.target) + 1;
27997 if (!ty.isAbiInt(zcu)) return false;
27998 return of_is.is.bitSize(cg.target) == (ty.intInfo(zcu).bits - 1) % of_is.of.bitSize(cg.target) + 1;
27999 },
28000 .signed_or_exact_remainder_int => |of_is| {
28001 if (ty.isPtrAtRuntime(zcu))
28002 return of_is.is.bitSize(cg.target) == (cg.target.ptrBitWidth() - 1) % of_is.of.bitSize(cg.target) + 1;
28003 if (!ty.isAbiInt(zcu)) return false;
28004 const int_info = ty.intInfo(zcu);
28005 return switch (int_info.signedness) {
28006 .signed => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
28007 .unsigned => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
28008 };
28009 },
28010 .unsigned_or_exact_remainder_int => |of_is| {
28011 if (ty.toIntern() == .bool_type) return true;
28012 if (ty.isPtrAtRuntime(zcu))
28013 return of_is.is.bitSize(cg.target) >= (cg.target.ptrBitWidth() - 1) % of_is.of.bitSize(cg.target) + 1;
28014 if (!ty.isAbiInt(zcu)) return false;
28015 const int_info = ty.intInfo(zcu);
28016 return switch (int_info.signedness) {
28017 .signed => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
28018 .unsigned => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1,
28019 };
28020 },
28021 .signed_int => |size| {
28022 if (!ty.isAbiInt(zcu)) return false;
28023 const int_info = ty.intInfo(zcu);
28024 return int_info.signedness == .signed and size.bitSize(cg.target) >= int_info.bits;
28025 },
28026 .unsigned_int => |size| {
28027 if (ty.toIntern() == .bool_type) return true;
28028 if (ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
28029 if (!ty.isAbiInt(zcu)) return false;
28030 const int_info = ty.intInfo(zcu);
28031 return int_info.signedness == .unsigned and size.bitSize(cg.target) >= int_info.bits;
2436528032 },
2436628033 }
2436728034 }
......@@ -24379,97 +28046,107 @@ const Select = struct {
2437928046 imm32,
2438028047 simm32,
2438128048 mem,
24382 mut_mem,
2438328049 to_mem,
28050 mut_mem,
28051 to_mut_mem,
2438428052 gpr,
28053 to_gpr,
2438528054 mut_gpr,
28055 to_mut_gpr,
2438628056 mm,
28057 to_mm,
2438728058 mut_mm,
28059 to_mut_mm,
2438828060 xmm,
28061 to_xmm,
2438928062 mut_xmm,
28063 to_mut_xmm,
2439028064 ymm,
28065 to_ymm,
2439128066 mut_ymm,
28067 to_mut_ymm,
2439228068
2439328069 fn matches(src: Src, temp: Temp, cg: *CodeGen) bool {
24394 switch (src) {
28070 return switch (src) {
2439528071 .none => unreachable,
24396 .any => return true,
24397 .imm8 => return switch (temp.tracking(cg).short) {
28072 .any => true,
28073 .imm8 => switch (temp.tracking(cg).short) {
2439828074 .immediate => |imm| std.math.cast(u8, imm) != null,
2439928075 else => false,
2440028076 },
24401 .imm16 => return switch (temp.tracking(cg).short) {
28077 .imm16 => switch (temp.tracking(cg).short) {
2440228078 .immediate => |imm| std.math.cast(u16, imm) != null,
2440328079 else => false,
2440428080 },
24405 .imm32 => return switch (temp.tracking(cg).short) {
28081 .imm32 => switch (temp.tracking(cg).short) {
2440628082 .immediate => |imm| std.math.cast(u32, imm) != null,
2440728083 else => false,
2440828084 },
24409 .simm32 => return switch (temp.tracking(cg).short) {
28085 .simm32 => switch (temp.tracking(cg).short) {
2441028086 .immediate => |imm| std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
2441128087 else => false,
2441228088 },
24413 .mem => return temp.tracking(cg).short.isMemory(),
24414 .mut_mem => return temp.isMut(cg) and temp.tracking(cg).short.isMemory(),
24415 .to_mem => return true,
24416 .gpr, .mut_gpr => {
24417 const mcv = temp.tracking(cg).short;
24418 const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu);
24419 return abi_size <= 8 and switch (mcv) {
24420 .register => |reg| reg.class() == .general_purpose,
24421 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and
24422 reg_off.off == 0,
24423 .register_pair, .register_triple, .register_quadruple => false,
24424 else => true,
24425 };
28089 .mem => temp.tracking(cg).short.isMemory(),
28090 .to_mem, .to_mut_mem => true,
28091 .mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(),
28092 .gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) {
28093 .register => |reg| reg.class() == .general_purpose,
28094 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and reg_off.off == 0,
28095 else => false,
2442628096 },
24427 .mm, .mut_mm => {
24428 const mcv = temp.tracking(cg).short;
24429 const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu);
24430 return abi_size <= 8 and switch (mcv) {
24431 .register => |reg| reg.class() == .mmx,
24432 .register_offset => |reg_off| reg_off.reg.class() == .mmx and
24433 reg_off.off == 0,
24434 else => false,
24435 };
28097 .mut_gpr => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) {
28098 .register => |reg| reg.class() == .general_purpose,
28099 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and reg_off.off == 0,
28100 else => false,
2443628101 },
24437 .xmm, .mut_xmm => {
24438 const mcv = temp.tracking(cg).short;
24439 const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu);
24440 return abi_size > 8 and abi_size <= 16 and switch (mcv) {
24441 .register => |reg| reg.class() == .sse,
24442 .register_offset => |reg_off| reg_off.reg.class() == .sse and
24443 reg_off.off == 0,
24444 .register_pair, .register_triple, .register_quadruple => false,
24445 else => true,
24446 };
28102 .to_gpr, .to_mut_gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8,
28103 .mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
28104 .register => |reg| reg.class() == .mmx,
28105 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
28106 else => false,
2444728107 },
24448 .ymm, .mut_ymm => {
24449 const mcv = temp.tracking(cg).short;
24450 const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu);
24451 return abi_size > 16 and abi_size <= 32 and switch (mcv) {
24452 .register => |reg| reg.class() == .sse,
24453 .register_offset => |reg_off| reg_off.reg.class() == .sse and
24454 reg_off.off == 0,
24455 .register_pair, .register_triple, .register_quadruple => false,
24456 else => true,
24457 };
28108 .mut_mm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
28109 .register => |reg| reg.class() == .mmx,
28110 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
28111 else => false,
2445828112 },
24459 }
28113 .to_mm, .to_mut_mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8,
28114 .xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
28115 .register => |reg| reg.class() == .sse,
28116 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
28117 else => false,
28118 },
28119 .mut_xmm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
28120 .register => |reg| reg.class() == .sse,
28121 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
28122 else => false,
28123 },
28124 .to_xmm, .to_mut_xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16,
28125 .ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
28126 .register => |reg| reg.class() == .sse,
28127 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
28128 else => false,
28129 },
28130 .mut_ymm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
28131 .register => |reg| reg.class() == .sse,
28132 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
28133 else => false,
28134 },
28135 .to_ymm, .to_mut_ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32,
28136 };
2446028137 }
2446128138
2446228139 fn convert(src: Src, temp: *Temp, cg: *CodeGen) !bool {
2446328140 return switch (src) {
2446428141 .none => unreachable,
2446528142 .any, .imm8, .imm16, .imm32, .simm32 => false,
24466 .mem, .mut_mem, .to_mem => try temp.toBase(cg),
24467 .gpr => try temp.toRegClass(false, .general_purpose, cg),
24468 .mut_gpr => try temp.toRegClass(true, .general_purpose, cg),
24469 .mm => try temp.toRegClass(false, .mmx, cg),
24470 .mut_mm => try temp.toRegClass(true, .mmx, cg),
24471 .xmm, .ymm => try temp.toRegClass(false, .sse, cg),
24472 .mut_xmm, .mut_ymm => try temp.toRegClass(true, .sse, cg),
28143 .mem, .to_mem, .mut_mem, .to_mut_mem => try temp.toBase(cg),
28144 .gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg),
28145 .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg),
28146 .mm, .to_mm => try temp.toRegClass(false, .mmx, cg),
28147 .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg),
28148 .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg),
28149 .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg),
2447328150 };
2447428151 }
2447528152 };
......@@ -24489,6 +28166,10 @@ const Select = struct {
2448928166 rc: Register.Class,
2449028167 rc_mask: struct { rc: Register.Class, info: MaskInfo },
2449128168 mem,
28169 smin_mem: Select.Operand.Ref,
28170 smax_mem: Select.Operand.Ref,
28171 umin_mem: Select.Operand.Ref,
28172 umax_mem: Select.Operand.Ref,
2449228173 ref: Select.Operand.Ref,
2449328174 ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo },
2449428175
......@@ -24501,14 +28182,81 @@ const Select = struct {
2450128182 };
2450228183
2450328184 fn create(spec: TempSpec, s: *Select) !?Temp {
28185 const cg = s.cg;
2450428186 return switch (spec.kind) {
2450528187 .unused => null,
24506 .any => try s.cg.tempAlloc(spec.type),
24507 .cc => |cc| try s.cg.tempFromValue(spec.type, .{ .eflags = cc }),
24508 .reg => |reg| try s.cg.tempFromValue(spec.type, .{ .register = reg }),
24509 .rc => |rc| try s.cg.tempAllocReg(spec.type, regSetForRegClass(rc)),
24510 .rc_mask => |rc_mask| try s.cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)),
24511 .mem => try s.cg.tempAllocMem(spec.type),
28188 .any => try cg.tempAlloc(spec.type),
28189 .cc => |cc| try cg.tempFromValue(spec.type, .{ .eflags = cc }),
28190 .reg => |reg| try cg.tempFromValue(spec.type, .{ .register = reg }),
28191 .rc => |rc| try cg.tempAllocReg(spec.type, regSetForRegClass(rc)),
28192 .rc_mask => |rc_mask| try cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)),
28193 .mem => try cg.tempAllocMem(spec.type),
28194 .smin_mem, .smax_mem, .umin_mem, .umax_mem => |ty_ref| {
28195 const pt = cg.pt;
28196 const zcu = pt.zcu;
28197 const ip = &zcu.intern_pool;
28198 const ty = ty_ref.deref(s).typeOf(s.cg);
28199 const vector_len, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) {
28200 else => .{ null, ty },
28201 .vector_type => |vector_type| .{ vector_type.len, .fromInterned(vector_type.child) },
28202 };
28203 const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) {
28204 .bool_type => .{
28205 scalar_ty,
28206 .fromInterned(switch (spec.kind) {
28207 else => unreachable,
28208 .smin_mem, .umax_mem => .bool_true,
28209 .smax_mem, .umin_mem => .bool_false,
28210 }),
28211 },
28212 else => {
28213 const scalar_info: InternPool.Key.IntType = if (scalar_ty.isAbiInt(zcu))
28214 scalar_ty.intInfo(zcu)
28215 else
28216 .{ .signedness = .unsigned, .bits = @intCast(scalar_ty.bitSize(zcu)) };
28217 const scalar_int_ty = try pt.intType(scalar_info.signedness, scalar_info.bits);
28218 if (scalar_info.bits <= 64) {
28219 const int_val: i64 = switch (spec.kind) {
28220 else => unreachable,
28221 .smin_mem => std.math.minInt(i64),
28222 .smax_mem => std.math.maxInt(i64),
28223 .umin_mem => 0,
28224 .umax_mem => -1,
28225 };
28226 const shift: u6 = @intCast(64 - scalar_info.bits);
28227 break :res_scalar .{ scalar_int_ty, switch (scalar_info.signedness) {
28228 .signed => try pt.intValue_i64(scalar_int_ty, int_val >> shift),
28229 .unsigned => try pt.intValue_u64(scalar_int_ty, @as(u64, @bitCast(int_val)) >> shift),
28230 } };
28231 }
28232 var big_int: std.math.big.int.Managed = try .init(cg.gpa);
28233 defer big_int.deinit();
28234 try big_int.setTwosCompIntLimit(switch (spec.kind) {
28235 else => unreachable,
28236 .smin_mem, .umin_mem => .min,
28237 .smax_mem, .umax_mem => .max,
28238 }, switch (spec.kind) {
28239 else => unreachable,
28240 .smin_mem, .smax_mem => .signed,
28241 .umin_mem, .umax_mem => .unsigned,
28242 }, scalar_info.bits);
28243 try big_int.truncate(&big_int, scalar_info.signedness, scalar_info.bits);
28244 break :res_scalar .{ scalar_int_ty, try pt.intValue_big(scalar_int_ty, big_int.toConst()) };
28245 },
28246 };
28247 const res_ty, const res_val: Value = if (vector_len) |len| res: {
28248 const vector_ty = try pt.vectorType(.{
28249 .len = len,
28250 .child = res_scalar_ty.toIntern(),
28251 });
28252 const vector_val = try pt.intern(.{ .aggregate = .{
28253 .ty = vector_ty.toIntern(),
28254 .storage = .{ .repeated_elem = res_scalar_val.toIntern() },
28255 } });
28256 break :res .{ vector_ty, .fromInterned(vector_val) };
28257 } else .{ res_scalar_ty, res_scalar_val };
28258 return try cg.tempFromValue(res_ty, try cg.genTypedValue(res_val));
28259 },
2451228260 .ref => |ref| ref.deref(s),
2451328261 .ref_mask => |ref_mask| ref_mask.ref.deref(s),
2451428262 };
......@@ -24541,21 +28289,51 @@ const Select = struct {
2454128289 forward_label,
2454228290 ref,
2454328291 simm,
28292 uimm,
2454428293 lea,
2454528294 mem,
2454628295 };
24547 const Adjust = enum {
24548 none,
24549 add_ptr_size,
24550 sub_ptr_size,
24551 add_ptr_bit_size,
24552 sub_ptr_bit_size,
24553 add_size,
24554 sub_size,
24555 add_len,
24556 sub_len,
24557 add_elem_limbs,
24558 sub_elem_limbs,
28296 const Adjust = packed struct(u8) {
28297 factor: i2,
28298 scale: Memory.Scale,
28299 amount: enum(u4) {
28300 none,
28301 ptr_size,
28302 ptr_bit_size,
28303 size,
28304 src0_size,
28305 bit_size,
28306 src0_bit_size,
28307 len,
28308 elem_limbs,
28309 src0_elem_size,
28310 smin,
28311 smax,
28312 umax,
28313 },
28314
28315 const none: Adjust = .{ .factor = 0, .scale = .@"1", .amount = .none };
28316 const sub_ptr_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .ptr_size };
28317 const add_ptr_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .ptr_bit_size };
28318 const add_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .size };
28319 const sub_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .size };
28320 const add_src0_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_size };
28321 const sub_src0_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_size };
28322 const add_2_bit_size: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .bit_size };
28323 const add_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .bit_size };
28324 const sub_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .bit_size };
28325 const add_src0_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_bit_size };
28326 const sub_src0_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_bit_size };
28327 const add_8_len: Adjust = .{ .factor = 1, .scale = .@"8", .amount = .len };
28328 const add_4_len: Adjust = .{ .factor = 1, .scale = .@"4", .amount = .len };
28329 const add_3_len: Adjust = .{ .factor = 1, .scale = .@"3", .amount = .len };
28330 const add_2_len: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .len };
28331 const add_len: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .len };
28332 const sub_len: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .len };
28333 const add_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_elem_size };
28334 const sub_src0_elem_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_elem_size };
28335 const add_elem_limbs: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .elem_limbs };
28336 const add_umax: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .umax };
2455928337 };
2456028338 const Ref = enum(u4) {
2456128339 tmp0,
......@@ -24741,15 +28519,24 @@ const Select = struct {
2474128519 const src1x: Select.Operand = .{ .tag = .ref, .base = .src1x };
2474228520 const src1y: Select.Operand = .{ .tag = .ref, .base = .src1y };
2474328521
24744 fn i(imm: i32) Select.Operand {
28522 fn si(imm: i32) Select.Operand {
2474528523 return .{ .tag = .simm, .imm = imm };
2474628524 }
24747 fn a(base: Ref.Sized, adjust: Adjust) Select.Operand {
28525 fn sa(base: Ref.Sized, adjust: Adjust) Select.Operand {
2474828526 return .{ .tag = .simm, .base = base, .adjust = adjust };
2474928527 }
24750 fn ia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
28528 fn sia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
2475128529 return .{ .tag = .simm, .base = base, .adjust = adjust, .imm = imm };
2475228530 }
28531 fn ui(imm: i32) Select.Operand {
28532 return .{ .tag = .uimm, .imm = imm };
28533 }
28534 fn ua(base: Ref.Sized, adjust: Adjust) Select.Operand {
28535 return .{ .tag = .uimm, .base = base, .adjust = adjust };
28536 }
28537 fn uia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand {
28538 return .{ .tag = .uimm, .base = base, .adjust = adjust, .imm = imm };
28539 }
2475328540
2475428541 fn lea(size: Memory.Size, base: Ref) Select.Operand {
2475528542 return .{
......@@ -24757,6 +28544,13 @@ const Select = struct {
2475728544 .base = .{ .ref = base, .size = size },
2475828545 };
2475928546 }
28547 fn leaa(size: Memory.Size, base: Ref, adjust: Adjust) Select.Operand {
28548 return .{
28549 .tag = .lea,
28550 .base = .{ .ref = base, .size = size },
28551 .adjust = adjust,
28552 };
28553 }
2476028554 fn lead(size: Memory.Size, base: Ref, disp: i32) Select.Operand {
2476128555 return .{
2476228556 .tag = .lea,
......@@ -24768,14 +28562,22 @@ const Select = struct {
2476828562 return .{
2476928563 .tag = .lea,
2477028564 .base = .{ .ref = base, .size = size },
24771 .index_ = .{ .ref = index, .scale = .@"1" },
28565 .index = .{ .ref = index, .scale = .@"1" },
28566 };
28567 }
28568 fn leaia(size: Memory.Size, base: Ref, index: Ref, adjust: Adjust) Select.Operand {
28569 return .{
28570 .tag = .lea,
28571 .base = .{ .ref = base, .size = size },
28572 .index = .{ .ref = index, .scale = .@"1" },
28573 .adjust = adjust,
2477228574 };
2477328575 }
2477428576 fn leaid(size: Memory.Size, base: Ref, index: Ref, disp: i32) Select.Operand {
2477528577 return .{
2477628578 .tag = .lea,
2477728579 .base = .{ .ref = base, .size = size },
24778 .index_ = .{ .ref = index, .scale = .@"1" },
28580 .index = .{ .ref = index, .scale = .@"1" },
2477928581 .imm = disp,
2478028582 };
2478128583 }
......@@ -24783,22 +28585,22 @@ const Select = struct {
2478328585 return .{
2478428586 .tag = .lea,
2478528587 .base = .{ .ref = base, .size = size },
24786 .index_ = .{ .ref = index, .scale = scale },
28588 .index = .{ .ref = index, .scale = scale },
2478728589 };
2478828590 }
2478928591 fn leasid(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
2479028592 return .{
2479128593 .tag = .lea,
2479228594 .base = .{ .ref = base, .size = size },
24793 .index_ = .{ .ref = index, .scale = scale },
28595 .index = .{ .ref = index, .scale = scale },
2479428596 .imm = disp,
2479528597 };
2479628598 }
24797 fn leasida(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32, adjust: Adjust) Select.Operand {
28599 fn leasiad(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
2479828600 return .{
2479928601 .tag = .lea,
2480028602 .base = .{ .ref = base, .size = size },
24801 .index_ = .{ .ref = index, .scale = scale },
28603 .index = .{ .ref = index, .scale = scale },
2480228604 .adjust = adjust,
2480328605 .imm = disp,
2480428606 };
......@@ -24817,6 +28619,21 @@ const Select = struct {
2481728619 .imm = disp,
2481828620 };
2481928621 }
28622 fn mema(base: Ref.Sized, adjust: Adjust) Select.Operand {
28623 return .{
28624 .tag = .mem,
28625 .base = base,
28626 .adjust = adjust,
28627 };
28628 }
28629 fn memad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand {
28630 return .{
28631 .tag = .mem,
28632 .base = base,
28633 .adjust = adjust,
28634 .imm = disp,
28635 };
28636 }
2482028637 fn memi(base: Ref.Sized, index: Ref) Select.Operand {
2482128638 return .{
2482228639 .tag = .mem,
......@@ -24832,6 +28649,15 @@ const Select = struct {
2483228649 .adjust = adjust,
2483328650 };
2483428651 }
28652 fn memiad(base: Ref.Sized, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
28653 return .{
28654 .tag = .mem,
28655 .base = base,
28656 .index = .{ .ref = index, .scale = .@"1" },
28657 .adjust = adjust,
28658 .imm = disp,
28659 };
28660 }
2483528661 fn memid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand {
2483628662 return .{
2483728663 .tag = .mem,
......@@ -24847,6 +28673,14 @@ const Select = struct {
2484728673 .index = .{ .ref = index, .scale = scale },
2484828674 };
2484928675 }
28676 fn memsia(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust) Select.Operand {
28677 return .{
28678 .tag = .mem,
28679 .base = base,
28680 .index = .{ .ref = index, .scale = scale },
28681 .adjust = adjust,
28682 };
28683 }
2485028684 fn memsid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand {
2485128685 return .{
2485228686 .tag = .mem,
......@@ -24855,7 +28689,7 @@ const Select = struct {
2485528689 .imm = disp,
2485628690 };
2485728691 }
24858 fn memsida(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32, adjust: Adjust) Select.Operand {
28692 fn memsiad(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand {
2485928693 return .{
2486028694 .tag = .mem,
2486128695 .base = base,
......@@ -24865,26 +28699,34 @@ const Select = struct {
2486528699 };
2486628700 }
2486728701
24868 fn adjustedImm(op: Select.Operand, s: *const Select) i32 {
24869 return switch (op.adjust) {
24870 .none => op.imm,
24871 .add_ptr_size => op.imm + @divExact(s.cg.target.ptrBitWidth(), 8),
24872 .sub_ptr_size => op.imm - @divExact(s.cg.target.ptrBitWidth(), 8),
24873 .add_ptr_bit_size => op.imm + s.cg.target.ptrBitWidth(),
24874 .sub_ptr_bit_size => op.imm - s.cg.target.ptrBitWidth(),
24875 .add_size => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu))),
24876 .sub_size => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu))),
24877 .add_len => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu))),
24878 .sub_len => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu))),
24879 .add_elem_limbs => op.imm + @as(i32, @intCast(@divExact(
24880 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
24881 @divExact(op.base.size.bitSize(s.cg.target), 8),
24882 ))),
24883 .sub_elem_limbs => op.imm - @as(i32, @intCast(@divExact(
28702 fn adjustedImm(op: Select.Operand, comptime SignedImm: type, s: *const Select) SignedImm {
28703 const UnsignedImm = @Type(.{
28704 .int = .{ .signedness = .unsigned, .bits = @typeInfo(SignedImm).int.bits },
28705 });
28706 return op.imm + @as(i5, op.adjust.factor) * op.adjust.scale.toFactor() * @as(SignedImm, switch (op.adjust.amount) {
28707 .none => 0,
28708 .ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8),
28709 .ptr_bit_size => s.cg.target.ptrBitWidth(),
28710 .size => @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)),
28711 .src0_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)),
28712 .bit_size => @intCast(op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
28713 .src0_bit_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
28714 .len => @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu)),
28715 .elem_limbs => @intCast(@divExact(
2488428716 op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
2488528717 @divExact(op.base.size.bitSize(s.cg.target), 8),
24886 ))),
24887 };
28718 )),
28719 .src0_elem_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)),
28720 .smin => @as(SignedImm, std.math.minInt(SignedImm)) >> @truncate(
28721 -%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
28722 ),
28723 .smax => @as(SignedImm, std.math.maxInt(SignedImm)) >> @truncate(
28724 -%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
28725 ),
28726 .umax => @bitCast(@as(UnsignedImm, std.math.maxInt(UnsignedImm)) >> @truncate(
28727 -%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
28728 )),
28729 });
2488828730 }
2488928731
2489028732 fn lower(op: Select.Operand, s: *Select) !CodeGen.Operand {
......@@ -24907,7 +28749,8 @@ const Select = struct {
2490728749 else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) },
2490828750 .register => |reg| .{ .reg = registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))) },
2490928751 },
24910 .simm => .{ .imm = .s(op.adjustedImm(s)) },
28752 .simm => .{ .imm = .s(op.adjustedImm(i32, s)) },
28753 .uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) },
2491128754 .lea => .{ .mem = .{
2491228755 .base = .{ .reg = registerAlias(op.base.ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)) },
2491328756 .mod = .{ .rm = .{
......@@ -24917,7 +28760,7 @@ const Select = struct {
2491728760 .none => .none,
2491828761 },
2491928762 .scale = op.index.scale,
24920 .disp = op.adjustedImm(s),
28763 .disp = op.adjustedImm(i32, s),
2492128764 } },
2492228765 } },
2492328766 .mem => .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, .{
......@@ -24927,7 +28770,7 @@ const Select = struct {
2492728770 .none => .none,
2492828771 },
2492928772 .scale = op.index.scale,
24930 .disp = op.adjustedImm(s),
28773 .disp = op.adjustedImm(i32, s),
2493128774 }) },
2493228775 };
2493328776 }
......@@ -24942,14 +28785,23 @@ fn select(
2494228785) !void {
2494328786 cases: for (cases) |case| {
2494428787 for (case.required_features) |required_feature| if (required_feature) |feature| if (!switch (feature) {
24945 .@"64bit" => cg.target.ptrBitWidth() == 64,
28788 .@"64bit" => switch (cg.target.cpu.arch) {
28789 else => unreachable,
28790 .x86 => false,
28791 .x86_64 => true,
28792 },
2494628793 .mmx => false,
2494728794 else => cg.hasFeature(feature),
2494828795 }) continue :cases;
2494928796 for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases;
2495028797 for (case.src_constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp.typeOf(cg), cg)) continue :cases;
28798 if (std.debug.runtime_safety) {
28799 for (case.dst_constraints[dst_temps.len..]) |dst_constraint| assert(dst_constraint == .any);
28800 for (case.src_constraints[src_temps.len..]) |src_constraint| assert(src_constraint == .any);
28801 }
2495128802 patterns: for (case.patterns) |pattern| {
24952 for (pattern.src, src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns;
28803 for (pattern.src[0..src_temps.len], src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns;
28804 if (std.debug.runtime_safety) for (pattern.src[src_temps.len..]) |src_pattern| assert(src_pattern == .none);
2495328805
2495428806 var s: Select = .{
2495528807 .cg = cg,
......@@ -24960,9 +28812,11 @@ fn select(
2496028812 const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)];
2496128813 const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)];
2496228814
28815 @memcpy(src_slots[0..src_temps.len], src_temps);
28816 std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
2496328817 for (tmp_slots, case.extra_temps) |*slot, spec| slot.* = try spec.create(&s) orelse continue;
2496428818
24965 while (true) for (pattern.src, src_temps) |src_pattern, *src_temp| {
28819 while (true) for (pattern.src[0..src_temps.len], src_temps) |src_pattern, *src_temp| {
2496628820 if (try src_pattern.convert(src_temp, cg)) break;
2496728821 } else break;
2496828822 @memcpy(src_slots[0..src_temps.len], src_temps);
src/arch/x86_64/Encoding.zig+12-5
......@@ -64,7 +64,7 @@ pub fn findByMnemonic(
6464 comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' ');
6565 comptime var features: []const std.Target.x86.Feature = &.{};
6666 inline while (comptime feature_it.next()) |feature| features = features ++ .{@field(std.Target.x86.Feature, feature)};
67 break :has_features std.Target.x86.featureSetHasAll(target.cpu.features, features[0..features.len].*);
67 break :has_features std.Target.x86.featureSetHasAll(target.cpu.features, features[0..].*);
6868 },
6969 }) continue;
7070
......@@ -250,7 +250,8 @@ pub const Mnemonic = enum {
250250 // General-purpose
251251 adc, add, @"and",
252252 bsf, bsr, bswap, bt, btc, btr, bts,
253 call, cbw, cdq, cdqe, clflush,
253 call, cbw, cdq, cdqe,
254 clac, clc, cld, clflush, cli, clts, clui,
254255 cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
255256 cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
256257 cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
......@@ -274,7 +275,9 @@ pub const Mnemonic = enum {
274275 rcl, rcr, ret, rol, ror, rorx,
275276 sal, sar, sarx, sbb,
276277 scas, scasb, scasd, scasq, scasw,
277 shl, shld, shlx, shr, shrd, shrx, sub, syscall,
278 shl, shld, shlx, shr, shrd, shrx,
279 stac, stc, std, sti, stui,
280 sub, syscall,
278281 seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
279282 setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
280283 setnz, seto, setp, setpe, setpo, sets, setz,
......@@ -307,7 +310,7 @@ pub const Mnemonic = enum {
307310 ldmxcsr,
308311 maxps, maxss,
309312 minps, minss,
310 movaps, movhlps, movlhps,
313 movaps, movhlps, movhps, movlhps, movlps,
311314 movmskps,
312315 movss, movups,
313316 mulps, mulss,
......@@ -333,6 +336,7 @@ pub const Mnemonic = enum {
333336 minpd, minsd,
334337 movapd,
335338 movdqa, movdqu,
339 movhpd, movlpd,
336340 movmskpd,
337341 //movsd,
338342 movupd,
......@@ -395,7 +399,7 @@ pub const Mnemonic = enum {
395399 vmovd,
396400 vmovddup,
397401 vmovdqa, vmovdqu,
398 vmovhlps, vmovlhps,
402 vmovhlps, vmovhpd, vmovhps, vmovlhps, vmovlpd, vmovlps,
399403 vmovmskpd, vmovmskps,
400404 vmovq,
401405 vmovsd,
......@@ -823,6 +827,7 @@ pub const Feature = enum {
823827 avx2,
824828 bmi,
825829 bmi2,
830 cmov,
826831 f16c,
827832 fma,
828833 lzcnt,
......@@ -830,6 +835,7 @@ pub const Feature = enum {
830835 pclmul,
831836 @"pclmul avx",
832837 popcnt,
838 smap,
833839 sse,
834840 sse2,
835841 sse3,
......@@ -837,6 +843,7 @@ pub const Feature = enum {
837843 sse4_2,
838844 ssse3,
839845 sha,
846 uintr,
840847 vaes,
841848 vpclmulqdq,
842849 x87,
src/arch/x86_64/Lower.zig+11-19
......@@ -418,8 +418,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
418418 // Here, we currently assume local dynamic TLS vars, and so
419419 // we emit LD model.
420420 _ = lower.reloc(.{ .linker_tlsld = sym_index }, 0);
421 lower.result_insts[lower.result_insts_len] =
422 try Instruction.new(.none, .lea, &[_]Operand{
421 lower.result_insts[lower.result_insts_len] = try .new(.none, .lea, &.{
423422 .{ .reg = .rdi },
424423 .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) },
425424 }, lower.target);
......@@ -427,8 +426,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
427426 _ = lower.reloc(.{
428427 .linker_extern_fn = try elf_file.getGlobalSymbol("__tls_get_addr", null),
429428 }, 0);
430 lower.result_insts[lower.result_insts_len] =
431 try Instruction.new(.none, .call, &[_]Operand{
429 lower.result_insts[lower.result_insts_len] = try .new(.none, .call, &.{
432430 .{ .imm = .s(0) },
433431 }, lower.target);
434432 lower.result_insts_len += 1;
......@@ -440,8 +438,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
440438 }) };
441439 } else {
442440 // Since we are linking statically, we emit LE model directly.
443 lower.result_insts[lower.result_insts_len] =
444 try Instruction.new(.none, .mov, &[_]Operand{
441 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
445442 .{ .reg = .rax },
446443 .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .fs } }) },
447444 }, lower.target);
......@@ -464,8 +461,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
464461 .mov => {
465462 if (elf_sym.flags.is_extern_ptr) {
466463 const reg = ops[0].reg;
467 lower.result_insts[lower.result_insts_len] =
468 try Instruction.new(.none, .mov, &[_]Operand{
464 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
469465 .{ .reg = reg.to64() },
470466 .{ .mem = Memory.initRip(.qword, 0) },
471467 }, lower.target);
......@@ -496,16 +492,14 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
496492
497493 if (macho_sym.flags.tlv) {
498494 _ = lower.reloc(.{ .linker_reloc = sym_index }, 0);
499 lower.result_insts[lower.result_insts_len] =
500 try Instruction.new(.none, .mov, &[_]Operand{
495 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
501496 .{ .reg = .rdi },
502497 .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) },
503 });
498 }, lower.target);
504499 lower.result_insts_len += 1;
505 lower.result_insts[lower.result_insts_len] =
506 try Instruction.new(.none, .call, &[_]Operand{
500 lower.result_insts[lower.result_insts_len] = try .new(.none, .call, &.{
507501 .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .rdi } }) },
508 });
502 }, lower.target);
509503 lower.result_insts_len += 1;
510504 emit_mnemonic = .mov;
511505 break :op .{ .reg = .rax };
......@@ -520,11 +514,10 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
520514 .mov => {
521515 if (macho_sym.flags.is_extern_ptr) {
522516 const reg = ops[0].reg;
523 lower.result_insts[lower.result_insts_len] =
524 try Instruction.new(.none, .mov, &[_]Operand{
517 lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{
525518 .{ .reg = reg.to64() },
526519 .{ .mem = Memory.initRip(.qword, 0) },
527 });
520 }, lower.target);
528521 lower.result_insts_len += 1;
529522 break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{ .base = .{
530523 .reg = reg.to64(),
......@@ -541,8 +534,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand)
541534 },
542535 };
543536 }
544 lower.result_insts[lower.result_insts_len] =
545 try Instruction.new(emit_prefix, emit_mnemonic, emit_ops, lower.target);
537 lower.result_insts[lower.result_insts_len] = try .new(emit_prefix, emit_mnemonic, emit_ops, lower.target);
546538 lower.result_insts_len += 1;
547539}
548540
src/arch/x86_64/Mir.zig+30-4
......@@ -38,6 +38,11 @@ pub const Inst = struct {
3838 /// ___ Right Without Affecting Flags
3939 _rx,
4040
41 /// ___ Forward
42 _f,
43 /// ___ Reverse
44 //_r,
45
4146 /// ___ Above
4247 _a,
4348 /// ___ Above Or Equal
......@@ -47,6 +52,7 @@ pub const Inst = struct {
4752 /// ___ Below Or Equal
4853 _be,
4954 /// ___ Carry
55 /// ___ Carry Flag
5056 _c,
5157 /// ___ Equal
5258 _e,
......@@ -98,6 +104,14 @@ pub const Inst = struct {
98104 _s,
99105 /// ___ Zero
100106 _z,
107 /// ___ Alignment Check Flag
108 _ac,
109 /// ___ Direction Flag
110 //_d,
111 /// ___ Interrupt Flag
112 _i,
113 /// ___ User Interrupt Flag
114 _ui,
101115
102116 /// ___ Byte
103117 //_b,
......@@ -299,9 +313,8 @@ pub const Inst = struct {
299313 /// Bitwise logical and of packed double-precision floating-point values
300314 @"and",
301315 /// Bit scan forward
302 bsf,
303316 /// Bit scan reverse
304 bsr,
317 bs,
305318 /// Byte swap
306319 bswap,
307320 /// Bit test
......@@ -317,6 +330,10 @@ pub const Inst = struct {
317330 cdq,
318331 /// Convert doubleword to quadword
319332 cdqe,
333 /// Clear carry flag
334 /// Clear direction flag
335 /// Clear interrupt flag
336 cl,
320337 /// Flush cache line
321338 clflush,
322339 /// Conditional move
......@@ -443,6 +460,11 @@ pub const Inst = struct {
443460 /// Subtract packed double-precision floating-point values
444461 /// Subtract scalar double-precision floating-point values
445462 sub,
463 /// Set carry flag
464 /// Set direction flag
465 /// Set interrupt flag
466 /// Store floating-point value
467 st,
446468 /// Store string
447469 sto,
448470 /// Syscall
......@@ -478,8 +500,6 @@ pub const Inst = struct {
478500 ldenv,
479501 /// Store x87 FPU environment
480502 nstenv,
481 /// Store floating-point value
482 st,
483503 /// Store x87 FPU environment
484504 stenv,
485505
......@@ -560,8 +580,14 @@ pub const Inst = struct {
560580 /// Move aligned packed single-precision floating-point values
561581 /// Move aligned packed double-precision floating-point values
562582 mova,
583 /// Move high packed single-precision floating-point values
584 /// Move high packed double-precision floating-point values
585 movh,
563586 /// Move packed single-precision floating-point values high to low
564587 movhl,
588 /// Move low packed single-precision floating-point values
589 /// Move low packed double-precision floating-point values
590 movl,
565591 /// Move packed single-precision floating-point values low to high
566592 movlh,
567593 /// Move unaligned packed single-precision floating-point values
src/arch/x86_64/bits.zig+8-4
......@@ -571,11 +571,15 @@ pub const Memory = struct {
571571 writer: anytype,
572572 ) @TypeOf(writer).Error!void {
573573 if (s == .none) return;
574 if (s != .ptr) {
575 try writer.writeAll(@tagName(s));
576 try writer.writeByte(' ');
574 try writer.writeAll(@tagName(s));
575 switch (s) {
576 .none => unreachable,
577 .ptr => {},
578 else => {
579 try writer.writeByte(' ');
580 try writer.writeAll("ptr");
581 },
577582 }
578 try writer.writeAll("ptr");
579583 }
580584 };
581585
src/arch/x86_64/encodings.zig+142-96
......@@ -132,98 +132,110 @@ pub const table = [_]Entry{
132132 .{ .cdq, .zo, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
133133 .{ .cqo, .zo, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
134134
135 .{ .clac, .zo, &.{}, &.{ 0x0f, 0x01, 0xca }, 0, .none, .smap },
136
137 .{ .clc, .zo, &.{}, &.{ 0xf8 }, 0, .none, .none },
138
139 .{ .cld, .zo, &.{}, &.{ 0xfc }, 0, .none, .none },
140
135141 .{ .clflush, .m, &.{ .m8 }, &.{ 0x0f, 0xae }, 7, .none, .none },
136142
137 .{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
138 .{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
139 .{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
140 .{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
141 .{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
142 .{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
143 .{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
144 .{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
145 .{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
146 .{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
147 .{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
148 .{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
149 .{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
150 .{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
151 .{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
152 .{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
153 .{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
154 .{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
155 .{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
156 .{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
157 .{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
158 .{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
159 .{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
160 .{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
161 .{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
162 .{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
163 .{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
164 .{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
165 .{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
166 .{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
167 .{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
168 .{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
169 .{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
170 .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
171 .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
172 .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
173 .{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
174 .{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
175 .{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
176 .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
177 .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
178 .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
179 .{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
180 .{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
181 .{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
182 .{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
183 .{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
184 .{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
185 .{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
186 .{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
187 .{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
188 .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
189 .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
190 .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
191 .{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
192 .{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
193 .{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
194 .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
195 .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
196 .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
197 .{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .none },
198 .{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
199 .{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .none },
200 .{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
201 .{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
202 .{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
203 .{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .none },
204 .{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
205 .{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .none },
206 .{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
207 .{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
208 .{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
209 .{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .none },
210 .{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
211 .{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .none },
212 .{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
213 .{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
214 .{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
215 .{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
216 .{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
217 .{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
218 .{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
219 .{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
220 .{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
221 .{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .none },
222 .{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
223 .{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .none },
224 .{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
225 .{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
226 .{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
143 .{ .cli, .zo, &.{}, &.{ 0xfa }, 0, .none, .none },
144
145 .{ .clts, .zo, &.{}, &.{ 0x0f, 0x06 }, 0, .none, .none },
146
147 .{ .clui, .zo, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xee }, 0, .none, .uintr },
148
149 .{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .cmov },
150 .{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .cmov },
151 .{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .cmov },
152 .{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
153 .{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
154 .{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
155 .{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
156 .{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
157 .{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
158 .{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .cmov },
159 .{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .cmov },
160 .{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .cmov },
161 .{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
162 .{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
163 .{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
164 .{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .cmov },
165 .{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .cmov },
166 .{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .cmov },
167 .{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .cmov },
168 .{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .cmov },
169 .{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .cmov },
170 .{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .cmov },
171 .{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .cmov },
172 .{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .cmov },
173 .{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .cmov },
174 .{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .cmov },
175 .{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .cmov },
176 .{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .cmov },
177 .{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .cmov },
178 .{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .cmov },
179 .{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .cmov },
180 .{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .cmov },
181 .{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .cmov },
182 .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .cmov },
183 .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .cmov },
184 .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .cmov },
185 .{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
186 .{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
187 .{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
188 .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .cmov },
189 .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .cmov },
190 .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .cmov },
191 .{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .cmov },
192 .{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .cmov },
193 .{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .cmov },
194 .{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .cmov },
195 .{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .cmov },
196 .{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .cmov },
197 .{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .cmov },
198 .{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .cmov },
199 .{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .cmov },
200 .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .cmov },
201 .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .cmov },
202 .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .cmov },
203 .{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .cmov },
204 .{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .cmov },
205 .{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .cmov },
206 .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .cmov },
207 .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .cmov },
208 .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .cmov },
209 .{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .cmov },
210 .{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .cmov },
211 .{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .cmov },
212 .{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .cmov },
213 .{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .cmov },
214 .{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .cmov },
215 .{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .cmov },
216 .{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .cmov },
217 .{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .cmov },
218 .{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .cmov },
219 .{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .cmov },
220 .{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .cmov },
221 .{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .cmov },
222 .{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .cmov },
223 .{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .cmov },
224 .{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .cmov },
225 .{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .cmov },
226 .{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .cmov },
227 .{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .cmov },
228 .{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .cmov },
229 .{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .cmov },
230 .{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .cmov },
231 .{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .cmov },
232 .{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .cmov },
233 .{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .cmov },
234 .{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .cmov },
235 .{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .cmov },
236 .{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .cmov },
237 .{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .cmov },
238 .{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .cmov },
227239
228240 .{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none, .none },
229241 .{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .short, .none },
......@@ -747,6 +759,16 @@ pub const table = [_]Entry{
747759 .{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none, .none },
748760 .{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long, .none },
749761
762 .{ .stac, .zo, &.{}, &.{ 0x0f, 0x01, 0xcb }, 0, .none, .smap },
763
764 .{ .stc, .zo, &.{}, &.{ 0xf9 }, 0, .none, .none },
765
766 .{ .std, .zo, &.{}, &.{ 0xfd }, 0, .none, .none },
767
768 .{ .sti, .zo, &.{}, &.{ 0xfb }, 0, .none, .none },
769
770 .{ .stui, .zo, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xef }, 0, .none, .uintr },
771
750772 .{ .stos, .zo, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
751773 .{ .stos, .zo, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
752774 .{ .stos, .zo, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
......@@ -927,8 +949,14 @@ pub const table = [_]Entry{
927949
928950 .{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
929951
952 .{ .movhps, .rm, &.{ .xmm, .m64 }, &.{ 0x0f, 0x16 }, 0, .none, .sse },
953 .{ .movhps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x17 }, 0, .none, .sse },
954
930955 .{ .movlhps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .none, .sse },
931956
957 .{ .movlps, .rm, &.{ .xmm, .m64 }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
958 .{ .movlps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x13 }, 0, .none, .sse },
959
932960 .{ .movmskps, .rm, &.{ .r32, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },
933961 .{ .movmskps, .rm, &.{ .r64, .xmm }, &.{ 0x0f, 0x50 }, 0, .none, .sse },
934962
......@@ -1037,6 +1065,12 @@ pub const table = [_]Entry{
10371065 .{ .movdqu, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x6f }, 0, .none, .sse2 },
10381066 .{ .movdqu, .mr, &.{ .xmm_m128, .xmm }, &.{ 0xf3, 0x0f, 0x7f }, 0, .none, .sse2 },
10391067
1068 .{ .movhpd, .rm, &.{ .xmm, .m64 }, &.{ 0x66, 0x0f, 0x16 }, 0, .none, .sse2 },
1069 .{ .movhpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x17 }, 0, .none, .sse2 },
1070
1071 .{ .movlpd, .rm, &.{ .xmm, .m64 }, &.{ 0x66, 0x0f, 0x12 }, 0, .none, .sse2 },
1072 .{ .movlpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x13 }, 0, .none, .sse2 },
1073
10401074 .{ .movmskpd, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },
10411075 .{ .movmskpd, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 },
10421076
......@@ -1486,8 +1520,20 @@ pub const table = [_]Entry{
14861520
14871521 .{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
14881522
1523 .{ .vmovhpd, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x66, 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
1524 .{ .vmovhpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x17 }, 0, .vex_128_wig, .avx },
1525
1526 .{ .vmovhps, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
1527 .{ .vmovhps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x17 }, 0, .vex_128_wig, .avx },
1528
14891529 .{ .vmovlhps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
14901530
1531 .{ .vmovlpd, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x66, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1532 .{ .vmovlpd, .mr, &.{ .m64, .xmm }, &.{ 0x66, 0x0f, 0x13 }, 0, .vex_128_wig, .avx },
1533
1534 .{ .vmovlps, .rvm, &.{ .xmm, .xmm, .m64 }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1535 .{ .vmovlps, .mr, &.{ .m64, .xmm }, &.{ 0x0f, 0x13 }, 0, .vex_128_wig, .avx },
1536
14911537 .{ .vmovq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .vex_128_wig, .avx },
14921538 .{ .vmovq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .vex_128_wig, .avx },
14931539
......@@ -1583,14 +1629,14 @@ pub const table = [_]Entry{
15831629 .{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
15841630 .{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
15851631
1586 .{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx },
1587 .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
1632 .{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .vex_128_w0, .avx },
1633 .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_w0, .avx },
15881634
1589 .{ .vpinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
1590 .{ .vpinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
1591 .{ .vpinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
1635 .{ .vpinsrb, .rvmi, &.{ .xmm, .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
1636 .{ .vpinsrd, .rvmi, &.{ .xmm, .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
1637 .{ .vpinsrq, .rvmi, &.{ .xmm, .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
15921638
1593 .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
1639 .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_w0, .avx },
15941640
15951641 .{ .vpmaxsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_128_wig, .avx },
15961642 .{ .vpmaxsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xee }, 0, .vex_128_wig, .avx },
src/dev.zig+1
......@@ -135,6 +135,7 @@ pub const Env = enum {
135135 else => Env.ast_gen.supports(feature),
136136 },
137137 .@"x86_64-linux" => switch (feature) {
138 .build_command,
138139 .stdio_listen,
139140 .incremental,
140141 .x86_64_backend,
src/link/Elf/Atom.zig+19-18
......@@ -1274,19 +1274,19 @@ const x86_64 = struct {
12741274 fn relaxGotpcrelx(code: []u8, t: *const std.Target) !void {
12751275 dev.check(.x86_64_backend);
12761276 const old_inst = disassemble(code) orelse return error.RelaxFailure;
1277 const inst = switch (old_inst.encoding.mnemonic) {
1278 .call => try Instruction.new(old_inst.prefix, .call, &.{
1277 const inst: Instruction = switch (old_inst.encoding.mnemonic) {
1278 .call => try .new(old_inst.prefix, .call, &.{
12791279 // TODO: hack to force imm32s in the assembler
1280 .{ .imm = Immediate.s(-129) },
1280 .{ .imm = .s(-129) },
12811281 }, t),
1282 .jmp => try Instruction.new(old_inst.prefix, .jmp, &.{
1282 .jmp => try .new(old_inst.prefix, .jmp, &.{
12831283 // TODO: hack to force imm32s in the assembler
1284 .{ .imm = Immediate.s(-129) },
1284 .{ .imm = .s(-129) },
12851285 }, t),
12861286 else => return error.RelaxFailure,
12871287 };
12881288 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
1289 const nop = try Instruction.new(.none, .nop, &.{}, t);
1289 const nop: Instruction = try .new(.none, .nop, &.{}, t);
12901290 try encode(&.{ nop, inst }, code);
12911291 }
12921292
......@@ -1295,7 +1295,7 @@ const x86_64 = struct {
12951295 const old_inst = disassemble(code) orelse return error.RelaxFailure;
12961296 switch (old_inst.encoding.mnemonic) {
12971297 .mov => {
1298 const inst = try Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t);
1298 const inst: Instruction = try .new(old_inst.prefix, .lea, &old_inst.ops, t);
12991299 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
13001300 try encode(&.{inst}, code);
13011301 },
......@@ -1404,14 +1404,15 @@ const x86_64 = struct {
14041404 dev.check(.x86_64_backend);
14051405 const old_inst = disassemble(code) orelse return false;
14061406 switch (old_inst.encoding.mnemonic) {
1407 .mov => if (Instruction.new(old_inst.prefix, .mov, &.{
1408 old_inst.ops[0],
1409 // TODO: hack to force imm32s in the assembler
1410 .{ .imm = Immediate.s(-129) },
1411 }, t)) |inst| {
1407 .mov => {
1408 const inst = Instruction.new(old_inst.prefix, .mov, &.{
1409 old_inst.ops[0],
1410 // TODO: hack to force imm32s in the assembler
1411 .{ .imm = .s(-129) },
1412 }, t) catch return false;
14121413 inst.encode(std.io.null_writer, .{}) catch return false;
14131414 return true;
1414 } else |_| return false,
1415 },
14151416 else => return false,
14161417 }
14171418 }
......@@ -1424,7 +1425,7 @@ const x86_64 = struct {
14241425 const inst = Instruction.new(old_inst.prefix, .mov, &.{
14251426 old_inst.ops[0],
14261427 // TODO: hack to force imm32s in the assembler
1427 .{ .imm = Immediate.s(-129) },
1428 .{ .imm = .s(-129) },
14281429 }, t) catch unreachable;
14291430 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
14301431 encode(&.{inst}, code) catch unreachable;
......@@ -1438,10 +1439,10 @@ const x86_64 = struct {
14381439 const old_inst = disassemble(code) orelse return error.RelaxFailure;
14391440 switch (old_inst.encoding.mnemonic) {
14401441 .lea => {
1441 const inst = try Instruction.new(old_inst.prefix, .mov, &.{
1442 const inst: Instruction = try .new(old_inst.prefix, .mov, &.{
14421443 old_inst.ops[0],
14431444 // TODO: hack to force imm32s in the assembler
1444 .{ .imm = Immediate.s(-129) },
1445 .{ .imm = .s(-129) },
14451446 }, target);
14461447 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
14471448 try encode(&.{inst}, code);
......@@ -1781,7 +1782,7 @@ const aarch64 = struct {
17811782 const off: u12 = @truncate(@as(u64, @bitCast(S_ + A)));
17821783 aarch64_util.writeAddImmInst(off, code);
17831784 } else {
1784 const old_inst = Instruction{
1785 const old_inst: Instruction = .{
17851786 .add_subtract_immediate = mem.bytesToValue(std.meta.TagPayload(
17861787 Instruction,
17871788 Instruction.add_subtract_immediate,
......@@ -1795,7 +1796,7 @@ const aarch64 = struct {
17951796 },
17961797
17971798 .TLSDESC_CALL => if (!target.flags.has_tlsdesc) {
1798 const old_inst = Instruction{
1799 const old_inst: Instruction = .{
17991800 .unconditional_branch_register = mem.bytesToValue(std.meta.TagPayload(
18001801 Instruction,
18011802 Instruction.unconditional_branch_register,
src/link/MachO/Atom.zig+7-5
......@@ -640,7 +640,8 @@ fn resolveRelocInner(
640640 macho_file: *MachO,
641641 writer: anytype,
642642) ResolveError!void {
643 const cpu_arch = macho_file.getTarget().cpu.arch;
643 const t = &macho_file.base.comp.root_mod.resolved_target.result;
644 const cpu_arch = t.cpu.arch;
644645 const rel_offset = math.cast(usize, rel.offset - self.off) orelse return error.Overflow;
645646 const P = @as(i64, @intCast(self.getAddress(macho_file))) + @as(i64, @intCast(rel_offset));
646647 const A = rel.addend + rel.getRelocAddend(cpu_arch);
......@@ -747,7 +748,7 @@ fn resolveRelocInner(
747748 const S_: i64 = @intCast(sym.getTlvPtrAddress(macho_file));
748749 try writer.writeInt(i32, @intCast(S_ + A - P), .little);
749750 } else {
750 try x86_64.relaxTlv(code[rel_offset - 3 ..]);
751 try x86_64.relaxTlv(code[rel_offset - 3 ..], t);
751752 try writer.writeInt(i32, @intCast(S + A - P), .little);
752753 }
753754 },
......@@ -893,11 +894,12 @@ fn resolveRelocInner(
893894const x86_64 = struct {
894895 fn relaxGotLoad(self: Atom, code: []u8, rel: Relocation, macho_file: *MachO) ResolveError!void {
895896 dev.check(.x86_64_backend);
897 const t = &macho_file.base.comp.root_mod.resolved_target.result;
896898 const diags = &macho_file.base.comp.link_diags;
897899 const old_inst = disassemble(code) orelse return error.RelaxFail;
898900 switch (old_inst.encoding.mnemonic) {
899901 .mov => {
900 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops) catch return error.RelaxFail;
902 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t) catch return error.RelaxFail;
901903 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
902904 encode(&.{inst}, code) catch return error.RelaxFail;
903905 },
......@@ -916,12 +918,12 @@ const x86_64 = struct {
916918 }
917919 }
918920
919 fn relaxTlv(code: []u8) error{RelaxFail}!void {
921 fn relaxTlv(code: []u8, t: *const std.Target) error{RelaxFail}!void {
920922 dev.check(.x86_64_backend);
921923 const old_inst = disassemble(code) orelse return error.RelaxFail;
922924 switch (old_inst.encoding.mnemonic) {
923925 .mov => {
924 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops) catch return error.RelaxFail;
926 const inst = Instruction.new(old_inst.prefix, .lea, &old_inst.ops, t) catch return error.RelaxFail;
925927 relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding });
926928 encode(&.{inst}, code) catch return error.RelaxFail;
927929 },
test/behavior/math.zig+7-2
......@@ -65,6 +65,8 @@ test "@clz" {
6565 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
6666 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
6767 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
68 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
69 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
6870
6971 try testClz();
7072 try comptime testClz();
......@@ -75,6 +77,7 @@ fn testClz() !void {
7577 try expect(testOneClz(u8, 0b00001010) == 4);
7678 try expect(testOneClz(u8, 0b00011010) == 3);
7779 try expect(testOneClz(u8, 0b00000000) == 8);
80 try expect(testOneClz(i8, -1) == 0);
7881}
7982
8083test "@clz big ints" {
......@@ -100,7 +103,7 @@ fn testOneClz(comptime T: type, x: T) u32 {
100103
101104test "@clz vectors" {
102105 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
103 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
106 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
104107 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
105108 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
106109 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -159,6 +162,8 @@ fn testCtz() !void {
159162 try expect(testOneCtz(u8, 0b10100000) == 5);
160163 try expect(testOneCtz(u8, 0b10001010) == 1);
161164 try expect(testOneCtz(u8, 0b00000000) == 8);
165 try expect(testOneCtz(i8, -1) == 0);
166 try expect(testOneCtz(i8, -2) == 1);
162167 try expect(testOneCtz(u16, 0b00000000) == 16);
163168}
164169
......@@ -1712,7 +1717,7 @@ test "mod lazy values" {
17121717
17131718test "@clz works on both vector and scalar inputs" {
17141719 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1715 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1720 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
17161721 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
17171722 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
17181723 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/vector.zig+1-1
......@@ -646,7 +646,7 @@ test "vector division operators" {
646646
647647test "vector bitwise not operator" {
648648 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
649 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
649 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
650650 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
651651 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
652652 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/x86_64.zig+1-2
......@@ -1,8 +1,7 @@
11//! CodeGen tests for the x86_64 backend.
22
3const builtin = @import("builtin");
4
53test {
4 const builtin = @import("builtin");
65 if (builtin.zig_backend != .stage2_x86_64) return error.SkipZigTest;
76 if (builtin.object_format == .coff) return error.SkipZigTest;
87 _ = @import("x86_64/math.zig");
test/behavior/x86_64/build.zig created+114
......@@ -0,0 +1,114 @@
1const std = @import("std");
2pub fn build(b: *std.Build) void {
3 const compiler_rt_lib = b.addStaticLibrary(.{
4 .name = "compiler_rt",
5 .use_llvm = false,
6 .use_lld = false,
7 .root_module = b.createModule(.{
8 .root_source_file = b.addWriteFiles().add("compiler_rt.zig", ""),
9 .target = b.resolveTargetQuery(.{ .cpu_arch = .x86_64 }),
10 }),
11 });
12 compiler_rt_lib.bundle_compiler_rt = true;
13
14 for ([_]std.Target.Query{
15 .{
16 .cpu_arch = .x86_64,
17 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
18 .cpu_features_add = std.Target.x86.featureSet(&.{.bsf_bsr_0_clobbers_result}),
19 //.cpu_features_sub = std.Target.x86.featureSet(&.{.sse}),
20 },
21 .{
22 .cpu_arch = .x86_64,
23 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
24 .cpu_features_add = std.Target.x86.featureSet(&.{.bsf_bsr_0_clobbers_result}),
25 .cpu_features_sub = std.Target.x86.featureSet(&.{
26 .cmov,
27 //.sse,
28 }),
29 },
30 //.{
31 // .cpu_arch = .x86_64,
32 // .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
33 // .cpu_features_sub = std.Target.x86.featureSet(&.{.sse}),
34 //},
35 .{
36 .cpu_arch = .x86_64,
37 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
38 .cpu_features_sub = std.Target.x86.featureSet(&.{.sse2}),
39 },
40 .{
41 .cpu_arch = .x86_64,
42 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
43 },
44 .{
45 .cpu_arch = .x86_64,
46 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
47 .cpu_features_add = std.Target.x86.featureSet(&.{.sse3}),
48 },
49 .{
50 .cpu_arch = .x86_64,
51 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
52 .cpu_features_add = std.Target.x86.featureSet(&.{.ssse3}),
53 },
54 .{
55 .cpu_arch = .x86_64,
56 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
57 .cpu_features_add = std.Target.x86.featureSet(&.{.sse4_1}),
58 },
59 .{
60 .cpu_arch = .x86_64,
61 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64 },
62 .cpu_features_add = std.Target.x86.featureSet(&.{.sse4_2}),
63 },
64 .{
65 .cpu_arch = .x86_64,
66 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v2 },
67 },
68 .{
69 .cpu_arch = .x86_64,
70 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v2 },
71 .cpu_features_add = std.Target.x86.featureSet(&.{.avx}),
72 },
73 .{
74 .cpu_arch = .x86_64,
75 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 },
76 .cpu_features_sub = std.Target.x86.featureSet(&.{.avx2}),
77 },
78 .{
79 .cpu_arch = .x86_64,
80 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 },
81 },
82 .{
83 .cpu_arch = .x86_64,
84 .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v4 },
85 },
86 }) |query| {
87 const target = b.resolveTargetQuery(query);
88 const cpu = query.serializeCpuAlloc(b.allocator) catch @panic("OOM");
89 for ([_][]const u8{
90 "math.zig",
91 }) |path| {
92 const test_mod = b.createModule(.{
93 .root_source_file = b.path(path),
94 .target = target,
95 });
96 const test_exe = b.addTest(.{
97 .name = std.fs.path.stem(path),
98 .use_llvm = false,
99 .use_lld = false,
100 .root_module = test_mod,
101 });
102 if (!std.Target.x86.featureSetHas(target.result.cpu.features, .sse2)) {
103 test_exe.bundle_compiler_rt = false;
104 test_mod.linkLibrary(compiler_rt_lib);
105 }
106 const test_run = b.addRunArtifact(test_exe);
107 b.default_step.dependOn(&test_run.step);
108 for ([_]*std.Build.Step{
109 &test_exe.step,
110 &test_run.step,
111 }) |step| step.name = b.fmt("{s} {s}", .{ step.name, cpu });
112 }
113 }
114}
test/behavior/x86_64/math.zig+763
......@@ -1,3 +1,709 @@
1fn testUnary(comptime op: anytype) !void {
2 const testType = struct {
3 fn testType(comptime Type: type, comptime imm_arg: Type) !void {
4 const expected = op(Type, imm_arg);
5 try struct {
6 fn testOne(actual: @TypeOf(expected)) !void {
7 if (switch (@typeInfo(@TypeOf(expected))) {
8 else => actual != expected,
9 .vector => @reduce(.Or, actual != expected),
10 }) return error.Unexpected;
11 }
12 noinline fn testOps(mem_arg: Type) !void {
13 var reg_arg = mem_arg;
14 _ = .{&reg_arg};
15 try testOne(op(Type, reg_arg));
16 try testOne(op(Type, mem_arg));
17 try testOne(op(Type, imm_arg));
18 }
19 }.testOps(imm_arg);
20 }
21 }.testType;
22
23 try testType(i0, 0);
24 try testType(u0, 0);
25
26 try testType(i1, -1);
27 try testType(i1, 0);
28 try testType(u1, 0);
29 try testType(u1, 1 << 0);
30
31 try testType(i2, -1 << 1);
32 try testType(i2, -1);
33 try testType(i2, 0);
34 try testType(u2, 0);
35 try testType(u2, 1 << 0);
36 try testType(u2, 1 << 1);
37
38 try testType(i3, -1 << 2);
39 try testType(i3, -1);
40 try testType(i3, 0);
41 try testType(u3, 0);
42 try testType(u3, 1 << 0);
43 try testType(u3, 1 << 1);
44 try testType(u3, 1 << 2);
45
46 try testType(i4, -1 << 3);
47 try testType(i4, -1);
48 try testType(i4, 0);
49 try testType(u4, 0);
50 try testType(u4, 1 << 0);
51 try testType(u4, 1 << 1);
52 try testType(u4, 1 << 2);
53 try testType(u4, 1 << 3);
54
55 try testType(i5, -1 << 4);
56 try testType(i5, -1);
57 try testType(i5, 0);
58 try testType(u5, 0);
59 try testType(u5, 1 << 0);
60 try testType(u5, 1 << 1);
61 try testType(u5, 1 << 3);
62 try testType(u5, 1 << 4);
63
64 try testType(i7, -1 << 6);
65 try testType(i7, -1);
66 try testType(i7, 0);
67 try testType(u7, 0);
68 try testType(u7, 1 << 0);
69 try testType(u7, 1 << 1);
70 try testType(u7, 1 << 5);
71 try testType(u7, 1 << 6);
72
73 try testType(i8, -1 << 7);
74 try testType(i8, -1);
75 try testType(i8, 0);
76 try testType(u8, 0);
77 try testType(u8, 1 << 0);
78 try testType(u8, 1 << 1);
79 try testType(u8, 1 << 6);
80 try testType(u8, 1 << 7);
81
82 try testType(i9, -1 << 8);
83 try testType(i9, -1);
84 try testType(i9, 0);
85 try testType(u9, 0);
86 try testType(u9, 1 << 0);
87 try testType(u9, 1 << 1);
88 try testType(u9, 1 << 7);
89 try testType(u9, 1 << 8);
90
91 try testType(i15, -1 << 14);
92 try testType(i15, -1);
93 try testType(i15, 0);
94 try testType(u15, 0);
95 try testType(u15, 1 << 0);
96 try testType(u15, 1 << 1);
97 try testType(u15, 1 << 13);
98 try testType(u15, 1 << 14);
99
100 try testType(i16, -1 << 15);
101 try testType(i16, -1);
102 try testType(i16, 0);
103 try testType(u16, 0);
104 try testType(u16, 1 << 0);
105 try testType(u16, 1 << 1);
106 try testType(u16, 1 << 14);
107 try testType(u16, 1 << 15);
108
109 try testType(i17, -1 << 16);
110 try testType(i17, -1);
111 try testType(i17, 0);
112 try testType(u17, 0);
113 try testType(u17, 1 << 0);
114 try testType(u17, 1 << 1);
115 try testType(u17, 1 << 15);
116 try testType(u17, 1 << 16);
117
118 try testType(i31, -1 << 30);
119 try testType(i31, -1);
120 try testType(i31, 0);
121 try testType(u31, 0);
122 try testType(u31, 1 << 0);
123 try testType(u31, 1 << 1);
124 try testType(u31, 1 << 29);
125 try testType(u31, 1 << 30);
126
127 try testType(i32, -1 << 31);
128 try testType(i32, -1);
129 try testType(i32, 0);
130 try testType(u32, 0);
131 try testType(u32, 1 << 0);
132 try testType(u32, 1 << 1);
133 try testType(u32, 1 << 30);
134 try testType(u32, 1 << 31);
135
136 try testType(i33, -1 << 32);
137 try testType(i33, -1);
138 try testType(i33, 0);
139 try testType(u33, 0);
140 try testType(u33, 1 << 0);
141 try testType(u33, 1 << 1);
142 try testType(u33, 1 << 31);
143 try testType(u33, 1 << 32);
144
145 try testType(i63, -1 << 62);
146 try testType(i63, -1);
147 try testType(i63, 0);
148 try testType(u63, 0);
149 try testType(u63, 1 << 0);
150 try testType(u63, 1 << 1);
151 try testType(u63, 1 << 61);
152 try testType(u63, 1 << 62);
153
154 try testType(i64, -1 << 63);
155 try testType(i64, -1);
156 try testType(i64, 0);
157 try testType(u64, 0);
158 try testType(u64, 1 << 0);
159 try testType(u64, 1 << 1);
160 try testType(u64, 1 << 62);
161 try testType(u64, 1 << 63);
162
163 try testType(i65, -1 << 64);
164 try testType(i65, -1);
165 try testType(i65, 0);
166 try testType(u65, 0);
167 try testType(u65, 1 << 0);
168 try testType(u65, 1 << 1);
169 try testType(u65, 1 << 63);
170 try testType(u65, 1 << 64);
171
172 try testType(i95, -1 << 94);
173 try testType(i95, -1);
174 try testType(i95, 0);
175 try testType(u95, 0);
176 try testType(u95, 1 << 0);
177 try testType(u95, 1 << 1);
178 try testType(u95, 1 << 93);
179 try testType(u95, 1 << 94);
180
181 try testType(i96, -1 << 95);
182 try testType(i96, -1);
183 try testType(i96, 0);
184 try testType(u96, 0);
185 try testType(u96, 1 << 0);
186 try testType(u96, 1 << 1);
187 try testType(u96, 1 << 94);
188 try testType(u96, 1 << 95);
189
190 try testType(i97, -1 << 96);
191 try testType(i97, -1);
192 try testType(i97, 0);
193 try testType(u97, 0);
194 try testType(u97, 1 << 0);
195 try testType(u97, 1 << 1);
196 try testType(u97, 1 << 95);
197 try testType(u97, 1 << 96);
198
199 try testType(i127, -1 << 126);
200 try testType(i127, -1);
201 try testType(i127, 0);
202 try testType(u127, 0);
203 try testType(u127, 1 << 0);
204 try testType(u127, 1 << 1);
205 try testType(u127, 1 << 125);
206 try testType(u127, 1 << 126);
207
208 try testType(i128, -1 << 127);
209 try testType(i128, -1);
210 try testType(i128, 0);
211 try testType(u128, 0);
212 try testType(u128, 1 << 0);
213 try testType(u128, 1 << 1);
214 try testType(u128, 1 << 126);
215 try testType(u128, 1 << 127);
216
217 try testType(i129, -1 << 128);
218 try testType(i129, -1);
219 try testType(i129, 0);
220 try testType(u129, 0);
221 try testType(u129, 1 << 0);
222 try testType(u129, 1 << 1);
223 try testType(u129, 1 << 127);
224 try testType(u129, 1 << 128);
225
226 try testType(i159, -1 << 158);
227 try testType(i159, -1);
228 try testType(i159, 0);
229 try testType(u159, 0);
230 try testType(u159, 1 << 0);
231 try testType(u159, 1 << 1);
232 try testType(u159, 1 << 157);
233 try testType(u159, 1 << 158);
234
235 try testType(i160, -1 << 159);
236 try testType(i160, -1);
237 try testType(i160, 0);
238 try testType(u160, 0);
239 try testType(u160, 1 << 0);
240 try testType(u160, 1 << 1);
241 try testType(u160, 1 << 158);
242 try testType(u160, 1 << 159);
243
244 try testType(i161, -1 << 160);
245 try testType(i161, -1);
246 try testType(i161, 0);
247 try testType(u161, 0);
248 try testType(u161, 1 << 0);
249 try testType(u161, 1 << 1);
250 try testType(u161, 1 << 159);
251 try testType(u161, 1 << 160);
252
253 try testType(i191, -1 << 190);
254 try testType(i191, -1);
255 try testType(i191, 0);
256 try testType(u191, 0);
257 try testType(u191, 1 << 0);
258 try testType(u191, 1 << 1);
259 try testType(u191, 1 << 189);
260 try testType(u191, 1 << 190);
261
262 try testType(i192, -1 << 191);
263 try testType(i192, -1);
264 try testType(i192, 0);
265 try testType(u192, 0);
266 try testType(u192, 1 << 0);
267 try testType(u192, 1 << 1);
268 try testType(u192, 1 << 190);
269 try testType(u192, 1 << 191);
270
271 try testType(i193, -1 << 192);
272 try testType(i193, -1);
273 try testType(i193, 0);
274 try testType(u193, 0);
275 try testType(u193, 1 << 0);
276 try testType(u193, 1 << 1);
277 try testType(u193, 1 << 191);
278 try testType(u193, 1 << 192);
279
280 try testType(i223, -1 << 222);
281 try testType(i223, -1);
282 try testType(i223, 0);
283 try testType(u223, 0);
284 try testType(u223, 1 << 0);
285 try testType(u223, 1 << 1);
286 try testType(u223, 1 << 221);
287 try testType(u223, 1 << 222);
288
289 try testType(i224, -1 << 223);
290 try testType(i224, -1);
291 try testType(i224, 0);
292 try testType(u224, 0);
293 try testType(u224, 1 << 0);
294 try testType(u224, 1 << 1);
295 try testType(u224, 1 << 222);
296 try testType(u224, 1 << 223);
297
298 try testType(i225, -1 << 224);
299 try testType(i225, -1);
300 try testType(i225, 0);
301 try testType(u225, 0);
302 try testType(u225, 1 << 0);
303 try testType(u225, 1 << 1);
304 try testType(u225, 1 << 223);
305 try testType(u225, 1 << 224);
306
307 try testType(i255, -1 << 254);
308 try testType(i255, -1);
309 try testType(i255, 0);
310 try testType(u255, 0);
311 try testType(u255, 1 << 0);
312 try testType(u255, 1 << 1);
313 try testType(u255, 1 << 253);
314 try testType(u255, 1 << 254);
315
316 try testType(i256, -1 << 255);
317 try testType(i256, -1);
318 try testType(i256, 0);
319 try testType(u256, 0);
320 try testType(u256, 1 << 0);
321 try testType(u256, 1 << 1);
322 try testType(u256, 1 << 254);
323 try testType(u256, 1 << 255);
324
325 try testType(i257, -1 << 256);
326 try testType(i257, -1);
327 try testType(i257, 0);
328 try testType(u257, 0);
329 try testType(u257, 1 << 0);
330 try testType(u257, 1 << 1);
331 try testType(u257, 1 << 255);
332 try testType(u257, 1 << 256);
333
334 try testType(i511, -1 << 510);
335 try testType(i511, -1);
336 try testType(i511, 0);
337 try testType(u511, 0);
338 try testType(u511, 1 << 0);
339 try testType(u511, 1 << 1);
340 try testType(u511, 1 << 509);
341 try testType(u511, 1 << 510);
342
343 try testType(i512, -1 << 511);
344 try testType(i512, -1);
345 try testType(i512, 0);
346 try testType(u512, 0);
347 try testType(u512, 1 << 0);
348 try testType(u512, 1 << 1);
349 try testType(u512, 1 << 510);
350 try testType(u512, 1 << 511);
351
352 try testType(i513, -1 << 512);
353 try testType(i513, -1);
354 try testType(i513, 0);
355 try testType(u513, 0);
356 try testType(u513, 1 << 0);
357 try testType(u513, 1 << 1);
358 try testType(u513, 1 << 511);
359 try testType(u513, 1 << 512);
360
361 try testType(i1023, -1 << 1022);
362 try testType(i1023, -1);
363 try testType(i1023, 0);
364 try testType(u1023, 0);
365 try testType(u1023, 1 << 0);
366 try testType(u1023, 1 << 1);
367 try testType(u1023, 1 << 1021);
368 try testType(u1023, 1 << 1022);
369
370 try testType(i1024, -1 << 1023);
371 try testType(i1024, -1);
372 try testType(i1024, 0);
373 try testType(u1024, 0);
374 try testType(u1024, 1 << 0);
375 try testType(u1024, 1 << 1);
376 try testType(u1024, 1 << 1022);
377 try testType(u1024, 1 << 1023);
378
379 try testType(i1025, -1 << 1024);
380 try testType(i1025, -1);
381 try testType(i1025, 0);
382 try testType(u1025, 0);
383 try testType(u1025, 1 << 0);
384 try testType(u1025, 1 << 1);
385 try testType(u1025, 1 << 1023);
386 try testType(u1025, 1 << 1024);
387
388 try testType(@Vector(3, i0), .{ 0 << 0, 0, 0 });
389 try testType(@Vector(3, u0), .{ 0, 0, 0 << 0 });
390
391 try testType(@Vector(3, i1), .{ -1 << 0, -1, 0 });
392 try testType(@Vector(3, u1), .{ 0, 1, 1 << 0 });
393
394 try testType(@Vector(3, i2), .{ -1 << 1, -1, 0 });
395 try testType(@Vector(3, u2), .{ 0, 1, 1 << 1 });
396
397 try testType(@Vector(3, i3), .{ -1 << 2, -1, 0 });
398 try testType(@Vector(3, u3), .{ 0, 1, 1 << 2 });
399
400 try testType(@Vector(3, i4), .{ -1 << 3, -1, 0 });
401 try testType(@Vector(3, u4), .{ 0, 1, 1 << 3 });
402 try testType(@Vector(1, u4), .{
403 0xb,
404 });
405 try testType(@Vector(2, u4), .{
406 0x3, 0x4,
407 });
408 try testType(@Vector(4, u4), .{
409 0x9, 0x2, 0xf, 0xe,
410 });
411 try testType(@Vector(8, u4), .{
412 0x8, 0x1, 0xb, 0x1, 0xf, 0x5, 0x9, 0x6,
413 });
414 try testType(@Vector(16, u4), .{
415 0xb, 0x6, 0x0, 0x7, 0x8, 0x5, 0x6, 0x9, 0xe, 0xb, 0x3, 0xa, 0xb, 0x5, 0x8, 0xc,
416 });
417 try testType(@Vector(32, u4), .{
418 0xe, 0x6, 0xe, 0xa, 0xb, 0x4, 0xa, 0xb, 0x1, 0x3, 0xb, 0xc, 0x0, 0xb, 0x9, 0x4, 0xd, 0xa, 0xd, 0xd, 0x4, 0x8, 0x8, 0x6, 0xb, 0xe, 0x9, 0x6, 0xc, 0xd, 0x5, 0xd,
419 });
420 try testType(@Vector(64, u4), .{
421 0x1, 0xc, 0xe, 0x9, 0x9, 0xf, 0x3, 0xf, 0x9, 0x9, 0x5, 0x3, 0xb, 0xd, 0xd, 0xf, 0x1, 0x2, 0xf, 0x9, 0x4, 0x4, 0x8, 0x9, 0x2, 0x9, 0x8, 0xe, 0x8, 0xa, 0x4, 0x3,
422 0x4, 0xc, 0xb, 0x6, 0x4, 0x0, 0xa, 0x5, 0x1, 0xa, 0x4, 0xe, 0xa, 0x7, 0xd, 0x0, 0x4, 0xe, 0xe, 0x7, 0x7, 0xa, 0x4, 0x5, 0x6, 0xc, 0x6, 0x2, 0x6, 0xa, 0xe, 0xa,
423 });
424 try testType(@Vector(128, u4), .{
425 0xd, 0x5, 0x6, 0xe, 0x3, 0x3, 0x3, 0xe, 0xd, 0xd, 0x9, 0x0, 0x0, 0xe, 0xa, 0x9, 0x8, 0x7, 0xb, 0x5, 0x7, 0xf, 0xb, 0x8, 0x0, 0xf, 0xb, 0x3, 0xa, 0x2, 0xb, 0xc,
426 0x1, 0x1, 0xc, 0x8, 0x8, 0x6, 0x9, 0x1, 0xb, 0x0, 0x2, 0xb, 0x2, 0x2, 0x7, 0x6, 0x1, 0x1, 0xb, 0x4, 0x6, 0x4, 0x7, 0xc, 0xd, 0xc, 0xa, 0x8, 0x1, 0x7, 0x8, 0xa,
427 0x9, 0xa, 0x1, 0x8, 0x1, 0x7, 0x9, 0x4, 0x5, 0x9, 0xd, 0x0, 0xa, 0xf, 0x3, 0x3, 0x9, 0x2, 0xf, 0x5, 0xb, 0x8, 0x6, 0xb, 0xf, 0x5, 0x8, 0x3, 0x9, 0xf, 0x6, 0x8,
428 0xc, 0x8, 0x3, 0x4, 0xa, 0xe, 0xc, 0x1, 0xe, 0x9, 0x1, 0x8, 0xf, 0x6, 0xc, 0xc, 0x6, 0xf, 0x6, 0xd, 0xb, 0x9, 0xc, 0x3, 0xd, 0xa, 0x6, 0x8, 0x4, 0xa, 0x6, 0x9,
429 });
430 try testType(@Vector(256, u4), .{
431 0x6, 0xc, 0xe, 0x3, 0x8, 0x2, 0xb, 0xd, 0x3, 0xa, 0x3, 0x8, 0xb, 0x8, 0x3, 0x0, 0xb, 0x5, 0x1, 0x3, 0x2, 0x2, 0xf, 0xc, 0x5, 0x1, 0x3, 0xb, 0x1, 0xc, 0x2, 0xd,
432 0xa, 0x8, 0x1, 0xc, 0xb, 0xa, 0x3, 0x1, 0xe, 0x4, 0xf, 0xb, 0xd, 0x8, 0xf, 0xa, 0xc, 0xb, 0xb, 0x0, 0xa, 0xc, 0xf, 0xe, 0x8, 0xd, 0x9, 0x3, 0xa, 0xe, 0x8, 0x7,
433 0x5, 0xa, 0x0, 0xe, 0x0, 0xd, 0x2, 0x2, 0x9, 0x4, 0x8, 0x9, 0x0, 0x4, 0x4, 0x8, 0xe, 0x1, 0xf, 0x1, 0x9, 0x3, 0xf, 0xc, 0xa, 0x0, 0x3, 0x2, 0x4, 0x1, 0x2, 0x3,
434 0xf, 0x2, 0x7, 0xb, 0x5, 0x0, 0xd, 0x3, 0x4, 0xf, 0xa, 0x3, 0xc, 0x2, 0x5, 0xe, 0x7, 0x5, 0xd, 0x7, 0x9, 0x0, 0xd, 0x7, 0x9, 0xd, 0x5, 0x7, 0xf, 0xd, 0xb, 0x4,
435 0x9, 0x6, 0xf, 0xb, 0x1, 0xb, 0x6, 0xb, 0xf, 0x7, 0xf, 0x0, 0x4, 0x7, 0x5, 0xa, 0x8, 0x1, 0xf, 0x9, 0x9, 0x0, 0x6, 0xb, 0x1, 0x2, 0x4, 0x3, 0x2, 0x0, 0x7, 0x0,
436 0x6, 0x7, 0xf, 0x1, 0xe, 0xa, 0x8, 0x2, 0x9, 0xc, 0x1, 0x5, 0x7, 0x1, 0xb, 0x0, 0x1, 0x3, 0xd, 0x3, 0x0, 0x1, 0xa, 0x0, 0x3, 0x7, 0x1, 0x2, 0xb, 0xc, 0x2, 0x9,
437 0x8, 0x8, 0x7, 0x0, 0xd, 0x5, 0x1, 0x5, 0x7, 0x7, 0x2, 0x3, 0x8, 0x7, 0xc, 0x8, 0xf, 0xa, 0xf, 0xf, 0x3, 0x2, 0x0, 0x4, 0x7, 0x5, 0x6, 0xd, 0x6, 0x3, 0xa, 0x4,
438 0x1, 0x1, 0x2, 0xc, 0x3, 0xe, 0x2, 0xc, 0x7, 0x6, 0xe, 0xf, 0xb, 0x8, 0x6, 0x6, 0x9, 0x0, 0x4, 0xb, 0xe, 0x4, 0x2, 0x7, 0xf, 0xc, 0x0, 0x6, 0xd, 0xa, 0xe, 0xc,
439 });
440
441 try testType(@Vector(3, i5), .{ -1 << 4, -1, 0 });
442 try testType(@Vector(3, u5), .{ 0, 1, 1 << 4 });
443
444 try testType(@Vector(3, i7), .{ -1 << 6, -1, 0 });
445 try testType(@Vector(3, u7), .{ 0, 1, 1 << 6 });
446
447 try testType(@Vector(3, i8), .{ -1 << 7, -1, 0 });
448 try testType(@Vector(3, u8), .{ 0, 1, 1 << 7 });
449 try testType(@Vector(1, u8), .{
450 0x33,
451 });
452 try testType(@Vector(2, u8), .{
453 0x66, 0x87,
454 });
455 try testType(@Vector(4, u8), .{
456 0x9d, 0xcb, 0x30, 0x7b,
457 });
458 try testType(@Vector(8, u8), .{
459 0x4b, 0x35, 0x3f, 0x5c, 0xa5, 0x91, 0x23, 0x6d,
460 });
461 try testType(@Vector(16, u8), .{
462 0xb7, 0x57, 0x27, 0x29, 0x58, 0xf8, 0xc9, 0x6c, 0xbe, 0x41, 0xf4, 0xd7, 0x4d, 0x01, 0xf0, 0x37,
463 });
464 try testType(@Vector(32, u8), .{
465 0x5f, 0x61, 0x34, 0xe8, 0x37, 0x12, 0xba, 0x5a, 0x85, 0xf3, 0x3e, 0xa2, 0x0f, 0xd0, 0x65, 0xae,
466 0xed, 0xf5, 0xe8, 0x65, 0x61, 0x28, 0x4a, 0x27, 0x2e, 0x01, 0x40, 0x8c, 0xe3, 0x36, 0x5d, 0xb6,
467 });
468 try testType(@Vector(64, u8), .{
469 0xb0, 0x19, 0x5c, 0xc2, 0x3b, 0x16, 0x70, 0xad, 0x26, 0x45, 0xf2, 0xe1, 0x4f, 0x0f, 0x01, 0x72,
470 0x7f, 0x1f, 0x07, 0x9e, 0xee, 0x9b, 0xb3, 0x38, 0x50, 0xf3, 0x56, 0x73, 0xd0, 0xd1, 0xee, 0xe3,
471 0xeb, 0xf3, 0x1b, 0xe0, 0x77, 0x78, 0x75, 0xc6, 0x19, 0xe4, 0x69, 0xaa, 0x73, 0x08, 0xcd, 0x0c,
472 0xf9, 0xed, 0x94, 0xf8, 0x79, 0x86, 0x63, 0x31, 0xbf, 0xd1, 0xe3, 0x17, 0x2b, 0xb9, 0xa1, 0x72,
473 });
474 try testType(@Vector(128, u8), .{
475 0x2e, 0x93, 0x87, 0x09, 0x4f, 0x68, 0x14, 0xab, 0x3f, 0x04, 0x86, 0xc1, 0x95, 0xe8, 0x74, 0x11,
476 0x57, 0x25, 0xe1, 0x88, 0xc0, 0x96, 0x33, 0x99, 0x15, 0x86, 0x2c, 0x84, 0x2e, 0xd7, 0x57, 0x21,
477 0xd3, 0x18, 0xd5, 0x0e, 0xb4, 0x60, 0xe2, 0x08, 0xce, 0xbc, 0xd5, 0x4d, 0x8f, 0x59, 0x01, 0x67,
478 0x71, 0x0a, 0x74, 0x48, 0xef, 0x39, 0x49, 0x7e, 0xa8, 0x39, 0x34, 0x75, 0x95, 0x3b, 0x38, 0xea,
479 0x60, 0xd7, 0xed, 0x8f, 0xbb, 0xc0, 0x7d, 0xc2, 0x79, 0x2d, 0xbf, 0xa5, 0x64, 0xf4, 0x09, 0x86,
480 0xfb, 0x29, 0xfe, 0xc7, 0xff, 0x62, 0x1a, 0x6f, 0xf8, 0xbd, 0xfe, 0xa4, 0xac, 0x24, 0xcf, 0x56,
481 0x82, 0x69, 0x81, 0x0d, 0xc1, 0x51, 0x8d, 0x85, 0xf4, 0x00, 0xe7, 0x25, 0xab, 0xa5, 0x33, 0x45,
482 0x66, 0x2e, 0x33, 0xc8, 0xf3, 0x35, 0x16, 0x7d, 0x1f, 0xc9, 0xf7, 0x44, 0xab, 0x66, 0x28, 0x0d,
483 });
484
485 try testType(@Vector(3, i9), .{ -1 << 8, -1, 0 });
486 try testType(@Vector(3, u9), .{ 0, 1, 1 << 8 });
487
488 try testType(@Vector(3, i15), .{ -1 << 14, -1, 0 });
489 try testType(@Vector(3, u15), .{ 0, 1, 1 << 14 });
490
491 try testType(@Vector(3, i16), .{ -1 << 15, -1, 0 });
492 try testType(@Vector(3, u16), .{ 0, 1, 1 << 15 });
493 try testType(@Vector(1, u16), .{
494 0x4da6,
495 });
496 try testType(@Vector(2, u16), .{
497 0x04d7, 0x50c6,
498 });
499 try testType(@Vector(4, u16), .{
500 0x4c06, 0xd71f, 0x4d8f, 0xe0a4,
501 });
502 try testType(@Vector(8, u16), .{
503 0xee9a, 0x881d, 0x31fb, 0xd3f7, 0x2c74, 0x6949, 0x4e04, 0x53d7,
504 });
505 try testType(@Vector(16, u16), .{
506 0xeafe, 0x9a7b, 0x0d6f, 0x18cb, 0xaf8f, 0x8ee4, 0xa47e, 0xd39a,
507 0x6572, 0x9c53, 0xf36e, 0x982e, 0x41c1, 0x8682, 0xf5dc, 0x7e01,
508 });
509 try testType(@Vector(32, u16), .{
510 0xdfb3, 0x7de6, 0xd9ed, 0xb42e, 0x95ac, 0x9b5b, 0x0422, 0xdfcd,
511 0x6196, 0x4dbe, 0x1818, 0x8816, 0x75e7, 0xc9b0, 0x92f7, 0x1f71,
512 0xe584, 0x576c, 0x043a, 0x0f31, 0xfc4c, 0x2c87, 0x6b02, 0x0229,
513 0x25b7, 0x53cd, 0x9bab, 0x866b, 0x9008, 0xf0f3, 0xeb21, 0x88e2,
514 });
515 try testType(@Vector(64, u16), .{
516 0x084c, 0x445f, 0xce89, 0xd3ee, 0xb399, 0x315d, 0x8ef8, 0x4f6f,
517 0xf9af, 0xcbc4, 0x0332, 0xcd55, 0xa4dc, 0xbc38, 0x6e33, 0x8ead,
518 0xd15a, 0x5057, 0x58ef, 0x657a, 0xe9f0, 0x1418, 0x2b62, 0x3387,
519 0x1c15, 0x04e1, 0x0276, 0x3783, 0xad9c, 0xea9a, 0x0e5e, 0xe803,
520 0x2ee7, 0x0cf1, 0x30f1, 0xb12a, 0x381b, 0x353d, 0xf637, 0xf853,
521 0x2ac1, 0x7ce8, 0x6a50, 0xcbb8, 0xc9b8, 0x9b25, 0xd1e9, 0xeff0,
522 0xc0a2, 0x8e51, 0xde7a, 0x4e58, 0x5685, 0xeb3f, 0xd29b, 0x66ed,
523 0x3dd5, 0xcb59, 0x6003, 0xf710, 0x943a, 0x7276, 0xe547, 0xe48f,
524 });
525
526 try testType(@Vector(3, i17), .{ -1 << 16, -1, 0 });
527 try testType(@Vector(3, u17), .{ 0, 1, 1 << 16 });
528
529 try testType(@Vector(3, i31), .{ -1 << 30, -1, 0 });
530 try testType(@Vector(3, u31), .{ 0, 1, 1 << 30 });
531
532 try testType(@Vector(3, i32), .{ -1 << 31, -1, 0 });
533 try testType(@Vector(3, u32), .{ 0, 1, 1 << 31 });
534 try testType(@Vector(1, u32), .{
535 0x17e2805c,
536 });
537 try testType(@Vector(2, u32), .{
538 0xdb6aadc5, 0xb1ff3754,
539 });
540 try testType(@Vector(4, u32), .{
541 0xf7897b31, 0x342e1af9, 0x190fd76b, 0x283b5374,
542 });
543 try testType(@Vector(8, u32), .{
544 0x81a0bd16, 0xc55da94e, 0x910f7e7c, 0x078d5ef7,
545 0x0bdb1e4a, 0xf1a96e99, 0xcdd729b5, 0xe6966a1c,
546 });
547 try testType(@Vector(16, u32), .{
548 0xfee812db, 0x29eacbed, 0xaed48136, 0x3053de13,
549 0xbbda20df, 0x6faa274a, 0xe0b5ec3a, 0x1878b0dc,
550 0x98204475, 0x810d8d05, 0x1e6996b6, 0xc543826a,
551 0x53b47d8c, 0xc72c3142, 0x12f7e1f9, 0xf6782e54,
552 });
553 try testType(@Vector(32, u32), .{
554 0xf0cf30d3, 0xe3c587b8, 0xcee44739, 0xe4a0bd72,
555 0x41d44cce, 0x6d7c4259, 0xd85580a5, 0xec4b02d7,
556 0xa366483d, 0x2d7b59d4, 0xe9c0ace4, 0x82cb441c,
557 0xa23958ba, 0x04a70148, 0x3f0d20a3, 0xf9e21e37,
558 0x009fce8b, 0x4a34a229, 0xf09c35cf, 0xc0977d4d,
559 0xcc4d4647, 0xa30f1363, 0x27a65b14, 0xe572c785,
560 0x8f42e320, 0x2b2cdeca, 0x11205bd4, 0x739d26aa,
561 0xcbcc2df0, 0x5f7a3649, 0xbde1b7aa, 0x180a169f,
562 });
563
564 try testType(@Vector(3, i33), .{ -1 << 32, -1, 0 });
565 try testType(@Vector(3, u33), .{ 0, 1, 1 << 32 });
566
567 try testType(@Vector(3, i63), .{ -1 << 62, -1, 0 });
568 try testType(@Vector(3, u63), .{ 0, 1, 1 << 62 });
569
570 try testType(@Vector(3, i64), .{ -1 << 63, -1, 0 });
571 try testType(@Vector(3, u64), .{ 0, 1, 1 << 63 });
572 try testType(@Vector(1, u64), .{
573 0x7d2e439abb0edba7,
574 });
575 try testType(@Vector(2, u64), .{
576 0x3749ee5a2d237b9f, 0x6d8f4c3e1378f389,
577 });
578 try testType(@Vector(4, u64), .{
579 0x03c127040e10d52b, 0xa86fe019072e27eb,
580 0x0a554a47b709cdba, 0xf4342cc597e196c3,
581 });
582 try testType(@Vector(8, u64), .{
583 0xea455c104375a055, 0x5c35d9d945edb2fa,
584 0xc11b73d9d9d546fc, 0x2a9d63aae838dd5b,
585 0xed6603f1f5d574b3, 0x2f37b354c81c1e56,
586 0xbe7f5e2476bc76bd, 0xb0c88eacfffa9a8f,
587 });
588 try testType(@Vector(16, u64), .{
589 0x2258fc04b31f8dbe, 0x3a2e5483003a10d8,
590 0xebf24b31c0460510, 0x15d5b4c09b53ffa5,
591 0x05abf6e744b17cc6, 0x9747b483f2d159fe,
592 0x4616d8b2c8673125, 0x8ae3f91d422447eb,
593 0x18da2f101a9e9776, 0x77a1197fb0441007,
594 0x4ba480c8ec2dd10b, 0xeb99b9c0a1725278,
595 0xd9d0acc5084ecdf0, 0xa0a23317fff4f515,
596 0x0901c59a9a6a408b, 0x7c77ca72e25df033,
597 });
598
599 try testType(@Vector(3, i65), .{ -1 << 64, -1, 0 });
600 try testType(@Vector(3, u65), .{ 0, 1, 1 << 64 });
601
602 try testType(@Vector(3, i127), .{ -1 << 126, -1, 0 });
603 try testType(@Vector(3, u127), .{ 0, 1, 1 << 126 });
604
605 try testType(@Vector(3, i128), .{ -1 << 127, -1, 0 });
606 try testType(@Vector(3, u128), .{ 0, 1, 1 << 127 });
607 try testType(@Vector(1, u128), .{
608 0x809f29e7fbafadc01145e1732590e7d9,
609 });
610 try testType(@Vector(2, u128), .{
611 0x5150ac3438aacd0d51132cc2723b2995,
612 0x151be9c47ad29cf719cf8358dd40165c,
613 });
614 try testType(@Vector(4, u128), .{
615 0x4bae22df929f2f7cb9bd84deaad3e7a8,
616 0x1ed46b2d6e1f3569f56b2ac33d8bc1cb,
617 0xae93ea459d2ccfd5fb794e6d5c31aabb,
618 0xb1177136acf099f550b70949ac202ec4,
619 });
620 try testType(@Vector(8, u128), .{
621 0x7cd78db6baed6bfdf8c5265136c4e0fd,
622 0xa41b8984c6bbde84640068194b7eba98,
623 0xd33102778f2ae1a48d1e9bf8801bbbf0,
624 0x0d59f6de003513a60055c86cbce2c200,
625 0x825579d90012afddfbf04851c0748561,
626 0xc2647c885e9d6f0ee1f5fac5da8ef7f5,
627 0xcb4bbc1f81aa8ee68aa4dc140745687b,
628 0x4ff10f914f74b46c694407f5bf7c7836,
629 });
630
631 try testType(@Vector(3, i129), .{ -1 << 128, -1, 0 });
632 try testType(@Vector(3, u129), .{ 0, 1, 1 << 128 });
633
634 try testType(@Vector(3, i191), .{ -1 << 190, -1, 0 });
635 try testType(@Vector(3, u191), .{ 0, 1, 1 << 190 });
636
637 try testType(@Vector(3, i192), .{ -1 << 191, -1, 0 });
638 try testType(@Vector(3, u192), .{ 0, 1, 1 << 191 });
639 try testType(@Vector(1, u192), .{
640 0xe7baafcb9781626a77571b0539b9471a60c97d6c02106c8b,
641 });
642 try testType(@Vector(2, u192), .{
643 0xbc9510913ed09e2c2aa50ffab9f1bc7b303a87f36e232a83,
644 0x1f37bee446d7712d1ad457c47a66812cb926198d052aee65,
645 });
646 try testType(@Vector(4, u192), .{
647 0xdca6a7cfc19c69efc34022062a8ca36f2569ab3dce001202,
648 0xd25a4529e621c9084181fdb6917c6a32eccc58b63601b35d,
649 0x0a258afd6debbaf8c158f1caa61fed63b31871d13f51b43d,
650 0x6b40a178674fcb82c623ac322f851623d5e993dac97a219a,
651 });
652
653 try testType(@Vector(3, i193), .{ -1 << 192, -1, 0 });
654 try testType(@Vector(3, u193), .{ 0, 1, 1 << 192 });
655
656 try testType(@Vector(3, i255), .{ -1 << 254, -1, 0 });
657 try testType(@Vector(3, u255), .{ 0, 1, 1 << 254 });
658
659 try testType(@Vector(3, i256), .{ -1 << 255, -1, 0 });
660 try testType(@Vector(3, u256), .{ 0, 1, 1 << 255 });
661 try testType(@Vector(1, u256), .{
662 0x230413bb481fa3a997796acf282010c560d1942e7339fd584a0f15a90c83fbda,
663 });
664 try testType(@Vector(2, u256), .{
665 0x3ad569f8d91fdbc9da8ec0e933565919f2feb90b996c90c352b461aa0908e62d,
666 0x0f109696d64647983f1f757042515510729ad1350e862cbf38cb73b5cf99f0f7,
667 });
668 try testType(@Vector(4, u256), .{
669 0x1717c6ded4ac6de282d59f75f068da47d5a47a30f2c5053d2d59e715f9d28b97,
670 0x3087189ce7540e2e0028b80af571ebc6353a00b2917f243a869ed29ecca0adaa,
671 0x1507c6a9d104684bf503cdb08841cf91adab4644306bd67aafff5326604833ce,
672 0x857e134ff9179733c871295b25f824bd3eb562977bad30890964fa0cdc15bb07,
673 });
674
675 try testType(@Vector(3, i257), .{ -1 << 256, -1, 0 });
676 try testType(@Vector(3, u257), .{ 0, 1, 1 << 256 });
677
678 try testType(@Vector(3, i511), .{ -1 << 510, -1, 0 });
679 try testType(@Vector(3, u511), .{ 0, 1, 1 << 510 });
680
681 try testType(@Vector(3, i512), .{ -1 << 511, -1, 0 });
682 try testType(@Vector(3, u512), .{ 0, 1, 1 << 511 });
683 try testType(@Vector(1, u512), .{
684 0xa3ff51a609f1370e5eeb96b05169bf7469e465cf76ac5b4ea8ffd166c1ba3cd94f2dedf0d647a1fe424f3a06e6d7940f03e257f28100970b00bd5528c52b9ae6,
685 });
686 try testType(@Vector(2, u512), .{
687 0xc6d43cd46ae31ab71f9468a895c83bf17516c6b2f1c9b04b9aa113bf7fe1b789eb7d95fcf951f12a9a6f2124589551efdd8c00f528b366a7bfb852faf8f3da53,
688 0xc9099d2bdf8d1a0d30485ec6db4a24cbc0d89a863de30e18313ee1d66f71dd2d26235caaa703286cf4a2b51e1a12ef96d2d944c66c0bd3f0d72dd4cf0fc8100e,
689 });
690
691 try testType(@Vector(3, i513), .{ -1 << 512, -1, 0 });
692 try testType(@Vector(3, u513), .{ 0, 1, 1 << 512 });
693
694 try testType(@Vector(3, i1023), .{ -1 << 1022, -1, 0 });
695 try testType(@Vector(3, u1023), .{ 0, 1, 1 << 1022 });
696
697 try testType(@Vector(3, i1024), .{ -1 << 1023, -1, 0 });
698 try testType(@Vector(3, u1024), .{ 0, 1, 1 << 1023 });
699 try testType(@Vector(1, u1024), .{
700 0xc6cfaa6571139552e1f067402dfc131d9b9a58aafda97198a78764b05138fb68cf26f085b7652f3d5ae0e56aa21732f296a581bb411d4a73795c213de793489fa49b173b9f5c089aa6295ff1fcdc14d491a05035b45d08fc35cd67a83d887a02b8db512f07518132e0ba56533c7d6fbe958255eddf5649bd8aba288c0dd84a25,
701 });
702
703 try testType(@Vector(3, i1025), .{ -1 << 1024, -1, 0 });
704 try testType(@Vector(3, u1025), .{ 0, 1, 1 << 1024 });
705}
706
1707fn testBinary(comptime op: anytype) !void {
2708 const testType = struct {
3709 fn testType(comptime Type: type, comptime imm_lhs: Type, comptime imm_rhs: Type) !void {
......@@ -306,6 +1012,63 @@ fn testBinary(comptime op: anytype) !void {
3061012 0x8b0b4a27fc94a0e90652d19bc755b63d,
3071013 0xa858bce5ad0e48c13588a4e170e8667c,
3081014 });
1015
1016 try testType(@Vector(1, u256), .{
1017 0x28df37e1f57a56133ba3f5b5b2164ce24eb6c29a8973a597fd91fbee8ab4bafb,
1018 }, .{
1019 0x63f725028cab082b5b1e6cb474428c8c3655cf438f3bb05c7a87f8270198f357,
1020 });
1021 try testType(@Vector(2, u256), .{
1022 0xcc79740b85597ef411e6d7e92049dfaa2328781ea4911540a3dcb512b71c7f3c,
1023 0x51ae46d2f93cbecff1578481f6ddc633dacee94ecaf81597c752c5c5db0ae766,
1024 }, .{
1025 0x257f0107305cb71cef582a9a58612a019f335e390d7998f51f5898f245874a6e,
1026 0x0a95a17323a4d16a715720f122b752785e9877e3dd3d3f9b72cdac3d1139a81f,
1027 });
1028 try testType(@Vector(4, u256), .{
1029 0x19667a6e269342cba437a8904c7ba42a762358d32723723ae2637b01124e63c5,
1030 0x14f7d3599a7edc7bcc46874f68d4291793e6ef72bd1f3763bc5e923f54f2f781,
1031 0x1c939de0ae980b80de773a04088ba45813441336cdfdc281ee356c98d71f653b,
1032 0x39f5d755965382fe13d1b1d6690b8e3827f153f8166768c4ad8a28a963b781f2,
1033 }, .{
1034 0xbe03de37cdcb8126083b4e86cd8a9803121d31b186fd5ce555ad77ce624dd6c7,
1035 0xa0c0730f0d7f141cc959849d09730b049f00693361539f1bc4758270554a60c1,
1036 0x2664bdba8de4eaa36ecee72f6bfec5b4daa6b4e00272d8116f2cc532c29490cc,
1037 0xe47a122bd45d5e7d69722d864a6b795ddee965a0993094f8791dd309d692de8b,
1038 });
1039
1040 try testType(@Vector(1, u512), .{
1041 0x651058c1d89a8f34cfc5e66b6d25294eecfcc4a7e1e4a356eb51ee7d7b2db25378e4afee51b7d18d16e520772a60c50a02d7966f40ced1870b32c658e5821397,
1042 }, .{
1043 0xd726e265ec80cb99510ba4f480ca64e959de5c528a7f54c386ecad22eeeefa845f0fd44b1bd64258a5f868197ee2d8fed59df9c9f0b72e74051a7ff20230880e,
1044 });
1045 try testType(@Vector(2, u512), .{
1046 0x22c8183c95cca8b09fdf541e431b73e9e4a1a5a00dff12381937fab52681d09d38ea25727d7025a2be08942cfa01535759e1644792e347c7901ec94b343c6337,
1047 0x292fdf644e75927e1aea9465ae2f60fb27550cd095f1afdea2cf7855286d26fbeed1c0b9c0474b73cb6b75621f7eadaa2f94ec358179ce2aaa0766df20da1ef3,
1048 }, .{
1049 0xe1cd8c0ca244c6626d4415e10b4ac43fa69e454c529c24fec4b13e6b945684d4ea833709c16c636ca78cffa5c5bf0fe945cd714a9ad695184a6bdad31dec9e31,
1050 0x8fa3d86099e9e2789d72f8e792290356d659ab20ac0414ff94745984c6ae7d986082197bb849889f912e896670aa2c1a11bd7e66e3f650710b0f0a18a1533f90,
1051 });
1052
1053 try testType(@Vector(1, u1024), .{
1054 0x0ca1a0dfaf8bb1da714b457d23c71aef948e66c7cd45c0aa941498a796fb18502ec32f34e885d0a107d44ae81595f8b52c2f0fb38e584b7139903a0e8a823ae20d01ca0662722dd474e7efc40f32d74cc065d97d8a09d0447f1ab6107fa0a57f3f8c866ae872506627ce82f18add79cee8dc69837f4ead3ca770c4d622d7e544,
1055 }, .{
1056 0xf1e3bbe031d59351770a7a501b6e969b2c00d144f17648db3f944b69dfeb7be72e5ff933a061eba4eaa422f8ca09e5a97d0b0dd740fd4076eba8c72d7a278523f399202dc2d043c4e0eb58a2bcd4066e2146e321810b1ee4d3afdddb4f026bcc7905ce17e033a7727b4e08f33b53c63d8c9f763fc6c31d0523eb38c30d5e40bc,
1057 });
1058}
1059
1060inline fn bitNot(comptime Type: type, rhs: Type) @TypeOf(~rhs) {
1061 return ~rhs;
1062}
1063test bitNot {
1064 try testUnary(bitNot);
1065}
1066
1067inline fn clz(comptime Type: type, rhs: Type) @TypeOf(@clz(rhs)) {
1068 return @clz(rhs);
1069}
1070test clz {
1071 try testUnary(clz);
3091072}
3101073
3111074inline fn bitAnd(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs & rhs) {
tools/update_cpu_features.zig+135-3
......@@ -902,8 +902,8 @@ const llvm_targets = [_]LlvmTarget{
902902 .features = &.{ "v8a", "exynos" },
903903 },
904904 },
905 // LLVM removed support for v2 and v3 but zig wants to support targeting old hardware
906905 .extra_features = &.{
906 // LLVM removed support for v2 and v3 but zig wants to support targeting old hardware
907907 .{
908908 .zig_name = "v2",
909909 .desc = "ARMv2 architecture",
......@@ -1043,10 +1043,22 @@ const llvm_targets = [_]LlvmTarget{
10431043 .llvm_name = "64bit-mode",
10441044 .omit = true,
10451045 },
1046 .{
1047 .llvm_name = "alderlake",
1048 .extra_deps = &.{ "smap", "smep" },
1049 },
10461050 .{
10471051 .llvm_name = "amdfam10",
10481052 .extra_deps = &.{"3dnowa"},
10491053 },
1054 .{
1055 .llvm_name = "arrowlake",
1056 .extra_deps = &.{ "smap", "smep" },
1057 },
1058 .{
1059 .llvm_name = "arrowlake-s",
1060 .extra_deps = &.{ "smap", "smep" },
1061 },
10501062 .{
10511063 .llvm_name = "athlon",
10521064 .extra_deps = &.{"3dnowa"},
......@@ -1081,16 +1093,64 @@ const llvm_targets = [_]LlvmTarget{
10811093 },
10821094 .{
10831095 .llvm_name = "barcelona",
1084 .extra_deps = &.{"3dnowa"},
1096 .extra_deps = &.{ "3dnowa", "smap", "smep" },
1097 },
1098 .{
1099 .llvm_name = "broadwell",
1100 .extra_deps = &.{ "smap", "smep" },
10851101 },
10861102 .{
10871103 .llvm_name = "c3",
10881104 .extra_deps = &.{"3dnow"},
10891105 },
1106 .{
1107 .llvm_name = "cannonlake",
1108 .extra_deps = &.{ "smap", "smep" },
1109 },
1110 .{
1111 .llvm_name = "cascadelake",
1112 .extra_deps = &.{ "smap", "smep" },
1113 },
1114 .{
1115 .llvm_name = "emeraldrapids",
1116 .extra_deps = &.{ "smap", "smep" },
1117 },
10901118 .{
10911119 .llvm_name = "geode",
10921120 .extra_deps = &.{"3dnowa"},
10931121 },
1122 .{
1123 .llvm_name = "goldmont",
1124 .extra_deps = &.{ "smap", "smep" },
1125 },
1126 .{
1127 .llvm_name = "goldmont_plus",
1128 .extra_deps = &.{ "smap", "smep" },
1129 },
1130 .{
1131 .llvm_name = "haswell",
1132 .extra_deps = &.{"smep"},
1133 },
1134 .{
1135 .llvm_name = "i386",
1136 .extra_deps = &.{"bsf_bsr_0_clobbers_result"},
1137 },
1138 .{
1139 .llvm_name = "i486",
1140 .extra_deps = &.{"bsf_bsr_0_clobbers_result"},
1141 },
1142 .{
1143 .llvm_name = "icelake_client",
1144 .extra_deps = &.{ "smap", "smep" },
1145 },
1146 .{
1147 .llvm_name = "icelake_server",
1148 .extra_deps = &.{ "smap", "smep" },
1149 },
1150 .{
1151 .llvm_name = "ivybridge",
1152 .extra_deps = &.{"smep"},
1153 },
10941154 .{
10951155 .llvm_name = "k6-2",
10961156 .extra_deps = &.{"3dnow"},
......@@ -1127,6 +1187,10 @@ const llvm_targets = [_]LlvmTarget{
11271187 .llvm_name = "lakemont",
11281188 .extra_deps = &.{"soft_float"},
11291189 },
1190 .{
1191 .llvm_name = "meteorlake",
1192 .extra_deps = &.{ "smap", "smep" },
1193 },
11301194 .{
11311195 .llvm_name = "opteron",
11321196 .extra_deps = &.{"3dnowa"},
......@@ -1135,6 +1199,38 @@ const llvm_targets = [_]LlvmTarget{
11351199 .llvm_name = "opteron-sse3",
11361200 .extra_deps = &.{"3dnowa"},
11371201 },
1202 .{
1203 .llvm_name = "raptorlake",
1204 .extra_deps = &.{ "smap", "smep" },
1205 },
1206 .{
1207 .llvm_name = "rocketlake",
1208 .extra_deps = &.{ "smap", "smep" },
1209 },
1210 .{
1211 .llvm_name = "sapphirerapids",
1212 .extra_deps = &.{ "smap", "smep" },
1213 },
1214 .{
1215 .llvm_name = "silvermont",
1216 .extra_deps = &.{"smep"},
1217 },
1218 .{
1219 .llvm_name = "skx",
1220 .extra_deps = &.{ "smap", "smep" },
1221 },
1222 .{
1223 .llvm_name = "skylake",
1224 .extra_deps = &.{ "smap", "smep" },
1225 },
1226 .{
1227 .llvm_name = "skylake_avx512",
1228 .extra_deps = &.{ "smap", "smep" },
1229 },
1230 .{
1231 .llvm_name = "tigerlake",
1232 .extra_deps = &.{ "smap", "smep" },
1233 },
11381234 .{
11391235 .llvm_name = "winchip2",
11401236 .extra_deps = &.{"3dnow"},
......@@ -1143,9 +1239,29 @@ const llvm_targets = [_]LlvmTarget{
11431239 .llvm_name = "sse4.2",
11441240 .extra_deps = &.{"crc32"},
11451241 },
1242 .{
1243 .llvm_name = "znver1",
1244 .extra_deps = &.{ "smap", "smep" },
1245 },
1246 .{
1247 .llvm_name = "znver2",
1248 .extra_deps = &.{ "smap", "smep" },
1249 },
1250 .{
1251 .llvm_name = "znver3",
1252 .extra_deps = &.{ "smap", "smep" },
1253 },
1254 .{
1255 .llvm_name = "znver4",
1256 .extra_deps = &.{ "smap", "smep" },
1257 },
1258 .{
1259 .llvm_name = "znver5",
1260 .extra_deps = &.{ "smap", "smep" },
1261 },
11461262 },
1147 // Features removed from LLVM
11481263 .extra_features = &.{
1264 // Features removed from LLVM
11491265 .{
11501266 .zig_name = "3dnow",
11511267 .desc = "Enable 3DNow! instructions",
......@@ -1171,6 +1287,22 @@ const llvm_targets = [_]LlvmTarget{
11711287 .desc = "Prefetch with Intent to Write and T1 Hint",
11721288 .deps = &.{},
11731289 },
1290 // Custom Zig features
1291 .{
1292 .zig_name = "bsf_bsr_0_clobbers_result",
1293 .desc = "BSF/BSR may clobber the lower 32-bits of the result register when the source is zero",
1294 .deps = &.{},
1295 },
1296 .{
1297 .zig_name = "smap",
1298 .desc = "Enable Supervisor Mode Access Prevention",
1299 .deps = &.{},
1300 },
1301 .{
1302 .zig_name = "smep",
1303 .desc = "Enable Supervisor Mode Execution Prevention",
1304 .deps = &.{},
1305 },
11741306 },
11751307 .omit_cpus = &.{
11761308 // LLVM defines a bunch of dumb aliases with foreach loops in X86.td.