authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-19 11:37:40-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-02-19 11:37:40-08:00
logbd237bced4c45228dadba07154660e91746d165b
tree8a69c32fe956408d85d4dd0797ee2fa01e09eb50
parent0cf6ae290b97e7fee7e37b5c487fd3e6b35248b3
parent7d70d7b215b1c388ee4c555ddd13839d7161d5de
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22932 from jacobly0/x86_64-rewrite

x86_64: start rewriting bit counting operations

4 files changed, 1543 insertions(+), 6 deletions(-)

lib/std/math/big/int.zig+2-2
...@@ -2544,8 +2544,7 @@ pub const Const = struct {...@@ -2544,8 +2544,7 @@ pub const Const = struct {
2544 const bits_per_limb = @bitSizeOf(Limb);2544 const bits_per_limb = @bitSizeOf(Limb);
2545 while (i != 0) {2545 while (i != 0) {
2546 i -= 1;2546 i -= 1;
2547 const limb = a.limbs[i];2547 const this_limb_lz = @clz(a.limbs[i]);
2548 const this_limb_lz = @clz(limb);
2549 total_limb_lz += this_limb_lz;2548 total_limb_lz += this_limb_lz;
2550 if (this_limb_lz != bits_per_limb) break;2549 if (this_limb_lz != bits_per_limb) break;
2551 }2550 }
...@@ -2557,6 +2556,7 @@ pub const Const = struct {...@@ -2557,6 +2556,7 @@ pub const Const = struct {
2557 pub fn ctz(a: Const, bits: Limb) Limb {2556 pub fn ctz(a: Const, bits: Limb) Limb {
2558 // Limbs are stored in little-endian order. Converting a negative number to twos-complement2557 // Limbs are stored in little-endian order. Converting a negative number to twos-complement
2559 // flips all bits above the lowest set bit, which does not affect the trailing zero count.2558 // flips all bits above the lowest set bit, which does not affect the trailing zero count.
2559 if (a.eqlZero()) return bits;
2560 var result: Limb = 0;2560 var result: Limb = 0;
2561 for (a.limbs) |limb| {2561 for (a.limbs) |limb| {
2562 const limb_tz = @ctz(limb);2562 const limb_tz = @ctz(limb);
lib/std/math/big/int_test.zig+224
...@@ -3332,3 +3332,227 @@ test "(BigInt) negative" {...@@ -3332,3 +3332,227 @@ test "(BigInt) negative" {
3332 try testing.expect(mem.eql(u8, a_fmt, "(BigInt)"));3332 try testing.expect(mem.eql(u8, a_fmt, "(BigInt)"));
3333 try testing.expect(!mem.eql(u8, b_fmt, "(BigInt)"));3333 try testing.expect(!mem.eql(u8, b_fmt, "(BigInt)"));
3334}3334}
3335
3336test "clz" {
3337 const neg_limb_max_squared: std.math.big.int.Const = .{
3338 .limbs = &.{ 1, maxInt(Limb) - 1 },
3339 .positive = false,
3340 };
3341 try testing.expect(neg_limb_max_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3342
3343 const neg_limb_max_squared_plus_one: std.math.big.int.Const = .{
3344 .limbs = &.{ 0, maxInt(Limb) - 1 },
3345 .positive = false,
3346 };
3347 try testing.expect(neg_limb_max_squared_plus_one.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3348
3349 const neg_limb_msb_squared: std.math.big.int.Const = .{
3350 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3351 .positive = false,
3352 };
3353 try testing.expect(neg_limb_msb_squared.clz(@bitSizeOf(Limb) * 2) == 0);
3354 try testing.expect(neg_limb_msb_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3355
3356 const neg_limb_max: std.math.big.int.Const = .{
3357 .limbs = &.{maxInt(Limb)},
3358 .positive = false,
3359 };
3360 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) + 1) == 0);
3361 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3362 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) * 2) == 0);
3363 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3364
3365 const neg_limb_msb: std.math.big.int.Const = .{
3366 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3367 .positive = false,
3368 };
3369 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb)) == 0);
3370 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) + 1) == 0);
3371 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3372 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) * 2) == 0);
3373 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3374
3375 const neg_one: std.math.big.int.Const = .{
3376 .limbs = &.{1},
3377 .positive = false,
3378 };
3379 try testing.expect(neg_one.clz(@bitSizeOf(Limb)) == 0);
3380 try testing.expect(neg_one.clz(@bitSizeOf(Limb) + 1) == 0);
3381 try testing.expect(neg_one.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3382 try testing.expect(neg_one.clz(@bitSizeOf(Limb) * 2) == 0);
3383 try testing.expect(neg_one.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3384
3385 const zero: std.math.big.int.Const = .{
3386 .limbs = &.{0},
3387 .positive = true,
3388 };
3389 try testing.expect(zero.clz(@bitSizeOf(Limb)) == @bitSizeOf(Limb));
3390 try testing.expect(zero.clz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) + 1);
3391 try testing.expect(zero.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 1);
3392 try testing.expect(zero.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2);
3393 try testing.expect(zero.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 + 1);
3394
3395 const one: std.math.big.int.Const = .{
3396 .limbs = &.{1},
3397 .positive = true,
3398 };
3399 try testing.expect(one.clz(@bitSizeOf(Limb)) == @bitSizeOf(Limb) - 1);
3400 try testing.expect(one.clz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb));
3401 try testing.expect(one.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 2);
3402 try testing.expect(one.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2 - 1);
3403 try testing.expect(one.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2);
3404
3405 const limb_msb: std.math.big.int.Const = .{
3406 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3407 .positive = true,
3408 };
3409 try testing.expect(limb_msb.clz(@bitSizeOf(Limb)) == 0);
3410 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) + 1) == 1);
3411 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3412 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb));
3413 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3414
3415 const limb_max: std.math.big.int.Const = .{
3416 .limbs = &.{maxInt(Limb)},
3417 .positive = true,
3418 };
3419 try testing.expect(limb_max.clz(@bitSizeOf(Limb)) == 0);
3420 try testing.expect(limb_max.clz(@bitSizeOf(Limb) + 1) == 1);
3421 try testing.expect(limb_max.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3422 try testing.expect(limb_max.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb));
3423 try testing.expect(limb_max.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3424
3425 const limb_msb_squared: std.math.big.int.Const = .{
3426 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3427 .positive = true,
3428 };
3429 try testing.expect(limb_msb_squared.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3430 try testing.expect(limb_msb_squared.clz(@bitSizeOf(Limb) * 2) == 1);
3431 try testing.expect(limb_msb_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 2);
3432
3433 const limb_max_squared_minus_one: std.math.big.int.Const = .{
3434 .limbs = &.{ 0, maxInt(Limb) - 1 },
3435 .positive = true,
3436 };
3437 try testing.expect(limb_max_squared_minus_one.clz(@bitSizeOf(Limb) * 2) == 0);
3438 try testing.expect(limb_max_squared_minus_one.clz(@bitSizeOf(Limb) * 2 + 1) == 1);
3439
3440 const limb_max_squared: std.math.big.int.Const = .{
3441 .limbs = &.{ 1, maxInt(Limb) - 1 },
3442 .positive = true,
3443 };
3444 try testing.expect(limb_max_squared.clz(@bitSizeOf(Limb) * 2) == 0);
3445 try testing.expect(limb_max_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 1);
3446}
3447
3448test "ctz" {
3449 const neg_limb_max_squared: std.math.big.int.Const = .{
3450 .limbs = &.{ 1, maxInt(Limb) - 1 },
3451 .positive = false,
3452 };
3453 try testing.expect(neg_limb_max_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3454
3455 const neg_limb_max_squared_plus_one: std.math.big.int.Const = .{
3456 .limbs = &.{ 0, maxInt(Limb) - 1 },
3457 .positive = false,
3458 };
3459 try testing.expect(neg_limb_max_squared_plus_one.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3460
3461 const neg_limb_msb_squared: std.math.big.int.Const = .{
3462 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3463 .positive = false,
3464 };
3465 try testing.expect(neg_limb_msb_squared.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2 - 2);
3466 try testing.expect(neg_limb_msb_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 - 2);
3467
3468 const neg_limb_max: std.math.big.int.Const = .{
3469 .limbs = &.{maxInt(Limb)},
3470 .positive = false,
3471 };
3472 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) + 1) == 0);
3473 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3474 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) * 2) == 0);
3475 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3476
3477 const neg_limb_msb: std.math.big.int.Const = .{
3478 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3479 .positive = false,
3480 };
3481 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb)) == @bitSizeOf(Limb) - 1);
3482 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) - 1);
3483 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3484 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) - 1);
3485 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) - 1);
3486
3487 const neg_one: std.math.big.int.Const = .{
3488 .limbs = &.{1},
3489 .positive = false,
3490 };
3491 try testing.expect(neg_one.ctz(@bitSizeOf(Limb)) == 0);
3492 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) + 1) == 0);
3493 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3494 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) * 2) == 0);
3495 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3496
3497 const zero: std.math.big.int.Const = .{
3498 .limbs = &.{0},
3499 .positive = true,
3500 };
3501 try testing.expect(zero.ctz(@bitSizeOf(Limb)) == @bitSizeOf(Limb));
3502 try testing.expect(zero.ctz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) + 1);
3503 try testing.expect(zero.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 1);
3504 try testing.expect(zero.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2);
3505 try testing.expect(zero.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 + 1);
3506
3507 const one: std.math.big.int.Const = .{
3508 .limbs = &.{1},
3509 .positive = true,
3510 };
3511 try testing.expect(one.ctz(@bitSizeOf(Limb)) == 0);
3512 try testing.expect(one.ctz(@bitSizeOf(Limb) + 1) == 0);
3513 try testing.expect(one.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3514 try testing.expect(one.ctz(@bitSizeOf(Limb) * 2) == 0);
3515 try testing.expect(one.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3516
3517 const limb_msb: std.math.big.int.Const = .{
3518 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3519 .positive = true,
3520 };
3521 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb)) == @bitSizeOf(Limb) - 1);
3522 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) - 1);
3523 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3524 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) - 1);
3525 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) - 1);
3526
3527 const limb_max: std.math.big.int.Const = .{
3528 .limbs = &.{maxInt(Limb)},
3529 .positive = true,
3530 };
3531 try testing.expect(limb_max.ctz(@bitSizeOf(Limb)) == 0);
3532 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) + 1) == 0);
3533 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3534 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) * 2) == 0);
3535 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3536
3537 const limb_msb_squared: std.math.big.int.Const = .{
3538 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3539 .positive = true,
3540 };
3541 try testing.expect(limb_msb_squared.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 2);
3542 try testing.expect(limb_msb_squared.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2 - 2);
3543 try testing.expect(limb_msb_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 - 2);
3544
3545 const limb_max_squared_minus_one: std.math.big.int.Const = .{
3546 .limbs = &.{ 0, maxInt(Limb) - 1 },
3547 .positive = true,
3548 };
3549 try testing.expect(limb_max_squared_minus_one.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) + 1);
3550 try testing.expect(limb_max_squared_minus_one.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3551
3552 const limb_max_squared: std.math.big.int.Const = .{
3553 .limbs = &.{ 1, maxInt(Limb) - 1 },
3554 .positive = true,
3555 };
3556 try testing.expect(limb_max_squared.ctz(@bitSizeOf(Limb) * 2) == 0);
3557 try testing.expect(limb_max_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3558}
src/arch/x86_64/CodeGen.zig+1301-4
...@@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2418}2418}
24192419
2420fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {2420fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2421 @setEvalBranchQuota(12_600);2421 @setEvalBranchQuota(12_900);
2422 const pt = cg.pt;2422 const pt = cg.pt;
2423 const zcu = pt.zcu;2423 const zcu = pt.zcu;
2424 const ip = &zcu.intern_pool;2424 const ip = &zcu.intern_pool;
...@@ -2475,8 +2475,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2475,8 +2475,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24752475
2476 .bitcast => try cg.airBitCast(inst),2476 .bitcast => try cg.airBitCast(inst),
24772477
2478 .ctz => try cg.airCtz(inst),
2479 .popcount => try cg.airPopCount(inst),
2480 .bit_reverse => try cg.airBitReverse(inst),2478 .bit_reverse => try cg.airBitReverse(inst),
2481 .splat => try cg.airSplat(inst),2479 .splat => try cg.airSplat(inst),
2482 .select => try cg.airSelect(inst),2480 .select => try cg.airSelect(inst),
...@@ -26978,7 +26976,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -26978,7 +26976,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26978 .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }),26976 .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }),
26979 .call_never_tail => try cg.airCall(inst, .never_tail, .{ .safety = true }),26977 .call_never_tail => try cg.airCall(inst, .never_tail, .{ .safety = true }),
26980 .call_never_inline => try cg.airCall(inst, .never_inline, .{ .safety = true }),26978 .call_never_inline => try cg.airCall(inst, .never_inline, .{ .safety = true }),
26981
26982 .clz => |air_tag| if (use_old) try cg.airClz(inst) else {26979 .clz => |air_tag| if (use_old) try cg.airClz(inst) else {
26983 const ty_op = air_datas[@intFromEnum(inst)].ty_op;26980 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
26984 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});26981 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
...@@ -30084,6 +30081,1293 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -30084,6 +30081,1293 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30084 };30081 };
30085 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);30082 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
30086 },30083 },
30084 .ctz => |air_tag| if (use_old) try cg.airCtz(inst) else fallback: {
30085 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
30086 if (ty_op.ty.toType().isVector(zcu)) break :fallback try cg.airCtz(inst);
30087 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
30088 var res: [1]Temp = undefined;
30089 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
30090 .required_features = .{ .slow_incdec, null, null, null },
30091 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any },
30092 .patterns = &.{
30093 .{ .src = .{ .mut_mem, .none, .none } },
30094 .{ .src = .{ .to_mut_gpr, .none, .none } },
30095 },
30096 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30097 .clobbers = .{ .eflags = true },
30098 .each = .{ .once = &.{
30099 .{ ._, ._, .add, .dst0b, .si(1), ._, ._ },
30100 } },
30101 }, .{
30102 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any },
30103 .patterns = &.{
30104 .{ .src = .{ .mut_mem, .none, .none } },
30105 .{ .src = .{ .to_mut_gpr, .none, .none } },
30106 },
30107 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30108 .clobbers = .{ .eflags = true },
30109 .each = .{ .once = &.{
30110 .{ ._, ._c, .in, .dst0b, ._, ._, ._ },
30111 } },
30112 }, .{
30113 .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any },
30114 .patterns = &.{
30115 .{ .src = .{ .mut_mem, .none, .none } },
30116 .{ .src = .{ .to_mut_gpr, .none, .none } },
30117 },
30118 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30119 .clobbers = .{ .eflags = true },
30120 .each = .{ .once = &.{
30121 .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ },
30122 } },
30123 }, .{
30124 .required_features = .{ .bmi, null, null, null },
30125 .src_constraints = .{ .{ .exact_int = 16 }, .any, .any },
30126 .patterns = &.{
30127 .{ .src = .{ .mem, .none, .none } },
30128 .{ .src = .{ .to_gpr, .none, .none } },
30129 },
30130 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30131 .clobbers = .{ .eflags = true },
30132 .each = .{ .once = &.{
30133 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30134 .{ ._, ._, .tzcnt, .dst0w, .src0w, ._, ._ },
30135 } },
30136 }, .{
30137 .required_features = .{ .bmi, .false_deps_lzcnt_tzcnt, null, null },
30138 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30139 .patterns = &.{
30140 .{ .src = .{ .mem, .none, .none } },
30141 .{ .src = .{ .to_gpr, .none, .none } },
30142 },
30143 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30144 .clobbers = .{ .eflags = true },
30145 .each = .{ .once = &.{
30146 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30147 .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ },
30148 } },
30149 }, .{
30150 .required_features = .{ .bmi, null, null, null },
30151 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30152 .patterns = &.{
30153 .{ .src = .{ .mem, .none, .none } },
30154 .{ .src = .{ .to_gpr, .none, .none } },
30155 },
30156 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30157 .clobbers = .{ .eflags = true },
30158 .each = .{ .once = &.{
30159 .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ },
30160 } },
30161 }, .{
30162 .required_features = .{ .bmi, null, null, null },
30163 .src_constraints = .{ .{ .int = .dword }, .any, .any },
30164 .patterns = &.{
30165 .{ .src = .{ .to_mut_gpr, .none, .none } },
30166 },
30167 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30168 .clobbers = .{ .eflags = true },
30169 .each = .{ .once = &.{
30170 .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ },
30171 .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ },
30172 } },
30173 }, .{
30174 .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null },
30175 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30176 .patterns = &.{
30177 .{ .src = .{ .mem, .none, .none } },
30178 .{ .src = .{ .to_gpr, .none, .none } },
30179 },
30180 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30181 .clobbers = .{ .eflags = true },
30182 .each = .{ .once = &.{
30183 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30184 .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ },
30185 } },
30186 }, .{
30187 .required_features = .{ .@"64bit", .bmi, null, null },
30188 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30189 .patterns = &.{
30190 .{ .src = .{ .mem, .none, .none } },
30191 .{ .src = .{ .to_gpr, .none, .none } },
30192 },
30193 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30194 .clobbers = .{ .eflags = true },
30195 .each = .{ .once = &.{
30196 .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ },
30197 } },
30198 }, .{
30199 .required_features = .{ .@"64bit", .bmi, null, null },
30200 .src_constraints = .{ .{ .int = .qword }, .any, .any },
30201 .patterns = &.{
30202 .{ .src = .{ .to_mut_gpr, .none, .none } },
30203 },
30204 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30205 .clobbers = .{ .eflags = true },
30206 .each = .{ .once = &.{
30207 .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ },
30208 .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ },
30209 } },
30210 }, .{
30211 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
30212 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30213 .patterns = &.{
30214 .{ .src = .{ .mem, .none, .none } },
30215 .{ .src = .{ .to_gpr, .none, .none } },
30216 },
30217 .extra_temps = .{
30218 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
30219 .unused,
30220 .unused,
30221 .unused,
30222 .unused,
30223 .unused,
30224 .unused,
30225 .unused,
30226 .unused,
30227 .unused,
30228 .unused,
30229 },
30230 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30231 .clobbers = .{ .eflags = true },
30232 .each = .{ .once = &.{
30233 .{ ._, ._, .mov, .tmp0d, .si(32), ._, ._ },
30234 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30235 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
30236 } },
30237 }, .{
30238 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
30239 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30240 .patterns = &.{
30241 .{ .src = .{ .mem, .none, .none } },
30242 .{ .src = .{ .to_gpr, .none, .none } },
30243 },
30244 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30245 .clobbers = .{ .eflags = true },
30246 .each = .{ .once = &.{
30247 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30248 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30249 .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ },
30250 } },
30251 }, .{
30252 .src_constraints = .{ .{ .exact_int = 16 }, .any, .any },
30253 .patterns = &.{
30254 .{ .src = .{ .mem, .none, .none } },
30255 .{ .src = .{ .to_gpr, .none, .none } },
30256 },
30257 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30258 .clobbers = .{ .eflags = true },
30259 .each = .{ .once = &.{
30260 .{ ._, ._, .mov, .dst0d, .si(16), ._, ._ },
30261 .{ ._, ._f, .bs, .dst0w, .src0w, ._, ._ },
30262 } },
30263 }, .{
30264 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30265 .patterns = &.{
30266 .{ .src = .{ .mem, .none, .none } },
30267 .{ .src = .{ .to_gpr, .none, .none } },
30268 },
30269 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30270 .clobbers = .{ .eflags = true },
30271 .each = .{ .once = &.{
30272 .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ },
30273 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30274 } },
30275 }, .{
30276 .src_constraints = .{ .{ .int = .dword }, .any, .any },
30277 .patterns = &.{
30278 .{ .src = .{ .to_mut_gpr, .none, .none } },
30279 },
30280 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30281 .clobbers = .{ .eflags = true },
30282 .each = .{ .once = &.{
30283 .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ },
30284 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30285 } },
30286 }, .{
30287 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
30288 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30289 .patterns = &.{
30290 .{ .src = .{ .mem, .none, .none } },
30291 .{ .src = .{ .to_gpr, .none, .none } },
30292 },
30293 .extra_temps = .{
30294 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
30295 .unused,
30296 .unused,
30297 .unused,
30298 .unused,
30299 .unused,
30300 .unused,
30301 .unused,
30302 .unused,
30303 .unused,
30304 .unused,
30305 },
30306 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30307 .clobbers = .{ .eflags = true },
30308 .each = .{ .once = &.{
30309 .{ ._, ._, .mov, .tmp0d, .si(64), ._, ._ },
30310 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30311 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
30312 } },
30313 }, .{
30314 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
30315 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30316 .patterns = &.{
30317 .{ .src = .{ .mem, .none, .none } },
30318 .{ .src = .{ .to_gpr, .none, .none } },
30319 },
30320 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30321 .clobbers = .{ .eflags = true },
30322 .each = .{ .once = &.{
30323 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30324 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30325 .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ },
30326 } },
30327 }, .{
30328 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30329 .patterns = &.{
30330 .{ .src = .{ .mem, .none, .none } },
30331 .{ .src = .{ .to_gpr, .none, .none } },
30332 },
30333 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30334 .clobbers = .{ .eflags = true },
30335 .each = .{ .once = &.{
30336 .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ },
30337 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30338 } },
30339 }, .{
30340 .src_constraints = .{ .{ .int = .qword }, .any, .any },
30341 .patterns = &.{
30342 .{ .src = .{ .to_mut_gpr, .none, .none } },
30343 },
30344 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30345 .clobbers = .{ .eflags = true },
30346 .each = .{ .once = &.{
30347 .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ },
30348 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30349 } },
30350 }, .{
30351 .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null },
30352 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30353 .patterns = &.{
30354 .{ .src = .{ .to_mem, .none, .none } },
30355 },
30356 .extra_temps = .{
30357 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30358 .unused,
30359 .unused,
30360 .unused,
30361 .unused,
30362 .unused,
30363 .unused,
30364 .unused,
30365 .unused,
30366 .unused,
30367 .unused,
30368 },
30369 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30370 .clobbers = .{ .eflags = true },
30371 .each = .{ .once = &.{
30372 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30373 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
30374 .{ ._, ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30375 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
30376 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30377 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30378 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30379 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30380 } },
30381 }, .{
30382 .required_features = .{ .@"64bit", .bmi, null, null },
30383 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30384 .patterns = &.{
30385 .{ .src = .{ .to_mem, .none, .none } },
30386 },
30387 .extra_temps = .{
30388 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30389 .unused,
30390 .unused,
30391 .unused,
30392 .unused,
30393 .unused,
30394 .unused,
30395 .unused,
30396 .unused,
30397 .unused,
30398 .unused,
30399 },
30400 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30401 .clobbers = .{ .eflags = true },
30402 .each = .{ .once = &.{
30403 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30404 .{ .@"0:", ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30405 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
30406 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30407 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30408 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30409 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30410 } },
30411 }, .{
30412 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
30413 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30414 .patterns = &.{
30415 .{ .src = .{ .to_mem, .none, .none } },
30416 },
30417 .extra_temps = .{
30418 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30419 .unused,
30420 .unused,
30421 .unused,
30422 .unused,
30423 .unused,
30424 .unused,
30425 .unused,
30426 .unused,
30427 .unused,
30428 .unused,
30429 },
30430 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30431 .clobbers = .{ .eflags = true },
30432 .each = .{ .once = &.{
30433 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30434 .{ .@"0:", ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30435 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30436 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30437 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30438 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30439 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30440 } },
30441 }, .{
30442 .required_features = .{ .@"64bit", null, null, null },
30443 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30444 .patterns = &.{
30445 .{ .src = .{ .to_mem, .none, .none } },
30446 },
30447 .extra_temps = .{
30448 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30449 .unused,
30450 .unused,
30451 .unused,
30452 .unused,
30453 .unused,
30454 .unused,
30455 .unused,
30456 .unused,
30457 .unused,
30458 .unused,
30459 },
30460 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30461 .clobbers = .{ .eflags = true },
30462 .each = .{ .once = &.{
30463 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30464 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
30465 .{ ._, ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30466 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30467 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30468 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30469 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30470 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30471 } },
30472 } }) catch |err| switch (err) {
30473 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
30474 @tagName(air_tag),
30475 cg.typeOf(ty_op.operand).fmt(pt),
30476 ops[0].tracking(cg),
30477 }),
30478 else => |e| return e,
30479 };
30480 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
30481 },
30482 .popcount => |air_tag| if (use_old) try cg.airPopCount(inst) else fallback: {
30483 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
30484 if (ty_op.ty.toType().isVector(zcu)) break :fallback try cg.airPopCount(inst);
30485 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
30486 var res: [1]Temp = undefined;
30487 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
30488 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any },
30489 .patterns = &.{
30490 .{ .src = .{ .mut_mem, .none, .none } },
30491 .{ .src = .{ .to_mut_gpr, .none, .none } },
30492 },
30493 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30494 .clobbers = .{ .eflags = true },
30495 .each = .{ .once = &.{
30496 .{ ._, ._, .neg, .dst0b, ._, ._, ._ },
30497 } },
30498 }, .{
30499 .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any },
30500 .patterns = &.{
30501 .{ .src = .{ .mut_mem, .none, .none } },
30502 .{ .src = .{ .to_mut_gpr, .none, .none } },
30503 },
30504 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30505 .each = .{ .once = &.{} },
30506 }, .{
30507 .required_features = .{ .popcnt, null, null, null },
30508 .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any },
30509 .patterns = &.{
30510 .{ .src = .{ .mem, .none, .none } },
30511 .{ .src = .{ .to_gpr, .none, .none } },
30512 },
30513 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30514 .clobbers = .{ .eflags = true },
30515 .each = .{ .once = &.{
30516 .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ },
30517 .{ ._, ._, .popcnt, .dst0d, .dst0d, ._, ._ },
30518 } },
30519 }, .{
30520 .required_features = .{ .popcnt, null, null, null },
30521 .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any },
30522 .patterns = &.{
30523 .{ .src = .{ .mem, .none, .none } },
30524 .{ .src = .{ .to_gpr, .none, .none } },
30525 },
30526 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30527 .clobbers = .{ .eflags = true },
30528 .each = .{ .once = &.{
30529 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30530 .{ ._, ._, .popcnt, .dst0w, .src0w, ._, ._ },
30531 } },
30532 }, .{
30533 .required_features = .{ .popcnt, null, null, null },
30534 .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any },
30535 .patterns = &.{
30536 .{ .src = .{ .mut_gpr, .none, .none } },
30537 },
30538 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30539 .clobbers = .{ .eflags = true },
30540 .each = .{ .once = &.{
30541 .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ },
30542 } },
30543 }, .{
30544 .required_features = .{ .popcnt, .false_deps_popcnt, null, null },
30545 .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any },
30546 .patterns = &.{
30547 .{ .src = .{ .mem, .none, .none } },
30548 .{ .src = .{ .to_gpr, .none, .none } },
30549 },
30550 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30551 .clobbers = .{ .eflags = true },
30552 .each = .{ .once = &.{
30553 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30554 .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ },
30555 } },
30556 }, .{
30557 .required_features = .{ .popcnt, null, null, null },
30558 .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any },
30559 .patterns = &.{
30560 .{ .src = .{ .mem, .none, .none } },
30561 .{ .src = .{ .to_gpr, .none, .none } },
30562 },
30563 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30564 .clobbers = .{ .eflags = true },
30565 .each = .{ .once = &.{
30566 .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ },
30567 } },
30568 }, .{
30569 .required_features = .{ .popcnt, null, null, null },
30570 .src_constraints = .{ .{ .signed_int = .dword }, .any, .any },
30571 .patterns = &.{
30572 .{ .src = .{ .to_mut_gpr, .none, .none } },
30573 },
30574 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30575 .clobbers = .{ .eflags = true },
30576 .each = .{ .once = &.{
30577 .{ ._, ._l, .sh, .src0d, .uia(32, .src0, .sub_bit_size), ._, ._ },
30578 .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ },
30579 } },
30580 }, .{
30581 .required_features = .{ .@"64bit", .popcnt, null, null },
30582 .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any },
30583 .patterns = &.{
30584 .{ .src = .{ .mut_gpr, .none, .none } },
30585 },
30586 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30587 .clobbers = .{ .eflags = true },
30588 .each = .{ .once = &.{
30589 .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ },
30590 } },
30591 }, .{
30592 .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null },
30593 .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any },
30594 .patterns = &.{
30595 .{ .src = .{ .mem, .none, .none } },
30596 .{ .src = .{ .to_gpr, .none, .none } },
30597 },
30598 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30599 .clobbers = .{ .eflags = true },
30600 .each = .{ .once = &.{
30601 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30602 .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ },
30603 } },
30604 }, .{
30605 .required_features = .{ .@"64bit", .popcnt, null, null },
30606 .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any },
30607 .patterns = &.{
30608 .{ .src = .{ .mem, .none, .none } },
30609 .{ .src = .{ .to_gpr, .none, .none } },
30610 },
30611 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30612 .clobbers = .{ .eflags = true },
30613 .each = .{ .once = &.{
30614 .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ },
30615 } },
30616 }, .{
30617 .required_features = .{ .@"64bit", .popcnt, null, null },
30618 .src_constraints = .{ .{ .signed_int = .qword }, .any, .any },
30619 .patterns = &.{
30620 .{ .src = .{ .to_mut_gpr, .none, .none } },
30621 },
30622 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30623 .clobbers = .{ .eflags = true },
30624 .each = .{ .once = &.{
30625 .{ ._, ._l, .sh, .src0q, .uia(64, .src0, .sub_bit_size), ._, ._ },
30626 .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ },
30627 } },
30628 }, .{
30629 .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any },
30630 .patterns = &.{
30631 .{ .src = .{ .to_mut_gpr, .none, .none } },
30632 },
30633 .extra_temps = .{
30634 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
30635 .unused,
30636 .unused,
30637 .unused,
30638 .unused,
30639 .unused,
30640 .unused,
30641 .unused,
30642 .unused,
30643 .unused,
30644 .unused,
30645 },
30646 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30647 .clobbers = .{ .eflags = true },
30648 .each = .{ .once = &.{
30649 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30650 .{ ._, ._r, .sh, .src0b, .ui(1), ._, ._ },
30651 .{ ._, ._, .@"and", .src0b, .ui(0x55), ._, ._ },
30652 .{ ._, ._, .sub, .tmp0b, .src0b, ._, ._ },
30653 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30654 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30655 .{ ._, ._, .@"and", .dst0b, .ui(0x33), ._, ._ },
30656 .{ ._, ._, .@"and", .tmp0b, .ui(0x33), ._, ._ },
30657 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30658 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30659 .{ ._, ._r, .sh, .dst0b, .ui(4), ._, ._ },
30660 .{ ._, ._, .@"and", .tmp0b, .ui(0x0f), ._, ._ },
30661 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30662 } },
30663 }, .{
30664 .src_constraints = .{ .{ .signed_int = .byte }, .any, .any },
30665 .patterns = &.{
30666 .{ .src = .{ .to_mut_gpr, .none, .none } },
30667 },
30668 .extra_temps = .{
30669 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
30670 .unused,
30671 .unused,
30672 .unused,
30673 .unused,
30674 .unused,
30675 .unused,
30676 .unused,
30677 .unused,
30678 .unused,
30679 .unused,
30680 },
30681 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30682 .clobbers = .{ .eflags = true },
30683 .each = .{ .once = &.{
30684 .{ ._, ._l, .sh, .src0b, .uia(8, .src0, .sub_bit_size), ._, ._ },
30685 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30686 .{ ._, ._r, .sh, .src0b, .ui(1), ._, ._ },
30687 .{ ._, ._, .@"and", .src0b, .ui(0x55), ._, ._ },
30688 .{ ._, ._, .sub, .tmp0b, .src0b, ._, ._ },
30689 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30690 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30691 .{ ._, ._, .@"and", .dst0b, .ui(0x33), ._, ._ },
30692 .{ ._, ._, .@"and", .tmp0b, .ui(0x33), ._, ._ },
30693 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30694 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30695 .{ ._, ._r, .sh, .dst0b, .ui(4), ._, ._ },
30696 .{ ._, ._, .@"and", .tmp0b, .ui(0x0f), ._, ._ },
30697 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30698 } },
30699 }, .{
30700 .required_features = .{ .fast_imm16, null, null, null },
30701 .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any },
30702 .patterns = &.{
30703 .{ .src = .{ .to_mut_gpr, .none, .none } },
30704 },
30705 .extra_temps = .{
30706 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
30707 .unused,
30708 .unused,
30709 .unused,
30710 .unused,
30711 .unused,
30712 .unused,
30713 .unused,
30714 .unused,
30715 .unused,
30716 .unused,
30717 },
30718 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30719 .clobbers = .{ .eflags = true },
30720 .each = .{ .once = &.{
30721 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30722 .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ },
30723 .{ ._, ._, .@"and", .src0w, .ui(0x5555), ._, ._ },
30724 .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ },
30725 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30726 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30727 .{ ._, ._, .@"and", .dst0w, .ui(0x3333), ._, ._ },
30728 .{ ._, ._, .@"and", .tmp0w, .ui(0x3333), ._, ._ },
30729 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ },
30730 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30731 .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ },
30732 .{ ._, ._, .@"and", .tmp0w, .ui(0x0f0f), ._, ._ },
30733 .{ ._, ._, .@"and", .dst0w, .ui(0x0f0f), ._, ._ },
30734 .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ },
30735 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30736 .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ },
30737 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30738 } },
30739 }, .{
30740 .required_features = .{ .fast_imm16, null, null, null },
30741 .src_constraints = .{ .{ .signed_int = .word }, .any, .any },
30742 .patterns = &.{
30743 .{ .src = .{ .to_mut_gpr, .none, .none } },
30744 },
30745 .extra_temps = .{
30746 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
30747 .unused,
30748 .unused,
30749 .unused,
30750 .unused,
30751 .unused,
30752 .unused,
30753 .unused,
30754 .unused,
30755 .unused,
30756 .unused,
30757 },
30758 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30759 .clobbers = .{ .eflags = true },
30760 .each = .{ .once = &.{
30761 .{ ._, ._l, .sh, .src0d, .uia(16, .src0, .sub_bit_size), ._, ._ },
30762 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30763 .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ },
30764 .{ ._, ._, .@"and", .src0w, .ui(0x5555), ._, ._ },
30765 .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ },
30766 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30767 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30768 .{ ._, ._, .@"and", .dst0w, .ui(0x3333), ._, ._ },
30769 .{ ._, ._, .@"and", .tmp0w, .ui(0x3333), ._, ._ },
30770 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ },
30771 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30772 .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ },
30773 .{ ._, ._, .@"and", .tmp0w, .ui(0x0f0f), ._, ._ },
30774 .{ ._, ._, .@"and", .dst0w, .ui(0x0f0f), ._, ._ },
30775 .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ },
30776 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30777 .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ },
30778 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30779 } },
30780 }, .{
30781 .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any },
30782 .patterns = &.{
30783 .{ .src = .{ .to_mut_gpr, .none, .none } },
30784 },
30785 .extra_temps = .{
30786 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
30787 .unused,
30788 .unused,
30789 .unused,
30790 .unused,
30791 .unused,
30792 .unused,
30793 .unused,
30794 .unused,
30795 .unused,
30796 .unused,
30797 },
30798 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30799 .clobbers = .{ .eflags = true },
30800 .each = .{ .once = &.{
30801 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30802 .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ },
30803 .{ ._, ._, .@"and", .src0d, .ui(0x5555), ._, ._ },
30804 .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ },
30805 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30806 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30807 .{ ._, ._, .@"and", .dst0d, .ui(0x3333), ._, ._ },
30808 .{ ._, ._, .@"and", .tmp0d, .ui(0x3333), ._, ._ },
30809 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ },
30810 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30811 .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ },
30812 .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f), ._, ._ },
30813 .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f), ._, ._ },
30814 .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ },
30815 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30816 .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ },
30817 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30818 } },
30819 }, .{
30820 .src_constraints = .{ .{ .signed_int = .word }, .any, .any },
30821 .patterns = &.{
30822 .{ .src = .{ .to_mut_gpr, .none, .none } },
30823 },
30824 .extra_temps = .{
30825 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
30826 .unused,
30827 .unused,
30828 .unused,
30829 .unused,
30830 .unused,
30831 .unused,
30832 .unused,
30833 .unused,
30834 .unused,
30835 .unused,
30836 },
30837 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30838 .clobbers = .{ .eflags = true },
30839 .each = .{ .once = &.{
30840 .{ ._, ._l, .sh, .src0d, .uia(16, .src0, .sub_bit_size), ._, ._ },
30841 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30842 .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ },
30843 .{ ._, ._, .@"and", .src0d, .ui(0x5555), ._, ._ },
30844 .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ },
30845 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30846 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30847 .{ ._, ._, .@"and", .dst0d, .ui(0x3333), ._, ._ },
30848 .{ ._, ._, .@"and", .tmp0d, .ui(0x3333), ._, ._ },
30849 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ },
30850 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30851 .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ },
30852 .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f), ._, ._ },
30853 .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f), ._, ._ },
30854 .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ },
30855 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30856 .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ },
30857 .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ },
30858 } },
30859 }, .{
30860 .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any },
30861 .patterns = &.{
30862 .{ .src = .{ .to_mut_gpr, .none, .none } },
30863 },
30864 .extra_temps = .{
30865 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
30866 .unused,
30867 .unused,
30868 .unused,
30869 .unused,
30870 .unused,
30871 .unused,
30872 .unused,
30873 .unused,
30874 .unused,
30875 .unused,
30876 },
30877 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30878 .clobbers = .{ .eflags = true },
30879 .each = .{ .once = &.{
30880 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30881 .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ },
30882 .{ ._, ._, .@"and", .src0d, .ui(0x55555555), ._, ._ },
30883 .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ },
30884 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30885 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30886 .{ ._, ._, .@"and", .dst0d, .ui(0x33333333), ._, ._ },
30887 .{ ._, ._, .@"and", .tmp0d, .ui(0x33333333), ._, ._ },
30888 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ },
30889 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30890 .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ },
30891 .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f0f0f), ._, ._ },
30892 .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f0f0f), ._, ._ },
30893 .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ },
30894 .{ ._, .i_, .mul, .dst0d, .tmp0d, .ui(0x01010101), ._ },
30895 .{ ._, ._r, .sh, .dst0d, .ui(32 - 8), ._, ._ },
30896 } },
30897 }, .{
30898 .src_constraints = .{ .{ .signed_int = .dword }, .any, .any },
30899 .patterns = &.{
30900 .{ .src = .{ .to_mut_gpr, .none, .none } },
30901 },
30902 .extra_temps = .{
30903 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
30904 .unused,
30905 .unused,
30906 .unused,
30907 .unused,
30908 .unused,
30909 .unused,
30910 .unused,
30911 .unused,
30912 .unused,
30913 .unused,
30914 },
30915 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30916 .clobbers = .{ .eflags = true },
30917 .each = .{ .once = &.{
30918 .{ ._, ._l, .sh, .src0d, .uia(32, .src0, .sub_bit_size), ._, ._ },
30919 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
30920 .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ },
30921 .{ ._, ._, .@"and", .src0d, .ui(0x55555555), ._, ._ },
30922 .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ },
30923 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
30924 .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ },
30925 .{ ._, ._, .@"and", .dst0d, .ui(0x33333333), ._, ._ },
30926 .{ ._, ._, .@"and", .tmp0d, .ui(0x33333333), ._, ._ },
30927 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ },
30928 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
30929 .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ },
30930 .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f0f0f), ._, ._ },
30931 .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f0f0f), ._, ._ },
30932 .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ },
30933 .{ ._, .i_, .mul, .dst0d, .tmp0d, .ui(0x01010101), ._ },
30934 .{ ._, ._r, .sh, .dst0d, .ui(32 - 8), ._, ._ },
30935 } },
30936 }, .{
30937 .required_features = .{ .@"64bit", null, null, null },
30938 .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any },
30939 .patterns = &.{
30940 .{ .src = .{ .to_mut_gpr, .none, .none } },
30941 },
30942 .extra_temps = .{
30943 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
30944 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
30945 .unused,
30946 .unused,
30947 .unused,
30948 .unused,
30949 .unused,
30950 .unused,
30951 .unused,
30952 .unused,
30953 .unused,
30954 },
30955 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30956 .clobbers = .{ .eflags = true },
30957 .each = .{ .once = &.{
30958 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
30959 .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ },
30960 .{ ._, ._, .mov, .tmp1q, .uia(0x55555555, .none, .repeat), ._, ._ },
30961 .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ },
30962 .{ ._, ._, .sub, .tmp0q, .src0q, ._, ._ },
30963 .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ },
30964 .{ ._, ._r, .sh, .tmp0q, .ui(2), ._, ._ },
30965 .{ ._, ._, .mov, .tmp1q, .uia(0x33333333, .none, .repeat), ._, ._ },
30966 .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ },
30967 .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ },
30968 .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ },
30969 .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ },
30970 .{ ._, ._r, .sh, .dst0q, .ui(4), ._, ._ },
30971 .{ ._, ._, .mov, .tmp1q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ },
30972 .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ },
30973 .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ },
30974 .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ },
30975 .{ ._, ._, .mov, .tmp1q, .uia(0x01010101, .none, .repeat), ._, ._ },
30976 .{ ._, .i_, .mul, .dst0q, .tmp1q, ._, ._ },
30977 .{ ._, ._r, .sh, .dst0q, .ui(64 - 8), ._, ._ },
30978 } },
30979 }, .{
30980 .required_features = .{ .@"64bit", null, null, null },
30981 .src_constraints = .{ .{ .signed_int = .qword }, .any, .any },
30982 .patterns = &.{
30983 .{ .src = .{ .to_mut_gpr, .none, .none } },
30984 },
30985 .extra_temps = .{
30986 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
30987 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
30988 .unused,
30989 .unused,
30990 .unused,
30991 .unused,
30992 .unused,
30993 .unused,
30994 .unused,
30995 .unused,
30996 .unused,
30997 },
30998 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30999 .clobbers = .{ .eflags = true },
31000 .each = .{ .once = &.{
31001 .{ ._, ._l, .sh, .src0q, .uia(64, .src0, .sub_bit_size), ._, ._ },
31002 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
31003 .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ },
31004 .{ ._, ._, .mov, .tmp1q, .uia(0x55555555, .none, .repeat), ._, ._ },
31005 .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ },
31006 .{ ._, ._, .sub, .tmp0q, .src0q, ._, ._ },
31007 .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ },
31008 .{ ._, ._r, .sh, .tmp0q, .ui(2), ._, ._ },
31009 .{ ._, ._, .mov, .tmp1q, .uia(0x33333333, .none, .repeat), ._, ._ },
31010 .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ },
31011 .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ },
31012 .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ },
31013 .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ },
31014 .{ ._, ._r, .sh, .dst0q, .ui(4), ._, ._ },
31015 .{ ._, ._, .mov, .tmp1q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ },
31016 .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ },
31017 .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ },
31018 .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ },
31019 .{ ._, ._, .mov, .tmp1q, .uia(0x01010101, .none, .repeat), ._, ._ },
31020 .{ ._, .i_, .mul, .dst0q, .tmp1q, ._, ._ },
31021 .{ ._, ._r, .sh, .dst0q, .ui(64 - 8), ._, ._ },
31022 } },
31023 }, .{
31024 .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null },
31025 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any },
31026 .patterns = &.{
31027 .{ .src = .{ .to_mem, .none, .none } },
31028 },
31029 .extra_temps = .{
31030 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31031 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31032 .unused,
31033 .unused,
31034 .unused,
31035 .unused,
31036 .unused,
31037 .unused,
31038 .unused,
31039 .unused,
31040 .unused,
31041 },
31042 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31043 .clobbers = .{ .eflags = true },
31044 .each = .{ .once = &.{
31045 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
31046 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
31047 .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ },
31048 .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31049 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31050 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31051 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31052 } },
31053 }, .{
31054 .required_features = .{ .@"64bit", .popcnt, null, null },
31055 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any },
31056 .patterns = &.{
31057 .{ .src = .{ .to_mem, .none, .none } },
31058 },
31059 .extra_temps = .{
31060 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31061 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31062 .unused,
31063 .unused,
31064 .unused,
31065 .unused,
31066 .unused,
31067 .unused,
31068 .unused,
31069 .unused,
31070 .unused,
31071 },
31072 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31073 .clobbers = .{ .eflags = true },
31074 .each = .{ .once = &.{
31075 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
31076 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
31077 .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31078 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31079 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31080 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31081 } },
31082 }, .{
31083 .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null },
31084 .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any },
31085 .patterns = &.{
31086 .{ .src = .{ .to_mem, .none, .none } },
31087 },
31088 .extra_temps = .{
31089 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31090 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31091 .unused,
31092 .unused,
31093 .unused,
31094 .unused,
31095 .unused,
31096 .unused,
31097 .unused,
31098 .unused,
31099 .unused,
31100 },
31101 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31102 .clobbers = .{ .eflags = true },
31103 .each = .{ .once = &.{
31104 .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -16), ._, ._ },
31105 .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ },
31106 .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ },
31107 .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ },
31108 .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ },
31109 .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31110 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31111 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31112 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31113 } },
31114 }, .{
31115 .required_features = .{ .@"64bit", .popcnt, null, null },
31116 .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any },
31117 .patterns = &.{
31118 .{ .src = .{ .to_mem, .none, .none } },
31119 },
31120 .extra_temps = .{
31121 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31122 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31123 .unused,
31124 .unused,
31125 .unused,
31126 .unused,
31127 .unused,
31128 .unused,
31129 .unused,
31130 .unused,
31131 .unused,
31132 },
31133 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31134 .clobbers = .{ .eflags = true },
31135 .each = .{ .once = &.{
31136 .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -16), ._, ._ },
31137 .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ },
31138 .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ },
31139 .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ },
31140 .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31141 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31142 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31143 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31144 } },
31145 }, .{
31146 .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null },
31147 .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any },
31148 .patterns = &.{
31149 .{ .src = .{ .to_mem, .none, .none } },
31150 },
31151 .extra_temps = .{
31152 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31153 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31154 .unused,
31155 .unused,
31156 .unused,
31157 .unused,
31158 .unused,
31159 .unused,
31160 .unused,
31161 .unused,
31162 .unused,
31163 },
31164 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31165 .clobbers = .{ .eflags = true },
31166 .each = .{ .once = &.{
31167 .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -8), ._, ._ },
31168 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
31169 .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ },
31170 .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ },
31171 .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ },
31172 .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31173 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31174 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31175 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31176 } },
31177 }, .{
31178 .required_features = .{ .@"64bit", .popcnt, null, null },
31179 .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any },
31180 .patterns = &.{
31181 .{ .src = .{ .to_mem, .none, .none } },
31182 },
31183 .extra_temps = .{
31184 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31185 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31186 .unused,
31187 .unused,
31188 .unused,
31189 .unused,
31190 .unused,
31191 .unused,
31192 .unused,
31193 .unused,
31194 .unused,
31195 },
31196 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31197 .clobbers = .{ .eflags = true },
31198 .each = .{ .once = &.{
31199 .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -8), ._, ._ },
31200 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
31201 .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ },
31202 .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ },
31203 .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31204 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31205 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31206 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31207 } },
31208 }, .{
31209 .required_features = .{ .@"64bit", null, null, null },
31210 .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any },
31211 .patterns = &.{
31212 .{ .src = .{ .to_mem, .none, .none } },
31213 },
31214 .extra_temps = .{
31215 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31216 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31217 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31218 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31219 .unused,
31220 .unused,
31221 .unused,
31222 .unused,
31223 .unused,
31224 .unused,
31225 .unused,
31226 },
31227 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31228 .clobbers = .{ .eflags = true },
31229 .each = .{ .once = &.{
31230 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
31231 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
31232 .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31233 .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ },
31234 .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ },
31235 .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ },
31236 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31237 .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ },
31238 .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ },
31239 .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ },
31240 .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ },
31241 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31242 .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ },
31243 .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ },
31244 .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ },
31245 .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ },
31246 .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ },
31247 .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ },
31248 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31249 .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ },
31250 .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ },
31251 .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ },
31252 .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ },
31253 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31254 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31255 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31256 } },
31257 }, .{
31258 .required_features = .{ .@"64bit", null, null, null },
31259 .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any },
31260 .patterns = &.{
31261 .{ .src = .{ .to_mem, .none, .none } },
31262 },
31263 .extra_temps = .{
31264 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31265 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31266 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31267 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31268 .unused,
31269 .unused,
31270 .unused,
31271 .unused,
31272 .unused,
31273 .unused,
31274 .unused,
31275 },
31276 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31277 .clobbers = .{ .eflags = true },
31278 .each = .{ .once = &.{
31279 .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ },
31280 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
31281 .{ ._, ._l, .sh, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ },
31282 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
31283 .{ ._, ._mp, .j, .@"1f", ._, ._, ._ },
31284 .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31285 .{ .@"1:", ._, .mov, .tmp2q, .tmp1q, ._, ._ },
31286 .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ },
31287 .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ },
31288 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31289 .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ },
31290 .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ },
31291 .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ },
31292 .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ },
31293 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31294 .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ },
31295 .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ },
31296 .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ },
31297 .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ },
31298 .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ },
31299 .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ },
31300 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31301 .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ },
31302 .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ },
31303 .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ },
31304 .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ },
31305 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31306 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31307 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31308 } },
31309 }, .{
31310 .required_features = .{ .@"64bit", null, null, null },
31311 .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any },
31312 .patterns = &.{
31313 .{ .src = .{ .to_mem, .none, .none } },
31314 },
31315 .extra_temps = .{
31316 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
31317 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31318 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31319 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
31320 .unused,
31321 .unused,
31322 .unused,
31323 .unused,
31324 .unused,
31325 .unused,
31326 .unused,
31327 },
31328 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
31329 .clobbers = .{ .eflags = true },
31330 .each = .{ .once = &.{
31331 .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ },
31332 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
31333 .{ ._, ._l, .sh, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ },
31334 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
31335 .{ ._, ._mp, .j, .@"1f", ._, ._, ._ },
31336 .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
31337 .{ .@"1:", ._, .mov, .tmp2q, .tmp1q, ._, ._ },
31338 .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ },
31339 .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ },
31340 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31341 .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ },
31342 .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ },
31343 .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ },
31344 .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ },
31345 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31346 .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ },
31347 .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ },
31348 .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ },
31349 .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ },
31350 .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ },
31351 .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ },
31352 .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ },
31353 .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ },
31354 .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ },
31355 .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ },
31356 .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ },
31357 .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ },
31358 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
31359 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
31360 } },
31361 } }) catch |err| switch (err) {
31362 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
31363 @tagName(air_tag),
31364 cg.typeOf(ty_op.operand).fmt(pt),
31365 ops[0].tracking(cg),
31366 }),
31367 else => |e| return e,
31368 };
31369 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
31370 },
30087 .byte_swap => |air_tag| if (use_old) try cg.airByteSwap(inst) else fallback: {31371 .byte_swap => |air_tag| if (use_old) try cg.airByteSwap(inst) else fallback: {
30088 const ty_op = air_datas[@intFromEnum(inst)].ty_op;31372 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
30089 if (ty_op.ty.toType().isVector(zcu)) break :fallback try cg.airByteSwap(inst);31373 if (ty_op.ty.toType().isVector(zcu)) break :fallback try cg.airByteSwap(inst);
...@@ -99931,6 +101215,7 @@ const Select = struct {...@@ -99931,6 +101215,7 @@ const Select = struct {
99931 unaligned_size,101215 unaligned_size,
99932 bit_size,101216 bit_size,
99933 src0_bit_size,101217 src0_bit_size,
101218 @"8_size_sub_bit_size",
99934 len,101219 len,
99935 elem_limbs,101220 elem_limbs,
99936 elem_size,101221 elem_size,
...@@ -99942,6 +101227,7 @@ const Select = struct {...@@ -99942,6 +101227,7 @@ const Select = struct {
99942 smin,101227 smin,
99943 smax,101228 smax,
99944 umax,101229 umax,
101230 repeat,
99945 },101231 },
99946 op: enum(u2) { mul, div, div_8_down, rem_8_mul },101232 op: enum(u2) { mul, div, div_8_down, rem_8_mul },
99947 rhs: Memory.Scale,101233 rhs: Memory.Scale,
...@@ -99958,6 +101244,7 @@ const Select = struct {...@@ -99958,6 +101244,7 @@ const Select = struct {
99958 const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" };101244 const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" };
99959 const sub_src0_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .div, .rhs = .@"8" };101245 const sub_src0_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .div, .rhs = .@"8" };
99960 const sub_src0_size: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .mul, .rhs = .@"1" };101246 const sub_src0_size: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .mul, .rhs = .@"1" };
101247 const add_8_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"8" };
99961 const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" };101248 const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" };
99962 const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" };101249 const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" };
99963 const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" };101250 const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" };
...@@ -99972,6 +101259,7 @@ const Select = struct {...@@ -99972,6 +101259,7 @@ const Select = struct {
99972 const sub_bit_size: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .mul, .rhs = .@"1" };101259 const sub_bit_size: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .mul, .rhs = .@"1" };
99973 const add_src0_bit_size: Adjust = .{ .sign = .pos, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" };101260 const add_src0_bit_size: Adjust = .{ .sign = .pos, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" };
99974 const sub_src0_bit_size: Adjust = .{ .sign = .neg, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" };101261 const sub_src0_bit_size: Adjust = .{ .sign = .neg, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" };
101262 const add_bit_size_sub_8_size: Adjust = .{ .sign = .neg, .lhs = .@"8_size_sub_bit_size", .op = .mul, .rhs = .@"1" };
99975 const add_8_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"8" };101263 const add_8_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"8" };
99976 const add_4_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"4" };101264 const add_4_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"4" };
99977 const add_3_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"3" };101265 const add_3_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"3" };
...@@ -100000,6 +101288,7 @@ const Select = struct {...@@ -100000,6 +101288,7 @@ const Select = struct {
100000 const add_elem_limbs: Adjust = .{ .sign = .pos, .lhs = .elem_limbs, .op = .mul, .rhs = .@"1" };101288 const add_elem_limbs: Adjust = .{ .sign = .pos, .lhs = .elem_limbs, .op = .mul, .rhs = .@"1" };
100001 const add_smin: Adjust = .{ .sign = .pos, .lhs = .smin, .op = .mul, .rhs = .@"1" };101289 const add_smin: Adjust = .{ .sign = .pos, .lhs = .smin, .op = .mul, .rhs = .@"1" };
100002 const add_umax: Adjust = .{ .sign = .pos, .lhs = .umax, .op = .mul, .rhs = .@"1" };101290 const add_umax: Adjust = .{ .sign = .pos, .lhs = .umax, .op = .mul, .rhs = .@"1" };
101291 const repeat: Adjust = .{ .sign = .pos, .lhs = .repeat, .op = .mul, .rhs = .@"1" };
100003 };101292 };
100004 const Ref = enum(u5) {101293 const Ref = enum(u5) {
100005 tmp0,101294 tmp0,
...@@ -100683,6 +101972,10 @@ const Select = struct {...@@ -100683,6 +101972,10 @@ const Select = struct {
100683 .unaligned_size => @intCast(s.cg.unalignedSize(op.flags.base.ref.typeOf(s))),101972 .unaligned_size => @intCast(s.cg.unalignedSize(op.flags.base.ref.typeOf(s))),
100684 .bit_size => @intCast(op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),101973 .bit_size => @intCast(op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
100685 .src0_bit_size => @intCast(Select.Operand.Ref.src0.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),101974 .src0_bit_size => @intCast(Select.Operand.Ref.src0.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
101975 .@"8_size_sub_bit_size" => {
101976 const ty = op.flags.base.ref.typeOf(s);
101977 break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu));
101978 },
100686 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),101979 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),
100687 .elem_limbs => @intCast(@divExact(101980 .elem_limbs => @intCast(@divExact(
100688 op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),101981 op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
...@@ -100704,6 +101997,10 @@ const Select = struct {...@@ -100704,6 +101997,10 @@ const Select = struct {
100704 .umax => @bitCast(@as(UnsignedImm, std.math.maxInt(UnsignedImm)) >> @truncate(101997 .umax => @bitCast(@as(UnsignedImm, std.math.maxInt(UnsignedImm)) >> @truncate(
100705 -%op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),101998 -%op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
100706 )),101999 )),
102000 .repeat => switch (SignedImm) {
102001 else => unreachable,
102002 i64 => return @as(i64, op.imm) << 32 | @as(u32, @bitCast(op.imm)),
102003 },
100707 };102004 };
100708 const rhs = op.flags.adjust.rhs.toLog2();102005 const rhs = op.flags.adjust.rhs.toLog2();
100709 const op_res = op_res: switch (op.flags.adjust.op) {102006 const op_res = op_res: switch (op.flags.adjust.op) {
test/behavior/x86_64/math.zig+16
...@@ -19273,6 +19273,22 @@ test clz {...@@ -19273,6 +19273,22 @@ test clz {
19273 try test_clz.testIntVectors();19273 try test_clz.testIntVectors();
19274}19274}
1927519275
19276inline fn ctz(comptime Type: type, rhs: Type) @TypeOf(@ctz(rhs)) {
19277 return @ctz(rhs);
19278}
19279test ctz {
19280 const test_ctz = unary(ctz, .{});
19281 try test_ctz.testInts();
19282}
19283
19284inline fn popCount(comptime Type: type, rhs: Type) @TypeOf(@popCount(rhs)) {
19285 return @popCount(rhs);
19286}
19287test popCount {
19288 const test_pop_count = unary(popCount, .{});
19289 try test_pop_count.testInts();
19290}
19291
19276inline fn byteSwap(comptime Type: type, rhs: Type) RoundBitsUp(Type, 8) {19292inline fn byteSwap(comptime Type: type, rhs: Type) RoundBitsUp(Type, 8) {
19277 return @byteSwap(@as(RoundBitsUp(Type, 8), rhs));19293 return @byteSwap(@as(RoundBitsUp(Type, 8), rhs));
19278}19294}