authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-02-18 04:45:44-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-02-18 09:47:44-05:00
logebea56d279ecb09fa829881ee49bb070b6c4d833
treec321a6911bdce14aa5d248ef2e7dd6e30f57c8f3
parent0779e847f79851419dfeb39595b1817ce72ea9fa

x86_64: rewrite scalar `@ctz`

4 files changed, 640 insertions(+), 5 deletions(-)

lib/std/math/big/int.zig+2-2
......@@ -2544,8 +2544,7 @@ pub const Const = struct {
25442544 const bits_per_limb = @bitSizeOf(Limb);
25452545 while (i != 0) {
25462546 i -= 1;
2547 const limb = a.limbs[i];
2548 const this_limb_lz = @clz(limb);
2547 const this_limb_lz = @clz(a.limbs[i]);
25492548 total_limb_lz += this_limb_lz;
25502549 if (this_limb_lz != bits_per_limb) break;
25512550 }
......@@ -2557,6 +2556,7 @@ pub const Const = struct {
25572556 pub fn ctz(a: Const, bits: Limb) Limb {
25582557 // Limbs are stored in little-endian order. Converting a negative number to twos-complement
25592558 // flips all bits above the lowest set bit, which does not affect the trailing zero count.
2559 if (a.eqlZero()) return bits;
25602560 var result: Limb = 0;
25612561 for (a.limbs) |limb| {
25622562 const limb_tz = @ctz(limb);
lib/std/math/big/int_test.zig+224
......@@ -3332,3 +3332,227 @@ test "(BigInt) negative" {
33323332 try testing.expect(mem.eql(u8, a_fmt, "(BigInt)"));
33333333 try testing.expect(!mem.eql(u8, b_fmt, "(BigInt)"));
33343334}
3335
3336test "clz" {
3337 const neg_limb_max_squared: std.math.big.int.Const = .{
3338 .limbs = &.{ 1, maxInt(Limb) - 1 },
3339 .positive = false,
3340 };
3341 try testing.expect(neg_limb_max_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3342
3343 const neg_limb_max_squared_plus_one: std.math.big.int.Const = .{
3344 .limbs = &.{ 0, maxInt(Limb) - 1 },
3345 .positive = false,
3346 };
3347 try testing.expect(neg_limb_max_squared_plus_one.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3348
3349 const neg_limb_msb_squared: std.math.big.int.Const = .{
3350 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3351 .positive = false,
3352 };
3353 try testing.expect(neg_limb_msb_squared.clz(@bitSizeOf(Limb) * 2) == 0);
3354 try testing.expect(neg_limb_msb_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3355
3356 const neg_limb_max: std.math.big.int.Const = .{
3357 .limbs = &.{maxInt(Limb)},
3358 .positive = false,
3359 };
3360 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) + 1) == 0);
3361 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3362 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) * 2) == 0);
3363 try testing.expect(neg_limb_max.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3364
3365 const neg_limb_msb: std.math.big.int.Const = .{
3366 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3367 .positive = false,
3368 };
3369 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb)) == 0);
3370 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) + 1) == 0);
3371 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3372 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) * 2) == 0);
3373 try testing.expect(neg_limb_msb.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3374
3375 const neg_one: std.math.big.int.Const = .{
3376 .limbs = &.{1},
3377 .positive = false,
3378 };
3379 try testing.expect(neg_one.clz(@bitSizeOf(Limb)) == 0);
3380 try testing.expect(neg_one.clz(@bitSizeOf(Limb) + 1) == 0);
3381 try testing.expect(neg_one.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3382 try testing.expect(neg_one.clz(@bitSizeOf(Limb) * 2) == 0);
3383 try testing.expect(neg_one.clz(@bitSizeOf(Limb) * 2 + 1) == 0);
3384
3385 const zero: std.math.big.int.Const = .{
3386 .limbs = &.{0},
3387 .positive = true,
3388 };
3389 try testing.expect(zero.clz(@bitSizeOf(Limb)) == @bitSizeOf(Limb));
3390 try testing.expect(zero.clz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) + 1);
3391 try testing.expect(zero.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 1);
3392 try testing.expect(zero.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2);
3393 try testing.expect(zero.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 + 1);
3394
3395 const one: std.math.big.int.Const = .{
3396 .limbs = &.{1},
3397 .positive = true,
3398 };
3399 try testing.expect(one.clz(@bitSizeOf(Limb)) == @bitSizeOf(Limb) - 1);
3400 try testing.expect(one.clz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb));
3401 try testing.expect(one.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 2);
3402 try testing.expect(one.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2 - 1);
3403 try testing.expect(one.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2);
3404
3405 const limb_msb: std.math.big.int.Const = .{
3406 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3407 .positive = true,
3408 };
3409 try testing.expect(limb_msb.clz(@bitSizeOf(Limb)) == 0);
3410 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) + 1) == 1);
3411 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3412 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb));
3413 try testing.expect(limb_msb.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3414
3415 const limb_max: std.math.big.int.Const = .{
3416 .limbs = &.{maxInt(Limb)},
3417 .positive = true,
3418 };
3419 try testing.expect(limb_max.clz(@bitSizeOf(Limb)) == 0);
3420 try testing.expect(limb_max.clz(@bitSizeOf(Limb) + 1) == 1);
3421 try testing.expect(limb_max.clz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3422 try testing.expect(limb_max.clz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb));
3423 try testing.expect(limb_max.clz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3424
3425 const limb_msb_squared: std.math.big.int.Const = .{
3426 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3427 .positive = true,
3428 };
3429 try testing.expect(limb_msb_squared.clz(@bitSizeOf(Limb) * 2 - 1) == 0);
3430 try testing.expect(limb_msb_squared.clz(@bitSizeOf(Limb) * 2) == 1);
3431 try testing.expect(limb_msb_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 2);
3432
3433 const limb_max_squared_minus_one: std.math.big.int.Const = .{
3434 .limbs = &.{ 0, maxInt(Limb) - 1 },
3435 .positive = true,
3436 };
3437 try testing.expect(limb_max_squared_minus_one.clz(@bitSizeOf(Limb) * 2) == 0);
3438 try testing.expect(limb_max_squared_minus_one.clz(@bitSizeOf(Limb) * 2 + 1) == 1);
3439
3440 const limb_max_squared: std.math.big.int.Const = .{
3441 .limbs = &.{ 1, maxInt(Limb) - 1 },
3442 .positive = true,
3443 };
3444 try testing.expect(limb_max_squared.clz(@bitSizeOf(Limb) * 2) == 0);
3445 try testing.expect(limb_max_squared.clz(@bitSizeOf(Limb) * 2 + 1) == 1);
3446}
3447
3448test "ctz" {
3449 const neg_limb_max_squared: std.math.big.int.Const = .{
3450 .limbs = &.{ 1, maxInt(Limb) - 1 },
3451 .positive = false,
3452 };
3453 try testing.expect(neg_limb_max_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3454
3455 const neg_limb_max_squared_plus_one: std.math.big.int.Const = .{
3456 .limbs = &.{ 0, maxInt(Limb) - 1 },
3457 .positive = false,
3458 };
3459 try testing.expect(neg_limb_max_squared_plus_one.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3460
3461 const neg_limb_msb_squared: std.math.big.int.Const = .{
3462 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3463 .positive = false,
3464 };
3465 try testing.expect(neg_limb_msb_squared.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2 - 2);
3466 try testing.expect(neg_limb_msb_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 - 2);
3467
3468 const neg_limb_max: std.math.big.int.Const = .{
3469 .limbs = &.{maxInt(Limb)},
3470 .positive = false,
3471 };
3472 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) + 1) == 0);
3473 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3474 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) * 2) == 0);
3475 try testing.expect(neg_limb_max.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3476
3477 const neg_limb_msb: std.math.big.int.Const = .{
3478 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3479 .positive = false,
3480 };
3481 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb)) == @bitSizeOf(Limb) - 1);
3482 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) - 1);
3483 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3484 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) - 1);
3485 try testing.expect(neg_limb_msb.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) - 1);
3486
3487 const neg_one: std.math.big.int.Const = .{
3488 .limbs = &.{1},
3489 .positive = false,
3490 };
3491 try testing.expect(neg_one.ctz(@bitSizeOf(Limb)) == 0);
3492 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) + 1) == 0);
3493 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3494 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) * 2) == 0);
3495 try testing.expect(neg_one.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3496
3497 const zero: std.math.big.int.Const = .{
3498 .limbs = &.{0},
3499 .positive = true,
3500 };
3501 try testing.expect(zero.ctz(@bitSizeOf(Limb)) == @bitSizeOf(Limb));
3502 try testing.expect(zero.ctz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) + 1);
3503 try testing.expect(zero.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 1);
3504 try testing.expect(zero.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2);
3505 try testing.expect(zero.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 + 1);
3506
3507 const one: std.math.big.int.Const = .{
3508 .limbs = &.{1},
3509 .positive = true,
3510 };
3511 try testing.expect(one.ctz(@bitSizeOf(Limb)) == 0);
3512 try testing.expect(one.ctz(@bitSizeOf(Limb) + 1) == 0);
3513 try testing.expect(one.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3514 try testing.expect(one.ctz(@bitSizeOf(Limb) * 2) == 0);
3515 try testing.expect(one.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3516
3517 const limb_msb: std.math.big.int.Const = .{
3518 .limbs = &.{1 << @bitSizeOf(Limb) - 1},
3519 .positive = true,
3520 };
3521 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb)) == @bitSizeOf(Limb) - 1);
3522 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) + 1) == @bitSizeOf(Limb) - 1);
3523 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) - 1);
3524 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) - 1);
3525 try testing.expect(limb_msb.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) - 1);
3526
3527 const limb_max: std.math.big.int.Const = .{
3528 .limbs = &.{maxInt(Limb)},
3529 .positive = true,
3530 };
3531 try testing.expect(limb_max.ctz(@bitSizeOf(Limb)) == 0);
3532 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) + 1) == 0);
3533 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) * 2 - 1) == 0);
3534 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) * 2) == 0);
3535 try testing.expect(limb_max.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3536
3537 const limb_msb_squared: std.math.big.int.Const = .{
3538 .limbs = &.{ 0, 1 << @bitSizeOf(Limb) - 2 },
3539 .positive = true,
3540 };
3541 try testing.expect(limb_msb_squared.ctz(@bitSizeOf(Limb) * 2 - 1) == @bitSizeOf(Limb) * 2 - 2);
3542 try testing.expect(limb_msb_squared.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) * 2 - 2);
3543 try testing.expect(limb_msb_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) * 2 - 2);
3544
3545 const limb_max_squared_minus_one: std.math.big.int.Const = .{
3546 .limbs = &.{ 0, maxInt(Limb) - 1 },
3547 .positive = true,
3548 };
3549 try testing.expect(limb_max_squared_minus_one.ctz(@bitSizeOf(Limb) * 2) == @bitSizeOf(Limb) + 1);
3550 try testing.expect(limb_max_squared_minus_one.ctz(@bitSizeOf(Limb) * 2 + 1) == @bitSizeOf(Limb) + 1);
3551
3552 const limb_max_squared: std.math.big.int.Const = .{
3553 .limbs = &.{ 1, maxInt(Limb) - 1 },
3554 .positive = true,
3555 };
3556 try testing.expect(limb_max_squared.ctz(@bitSizeOf(Limb) * 2) == 0);
3557 try testing.expect(limb_max_squared.ctz(@bitSizeOf(Limb) * 2 + 1) == 0);
3558}
src/arch/x86_64/CodeGen.zig+406-3
......@@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24182418}
24192419
24202420fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2421 @setEvalBranchQuota(12_600);
2421 @setEvalBranchQuota(12_700);
24222422 const pt = cg.pt;
24232423 const zcu = pt.zcu;
24242424 const ip = &zcu.intern_pool;
......@@ -2475,7 +2475,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24752475
24762476 .bitcast => try cg.airBitCast(inst),
24772477
2478 .ctz => try cg.airCtz(inst),
24792478 .popcount => try cg.airPopCount(inst),
24802479 .bit_reverse => try cg.airBitReverse(inst),
24812480 .splat => try cg.airSplat(inst),
......@@ -26978,7 +26977,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2697826977 .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }),
2697926978 .call_never_tail => try cg.airCall(inst, .never_tail, .{ .safety = true }),
2698026979 .call_never_inline => try cg.airCall(inst, .never_inline, .{ .safety = true }),
26981
2698226980 .clz => |air_tag| if (use_old) try cg.airClz(inst) else {
2698326981 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
2698426982 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
......@@ -30084,6 +30082,404 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3008430082 };
3008530083 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
3008630084 },
30085 .ctz => |air_tag| if (use_old) try cg.airCtz(inst) else fallback: {
30086 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
30087 if (ty_op.ty.toType().isVector(zcu)) break :fallback try cg.airCtz(inst);
30088 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
30089 var res: [1]Temp = undefined;
30090 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
30091 .required_features = .{ .slow_incdec, null, null, null },
30092 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any },
30093 .patterns = &.{
30094 .{ .src = .{ .mut_mem, .none, .none } },
30095 .{ .src = .{ .to_mut_gpr, .none, .none } },
30096 },
30097 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30098 .clobbers = .{ .eflags = true },
30099 .each = .{ .once = &.{
30100 .{ ._, ._, .add, .dst0b, .si(1), ._, ._ },
30101 } },
30102 }, .{
30103 .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any },
30104 .patterns = &.{
30105 .{ .src = .{ .mut_mem, .none, .none } },
30106 .{ .src = .{ .to_mut_gpr, .none, .none } },
30107 },
30108 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30109 .clobbers = .{ .eflags = true },
30110 .each = .{ .once = &.{
30111 .{ ._, ._c, .in, .dst0b, ._, ._, ._ },
30112 } },
30113 }, .{
30114 .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any },
30115 .patterns = &.{
30116 .{ .src = .{ .mut_mem, .none, .none } },
30117 .{ .src = .{ .to_mut_gpr, .none, .none } },
30118 },
30119 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30120 .clobbers = .{ .eflags = true },
30121 .each = .{ .once = &.{
30122 .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ },
30123 } },
30124 }, .{
30125 .required_features = .{ .bmi, null, null, null },
30126 .src_constraints = .{ .{ .exact_int = 16 }, .any, .any },
30127 .patterns = &.{
30128 .{ .src = .{ .mem, .none, .none } },
30129 .{ .src = .{ .to_gpr, .none, .none } },
30130 },
30131 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30132 .clobbers = .{ .eflags = true },
30133 .each = .{ .once = &.{
30134 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30135 .{ ._, ._, .tzcnt, .dst0w, .src0w, ._, ._ },
30136 } },
30137 }, .{
30138 .required_features = .{ .bmi, .false_deps_lzcnt_tzcnt, null, null },
30139 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30140 .patterns = &.{
30141 .{ .src = .{ .mem, .none, .none } },
30142 .{ .src = .{ .to_gpr, .none, .none } },
30143 },
30144 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30145 .clobbers = .{ .eflags = true },
30146 .each = .{ .once = &.{
30147 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30148 .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ },
30149 } },
30150 }, .{
30151 .required_features = .{ .bmi, null, null, null },
30152 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30153 .patterns = &.{
30154 .{ .src = .{ .mem, .none, .none } },
30155 .{ .src = .{ .to_gpr, .none, .none } },
30156 },
30157 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30158 .clobbers = .{ .eflags = true },
30159 .each = .{ .once = &.{
30160 .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ },
30161 } },
30162 }, .{
30163 .required_features = .{ .bmi, null, null, null },
30164 .src_constraints = .{ .{ .int = .dword }, .any, .any },
30165 .patterns = &.{
30166 .{ .src = .{ .to_mut_gpr, .none, .none } },
30167 },
30168 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30169 .clobbers = .{ .eflags = true },
30170 .each = .{ .once = &.{
30171 .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ },
30172 .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ },
30173 } },
30174 }, .{
30175 .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null },
30176 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30177 .patterns = &.{
30178 .{ .src = .{ .mem, .none, .none } },
30179 .{ .src = .{ .to_gpr, .none, .none } },
30180 },
30181 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30182 .clobbers = .{ .eflags = true },
30183 .each = .{ .once = &.{
30184 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
30185 .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ },
30186 } },
30187 }, .{
30188 .required_features = .{ .@"64bit", .bmi, null, null },
30189 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30190 .patterns = &.{
30191 .{ .src = .{ .mem, .none, .none } },
30192 .{ .src = .{ .to_gpr, .none, .none } },
30193 },
30194 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30195 .clobbers = .{ .eflags = true },
30196 .each = .{ .once = &.{
30197 .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ },
30198 } },
30199 }, .{
30200 .required_features = .{ .@"64bit", .bmi, null, null },
30201 .src_constraints = .{ .{ .int = .qword }, .any, .any },
30202 .patterns = &.{
30203 .{ .src = .{ .to_mut_gpr, .none, .none } },
30204 },
30205 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30206 .clobbers = .{ .eflags = true },
30207 .each = .{ .once = &.{
30208 .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ },
30209 .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ },
30210 } },
30211 }, .{
30212 .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null },
30213 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30214 .patterns = &.{
30215 .{ .src = .{ .mem, .none, .none } },
30216 .{ .src = .{ .to_gpr, .none, .none } },
30217 },
30218 .extra_temps = .{
30219 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
30220 .unused,
30221 .unused,
30222 .unused,
30223 .unused,
30224 .unused,
30225 .unused,
30226 .unused,
30227 .unused,
30228 .unused,
30229 .unused,
30230 },
30231 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30232 .clobbers = .{ .eflags = true },
30233 .each = .{ .once = &.{
30234 .{ ._, ._, .mov, .tmp0d, .si(32), ._, ._ },
30235 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30236 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
30237 } },
30238 }, .{
30239 .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null },
30240 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30241 .patterns = &.{
30242 .{ .src = .{ .mem, .none, .none } },
30243 .{ .src = .{ .to_gpr, .none, .none } },
30244 },
30245 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30246 .clobbers = .{ .eflags = true },
30247 .each = .{ .once = &.{
30248 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30249 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30250 .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ },
30251 } },
30252 }, .{
30253 .src_constraints = .{ .{ .exact_int = 16 }, .any, .any },
30254 .patterns = &.{
30255 .{ .src = .{ .mem, .none, .none } },
30256 .{ .src = .{ .to_gpr, .none, .none } },
30257 },
30258 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30259 .clobbers = .{ .eflags = true },
30260 .each = .{ .once = &.{
30261 .{ ._, ._, .mov, .dst0d, .si(16), ._, ._ },
30262 .{ ._, ._f, .bs, .dst0w, .src0w, ._, ._ },
30263 } },
30264 }, .{
30265 .src_constraints = .{ .{ .exact_int = 32 }, .any, .any },
30266 .patterns = &.{
30267 .{ .src = .{ .mem, .none, .none } },
30268 .{ .src = .{ .to_gpr, .none, .none } },
30269 },
30270 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30271 .clobbers = .{ .eflags = true },
30272 .each = .{ .once = &.{
30273 .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ },
30274 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30275 } },
30276 }, .{
30277 .src_constraints = .{ .{ .int = .dword }, .any, .any },
30278 .patterns = &.{
30279 .{ .src = .{ .to_mut_gpr, .none, .none } },
30280 },
30281 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30282 .clobbers = .{ .eflags = true },
30283 .each = .{ .once = &.{
30284 .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ },
30285 .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ },
30286 } },
30287 }, .{
30288 .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null },
30289 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30290 .patterns = &.{
30291 .{ .src = .{ .mem, .none, .none } },
30292 .{ .src = .{ .to_gpr, .none, .none } },
30293 },
30294 .extra_temps = .{
30295 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
30296 .unused,
30297 .unused,
30298 .unused,
30299 .unused,
30300 .unused,
30301 .unused,
30302 .unused,
30303 .unused,
30304 .unused,
30305 .unused,
30306 },
30307 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30308 .clobbers = .{ .eflags = true },
30309 .each = .{ .once = &.{
30310 .{ ._, ._, .mov, .tmp0d, .si(64), ._, ._ },
30311 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30312 .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ },
30313 } },
30314 }, .{
30315 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
30316 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30317 .patterns = &.{
30318 .{ .src = .{ .mem, .none, .none } },
30319 .{ .src = .{ .to_gpr, .none, .none } },
30320 },
30321 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
30322 .clobbers = .{ .eflags = true },
30323 .each = .{ .once = &.{
30324 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30325 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30326 .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ },
30327 } },
30328 }, .{
30329 .src_constraints = .{ .{ .exact_int = 64 }, .any, .any },
30330 .patterns = &.{
30331 .{ .src = .{ .mem, .none, .none } },
30332 .{ .src = .{ .to_gpr, .none, .none } },
30333 },
30334 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30335 .clobbers = .{ .eflags = true },
30336 .each = .{ .once = &.{
30337 .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ },
30338 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30339 } },
30340 }, .{
30341 .src_constraints = .{ .{ .int = .qword }, .any, .any },
30342 .patterns = &.{
30343 .{ .src = .{ .to_mut_gpr, .none, .none } },
30344 },
30345 .dst_temps = .{ .{ .ref = .src0 }, .unused },
30346 .clobbers = .{ .eflags = true },
30347 .each = .{ .once = &.{
30348 .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ },
30349 .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ },
30350 } },
30351 }, .{
30352 .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null },
30353 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30354 .patterns = &.{
30355 .{ .src = .{ .to_mem, .none, .none } },
30356 },
30357 .extra_temps = .{
30358 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30359 .unused,
30360 .unused,
30361 .unused,
30362 .unused,
30363 .unused,
30364 .unused,
30365 .unused,
30366 .unused,
30367 .unused,
30368 .unused,
30369 },
30370 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30371 .clobbers = .{ .eflags = true },
30372 .each = .{ .once = &.{
30373 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30374 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
30375 .{ ._, ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30376 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
30377 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30378 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30379 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30380 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30381 } },
30382 }, .{
30383 .required_features = .{ .@"64bit", .bmi, null, null },
30384 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30385 .patterns = &.{
30386 .{ .src = .{ .to_mem, .none, .none } },
30387 },
30388 .extra_temps = .{
30389 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30390 .unused,
30391 .unused,
30392 .unused,
30393 .unused,
30394 .unused,
30395 .unused,
30396 .unused,
30397 .unused,
30398 .unused,
30399 .unused,
30400 },
30401 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30402 .clobbers = .{ .eflags = true },
30403 .each = .{ .once = &.{
30404 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30405 .{ .@"0:", ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30406 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
30407 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30408 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30409 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30410 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30411 } },
30412 }, .{
30413 .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null },
30414 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30415 .patterns = &.{
30416 .{ .src = .{ .to_mem, .none, .none } },
30417 },
30418 .extra_temps = .{
30419 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30420 .unused,
30421 .unused,
30422 .unused,
30423 .unused,
30424 .unused,
30425 .unused,
30426 .unused,
30427 .unused,
30428 .unused,
30429 .unused,
30430 },
30431 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30432 .clobbers = .{ .eflags = true },
30433 .each = .{ .once = &.{
30434 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30435 .{ .@"0:", ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30436 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30437 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30438 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30439 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30440 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30441 } },
30442 }, .{
30443 .required_features = .{ .@"64bit", null, null, null },
30444 .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
30445 .patterns = &.{
30446 .{ .src = .{ .to_mem, .none, .none } },
30447 },
30448 .extra_temps = .{
30449 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
30450 .unused,
30451 .unused,
30452 .unused,
30453 .unused,
30454 .unused,
30455 .unused,
30456 .unused,
30457 .unused,
30458 .unused,
30459 .unused,
30460 },
30461 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
30462 .clobbers = .{ .eflags = true },
30463 .each = .{ .once = &.{
30464 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
30465 .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ },
30466 .{ ._, ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
30467 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
30468 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
30469 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
30470 .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ },
30471 .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ },
30472 } },
30473 } }) catch |err| switch (err) {
30474 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
30475 @tagName(air_tag),
30476 cg.typeOf(ty_op.operand).fmt(pt),
30477 ops[0].tracking(cg),
30478 }),
30479 else => |e| return e,
30480 };
30481 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
30482 },
3008730483 .byte_swap => |air_tag| if (use_old) try cg.airByteSwap(inst) else fallback: {
3008830484 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
3008930485 if (ty_op.ty.toType().isVector(zcu)) break :fallback try cg.airByteSwap(inst);
......@@ -99931,6 +100327,7 @@ const Select = struct {
99931100327 unaligned_size,
99932100328 bit_size,
99933100329 src0_bit_size,
100330 @"8_size_sub_bit_size",
99934100331 len,
99935100332 elem_limbs,
99936100333 elem_size,
......@@ -99958,6 +100355,7 @@ const Select = struct {
99958100355 const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" };
99959100356 const sub_src0_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .div, .rhs = .@"8" };
99960100357 const sub_src0_size: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .mul, .rhs = .@"1" };
100358 const add_8_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"8" };
99961100359 const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" };
99962100360 const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" };
99963100361 const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" };
......@@ -99972,6 +100370,7 @@ const Select = struct {
99972100370 const sub_bit_size: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .mul, .rhs = .@"1" };
99973100371 const add_src0_bit_size: Adjust = .{ .sign = .pos, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" };
99974100372 const sub_src0_bit_size: Adjust = .{ .sign = .neg, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" };
100373 const add_bit_size_sub_8_size: Adjust = .{ .sign = .neg, .lhs = .@"8_size_sub_bit_size", .op = .mul, .rhs = .@"1" };
99975100374 const add_8_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"8" };
99976100375 const add_4_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"4" };
99977100376 const add_3_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"3" };
......@@ -100683,6 +101082,10 @@ const Select = struct {
100683101082 .unaligned_size => @intCast(s.cg.unalignedSize(op.flags.base.ref.typeOf(s))),
100684101083 .bit_size => @intCast(op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
100685101084 .src0_bit_size => @intCast(Select.Operand.Ref.src0.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)),
101085 .@"8_size_sub_bit_size" => {
101086 const ty = op.flags.base.ref.typeOf(s);
101087 break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu));
101088 },
100686101089 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),
100687101090 .elem_limbs => @intCast(@divExact(
100688101091 op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
test/behavior/x86_64/math.zig+8
......@@ -19273,6 +19273,14 @@ test clz {
1927319273 try test_clz.testIntVectors();
1927419274}
1927519275
19276inline fn ctz(comptime Type: type, rhs: Type) @TypeOf(@ctz(rhs)) {
19277 return @ctz(rhs);
19278}
19279test ctz {
19280 const test_ctz = unary(ctz, .{});
19281 try test_ctz.testInts();
19282}
19283
1927619284inline fn byteSwap(comptime Type: type, rhs: Type) RoundBitsUp(Type, 8) {
1927719285 return @byteSwap(@as(RoundBitsUp(Type, 8), rhs));
1927819286}