authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-18 08:02:17-04:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-03-21 08:49:54+01:00
log53ec2a955eb02dc829af8610f17af42717d512fb
tree68b2ef11487f4415186efbd6bb1fed8bf0bb8add
parentedd63f9abaa7dae1786a4cd91cc3031264bd3ef0

x86_64: implement clz, ctz, and popCount

5 files changed, 169 insertions(+), 17 deletions(-)

src/arch/x86_64/CodeGen.zig+128-13
......@@ -2597,28 +2597,143 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
25972597
25982598fn airClz(self: *Self, inst: Air.Inst.Index) !void {
25992599 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2600 const result: MCValue = if (self.liveness.isUnused(inst))
2601 .dead
2602 else
2603 return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
2600 const result = result: {
2601 if (self.liveness.isUnused(inst)) break :result .dead;
2602
2603 const dst_ty = self.air.typeOfIndex(inst);
2604 const src_ty = self.air.typeOf(ty_op.operand);
2605 const src_bits = src_ty.bitSize(self.target.*);
2606
2607 const src_mcv = try self.resolveInst(ty_op.operand);
2608 const mat_src_mcv = switch (src_mcv) {
2609 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
2610 else => src_mcv,
2611 };
2612 const mat_src_lock = switch (mat_src_mcv) {
2613 .register => |reg| self.register_manager.lockReg(reg),
2614 else => null,
2615 };
2616 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
2617
2618 const dst_reg = try self.register_manager.allocReg(inst, gp);
2619 const dst_mcv = MCValue{ .register = dst_reg };
2620 const dst_lock = self.register_manager.lockReg(dst_reg);
2621 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
2622
2623 if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
2624 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
2625 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
2626 const extra_bits = registerAlias(dst_reg, src_abi_size).bitSize() - src_bits;
2627 if (extra_bits > 0) {
2628 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
2629 }
2630 break :result dst_mcv;
2631 }
2632
2633 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
2634 const width_mcv = MCValue{ .register = width_reg };
2635 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
2636
2637 const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2));
2638 try self.asmCmovccRegisterRegister(
2639 registerAlias(dst_reg, dst_abi_size),
2640 registerAlias(width_reg, dst_abi_size),
2641 .z,
2642 );
2643
2644 try self.genBinOpMir(.sub, dst_ty, width_mcv, dst_mcv);
2645 break :result width_mcv;
2646 };
26042647 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
26052648}
26062649
26072650fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
26082651 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2609 const result: MCValue = if (self.liveness.isUnused(inst))
2610 .dead
2611 else
2612 return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
2652 const result = result: {
2653 if (self.liveness.isUnused(inst)) break :result .dead;
2654
2655 const dst_ty = self.air.typeOfIndex(inst);
2656 const src_ty = self.air.typeOf(ty_op.operand);
2657 const src_bits = src_ty.bitSize(self.target.*);
2658
2659 const src_mcv = try self.resolveInst(ty_op.operand);
2660 const mat_src_mcv = switch (src_mcv) {
2661 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
2662 else => src_mcv,
2663 };
2664 const mat_src_lock = switch (mat_src_mcv) {
2665 .register => |reg| self.register_manager.lockReg(reg),
2666 else => null,
2667 };
2668 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
2669
2670 const dst_reg = try self.register_manager.allocReg(inst, gp);
2671 const dst_mcv = MCValue{ .register = dst_reg };
2672 const dst_lock = self.register_manager.lockReg(dst_reg);
2673 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
2674
2675 if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
2676 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
2677 const extra_bits = registerAlias(dst_reg, src_abi_size).bitSize() - src_bits;
2678 const masked_mcv = if (extra_bits > 0) masked: {
2679 const mask_mcv = MCValue{
2680 .immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) << @intCast(u6, src_bits),
2681 };
2682 const tmp_mcv = tmp: {
2683 if (src_mcv.isImmediate() or self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
2684 break :tmp src_mcv;
2685 }
2686 try self.genSetReg(src_ty, dst_reg, src_mcv);
2687 break :tmp dst_mcv;
2688 };
2689 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
2690 break :masked tmp_mcv;
2691 } else mat_src_mcv;
2692 try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
2693 break :result dst_mcv;
2694 }
2695
2696 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
2697 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
2698
2699 const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
2700 try self.asmCmovccRegisterRegister(
2701 registerAlias(dst_reg, abi_size),
2702 registerAlias(width_reg, abi_size),
2703 .z,
2704 );
2705
2706 break :result dst_mcv;
2707 };
26132708 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
26142709}
26152710
26162711fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
26172712 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2618 const result: MCValue = if (self.liveness.isUnused(inst))
2619 .dead
2620 else
2621 return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
2713 const result = result: {
2714 if (self.liveness.isUnused(inst)) break :result .dead;
2715
2716 const op_ty = self.air.typeOf(ty_op.operand);
2717
2718 if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {
2719 const op_mcv = try self.resolveInst(ty_op.operand);
2720 const mat_op_mcv = switch (op_mcv) {
2721 .immediate => MCValue{ .register = try self.copyToTmpRegister(op_ty, op_mcv) },
2722 else => op_mcv,
2723 };
2724 const mat_op_lock = switch (mat_op_mcv) {
2725 .register => |reg| self.register_manager.lockReg(reg),
2726 else => null,
2727 };
2728 defer if (mat_op_lock) |lock| self.register_manager.unlockReg(lock);
2729
2730 const dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, gp) };
2731 try self.genBinOpMir(.popcnt, op_ty, dst_mcv, mat_op_mcv);
2732 break :result dst_mcv;
2733 }
2734
2735 return self.fail("TODO implement airPopcount for {}", .{op_ty.fmt(self.bin_file.options.module.?)});
2736 };
26222737 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
26232738}
26242739
......@@ -3491,7 +3606,7 @@ fn genBinOp(
34913606 if (lhs.isRegister() and self.reuseOperand(inst, lhs_air, 0, lhs)) {
34923607 break :blk lhs;
34933608 }
3494 if (rhs.isRegister() and is_commutative and self.reuseOperand(inst, rhs_air, 1, rhs)) {
3609 if (is_commutative and rhs.isRegister() and self.reuseOperand(inst, rhs_air, 1, rhs)) {
34953610 flipped = true;
34963611 break :blk rhs;
34973612 }
src/arch/x86_64/Emit.zig+5
......@@ -73,6 +73,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
7373 .adc,
7474 .add,
7575 .@"and",
76 .bsf,
77 .bsr,
7678 .call,
7779 .cbw,
7880 .cwde,
......@@ -89,12 +91,14 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
8991 .int3,
9092 .jmp,
9193 .lea,
94 .lzcnt,
9295 .mov,
9396 .movzx,
9497 .mul,
9598 .nop,
9699 .@"or",
97100 .pop,
101 .popcnt,
98102 .push,
99103 .ret,
100104 .sal,
......@@ -105,6 +109,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
105109 .sub,
106110 .syscall,
107111 .@"test",
112 .tzcnt,
108113 .ud2,
109114 .xor,
110115
src/arch/x86_64/Encoding.zig+4-2
......@@ -307,6 +307,7 @@ pub const Mnemonic = enum {
307307 // zig fmt: off
308308 // General-purpose
309309 adc, add, @"and",
310 bsf, bsr,
310311 call, cbw, cdq, cdqe,
311312 cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
312313 cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
......@@ -322,12 +323,13 @@ pub const Mnemonic = enum {
322323 jmp,
323324 lea,
324325 lods, lodsb, lodsd, lodsq, lodsw,
326 lzcnt,
325327 mov,
326328 movs, movsb, movsd, movsq, movsw,
327329 movsx, movsxd, movzx, mul,
328330 nop,
329331 @"or",
330 pop, push,
332 pop, popcnt, push,
331333 ret,
332334 sal, sar, sbb,
333335 scas, scasb, scasd, scasq, scasw,
......@@ -336,7 +338,7 @@ pub const Mnemonic = enum {
336338 setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
337339 setnz, seto, setp, setpe, setpo, sets, setz,
338340 stos, stosb, stosd, stosq, stosw,
339 @"test",
341 @"test", tzcnt,
340342 ud2,
341343 xor,
342344 // SSE
src/arch/x86_64/Mir.zig+10
......@@ -38,6 +38,10 @@ pub const Inst = struct {
3838 add,
3939 /// Logical and
4040 @"and",
41 /// Bit scan forward
42 bsf,
43 /// Bit scan reverse
44 bsr,
4145 /// Call
4246 call,
4347 /// Convert byte to word
......@@ -70,6 +74,8 @@ pub const Inst = struct {
7074 jmp,
7175 /// Load effective address
7276 lea,
77 /// Count the number of leading zero bits
78 lzcnt,
7379 /// Move
7480 mov,
7581 /// Move with sign extension
......@@ -84,6 +90,8 @@ pub const Inst = struct {
8490 @"or",
8591 /// Pop
8692 pop,
93 /// Return the count of number of bits set to 1
94 popcnt,
8795 /// Push
8896 push,
8997 /// Return
......@@ -104,6 +112,8 @@ pub const Inst = struct {
104112 syscall,
105113 /// Test condition
106114 @"test",
115 /// Count the number of trailing zero bits
116 tzcnt,
107117 /// Undefined instruction
108118 ud2,
109119 /// Logical exclusive-or
src/arch/x86_64/encodings.zig+22-2
......@@ -81,6 +81,14 @@ pub const table = &[_]Entry{
8181 .{ .@"and", .rm, .r32, .rm32, .none, .none, &.{ 0x23 }, 0, .none },
8282 .{ .@"and", .rm, .r64, .rm64, .none, .none, &.{ 0x23 }, 0, .long },
8383
84 .{ .bsf, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
85 .{ .bsf, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
86 .{ .bsf, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbc }, 0, .long },
87
88 .{ .bsr, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
89 .{ .bsr, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
90 .{ .bsr, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbd }, 0, .long },
91
8492 // This is M encoding according to Intel, but D makes more sense here.
8593 .{ .call, .d, .rel32, .none, .none, .none, &.{ 0xe8 }, 0, .none },
8694 .{ .call, .m, .rm64, .none, .none, .none, &.{ 0xff }, 2, .none },
......@@ -301,6 +309,10 @@ pub const table = &[_]Entry{
301309 .{ .lodsd, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .none },
302310 .{ .lodsq, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .long },
303311
312 .{ .lzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
313 .{ .lzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
314 .{ .lzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },
315
304316 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .none },
305317 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .rex },
306318 .{ .mov, .mr, .rm16, .r16, .none, .none, &.{ 0x89 }, 0, .none },
......@@ -397,6 +409,10 @@ pub const table = &[_]Entry{
397409 .{ .pop, .m, .rm16, .none, .none, .none, &.{ 0x8f }, 0, .none },
398410 .{ .pop, .m, .rm64, .none, .none, .none, &.{ 0x8f }, 0, .none },
399411
412 .{ .popcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
413 .{ .popcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
414 .{ .popcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long },
415
400416 .{ .push, .o, .r16, .none, .none, .none, &.{ 0x50 }, 0, .none },
401417 .{ .push, .o, .r64, .none, .none, .none, &.{ 0x50 }, 0, .none },
402418 .{ .push, .m, .rm16, .none, .none, .none, &.{ 0xff }, 6, .none },
......@@ -596,8 +612,8 @@ pub const table = &[_]Entry{
596612 .{ .sub, .rm, .r32, .rm32, .none, .none, &.{ 0x2b }, 0, .none },
597613 .{ .sub, .rm, .r64, .rm64, .none, .none, &.{ 0x2b }, 0, .long },
598614
599 .{ .syscall, .np, .none, .none, .none, .none, &.{ 0x0f, 0x05 }, 0, .none },
600
615 .{ .syscall, .np, .none, .none, .none, .none, &.{ 0x0f, 0x05 }, 0, .none }
616,
601617 .{ .@"test", .zi, .al, .imm8, .none, .none, &.{ 0xa8 }, 0, .none },
602618 .{ .@"test", .zi, .ax, .imm16, .none, .none, &.{ 0xa9 }, 0, .none },
603619 .{ .@"test", .zi, .eax, .imm32, .none, .none, &.{ 0xa9 }, 0, .none },
......@@ -613,6 +629,10 @@ pub const table = &[_]Entry{
613629 .{ .@"test", .mr, .rm32, .r32, .none, .none, &.{ 0x85 }, 0, .none },
614630 .{ .@"test", .mr, .rm64, .r64, .none, .none, &.{ 0x85 }, 0, .long },
615631
632 .{ .tzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
633 .{ .tzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
634 .{ .tzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .long },
635
616636 .{ .ud2, .np, .none, .none, .none, .none, &.{ 0x0f, 0x0b }, 0, .none },
617637
618638 .{ .xor, .zi, .al, .imm8, .none, .none, &.{ 0x34 }, 0, .none },