| ... | @@ -2771,29 +2771,112 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2771,29 +2771,112 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { |
| 2771 | | 2771 | |
| 2772 | fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { | 2772 | fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { |
| 2773 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; | 2773 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 2774 | const result = result: { | 2774 | const result: MCValue = result: { |
| 2775 | if (self.liveness.isUnused(inst)) break :result .dead; | 2775 | if (self.liveness.isUnused(inst)) break :result .dead; |
| 2776 | | 2776 | |
| 2777 | const op_ty = self.air.typeOf(ty_op.operand); | 2777 | const src_ty = self.air.typeOf(ty_op.operand); |
| | 2778 | const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); |
| | 2779 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 2778 | | 2780 | |
| 2779 | if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) { | 2781 | if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) { |
| 2780 | const op_mcv = try self.resolveInst(ty_op.operand); | 2782 | const mat_src_mcv = switch (src_mcv) { |
| 2781 | const mat_op_mcv = switch (op_mcv) { | 2783 | .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, |
| 2782 | .immediate => MCValue{ .register = try self.copyToTmpRegister(op_ty, op_mcv) }, | 2784 | else => src_mcv, |
| 2783 | else => op_mcv, | | |
| 2784 | }; | 2785 | }; |
| 2785 | const mat_op_lock = switch (mat_op_mcv) { | 2786 | const mat_src_lock = switch (mat_src_mcv) { |
| 2786 | .register => |reg| self.register_manager.lockReg(reg), | 2787 | .register => |reg| self.register_manager.lockReg(reg), |
| 2787 | else => null, | 2788 | else => null, |
| 2788 | }; | 2789 | }; |
| 2789 | defer if (mat_op_lock) |lock| self.register_manager.unlockReg(lock); | 2790 | defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); |
| | 2791 | |
| | 2792 | const dst_mcv: MCValue = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| | 2793 | src_mcv |
| | 2794 | else |
| | 2795 | .{ .register = try self.register_manager.allocReg(inst, gp) }; |
| 2790 | | 2796 | |
| 2791 | const dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, gp) }; | 2797 | const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16; |
| 2792 | try self.genBinOpMir(.popcnt, op_ty, dst_mcv, mat_op_mcv); | 2798 | try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv); |
| 2793 | break :result dst_mcv; | 2799 | break :result dst_mcv; |
| 2794 | } | 2800 | } |
| 2795 | | 2801 | |
| 2796 | return self.fail("TODO implement airPopcount for {}", .{op_ty.fmt(self.bin_file.options.module.?)}); | 2802 | const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8); |
| | 2803 | const imm_0_1 = Immediate.u(mask / 0b1_1); |
| | 2804 | const imm_00_11 = Immediate.u(mask / 0b01_01); |
| | 2805 | const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); |
| | 2806 | const imm_0000_0001 = Immediate.u(mask / 0b1111_1111); |
| | 2807 | |
| | 2808 | const tmp_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| | 2809 | src_mcv.register |
| | 2810 | else |
| | 2811 | try self.copyToTmpRegister(src_ty, src_mcv); |
| | 2812 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| | 2813 | defer self.register_manager.unlockReg(tmp_lock); |
| | 2814 | |
| | 2815 | const dst_reg = try self.register_manager.allocReg(inst, gp); |
| | 2816 | const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); |
| | 2817 | defer self.register_manager.unlockReg(dst_lock); |
| | 2818 | |
| | 2819 | { |
| | 2820 | const dst = registerAlias(dst_reg, src_abi_size); |
| | 2821 | const tmp = registerAlias(tmp_reg, src_abi_size); |
| | 2822 | const imm = if (src_abi_size > 4) |
| | 2823 | try self.register_manager.allocReg(null, gp) |
| | 2824 | else |
| | 2825 | undefined; |
| | 2826 | |
| | 2827 | // tmp = operand |
| | 2828 | try self.asmRegisterRegister(.mov, dst, tmp); |
| | 2829 | // dst = operand |
| | 2830 | try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1)); |
| | 2831 | // tmp = operand >> 1 |
| | 2832 | if (src_abi_size > 4) { |
| | 2833 | try self.asmRegisterImmediate(.mov, imm, imm_0_1); |
| | 2834 | try self.asmRegisterRegister(.@"and", tmp, imm); |
| | 2835 | } else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1); |
| | 2836 | // tmp = (operand >> 1) & 0x55...55 |
| | 2837 | try self.asmRegisterRegister(.sub, dst, tmp); |
| | 2838 | // dst = temp1 = operand - ((operand >> 1) & 0x55...55) |
| | 2839 | try self.asmRegisterRegister(.mov, tmp, dst); |
| | 2840 | // tmp = temp1 |
| | 2841 | try self.asmRegisterImmediate(.shr, dst, Immediate.u(2)); |
| | 2842 | // dst = temp1 >> 2 |
| | 2843 | if (src_abi_size > 4) { |
| | 2844 | try self.asmRegisterImmediate(.mov, imm, imm_00_11); |
| | 2845 | try self.asmRegisterRegister(.@"and", tmp, imm); |
| | 2846 | try self.asmRegisterRegister(.@"and", dst, imm); |
| | 2847 | } else { |
| | 2848 | try self.asmRegisterImmediate(.@"and", tmp, imm_00_11); |
| | 2849 | try self.asmRegisterImmediate(.@"and", dst, imm_00_11); |
| | 2850 | } |
| | 2851 | // tmp = temp1 & 0x33...33 |
| | 2852 | // dst = (temp1 >> 2) & 0x33...33 |
| | 2853 | try self.asmRegisterRegister(.add, tmp, dst); |
| | 2854 | // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33) |
| | 2855 | try self.asmRegisterRegister(.mov, dst, tmp); |
| | 2856 | // dst = temp2 |
| | 2857 | try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4)); |
| | 2858 | // tmp = temp2 >> 4 |
| | 2859 | try self.asmRegisterRegister(.add, dst, tmp); |
| | 2860 | // dst = temp2 + (temp2 >> 4) |
| | 2861 | if (src_abi_size > 4) { |
| | 2862 | try self.asmRegisterImmediate(.mov, imm, imm_0000_1111); |
| | 2863 | try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001); |
| | 2864 | try self.asmRegisterRegister(.@"and", dst, imm); |
| | 2865 | try self.asmRegisterRegister(.imul, dst, tmp); |
| | 2866 | } else { |
| | 2867 | try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111); |
| | 2868 | if (src_abi_size > 1) { |
| | 2869 | try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001); |
| | 2870 | } |
| | 2871 | } |
| | 2872 | // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f |
| | 2873 | // dst = temp3 * 0x01...01 |
| | 2874 | if (src_abi_size > 1) { |
| | 2875 | try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8)); |
| | 2876 | } |
| | 2877 | // dst = (temp3 * 0x01...01) >> (bits - 8) |
| | 2878 | } |
| | 2879 | break :result .{ .register = dst_reg }; |
| 2797 | }; | 2880 | }; |
| 2798 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); | 2881 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 2799 | } | 2882 | } |
Jacob Young <jacobly@ziglang.org> 2023-03-19 08:26:05-04:00
Jakub Konka <kubkon@jakubkonka.com> 2023-03-21 08:49:54+01:00
src