authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-04 03:36:04-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:36:19-04:00
log1a261917ce41efb49fe41ea0c6d9083212c17797
treee9e3ef244c9c659b5e5d512e782bb038e3662a16
parent9bea854dc2af293cba1d000b31f6e82d9c431285

x86_64: implement `@ctz` and `@clz` for `u128`

3 files changed, 74 insertions(+), 25 deletions(-)

src/arch/x86_64/CodeGen.zig+70-23
......@@ -3798,19 +3798,38 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
37983798
37993799 const dst_reg = try self.register_manager.allocReg(inst, gp);
38003800 const dst_mcv = MCValue{ .register = dst_reg };
3801 const dst_lock = self.register_manager.lockReg(dst_reg);
3802 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
3801 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3802 defer self.register_manager.unlockReg(dst_lock);
38033803
3804 const src_bits = src_ty.bitSize(self.target.*);
38043805 if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
3805 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
3806 const extra_bits = self.regExtraBits(src_ty);
3807 if (extra_bits > 0) {
3808 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
3809 }
3806 if (src_bits <= 64) {
3807 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
3808
3809 const extra_bits = self.regExtraBits(src_ty);
3810 if (extra_bits > 0) {
3811 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
3812 }
3813 } else if (src_bits <= 128) {
3814 const tmp_reg = try self.register_manager.allocReg(null, gp);
3815 const tmp_mcv = MCValue{ .register = tmp_reg };
3816 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3817 defer self.register_manager.unlockReg(tmp_lock);
3818
3819 try self.genBinOpMir(.lzcnt, Type.u64, dst_mcv, mat_src_mcv);
3820 try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
3821 try self.genBinOpMir(.lzcnt, Type.u64, tmp_mcv, mat_src_mcv.address().offset(8).deref());
3822 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
3823
3824 if (src_bits < 128) {
3825 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = 128 - src_bits });
3826 }
3827 } else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
38103828 break :result dst_mcv;
38113829 }
38123830
3813 const src_bits = src_ty.bitSize(self.target.*);
3831 if (src_bits > 64)
3832 return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
38143833 if (math.isPowerOfTwo(src_bits)) {
38153834 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
38163835 .immediate = src_bits ^ (src_bits - 1),
......@@ -3870,24 +3889,52 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
38703889 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
38713890
38723891 if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
3873 const extra_bits = self.regExtraBits(src_ty);
3874 const masked_mcv = if (extra_bits > 0) masked: {
3875 const mask_mcv = MCValue{
3876 .immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) <<
3877 @intCast(u6, src_bits),
3878 };
3879 const tmp_mcv = tmp: {
3880 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;
3881 try self.genSetReg(dst_reg, src_ty, src_mcv);
3882 break :tmp dst_mcv;
3883 };
3884 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
3885 break :masked tmp_mcv;
3886 } else mat_src_mcv;
3887 try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
3892 if (src_bits <= 64) {
3893 const extra_bits = self.regExtraBits(src_ty);
3894 const masked_mcv = if (extra_bits > 0) masked: {
3895 const tmp_mcv = tmp: {
3896 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
3897 break :tmp src_mcv;
3898 try self.genSetReg(dst_reg, src_ty, src_mcv);
3899 break :tmp dst_mcv;
3900 };
3901 try self.genBinOpMir(
3902 .@"or",
3903 src_ty,
3904 tmp_mcv,
3905 .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
3906 @intCast(u6, src_bits) },
3907 );
3908 break :masked tmp_mcv;
3909 } else mat_src_mcv;
3910 try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
3911 } else if (src_bits <= 128) {
3912 const tmp_reg = try self.register_manager.allocReg(null, gp);
3913 const tmp_mcv = MCValue{ .register = tmp_reg };
3914 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3915 defer self.register_manager.unlockReg(tmp_lock);
3916
3917 const masked_mcv = if (src_bits < 128) masked: {
3918 try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
3919 try self.genBinOpMir(
3920 .@"or",
3921 Type.u64,
3922 dst_mcv,
3923 .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
3924 );
3925 break :masked dst_mcv;
3926 } else mat_src_mcv.address().offset(8).deref();
3927 try self.genBinOpMir(.tzcnt, Type.u64, dst_mcv, masked_mcv);
3928 try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
3929 try self.genBinOpMir(.tzcnt, Type.u64, tmp_mcv, mat_src_mcv);
3930 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
3931 } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
38883932 break :result dst_mcv;
38893933 }
38903934
3935 if (src_bits > 64)
3936 return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3937
38913938 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
38923939 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
38933940
test/behavior/bugs/2114.zig+2-1
......@@ -9,7 +9,8 @@ fn ctz(x: anytype) usize {
99
1010test "fixed" {
1111 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
12 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
12 if (builtin.zig_backend == .stage2_x86_64 and
13 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .bmi)) return error.SkipZigTest; // TODO
1314 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1415 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1516 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/math.zig+2-1
......@@ -77,7 +77,8 @@ fn testClz() !void {
7777}
7878
7979test "@clz big ints" {
80 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
80 if (builtin.zig_backend == .stage2_x86_64 and
81 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .lzcnt)) return error.SkipZigTest; // TODO
8182 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8283 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
8384 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO