authorgravatar for sinon@vortan.devDavid Rubin <sinon@vortan.dev> 2026-06-16 19:58:06-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-06-24 08:05:03+02:00
logf26cdb2771a4bb4d5f1d5acc446ec51c3e177f75
tree95ee0f965e3ce1e1bc6d26af418ad2ee39909ef6
parentb40b1178ef29ea9b013f73cc3cd4f9b976d0e120

x86_64: better support for splatting bool vectors

2 files changed, 67 insertions(+), 52 deletions(-)

src/codegen/x86_64/CodeGen.zig+66-52
...@@ -169090,12 +169090,56 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -169090,12 +169090,56 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
169090 };169090 };
169091 try res[0].finish(inst, &.{reduce.operand}, &ops, cg);169091 try res[0].finish(inst, &.{reduce.operand}, &ops, cg);
169092 },169092 },
169093 .splat => |air_tag| fallback: {169093 .splat => |air_tag| {
169094 const ty_op = air_datas[@intFromEnum(inst)].ty_op;169094 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
169095 if (cg.typeOf(ty_op.operand).toIntern() == .bool_type) break :fallback try cg.airSplat(inst);
169096 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});169095 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
169097 var res: [1]Temp = undefined;169096 var res: [1]Temp = undefined;
169098 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{169097 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
169098 .dst_constraints = .{ .{ .bool_vec = .qword }, .any },
169099 .src_constraints = .{ .bool, .any, .any },
169100 .patterns = &.{
169101 .{ .src = .{ .to_gpr, .none, .none } },
169102 },
169103 .dst_temps = .{ .{ .rc = .general_purpose }, .unused },
169104 .clobbers = .{ .eflags = true },
169105 .each = .{ .once = &.{
169106 .{ ._, ._, .bt, .src0d, .si(0), ._, ._ },
169107 .{ ._, ._, .sbb, .dst0q, .dst0q, ._, ._ },
169108 .{ ._, ._r, .sh, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ },
169109 } },
169110 }, .{
169111 .dst_constraints = .{ .any_bool_vec, .any },
169112 .src_constraints = .{ .bool, .any, .any },
169113 .patterns = &.{
169114 .{ .src = .{ .to_gpr, .none, .none } },
169115 },
169116 .dst_temps = .{ .mem, .unused },
169117 .extra_temps = .{
169118 .{ .type = .isize, .kind = .{ .reg = .rdi } },
169119 .{ .type = .u8, .kind = .{ .reg = .rax } },
169120 .{ .type = .u32, .kind = .{ .reg = .rcx } },
169121 .unused,
169122 .unused,
169123 .unused,
169124 .unused,
169125 .unused,
169126 .unused,
169127 .unused,
169128 .unused,
169129 },
169130 .clobbers = .{ .eflags = true },
169131 .each = .{ .once = &.{
169132 .{ ._, ._, .bt, .src0d, .si(0), ._, ._ },
169133 .{ ._, ._, .sbb, .tmp1b, .tmp1b, ._, ._ },
169134 .{ ._, ._, .lea, .tmp0q, .dst0b, ._, ._ },
169135 .{ ._, ._, .mov, .tmp2d, .sia(1, .dst0, .add_bit_size_sub_1_div_8_down_1), ._, ._ },
169136 .{ ._, .@"rep _sb", .sto, ._, ._, ._, ._ },
169137 .{ ._, ._, .@"and", .memad(.dst0b, .add_bit_size_sub_1_div_8_down_1, 0), .ua(.dst0, .bit_size_last_byte_mask), ._, ._ },
169138 .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_sub_bit_size_div_8_down_1_sub_1), ._, ._ },
169139 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
169140 .{ ._, .@"rep _sb", .sto, ._, ._, ._, ._ },
169141 } },
169142 }, .{
169099 .required_features = .{ .avx2, null, null, null },169143 .required_features = .{ .avx2, null, null, null },
169100 .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },169144 .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },
169101 .src_constraints = .{ .{ .int = .byte }, .any, .any },169145 .src_constraints = .{ .{ .int = .byte }, .any, .any },
...@@ -180803,56 +180847,6 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {...@@ -180803,56 +180847,6 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
180803 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });180847 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
180804}180848}
180805180849
180806fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void {
180807 const pt = self.pt;
180808 const zcu = pt.zcu;
180809 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
180810 const vector_ty = self.typeOfIndex(inst);
180811 const vector_len = vector_ty.vectorLen(zcu);
180812 const scalar_ty = self.typeOf(ty_op.operand);
180813
180814 const result: MCValue = result: {
180815 if (scalar_ty.toIntern() != .bool_type) return self.fail("TODO implement airSplat for {f}", .{
180816 vector_ty.fmt(pt),
180817 });
180818 const regs =
180819 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
180820 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
180821 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
180822
180823 try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 }, .{});
180824 try self.genSetReg(
180825 regs[1],
180826 vector_ty,
180827 .{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) },
180828 .{},
180829 );
180830 const src_mcv = try self.resolveInst(ty_op.operand);
180831 const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4);
180832 try self.asmCmovccRegisterRegister(
180833 switch (src_mcv) {
180834 .eflags => |cc| cc,
180835 .register => |src_reg| cc: {
180836 try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1));
180837 break :cc .nz;
180838 },
180839 else => cc: {
180840 try self.asmMemoryImmediate(
180841 .{ ._, .@"test" },
180842 try src_mcv.mem(self, .{ .size = .byte }),
180843 .u(1),
180844 );
180845 break :cc .nz;
180846 },
180847 },
180848 registerAlias(regs[0], abi_size),
180849 registerAlias(regs[1], abi_size),
180850 );
180851 break :result .{ .register = regs[0] };
180852 };
180853 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
180854}
180855
180856fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {180850fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
180857 const pt = self.pt;180851 const pt = self.pt;
180858 const zcu = pt.zcu;180852 const zcu = pt.zcu;
...@@ -191236,6 +191230,9 @@ const Select = struct {...@@ -191236,6 +191230,9 @@ const Select = struct {
191236 unaligned_size_add_elem_size,191230 unaligned_size_add_elem_size,
191237 unaligned_size_sub_elem_size,191231 unaligned_size_sub_elem_size,
191238 unaligned_size_sub_2_elem_size,191232 unaligned_size_sub_2_elem_size,
191233 size_sub_bit_size_div_8_down_1_sub_1,
191234 bit_size_sub_1_div_8_down_1,
191235 bit_size_last_byte_mask,
191239 bit_size,191236 bit_size,
191240 src0_bit_size,191237 src0_bit_size,
191241 @"8_size_sub_bit_size",191238 @"8_size_sub_bit_size",
...@@ -191288,6 +191285,9 @@ const Select = struct {...@@ -191288,6 +191285,9 @@ const Select = struct {
191288 const add_unaligned_size_add_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_add_elem_size, .op = .mul, .rhs = .@"1" };191285 const add_unaligned_size_add_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_add_elem_size, .op = .mul, .rhs = .@"1" };
191289 const add_unaligned_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_elem_size, .op = .mul, .rhs = .@"1" };191286 const add_unaligned_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_elem_size, .op = .mul, .rhs = .@"1" };
191290 const add_unaligned_size_sub_2_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_2_elem_size, .op = .mul, .rhs = .@"1" };191287 const add_unaligned_size_sub_2_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_2_elem_size, .op = .mul, .rhs = .@"1" };
191288 const add_size_sub_bit_size_div_8_down_1_sub_1: Adjust = .{ .sign = .pos, .lhs = .size_sub_bit_size_div_8_down_1_sub_1, .op = .mul, .rhs = .@"1" };
191289 const add_bit_size_sub_1_div_8_down_1: Adjust = .{ .sign = .pos, .lhs = .bit_size_sub_1_div_8_down_1, .op = .mul, .rhs = .@"1" };
191290 const bit_size_last_byte_mask: Adjust = .{ .sign = .pos, .lhs = .bit_size_last_byte_mask, .op = .mul, .rhs = .@"1" };
191291 const add_2_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"2" };191291 const add_2_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"2" };
191292 const add_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"1" };191292 const add_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"1" };
191293 const add_bit_size_rem_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"1" };191293 const add_bit_size_rem_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"1" };
...@@ -192224,6 +192224,20 @@ const Select = struct {...@@ -192224,6 +192224,20 @@ const Select = struct {
192224 const ty = op.flags.base.ref.typeOf(s);192224 const ty = op.flags.base.ref.typeOf(s);
192225 break :lhs @intCast(s.cg.unalignedSize(ty) - ty.scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * 2);192225 break :lhs @intCast(s.cg.unalignedSize(ty) - ty.scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * 2);
192226 },192226 },
192227 .size_sub_bit_size_div_8_down_1_sub_1 => {
192228 const ty = op.flags.base.ref.typeOf(s);
192229 const size: SignedImm = @intCast(ty.abiSize(s.cg.pt.zcu));
192230 const bit_size: SignedImm = @intCast(s.cg.nonBoolScalarBitSize(ty));
192231 break :lhs size - @divFloor(bit_size - 1, 8) - 1;
192232 },
192233 .bit_size_sub_1_div_8_down_1 => {
192234 const bit_size: SignedImm = @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s)));
192235 break :lhs @divFloor(bit_size - 1, 8);
192236 },
192237 .bit_size_last_byte_mask => {
192238 const bit_size = s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s));
192239 break :lhs @as(u8, std.math.maxInt(u8)) >> @intCast(7 - (bit_size - 1) % 8);
192240 },
192227 .bit_size => @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s))),192241 .bit_size => @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s))),
192228 .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),192242 .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),
192229 .@"8_size_sub_bit_size" => {192243 .@"8_size_sub_bit_size" => {
test/behavior/x86_64/unary.zig+1
...@@ -5262,6 +5262,7 @@ inline fn splat(comptime Type: type, rhs: Type) Type {...@@ -5262,6 +5262,7 @@ inline fn splat(comptime Type: type, rhs: Type) Type {
5262}5262}
5263test splat {5263test splat {
5264 const test_splat = unary(splat, .{});5264 const test_splat = unary(splat, .{});
5265 try test_splat.testBoolVectors();
5265 try test_splat.testIntVectors();5266 try test_splat.testIntVectors();
5266 try test_splat.testFloatVectors();5267 try test_splat.testFloatVectors();
5267}5268}