authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2024-06-02 15:57:18+02:00
committergravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2024-06-10 20:32:43+02:00
log4bd9d9b7e0d769dd8d7701b73e54ae249ac7f1da
treeff9124805c74888ba8d08df4ae406caba37ff4ba
parentb9d738a5cff0ab896c25f1c8abe15757bcd6a0ba
signaturebadge-check Signed by SSH key SHA256:ZS52FNyUv2WUXvO4njmVaFVO46RHojFuOrxRc4LuKzg

spirv: change direct vector child repr to direct

Previously the child type of a vector was always in indirect representation. Concretely, this meant that vectors of bools are represented by vectors of u8. This was undesirable because it introduced a difference between vectorizable operations with a scalar bool and a vector of bool. This commit changes the representation to be the same for vectors and scalars everywhere. Some issues arised with constructing vectors: it seems the previous temporary- and-pointer approach does not work properly with vectors of bool. To work around this, simply use OpCompositeConstruct. This is the proper instruction for this, but it was previously not used because of a now-solved limitation in the SPIRV-LLVM-Translator. It was not yet applied to Zig because the Intel OpenCL CPU runtime does not have a recent enough version of the translator yet, but to solve that we just switch to testing with POCL instead.

1 files changed, 207 insertions(+), 63 deletions(-)

src/codegen/spirv.zig+207-63
......@@ -22,6 +22,7 @@ const IdResultType = spec.IdResultType;
2222const StorageClass = spec.StorageClass;
2323
2424const SpvModule = @import("spirv/Module.zig");
25const IdRange = SpvModule.IdRange;
2526
2627const SpvSection = @import("spirv/Section.zig");
2728const SpvAssembler = @import("spirv/Assembler.zig");
......@@ -32,7 +33,7 @@ pub const zig_call_abi_ver = 3;
3233
3334const InternMap = std.AutoHashMapUnmanaged(struct { InternPool.Index, DeclGen.Repr }, IdResult);
3435const PtrTypeMap = std.AutoHashMapUnmanaged(
35 struct { InternPool.Index, StorageClass },
36 struct { InternPool.Index, StorageClass, DeclGen.Repr },
3637 struct { ty_id: IdRef, fwd_emitted: bool },
3738);
3839
......@@ -626,7 +627,7 @@ const DeclGen = struct {
626627 }
627628
628629 /// Checks whether the type can be directly translated to SPIR-V vectors
629 fn isVector(self: *DeclGen, ty: Type) bool {
630 fn isSpvVector(self: *DeclGen, ty: Type) bool {
630631 const mod = self.module;
631632 const target = self.getTarget();
632633 if (ty.zigTypeTag(mod) != .Vector) return false;
......@@ -798,26 +799,39 @@ const DeclGen = struct {
798799
799800 /// Construct a vector at runtime.
800801 /// ty must be an vector type.
801 /// Constituents should be in `indirect` representation (as the elements of an vector should be).
802 /// Result is in `direct` representation.
803802 fn constructVector(self: *DeclGen, ty: Type, constituents: []const IdRef) !IdRef {
804 // The Khronos LLVM-SPIRV translator crashes because it cannot construct structs which'
805 // operands are not constant.
803 const mod = self.module;
804 assert(ty.vectorLen(mod) == constituents.len);
805
806 // Note: older versions of the Khronos SPRIV-LLVM translator crash on this instruction
807 // because it cannot construct structs which' operands are not constant.
806808 // See https://github.com/KhronosGroup/SPIRV-LLVM-Translator/issues/1349
807 // For now, just initialize the struct by setting the fields manually...
808 // TODO: Make this OpCompositeConstruct when we can
809 // Currently this is the case for Intel OpenCL CPU runtime (2023-WW46), but the
810 // alternatives dont work properly:
811 // - using temporaries/pointers doesn't work properly with vectors of bool, causes
812 // backends that use llvm to crash
813 // - using OpVectorInsertDynamic doesn't work for non-spirv-vectors of bool.
814
815 const result_id = self.spv.allocId();
816 try self.func.body.emit(self.spv.gpa, .OpCompositeConstruct, .{
817 .id_result_type = try self.resolveType(ty, .direct),
818 .id_result = result_id,
819 .constituents = constituents,
820 });
821 return result_id;
822 }
823
824 /// Construct a vector at runtime with all lanes set to the same value.
825 /// ty must be an vector type.
826 fn constructVectorSplat(self: *DeclGen, ty: Type, constituent: IdRef) !IdRef {
809827 const mod = self.module;
810 const ptr_composite_id = try self.alloc(ty, .{ .storage_class = .Function });
811 const ptr_elem_ty_id = try self.ptrType(ty.elemType2(mod), .Function);
812 for (constituents, 0..) |constitent_id, index| {
813 const ptr_id = try self.accessChain(ptr_elem_ty_id, ptr_composite_id, &.{@as(u32, @intCast(index))});
814 try self.func.body.emit(self.spv.gpa, .OpStore, .{
815 .pointer = ptr_id,
816 .object = constitent_id,
817 });
818 }
828 const n = ty.vectorLen(mod);
819829
820 return try self.load(ty, ptr_composite_id, .{});
830 const constituents = try self.gpa.alloc(IdRef, n);
831 defer self.gpa.free(constituents);
832 @memset(constituents, constituent);
833
834 return try self.constructVector(ty, constituents);
821835 }
822836
823837 /// Construct an array at runtime.
......@@ -1031,21 +1045,27 @@ const DeclGen = struct {
10311045 const constituents = try self.gpa.alloc(IdRef, @intCast(ty.arrayLenIncludingSentinel(mod)));
10321046 defer self.gpa.free(constituents);
10331047
1048 const child_repr: Repr = switch (tag) {
1049 .array_type => .indirect,
1050 .vector_type => .direct,
1051 else => unreachable,
1052 };
1053
10341054 switch (aggregate.storage) {
10351055 .bytes => |bytes| {
10361056 // TODO: This is really space inefficient, perhaps there is a better
10371057 // way to do it?
10381058 for (constituents, bytes.toSlice(constituents.len, ip)) |*constituent, byte| {
1039 constituent.* = try self.constInt(elem_ty, byte, .indirect);
1059 constituent.* = try self.constInt(elem_ty, byte, child_repr);
10401060 }
10411061 },
10421062 .elems => |elems| {
10431063 for (constituents, elems) |*constituent, elem| {
1044 constituent.* = try self.constant(elem_ty, Value.fromInterned(elem), .indirect);
1064 constituent.* = try self.constant(elem_ty, Value.fromInterned(elem), child_repr);
10451065 }
10461066 },
10471067 .repeated_elem => |elem| {
1048 @memset(constituents, try self.constant(elem_ty, Value.fromInterned(elem), .indirect));
1068 @memset(constituents, try self.constant(elem_ty, Value.fromInterned(elem), child_repr));
10491069 },
10501070 }
10511071
......@@ -1334,7 +1354,11 @@ const DeclGen = struct {
13341354 }
13351355
13361356 fn ptrType(self: *DeclGen, child_ty: Type, storage_class: StorageClass) !IdRef {
1337 const key = .{ child_ty.toIntern(), storage_class };
1357 return try self.ptrType2(child_ty, storage_class, .indirect);
1358 }
1359
1360 fn ptrType2(self: *DeclGen, child_ty: Type, storage_class: StorageClass, child_repr: Repr) !IdRef {
1361 const key = .{ child_ty.toIntern(), storage_class, child_repr };
13381362 const entry = try self.ptr_types.getOrPut(self.gpa, key);
13391363 if (entry.found_existing) {
13401364 const fwd_id = entry.value_ptr.ty_id;
......@@ -1354,7 +1378,7 @@ const DeclGen = struct {
13541378 .fwd_emitted = false,
13551379 };
13561380
1357 const child_ty_id = try self.resolveType(child_ty, .indirect);
1381 const child_ty_id = try self.resolveType(child_ty, child_repr);
13581382
13591383 try self.spv.sections.types_globals_constants.emit(self.spv.gpa, .OpTypePointer, .{
13601384 .id_result = result_id,
......@@ -1645,11 +1669,10 @@ const DeclGen = struct {
16451669 },
16461670 .Vector => {
16471671 const elem_ty = ty.childType(mod);
1648 // TODO: Make `.direct`.
1649 const elem_ty_id = try self.resolveType(elem_ty, .indirect);
1672 const elem_ty_id = try self.resolveType(elem_ty, repr);
16501673 const len = ty.vectorLen(mod);
16511674
1652 if (self.isVector(ty)) {
1675 if (self.isSpvVector(ty)) {
16531676 return try self.spv.vectorType(len, elem_ty_id);
16541677 } else {
16551678 return try self.arrayType(len, elem_ty_id);
......@@ -1948,7 +1971,7 @@ const DeclGen = struct {
19481971 const mod = wip.dg.module;
19491972 if (wip.is_array) {
19501973 assert(ty.isVector(mod));
1951 return try wip.dg.extractField(ty.childType(mod), value, @intCast(index));
1974 return try wip.dg.extractVectorComponent(ty.childType(mod), value, @intCast(index));
19521975 } else {
19531976 assert(index == 0);
19541977 return value;
......@@ -1961,11 +1984,7 @@ const DeclGen = struct {
19611984 /// Results is in `direct` representation.
19621985 fn finalize(wip: *WipElementWise) !IdRef {
19631986 if (wip.is_array) {
1964 // Convert all the constituents to indirect, as required for the array.
1965 for (wip.results) |*result| {
1966 result.* = try wip.dg.convertToIndirect(wip.ty, result.*);
1967 }
1968 return try wip.dg.constructArray(wip.result_ty, wip.results);
1987 return try wip.dg.constructVector(wip.result_ty, wip.results);
19691988 } else {
19701989 return wip.results[0];
19711990 }
......@@ -1982,7 +2001,7 @@ const DeclGen = struct {
19822001 /// Create a new element-wise operation.
19832002 fn elementWise(self: *DeclGen, result_ty: Type, force_element_wise: bool) !WipElementWise {
19842003 const mod = self.module;
1985 const is_array = result_ty.isVector(mod) and (!self.isVector(result_ty) or force_element_wise);
2004 const is_array = result_ty.isVector(mod) and (!self.isSpvVector(result_ty) or force_element_wise);
19862005 const num_results = if (is_array) result_ty.vectorLen(mod) else 1;
19872006 const results = try self.gpa.alloc(IdRef, num_results);
19882007 @memset(results, undefined);
......@@ -2253,29 +2272,102 @@ const DeclGen = struct {
22532272 /// This converts the argument type from resolveType(ty, .indirect) to resolveType(ty, .direct).
22542273 fn convertToDirect(self: *DeclGen, ty: Type, operand_id: IdRef) !IdRef {
22552274 const mod = self.module;
2256 return switch (ty.zigTypeTag(mod)) {
2257 .Bool => blk: {
2258 const result_id = self.spv.allocId();
2259 try self.func.body.emit(self.spv.gpa, .OpINotEqual, .{
2260 .id_result_type = try self.resolveType(Type.bool, .direct),
2261 .id_result = result_id,
2262 .operand_1 = operand_id,
2263 .operand_2 = try self.constBool(false, .indirect),
2264 });
2265 break :blk result_id;
2275 const scalar_ty = ty.scalarType(mod);
2276 const is_spv_vector = self.isSpvVector(ty);
2277 switch (scalar_ty.zigTypeTag(mod)) {
2278 .Bool => {
2279 // TODO: We may want to use something like elementWise in this function.
2280 // First we need to audit whether this would recursively call into itself.
2281 if (!ty.isVector(mod) or is_spv_vector) {
2282 const result_id = self.spv.allocId();
2283 const scalar_false_id = try self.constBool(false, .indirect);
2284 const false_id = if (is_spv_vector) blk: {
2285 const index = try mod.intern_pool.get(mod.gpa, .{
2286 .vector_type = .{
2287 .len = ty.vectorLen(mod),
2288 .child = Type.u1.toIntern(),
2289 },
2290 });
2291 const vec_ty = Type.fromInterned(index);
2292 break :blk try self.constructVectorSplat(vec_ty, scalar_false_id);
2293 } else scalar_false_id;
2294
2295 try self.func.body.emit(self.spv.gpa, .OpINotEqual, .{
2296 .id_result_type = try self.resolveType(ty, .direct),
2297 .id_result = result_id,
2298 .operand_1 = operand_id,
2299 .operand_2 = false_id,
2300 });
2301 return result_id;
2302 }
2303
2304 const constituents = try self.gpa.alloc(IdRef, ty.vectorLen(mod));
2305 for (constituents, 0..) |*id, i| {
2306 const element = try self.extractVectorComponent(scalar_ty, operand_id, @intCast(i));
2307 id.* = try self.convertToDirect(scalar_ty, element);
2308 }
2309 return try self.constructVector(ty, constituents);
22662310 },
2267 else => operand_id,
2268 };
2311 else => return operand_id,
2312 }
22692313 }
22702314
22712315 /// Convert representation from direct (in 'register) to direct (in memory)
22722316 /// This converts the argument type from resolveType(ty, .direct) to resolveType(ty, .indirect).
22732317 fn convertToIndirect(self: *DeclGen, ty: Type, operand_id: IdRef) !IdRef {
22742318 const mod = self.module;
2275 return switch (ty.zigTypeTag(mod)) {
2276 .Bool => try self.intFromBool(Type.u1, operand_id),
2277 else => operand_id,
2278 };
2319 const scalar_ty = ty.scalarType(mod);
2320 const is_spv_vector = self.isSpvVector(ty);
2321 switch (scalar_ty.zigTypeTag(mod)) {
2322 .Bool => {
2323 const result_ty = if (is_spv_vector) blk: {
2324 const index = try mod.intern_pool.get(mod.gpa, .{
2325 .vector_type = .{
2326 .len = ty.vectorLen(mod),
2327 .child = Type.u1.toIntern(),
2328 },
2329 });
2330 break :blk Type.fromInterned(index);
2331 } else Type.u1;
2332
2333 if (!ty.isVector(mod) or is_spv_vector) {
2334 // TODO: We may want to use something like elementWise in this function.
2335 // First we need to audit whether this would recursively call into itself.
2336 // Also unify it with intFromBool
2337
2338 const scalar_zero_id = try self.constInt(Type.u1, 0, .direct);
2339 const scalar_one_id = try self.constInt(Type.u1, 1, .direct);
2340
2341 const zero_id = if (is_spv_vector)
2342 try self.constructVectorSplat(result_ty, scalar_zero_id)
2343 else
2344 scalar_zero_id;
2345
2346 const one_id = if (is_spv_vector)
2347 try self.constructVectorSplat(result_ty, scalar_one_id)
2348 else
2349 scalar_one_id;
2350
2351 const result_id = self.spv.allocId();
2352 try self.func.body.emit(self.spv.gpa, .OpSelect, .{
2353 .id_result_type = try self.resolveType(result_ty, .direct),
2354 .id_result = result_id,
2355 .condition = operand_id,
2356 .object_1 = one_id,
2357 .object_2 = zero_id,
2358 });
2359 return result_id;
2360 }
2361
2362 const constituents = try self.gpa.alloc(IdRef, ty.vectorLen(mod));
2363 for (constituents, 0..) |*id, i| {
2364 const element = try self.extractVectorComponent(scalar_ty, operand_id, @intCast(i));
2365 id.* = try self.convertToIndirect(scalar_ty, element);
2366 }
2367 return try self.constructVector(result_ty, constituents);
2368 },
2369 else => return operand_id,
2370 }
22792371 }
22802372
22812373 fn extractField(self: *DeclGen, result_ty: Type, object: IdRef, field: u32) !IdRef {
......@@ -2292,6 +2384,21 @@ const DeclGen = struct {
22922384 return try self.convertToDirect(result_ty, result_id);
22932385 }
22942386
2387 fn extractVectorComponent(self: *DeclGen, result_ty: Type, vector_id: IdRef, field: u32) !IdRef {
2388 // Whether this is an OpTypeVector or OpTypeArray, we need to emit the same instruction regardless.
2389 const result_ty_id = try self.resolveType(result_ty, .direct);
2390 const result_id = self.spv.allocId();
2391 const indexes = [_]u32{field};
2392 try self.func.body.emit(self.spv.gpa, .OpCompositeExtract, .{
2393 .id_result_type = result_ty_id,
2394 .id_result = result_id,
2395 .composite = vector_id,
2396 .indexes = &indexes,
2397 });
2398 // Vector components are already stored in direct representation.
2399 return result_id;
2400 }
2401
22952402 const MemoryOptions = struct {
22962403 is_volatile: bool = false,
22972404 };
......@@ -2926,7 +3033,7 @@ const DeclGen = struct {
29263033 const ov_ty = result_ty.structFieldType(1, self.module);
29273034
29283035 const bool_ty_id = try self.resolveType(Type.bool, .direct);
2929 const cmp_ty_id = if (self.isVector(operand_ty))
3036 const cmp_ty_id = if (self.isSpvVector(operand_ty))
29303037 // TODO: Resolving a vector type with .direct should return a SPIR-V vector
29313038 try self.spv.vectorType(operand_ty.vectorLen(mod), try self.resolveType(Type.bool, .direct))
29323039 else
......@@ -3100,7 +3207,7 @@ const DeclGen = struct {
31003207 const ov_ty = result_ty.structFieldType(1, self.module);
31013208
31023209 const bool_ty_id = try self.resolveType(Type.bool, .direct);
3103 const cmp_ty_id = if (self.isVector(operand_ty))
3210 const cmp_ty_id = if (self.isSpvVector(operand_ty))
31043211 // TODO: Resolving a vector type with .direct should return a SPIR-V vector
31053212 try self.spv.vectorType(operand_ty.vectorLen(mod), try self.resolveType(Type.bool, .direct))
31063213 else
......@@ -3312,7 +3419,7 @@ const DeclGen = struct {
33123419
33133420 const info = self.arithmeticTypeInfo(operand_ty);
33143421
3315 var result_id = try self.extractField(scalar_ty, operand, 0);
3422 var result_id = try self.extractVectorComponent(scalar_ty, operand, 0);
33163423 const len = operand_ty.vectorLen(mod);
33173424
33183425 switch (reduce.operation) {
......@@ -3320,7 +3427,7 @@ const DeclGen = struct {
33203427 const cmp_op: std.math.CompareOperator = if (op == .Max) .gt else .lt;
33213428 for (1..len) |i| {
33223429 const lhs = result_id;
3323 const rhs = try self.extractField(scalar_ty, operand, @intCast(i));
3430 const rhs = try self.extractVectorComponent(scalar_ty, operand, @intCast(i));
33243431 result_id = try self.minMax(scalar_ty, cmp_op, lhs, rhs);
33253432 }
33263433
......@@ -3354,7 +3461,7 @@ const DeclGen = struct {
33543461
33553462 for (1..len) |i| {
33563463 const lhs = result_id;
3357 const rhs = try self.extractField(scalar_ty, operand, @intCast(i));
3464 const rhs = try self.extractVectorComponent(scalar_ty, operand, @intCast(i));
33583465 result_id = self.spv.allocId();
33593466
33603467 try self.func.body.emitRaw(self.spv.gpa, opcode, 4);
......@@ -3388,9 +3495,9 @@ const DeclGen = struct {
33883495
33893496 const index = elem.toSignedInt(mod);
33903497 if (index >= 0) {
3391 result_id.* = try self.extractField(wip.ty, a, @intCast(index));
3498 result_id.* = try self.extractVectorComponent(wip.ty, a, @intCast(index));
33923499 } else {
3393 result_id.* = try self.extractField(wip.ty, b, @intCast(~index));
3500 result_id.* = try self.extractVectorComponent(wip.ty, b, @intCast(~index));
33943501 }
33953502 }
33963503 return try wip.finalize();
......@@ -4086,8 +4193,7 @@ const DeclGen = struct {
40864193 defer self.gpa.free(elem_ids);
40874194
40884195 for (elements, 0..) |element, i| {
4089 const id = try self.resolve(element);
4090 elem_ids[i] = try self.convertToIndirect(result_ty.childType(mod), id);
4196 elem_ids[i] = try self.resolve(element);
40914197 }
40924198
40934199 return try self.constructVector(result_ty, elem_ids);
......@@ -4234,16 +4340,54 @@ const DeclGen = struct {
42344340 const array_id = try self.resolve(bin_op.lhs);
42354341 const index_id = try self.resolve(bin_op.rhs);
42364342
4343 if (self.isSpvVector(array_ty)) {
4344 const result_id = self.spv.allocId();
4345 try self.func.body.emit(self.spv.gpa, .OpVectorExtractDynamic, .{
4346 .id_result_type = try self.resolveType(elem_ty, .direct),
4347 .id_result = result_id,
4348 .vector = array_id,
4349 .index = index_id,
4350 });
4351 return result_id;
4352 }
4353
42374354 // SPIR-V doesn't have an array indexing function for some damn reason.
42384355 // For now, just generate a temporary and use that.
42394356 // TODO: This backend probably also should use isByRef from llvm...
42404357
4241 const elem_ptr_ty_id = try self.ptrType(elem_ty, .Function);
4358 const ptr_array_ty_id = try self.ptrType2(array_ty, .Function, .direct);
4359 const ptr_elem_ty_id = try self.ptrType2(elem_ty, .Function, .direct);
4360
4361 const tmp_id = self.spv.allocId();
4362 try self.func.prologue.emit(self.spv.gpa, .OpVariable, .{
4363 .id_result_type = ptr_array_ty_id,
4364 .id_result = tmp_id,
4365 .storage_class = .Function,
4366 });
4367
4368 try self.func.body.emit(self.spv.gpa, .OpStore, .{
4369 .pointer = tmp_id,
4370 .object = array_id,
4371 });
4372
4373 const elem_ptr_id = try self.accessChainId(ptr_elem_ty_id, tmp_id, &.{index_id});
4374
4375 const result_id = self.spv.allocId();
4376 try self.func.body.emit(self.spv.gpa, .OpLoad, .{
4377 .id_result_type = try self.resolveType(elem_ty, .direct),
4378 .id_result = result_id,
4379 .pointer = elem_ptr_id,
4380 });
4381
4382 if (array_ty.isVector(mod)) {
4383 // Result is already in direct representation
4384 return result_id;
4385 }
4386
4387 // This is an array type; the elements are stored in indirect representation.
4388 // We have to convert the type to direct.
42424389
4243 const tmp_id = try self.alloc(array_ty, .{ .storage_class = .Function });
4244 try self.store(array_ty, tmp_id, array_id, .{});
4245 const elem_ptr_id = try self.accessChainId(elem_ptr_ty_id, tmp_id, &.{index_id});
4246 return try self.load(elem_ty, elem_ptr_id, .{});
4390 return try self.convertToDirect(elem_ty, result_id);
42474391 }
42484392
42494393 fn airPtrElemVal(self: *DeclGen, inst: Air.Inst.Index) !?IdRef {