authorgravatar for alichraghi@proton.meAli Chraghi <alichraghi@proton.me> 2025-05-06 16:25:08+03:30
committergravatar for alichraghi@proton.meAli Chraghi <alichraghi@proton.me> 2025-05-21 12:57:40+03:30
logfca5f3602d697bd3de6a36d4504703693133144c
tree83135229630e0fb4e22b8d4e648ef6e668a965e3
parentf925e1379aa53228610df9b7ffc3d87dbcce0dbb

spirv: unroll all vector operations


2 files changed, 87 insertions(+), 328 deletions(-)

src/codegen/spirv.zig+87-326
......@@ -344,8 +344,7 @@ const NavGen = struct {
344344
345345 /// This structure is used to return information about a type typically used for
346346 /// arithmetic operations. These types may either be integers, floats, or a vector
347 /// of these. Most scalar operations also work on vectors, so we can easily represent
348 /// those as arithmetic types. If the type is a scalar, 'inner type' refers to the
347 /// of these. If the type is a scalar, 'inner type' refers to the
349348 /// scalar type. Otherwise, if its a vector, it refers to the vector's element type.
350349 const ArithmeticTypeInfo = struct {
351350 /// A classification of the inner type.
......@@ -615,41 +614,6 @@ const NavGen = struct {
615614 return if (self.spv.hasFeature(.int64)) 64 else 32;
616615 }
617616
618 /// Checks whether the type is "composite int", an integer consisting of multiple native integers. These are represented by
619 /// arrays of largestSupportedIntBits().
620 /// Asserts `ty` is an integer.
621 fn isCompositeInt(self: *NavGen, ty: Type) bool {
622 return self.backingIntBits(ty) == null;
623 }
624
625 /// Checks whether the type can be directly translated to SPIR-V vectors
626 fn isSpvVector(self: *NavGen, ty: Type) bool {
627 const zcu = self.pt.zcu;
628 if (ty.zigTypeTag(zcu) != .vector) return false;
629
630 // TODO: This check must be expanded for types that can be represented
631 // as integers (enums / packed structs?) and types that are represented
632 // by multiple SPIR-V values.
633 const scalar_ty = ty.scalarType(zcu);
634 switch (scalar_ty.zigTypeTag(zcu)) {
635 .bool,
636 .int,
637 .float,
638 => {},
639 else => return false,
640 }
641
642 const elem_ty = ty.childType(zcu);
643 const len = ty.vectorLen(zcu);
644
645 if (elem_ty.isNumeric(zcu) or elem_ty.toIntern() == .bool_type) {
646 if (len > 1 and len <= 4) return true;
647 if (self.spv.hasFeature(.vector16)) return (len == 8 or len == 16);
648 }
649
650 return false;
651 }
652
653617 fn arithmeticTypeInfo(self: *NavGen, ty: Type) ArithmeticTypeInfo {
654618 const zcu = self.pt.zcu;
655619 const target = self.spv.target;
......@@ -659,14 +623,14 @@ const NavGen = struct {
659623 }
660624 const vector_len = if (ty.isVector(zcu)) ty.vectorLen(zcu) else null;
661625 return switch (scalar_ty.zigTypeTag(zcu)) {
662 .bool => ArithmeticTypeInfo{
626 .bool => .{
663627 .bits = 1, // Doesn't matter for this class.
664628 .backing_bits = self.backingIntBits(1).?,
665629 .vector_len = vector_len,
666630 .signedness = .unsigned, // Technically, but doesn't matter for this class.
667631 .class = .bool,
668632 },
669 .float => ArithmeticTypeInfo{
633 .float => .{
670634 .bits = scalar_ty.floatBits(target),
671635 .backing_bits = scalar_ty.floatBits(target), // TODO: F80?
672636 .vector_len = vector_len,
......@@ -677,16 +641,16 @@ const NavGen = struct {
677641 const int_info = scalar_ty.intInfo(zcu);
678642 // TODO: Maybe it's useful to also return this value.
679643 const maybe_backing_bits = self.backingIntBits(int_info.bits);
680 break :blk ArithmeticTypeInfo{
644 break :blk .{
681645 .bits = int_info.bits,
682646 .backing_bits = maybe_backing_bits orelse 0,
683647 .vector_len = vector_len,
684648 .signedness = int_info.signedness,
685649 .class = if (maybe_backing_bits) |backing_bits|
686650 if (backing_bits == int_info.bits)
687 ArithmeticTypeInfo.Class.integer
651 .integer
688652 else
689 ArithmeticTypeInfo.Class.strange_integer
653 .strange_integer
690654 else
691655 .composite_integer,
692656 };
......@@ -1338,19 +1302,6 @@ const NavGen = struct {
13381302 return self.spv.functionType(return_ty_id, param_ids);
13391303 }
13401304
1341 fn zigScalarOrVectorTypeLike(self: *NavGen, new_ty: Type, base_ty: Type) !Type {
1342 const pt = self.pt;
1343 const new_scalar_ty = new_ty.scalarType(pt.zcu);
1344 if (!base_ty.isVector(pt.zcu)) {
1345 return new_scalar_ty;
1346 }
1347
1348 return try pt.vectorType(.{
1349 .len = base_ty.vectorLen(pt.zcu),
1350 .child = new_scalar_ty.toIntern(),
1351 });
1352 }
1353
13541305 /// Generate a union type. Union types are always generated with the
13551306 /// most aligned field active. If the tag alignment is greater
13561307 /// than that of the payload, a regular union (non-packed, with both tag and
......@@ -1632,12 +1583,7 @@ const NavGen = struct {
16321583 const elem_ty = ty.childType(zcu);
16331584 const elem_ty_id = try self.resolveType(elem_ty, repr);
16341585 const len = ty.vectorLen(zcu);
1635
1636 if (self.isSpvVector(ty)) {
1637 return try self.spv.vectorType(len, elem_ty_id);
1638 } else {
1639 return try self.arrayType(len, elem_ty_id);
1640 }
1586 return self.arrayType(len, elem_ty_id);
16411587 },
16421588 .@"struct" => {
16431589 const struct_type = switch (ip.indexToKey(ty.toIntern())) {
......@@ -2035,69 +1981,32 @@ const NavGen = struct {
20351981 const Vectorization = union(enum) {
20361982 /// This is an operation between scalars.
20371983 scalar,
2038 /// This is an operation between SPIR-V vectors.
2039 /// Value is number of components.
2040 spv_vectorized: u32,
20411984 /// This operation is unrolled into separate operations.
20421985 /// Inputs may still be SPIR-V vectors, for example,
20431986 /// when the operation can't be vectorized in SPIR-V.
20441987 /// Value is number of components.
20451988 unrolled: u32,
20461989
2047 /// Derive a vectorization from a particular type. This usually
2048 /// only checks the size, but the source-of-truth is implemented
2049 /// by `isSpvVector()`.
1990 /// Derive a vectorization from a particular type
20501991 fn fromType(ty: Type, ng: *NavGen) Vectorization {
20511992 const zcu = ng.pt.zcu;
2052 if (!ty.isVector(zcu)) {
2053 return .scalar;
2054 } else if (ng.isSpvVector(ty)) {
2055 return .{ .spv_vectorized = ty.vectorLen(zcu) };
2056 } else {
2057 return .{ .unrolled = ty.vectorLen(zcu) };
2058 }
1993 if (!ty.isVector(zcu)) return .scalar;
1994 return .{ .unrolled = ty.vectorLen(zcu) };
20591995 }
20601996
20611997 /// Given two vectorization methods, compute a "unification": a fallback
20621998 /// that works for both, according to the following rules:
20631999 /// - Scalars may broadcast
2064 /// - SPIR-V vectorized operations may unroll
2065 /// - Prefer scalar > SPIR-V vectorized > unrolled
2000 /// - SPIR-V vectorized operations will unroll
2001 /// - Prefer scalar > unrolled
20662002 fn unify(a: Vectorization, b: Vectorization) Vectorization {
2067 if (a == .scalar and b == .scalar) {
2068 return .scalar;
2069 } else if (a == .spv_vectorized and b == .spv_vectorized) {
2070 assert(a.components() == b.components());
2071 return .{ .spv_vectorized = a.components() };
2072 } else if (a == .unrolled or b == .unrolled) {
2073 if (a == .unrolled and b == .unrolled) {
2074 assert(a.components() == b.components());
2075 return .{ .unrolled = a.components() };
2076 } else if (a == .unrolled) {
2077 return .{ .unrolled = a.components() };
2078 } else if (b == .unrolled) {
2079 return .{ .unrolled = b.components() };
2080 } else {
2081 unreachable;
2082 }
2083 } else {
2084 if (a == .spv_vectorized) {
2085 return .{ .spv_vectorized = a.components() };
2086 } else if (b == .spv_vectorized) {
2087 return .{ .spv_vectorized = b.components() };
2088 } else {
2089 unreachable;
2090 }
2003 if (a == .scalar and b == .scalar) return .scalar;
2004 if (a == .unrolled or b == .unrolled) {
2005 if (a == .unrolled and b == .unrolled) assert(a.components() == b.components());
2006 if (a == .unrolled) return .{ .unrolled = a.components() };
2007 return .{ .unrolled = b.components() };
20912008 }
2092 }
2093
2094 /// Force this vectorization to be unrolled, if its
2095 /// an operation involving vectors.
2096 fn unroll(self: Vectorization) Vectorization {
2097 return switch (self) {
2098 .scalar, .unrolled => self,
2099 .spv_vectorized => |n| .{ .unrolled = n },
2100 };
2009 unreachable;
21012010 }
21022011
21032012 /// Query the number of components that inputs of this operation have.
......@@ -2106,35 +2015,10 @@ const NavGen = struct {
21062015 fn components(self: Vectorization) u32 {
21072016 return switch (self) {
21082017 .scalar => 1,
2109 .spv_vectorized => |n| n,
21102018 .unrolled => |n| n,
21112019 };
21122020 }
21132021
2114 /// Query the number of operations involving this vectorization.
2115 /// This is basically the number of components, except that SPIR-V vectorized
2116 /// operations only need a single SPIR-V instruction.
2117 fn operations(self: Vectorization) u32 {
2118 return switch (self) {
2119 .scalar, .spv_vectorized => 1,
2120 .unrolled => |n| n,
2121 };
2122 }
2123
2124 /// Turns `ty` into the result-type of an individual vector operation.
2125 /// `ty` may be a scalar or vector, it doesn't matter.
2126 fn operationType(self: Vectorization, ng: *NavGen, ty: Type) !Type {
2127 const pt = ng.pt;
2128 const scalar_ty = ty.scalarType(pt.zcu);
2129 return switch (self) {
2130 .scalar, .unrolled => scalar_ty,
2131 .spv_vectorized => |n| try pt.vectorType(.{
2132 .len = n,
2133 .child = scalar_ty.toIntern(),
2134 }),
2135 };
2136 }
2137
21382022 /// Turns `ty` into the result-type of the entire operation.
21392023 /// `ty` may be a scalar or vector, it doesn't matter.
21402024 fn resultType(self: Vectorization, ng: *NavGen, ty: Type) !Type {
......@@ -2142,10 +2026,7 @@ const NavGen = struct {
21422026 const scalar_ty = ty.scalarType(pt.zcu);
21432027 return switch (self) {
21442028 .scalar => scalar_ty,
2145 .unrolled, .spv_vectorized => |n| try pt.vectorType(.{
2146 .len = n,
2147 .child = scalar_ty.toIntern(),
2148 }),
2029 .unrolled => |n| try pt.vectorType(.{ .len = n, .child = scalar_ty.toIntern() }),
21492030 };
21502031 }
21512032
......@@ -2155,51 +2036,19 @@ const NavGen = struct {
21552036 fn prepare(self: Vectorization, ng: *NavGen, tmp: Temporary) !PreparedOperand {
21562037 const pt = ng.pt;
21572038 const is_vector = tmp.ty.isVector(pt.zcu);
2158 const is_spv_vector = ng.isSpvVector(tmp.ty);
21592039 const value: PreparedOperand.Value = switch (tmp.value) {
21602040 .singleton => |id| switch (self) {
21612041 .scalar => blk: {
21622042 assert(!is_vector);
21632043 break :blk .{ .scalar = id };
21642044 },
2165 .spv_vectorized => blk: {
2166 if (is_vector) {
2167 assert(is_spv_vector);
2168 break :blk .{ .spv_vectorwise = id };
2169 }
2170
2171 // Broadcast scalar into vector.
2172 const vector_ty = try pt.vectorType(.{
2173 .len = self.components(),
2174 .child = tmp.ty.toIntern(),
2175 });
2176
2177 const vector = try ng.constructCompositeSplat(vector_ty, id);
2178 return .{
2179 .ty = vector_ty,
2180 .value = .{ .spv_vectorwise = vector },
2181 };
2182 },
21832045 .unrolled => blk: {
2184 if (is_vector) {
2185 break :blk .{ .vector_exploded = try tmp.explode(ng) };
2186 } else {
2187 break :blk .{ .scalar_broadcast = id };
2188 }
2046 if (is_vector) break :blk .{ .vector_exploded = try tmp.explode(ng) };
2047 break :blk .{ .scalar_broadcast = id };
21892048 },
21902049 },
21912050 .exploded_vector => |range| switch (self) {
21922051 .scalar => unreachable,
2193 .spv_vectorized => |n| blk: {
2194 // We can vectorize this operation, but we have an exploded vector. This can happen
2195 // when a vectorizable operation succeeds a non-vectorizable operation. In this case,
2196 // pack up the IDs into a SPIR-V vector. This path should not be able to be hit with
2197 // a type that cannot do that.
2198 assert(is_spv_vector);
2199 assert(range.len == n);
2200 const vec = try tmp.materialize(ng);
2201 break :blk .{ .spv_vectorwise = vec };
2202 },
22032052 .unrolled => |n| blk: {
22042053 assert(range.len == n);
22052054 break :blk .{ .vector_exploded = range };
......@@ -2216,17 +2065,14 @@ const NavGen = struct {
22162065 /// Finalize the results of an operation back into a temporary. `results` is
22172066 /// a list of result-ids of the operation.
22182067 fn finalize(self: Vectorization, ty: Type, results: IdRange) Temporary {
2219 assert(self.operations() == results.len);
2220 const value: Temporary.Value = switch (self) {
2221 .scalar, .spv_vectorized => blk: {
2222 break :blk .{ .singleton = results.at(0) };
2223 },
2224 .unrolled => blk: {
2225 break :blk .{ .exploded_vector = results };
2068 assert(self.components() == results.len);
2069 return .{
2070 .ty = ty,
2071 .value = switch (self) {
2072 .scalar => .{ .singleton = results.at(0) },
2073 .unrolled => .{ .exploded_vector = results },
22262074 },
22272075 };
2228
2229 return .{ .ty = ty, .value = value };
22302076 }
22312077
22322078 /// This struct represents an operand that has gone through some setup, and is
......@@ -2242,32 +2088,20 @@ const NavGen = struct {
22422088 scalar: IdResult,
22432089 /// A single scalar that is broadcasted in an unrolled operation.
22442090 scalar_broadcast: IdResult,
2245 /// A SPIR-V vector that is used in SPIR-V vectorize operation.
2246 spv_vectorwise: IdResult,
22472091 /// A vector represented by a consecutive list of IDs that is used in an unrolled operation.
22482092 vector_exploded: IdRange,
22492093 };
22502094
22512095 /// Query the value at a particular index of the operation. Note that
2252 /// the index is *not* the component/lane, but the index of the *operation*. When
2253 /// this operation is vectorized, the return value of this function is a SPIR-V vector.
2254 /// See also `Vectorization.operations()`.
2096 /// the index is *not* the component/lane, but the index of the *operation*.
22552097 fn at(self: PreparedOperand, i: usize) IdResult {
22562098 switch (self.value) {
22572099 .scalar => |id| {
22582100 assert(i == 0);
22592101 return id;
22602102 },
2261 .scalar_broadcast => |id| {
2262 return id;
2263 },
2264 .spv_vectorwise => |id| {
2265 assert(i == 0);
2266 return id;
2267 },
2268 .vector_exploded => |range| {
2269 return range.at(i);
2270 },
2103 .scalar_broadcast => |id| return id,
2104 .vector_exploded => |range| return range.at(i),
22712105 }
22722106 }
22732107 };
......@@ -2299,7 +2133,7 @@ const NavGen = struct {
22992133
23002134 /// This function builds an OpSConvert of OpUConvert depending on the
23012135 /// signedness of the types.
2302 fn buildIntConvert(self: *NavGen, dst_ty: Type, src: Temporary) !Temporary {
2136 fn buildConvert(self: *NavGen, dst_ty: Type, src: Temporary) !Temporary {
23032137 const zcu = self.pt.zcu;
23042138
23052139 const dst_ty_id = try self.resolveType(dst_ty.scalarType(zcu), .direct);
......@@ -2318,13 +2152,17 @@ const NavGen = struct {
23182152 return src.pun(result_ty);
23192153 }
23202154
2321 const ops = v.operations();
2155 const ops = v.components();
23222156 const results = self.spv.allocIds(ops);
23232157
2324 const op_result_ty = try v.operationType(self, dst_ty);
2158 const op_result_ty = dst_ty.scalarType(zcu);
23252159 const op_result_ty_id = try self.resolveType(op_result_ty, .direct);
23262160
2327 const opcode: Opcode = if (dst_ty.isSignedInt(zcu)) .OpSConvert else .OpUConvert;
2161 const opcode: Opcode = blk: {
2162 if (dst_ty.scalarType(zcu).isAnyFloat()) break :blk .OpFConvert;
2163 if (dst_ty.scalarType(zcu).isSignedInt(zcu)) break :blk .OpSConvert;
2164 break :blk .OpUConvert;
2165 };
23282166
23292167 const op_src = try v.prepare(self, src);
23302168
......@@ -2339,13 +2177,14 @@ const NavGen = struct {
23392177 }
23402178
23412179 fn buildFma(self: *NavGen, a: Temporary, b: Temporary, c: Temporary) !Temporary {
2180 const zcu = self.pt.zcu;
23422181 const target = self.spv.target;
23432182
23442183 const v = self.vectorization(.{ a, b, c });
2345 const ops = v.operations();
2184 const ops = v.components();
23462185 const results = self.spv.allocIds(ops);
23472186
2348 const op_result_ty = try v.operationType(self, a.ty);
2187 const op_result_ty = a.ty.scalarType(zcu);
23492188 const op_result_ty_id = try self.resolveType(op_result_ty, .direct);
23502189 const result_ty = try v.resultType(self, a.ty);
23512190
......@@ -2382,10 +2221,10 @@ const NavGen = struct {
23822221 const zcu = self.pt.zcu;
23832222
23842223 const v = self.vectorization(.{ condition, lhs, rhs });
2385 const ops = v.operations();
2224 const ops = v.components();
23862225 const results = self.spv.allocIds(ops);
23872226
2388 const op_result_ty = try v.operationType(self, lhs.ty);
2227 const op_result_ty = lhs.ty.scalarType(zcu);
23892228 const op_result_ty_id = try self.resolveType(op_result_ty, .direct);
23902229 const result_ty = try v.resultType(self, lhs.ty);
23912230
......@@ -2431,10 +2270,10 @@ const NavGen = struct {
24312270
24322271 fn buildCmp(self: *NavGen, pred: CmpPredicate, lhs: Temporary, rhs: Temporary) !Temporary {
24332272 const v = self.vectorization(.{ lhs, rhs });
2434 const ops = v.operations();
2273 const ops = v.components();
24352274 const results = self.spv.allocIds(ops);
24362275
2437 const op_result_ty = try v.operationType(self, Type.bool);
2276 const op_result_ty: Type = .bool;
24382277 const op_result_ty_id = try self.resolveType(op_result_ty, .direct);
24392278 const result_ty = try v.resultType(self, Type.bool);
24402279
......@@ -2498,22 +2337,12 @@ const NavGen = struct {
24982337 };
24992338
25002339 fn buildUnary(self: *NavGen, op: UnaryOp, operand: Temporary) !Temporary {
2340 const zcu = self.pt.zcu;
25012341 const target = self.spv.target;
2502 const v = blk: {
2503 const v = self.vectorization(.{operand});
2504 break :blk switch (op) {
2505 // TODO: These instructions don't seem to be working
2506 // properly for LLVM-based backends on OpenCL for 8- and
2507 // 16-component vectors.
2508 .i_abs => if (self.spv.hasFeature(.vector16) and v.components() >= 8) v.unroll() else v,
2509 else => v,
2510 };
2511 };
2512
2513 const ops = v.operations();
2342 const v = self.vectorization(.{operand});
2343 const ops = v.components();
25142344 const results = self.spv.allocIds(ops);
2515
2516 const op_result_ty = try v.operationType(self, operand.ty);
2345 const op_result_ty = operand.ty.scalarType(zcu);
25172346 const op_result_ty_id = try self.resolveType(op_result_ty, .direct);
25182347 const result_ty = try v.resultType(self, operand.ty);
25192348
......@@ -2628,13 +2457,14 @@ const NavGen = struct {
26282457 };
26292458
26302459 fn buildBinary(self: *NavGen, op: BinaryOp, lhs: Temporary, rhs: Temporary) !Temporary {
2460 const zcu = self.pt.zcu;
26312461 const target = self.spv.target;
26322462
26332463 const v = self.vectorization(.{ lhs, rhs });
2634 const ops = v.operations();
2464 const ops = v.components();
26352465 const results = self.spv.allocIds(ops);
26362466
2637 const op_result_ty = try v.operationType(self, lhs.ty);
2467 const op_result_ty = lhs.ty.scalarType(zcu);
26382468 const op_result_ty_id = try self.resolveType(op_result_ty, .direct);
26392469 const result_ty = try v.resultType(self, lhs.ty);
26402470
......@@ -2730,9 +2560,9 @@ const NavGen = struct {
27302560 const ip = &zcu.intern_pool;
27312561
27322562 const v = lhs.vectorization(self).unify(rhs.vectorization(self));
2733 const ops = v.operations();
2563 const ops = v.components();
27342564
2735 const arith_op_ty = try v.operationType(self, lhs.ty);
2565 const arith_op_ty = lhs.ty.scalarType(zcu);
27362566 const arith_op_ty_id = try self.resolveType(arith_op_ty, .direct);
27372567
27382568 const lhs_op = try v.prepare(self, lhs);
......@@ -3175,17 +3005,18 @@ const NavGen = struct {
31753005 /// Convert representation from indirect (in memory) to direct (in 'register')
31763006 /// This converts the argument type from resolveType(ty, .indirect) to resolveType(ty, .direct).
31773007 fn convertToDirect(self: *NavGen, ty: Type, operand_id: IdRef) !IdRef {
3178 const zcu = self.pt.zcu;
3008 const pt = self.pt;
3009 const zcu = pt.zcu;
31793010 switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
31803011 .bool => {
31813012 const false_id = try self.constBool(false, .indirect);
3182 // The operation below requires inputs in direct representation, but the operand
3183 // is actually in indirect representation.
3184 // Cheekily swap out the type to the direct equivalent of the indirect type here, they have the
3185 // same representation when converted to SPIR-V.
3186 const operand_ty = try self.zigScalarOrVectorTypeLike(Type.u1, ty);
3187 // Note: We can guarantee that these are the same ID due to the SPIR-V Module's `vector_types` cache!
3188 assert(try self.resolveType(operand_ty, .direct) == try self.resolveType(ty, .indirect));
3013 const operand_ty = blk: {
3014 if (!ty.isVector(pt.zcu)) break :blk Type.u1;
3015 break :blk try pt.vectorType(.{
3016 .len = ty.vectorLen(pt.zcu),
3017 .child = Type.u1.toIntern(),
3018 });
3019 };
31893020
31903021 const result = try self.buildCmp(
31913022 .i_ne,
......@@ -3226,7 +3057,6 @@ const NavGen = struct {
32263057 }
32273058
32283059 fn extractVectorComponent(self: *NavGen, result_ty: Type, vector_id: IdRef, field: u32) !IdRef {
3229 // Whether this is an OpTypeVector or OpTypeArray, we need to emit the same instruction regardless.
32303060 const result_ty_id = try self.resolveType(result_ty, .direct);
32313061 const result_id = self.spv.allocId();
32323062 const indexes = [_]u32{field};
......@@ -3485,7 +3315,7 @@ const NavGen = struct {
34853315 // Note: The sign may differ here between the shift and the base type, in case
34863316 // of an arithmetic right shift. SPIR-V still expects the same type,
34873317 // so in that case we have to cast convert to signed.
3488 const casted_shift = try self.buildIntConvert(base.ty.scalarType(zcu), shift);
3318 const casted_shift = try self.buildConvert(base.ty.scalarType(zcu), shift);
34893319
34903320 const shifted = switch (info.signedness) {
34913321 .unsigned => try self.buildBinary(unsigned, base, casted_shift),
......@@ -3815,12 +3645,12 @@ const NavGen = struct {
38153645 .unsigned => blk: {
38163646 if (maybe_op_ty_bits) |op_ty_bits| {
38173647 const op_ty = try pt.intType(.unsigned, op_ty_bits);
3818 const casted_lhs = try self.buildIntConvert(op_ty, lhs);
3819 const casted_rhs = try self.buildIntConvert(op_ty, rhs);
3648 const casted_lhs = try self.buildConvert(op_ty, lhs);
3649 const casted_rhs = try self.buildConvert(op_ty, rhs);
38203650
38213651 const full_result = try self.buildBinary(.i_mul, casted_lhs, casted_rhs);
38223652
3823 const low_bits = try self.buildIntConvert(lhs.ty, full_result);
3653 const low_bits = try self.buildConvert(lhs.ty, full_result);
38243654 const result = try self.normalize(low_bits, info);
38253655
38263656 // Shift the result bits away to get the overflow bits.
......@@ -3846,9 +3676,7 @@ const NavGen = struct {
38463676 const high_overflowed = try self.buildCmp(.i_ne, zero, high_bits);
38473677
38483678 // If no overflow bits in low_bits, no extra work needs to be done.
3849 if (info.backing_bits == info.bits) {
3850 break :blk .{ result, high_overflowed };
3851 }
3679 if (info.backing_bits == info.bits) break :blk .{ result, high_overflowed };
38523680
38533681 // Shift the result bits away to get the overflow bits.
38543682 const shift = Temporary.init(lhs.ty, try self.constInt(lhs.ty, info.bits));
......@@ -3886,13 +3714,13 @@ const NavGen = struct {
38863714 if (maybe_op_ty_bits) |op_ty_bits| {
38873715 const op_ty = try pt.intType(.signed, op_ty_bits);
38883716 // Assume normalized; sign bit is set. We want a sign extend.
3889 const casted_lhs = try self.buildIntConvert(op_ty, lhs);
3890 const casted_rhs = try self.buildIntConvert(op_ty, rhs);
3717 const casted_lhs = try self.buildConvert(op_ty, lhs);
3718 const casted_rhs = try self.buildConvert(op_ty, rhs);
38913719
38923720 const full_result = try self.buildBinary(.i_mul, casted_lhs, casted_rhs);
38933721
38943722 // Truncate to the result type.
3895 const low_bits = try self.buildIntConvert(lhs.ty, full_result);
3723 const low_bits = try self.buildConvert(lhs.ty, full_result);
38963724 const result = try self.normalize(low_bits, info);
38973725
38983726 // Now, we need to check the overflow bits AND the sign
......@@ -3929,9 +3757,7 @@ const NavGen = struct {
39293757 // If no overflow bits in low_bits, no extra work needs to be done.
39303758 // Careful, we still have to check the sign bit, so this branch
39313759 // only goes for i33 and such.
3932 if (info.backing_bits == info.bits + 1) {
3933 break :blk .{ result, high_overflowed };
3934 }
3760 if (info.backing_bits == info.bits + 1) break :blk .{ result, high_overflowed };
39353761
39363762 // Shift the result bits away to get the overflow bits.
39373763 const shift = Temporary.init(lhs.ty, try self.constInt(lhs.ty, info.bits - 1));
......@@ -3972,7 +3798,7 @@ const NavGen = struct {
39723798
39733799 // Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that,
39743800 // so just manually upcast it if required.
3975 const casted_shift = try self.buildIntConvert(base.ty.scalarType(zcu), shift);
3801 const casted_shift = try self.buildConvert(base.ty.scalarType(zcu), shift);
39763802
39773803 const left = try self.buildBinary(.sll, base, casted_shift);
39783804 const result = try self.normalize(left, info);
......@@ -4026,7 +3852,7 @@ const NavGen = struct {
40263852 // Result of OpenCL ctz/clz returns operand.ty, and we want result_ty.
40273853 // result_ty is always large enough to hold the result, so we might have to down
40283854 // cast it.
4029 const result = try self.buildIntConvert(scalar_result_ty, count);
3855 const result = try self.buildConvert(scalar_result_ty, count);
40303856 return try result.materialize(self);
40313857 }
40323858
......@@ -4057,11 +3883,8 @@ const NavGen = struct {
40573883 const operand_ty = self.typeOf(reduce.operand);
40583884 const scalar_ty = operand_ty.scalarType(zcu);
40593885 const scalar_ty_id = try self.resolveType(scalar_ty, .direct);
4060
40613886 const info = self.arithmeticTypeInfo(operand_ty);
4062
40633887 const len = operand_ty.vectorLen(zcu);
4064
40653888 const first = try self.extractVectorComponent(scalar_ty, operand, 0);
40663889
40673890 switch (reduce.operation) {
......@@ -4136,51 +3959,9 @@ const NavGen = struct {
41363959
41373960 // Note: number of components in the result, a, and b may differ.
41383961 const result_ty = self.typeOfIndex(inst);
4139 const a_ty = self.typeOf(extra.a);
4140 const b_ty = self.typeOf(extra.b);
4141
41423962 const scalar_ty = result_ty.scalarType(zcu);
41433963 const scalar_ty_id = try self.resolveType(scalar_ty, .direct);
41443964
4145 // If all of the types are SPIR-V vectors, we can use OpVectorShuffle.
4146 if (self.isSpvVector(result_ty) and self.isSpvVector(a_ty) and self.isSpvVector(b_ty)) {
4147 // The SPIR-V shuffle instruction is similar to the Air instruction, except that the elements are
4148 // numbered consecutively instead of using negatives.
4149
4150 const components = try self.gpa.alloc(Word, result_ty.vectorLen(zcu));
4151 defer self.gpa.free(components);
4152
4153 const a_len = a_ty.vectorLen(zcu);
4154
4155 for (components, 0..) |*component, i| {
4156 const elem = try mask.elemValue(pt, i);
4157 if (elem.isUndef(zcu)) {
4158 // This is explicitly valid for OpVectorShuffle, it indicates undefined.
4159 component.* = 0xFFFF_FFFF;
4160 continue;
4161 }
4162
4163 const index = elem.toSignedInt(zcu);
4164 if (index >= 0) {
4165 component.* = @intCast(index);
4166 } else {
4167 component.* = @intCast(~index + a_len);
4168 }
4169 }
4170
4171 const result_id = self.spv.allocId();
4172 try self.func.body.emit(self.spv.gpa, .OpVectorShuffle, .{
4173 .id_result_type = try self.resolveType(result_ty, .direct),
4174 .id_result = result_id,
4175 .vector_1 = a,
4176 .vector_2 = b,
4177 .components = components,
4178 });
4179 return result_id;
4180 }
4181
4182 // Fall back to manually extracting and inserting components.
4183
41843965 const constituents = try self.gpa.alloc(IdRef, result_ty.vectorLen(zcu));
41853966 defer self.gpa.free(constituents);
41863967
......@@ -4535,9 +4316,7 @@ const NavGen = struct {
45354316 const dst_ty_id = try self.resolveType(dst_ty, .direct);
45364317
45374318 const result_id = blk: {
4538 if (src_ty_id == dst_ty_id) {
4539 break :blk src_id;
4540 }
4319 if (src_ty_id == dst_ty_id) break :blk src_id;
45414320
45424321 // TODO: Some more cases are missing here
45434322 // See fn bitCast in llvm.zig
......@@ -4618,7 +4397,7 @@ const NavGen = struct {
46184397 return try src.materialize(self);
46194398 }
46204399
4621 const converted = try self.buildIntConvert(dst_ty, src);
4400 const converted = try self.buildConvert(dst_ty, src);
46224401
46234402 // Make sure to normalize the result if shrinking.
46244403 // Because strange ints are sign extended in their backing
......@@ -4698,17 +4477,10 @@ const NavGen = struct {
46984477
46994478 fn airFloatCast(self: *NavGen, inst: Air.Inst.Index) !?IdRef {
47004479 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4701 const operand_id = try self.resolve(ty_op.operand);
4480 const operand = try self.temporary(ty_op.operand);
47024481 const dest_ty = self.typeOfIndex(inst);
4703 const dest_ty_id = try self.resolveType(dest_ty, .direct);
4704
4705 const result_id = self.spv.allocId();
4706 try self.func.body.emit(self.spv.gpa, .OpFConvert, .{
4707 .id_result_type = dest_ty_id,
4708 .id_result = result_id,
4709 .float_value = operand_id,
4710 });
4711 return result_id;
4482 const result = try self.buildConvert(dest_ty, operand);
4483 return try result.materialize(self);
47124484 }
47134485
47144486 fn airNot(self: *NavGen, inst: Air.Inst.Index) !?IdRef {
......@@ -4796,7 +4568,7 @@ const NavGen = struct {
47964568 break :blk try self.bitCast(field_int_ty, field_ty, field_id);
47974569 };
47984570 const shift_rhs = try self.constInt(backing_int_ty, running_bits);
4799 const extended_int_conv = try self.buildIntConvert(backing_int_ty, .{
4571 const extended_int_conv = try self.buildConvert(backing_int_ty, .{
48004572 .ty = field_int_ty,
48014573 .value = .{ .singleton = field_int_id },
48024574 });
......@@ -5016,17 +4788,6 @@ const NavGen = struct {
50164788 const array_id = try self.resolve(bin_op.lhs);
50174789 const index_id = try self.resolve(bin_op.rhs);
50184790
5019 if (self.isSpvVector(array_ty)) {
5020 const result_id = self.spv.allocId();
5021 try self.func.body.emit(self.spv.gpa, .OpVectorExtractDynamic, .{
5022 .id_result_type = try self.resolveType(elem_ty, .direct),
5023 .id_result = result_id,
5024 .vector = array_id,
5025 .index = index_id,
5026 });
5027 return result_id;
5028 }
5029
50304791 // SPIR-V doesn't have an array indexing function for some damn reason.
50314792 // For now, just generate a temporary and use that.
50324793 // TODO: This backend probably also should use isByRef from llvm...
......@@ -5173,7 +4934,7 @@ const NavGen = struct {
51734934 return self.bitCast(ty, payload_ty, payload.?);
51744935 }
51754936
5176 const trunc = try self.buildIntConvert(ty, .{ .ty = payload_ty, .value = .{ .singleton = payload.? } });
4937 const trunc = try self.buildConvert(ty, .{ .ty = payload_ty, .value = .{ .singleton = payload.? } });
51774938 return try trunc.materialize(self);
51784939 }
51794940
......@@ -5182,7 +4943,7 @@ const NavGen = struct {
51824943 try self.convertToIndirect(payload_ty, payload.?)
51834944 else
51844945 try self.bitCast(payload_int_ty, payload_ty, payload.?);
5185 const trunc = try self.buildIntConvert(ty, .{ .ty = payload_int_ty, .value = .{ .singleton = payload_int } });
4946 const trunc = try self.buildConvert(ty, .{ .ty = payload_int_ty, .value = .{ .singleton = payload_int } });
51864947 return try trunc.materialize(self);
51874948 }
51884949
......@@ -5273,7 +5034,7 @@ const NavGen = struct {
52735034 const result_id = blk: {
52745035 if (self.backingIntBits(field_bit_size).? == self.backingIntBits(@intCast(object_ty.bitSize(zcu))).?)
52755036 break :blk try self.bitCast(field_int_ty, object_ty, try masked.materialize(self));
5276 const trunc = try self.buildIntConvert(field_int_ty, masked);
5037 const trunc = try self.buildConvert(field_int_ty, masked);
52775038 break :blk try trunc.materialize(self);
52785039 };
52795040 if (field_ty.ip_index == .bool_type) return try self.convertToDirect(.bool, result_id);
......@@ -5297,7 +5058,7 @@ const NavGen = struct {
52975058 const result_id = blk: {
52985059 if (self.backingIntBits(field_bit_size).? == self.backingIntBits(@intCast(backing_int_ty.bitSize(zcu))).?)
52995060 break :blk try self.bitCast(int_ty, backing_int_ty, try masked.materialize(self));
5300 const trunc = try self.buildIntConvert(int_ty, masked);
5061 const trunc = try self.buildConvert(int_ty, masked);
53015062 break :blk try trunc.materialize(self);
53025063 };
53035064 if (field_ty.ip_index == .bool_type) return try self.convertToDirect(.bool, result_id);
......@@ -6752,7 +6513,7 @@ const NavGen = struct {
67526513 // TODO: Should we make these builtins return usize?
67536514 const result_id = try self.builtin3D(Type.u64, .LocalInvocationId, dimension, 0);
67546515 const tmp = Temporary.init(Type.u64, result_id);
6755 const result = try self.buildIntConvert(Type.u32, tmp);
6516 const result = try self.buildConvert(Type.u32, tmp);
67566517 return try result.materialize(self);
67576518 }
67586519
......@@ -6763,7 +6524,7 @@ const NavGen = struct {
67636524 // TODO: Should we make these builtins return usize?
67646525 const result_id = try self.builtin3D(Type.u64, .WorkgroupSize, dimension, 0);
67656526 const tmp = Temporary.init(Type.u64, result_id);
6766 const result = try self.buildIntConvert(Type.u32, tmp);
6527 const result = try self.buildConvert(Type.u32, tmp);
67676528 return try result.materialize(self);
67686529 }
67696530
......@@ -6774,7 +6535,7 @@ const NavGen = struct {
67746535 // TODO: Should we make these builtins return usize?
67756536 const result_id = try self.builtin3D(Type.u64, .WorkgroupId, dimension, 0);
67766537 const tmp = Temporary.init(Type.u64, result_id);
6777 const result = try self.buildIntConvert(Type.u32, tmp);
6538 const result = try self.buildConvert(Type.u32, tmp);
67786539 return try result.materialize(self);
67796540 }
67806541
src/codegen/spirv/Module.zig-2
......@@ -164,8 +164,6 @@ cache: struct {
164164 void_type: ?IdRef = null,
165165 int_types: std.AutoHashMapUnmanaged(std.builtin.Type.Int, IdRef) = .empty,
166166 float_types: std.AutoHashMapUnmanaged(std.builtin.Type.Float, IdRef) = .empty,
167 // This cache is required so that @Vector(X, u1) in direct representation has the
168 // same ID as @Vector(X, bool) in indirect representation.
169167 vector_types: std.AutoHashMapUnmanaged(struct { IdRef, u32 }, IdRef) = .empty,
170168 array_types: std.AutoHashMapUnmanaged(struct { IdRef, IdRef }, IdRef) = .empty,
171169