| ... | @@ -344,8 +344,7 @@ const NavGen = struct { | ... | @@ -344,8 +344,7 @@ const NavGen = struct { |
| 344 | | 344 | |
| 345 | /// This structure is used to return information about a type typically used for | 345 | /// This structure is used to return information about a type typically used for |
| 346 | /// arithmetic operations. These types may either be integers, floats, or a vector | 346 | /// arithmetic operations. These types may either be integers, floats, or a vector |
| 347 | /// of these. Most scalar operations also work on vectors, so we can easily represent | 347 | /// of these. If the type is a scalar, 'inner type' refers to the |
| 348 | /// those as arithmetic types. If the type is a scalar, 'inner type' refers to the | | |
| 349 | /// scalar type. Otherwise, if its a vector, it refers to the vector's element type. | 348 | /// scalar type. Otherwise, if its a vector, it refers to the vector's element type. |
| 350 | const ArithmeticTypeInfo = struct { | 349 | const ArithmeticTypeInfo = struct { |
| 351 | /// A classification of the inner type. | 350 | /// A classification of the inner type. |
| ... | @@ -615,41 +614,6 @@ const NavGen = struct { | ... | @@ -615,41 +614,6 @@ const NavGen = struct { |
| 615 | return if (self.spv.hasFeature(.int64)) 64 else 32; | 614 | return if (self.spv.hasFeature(.int64)) 64 else 32; |
| 616 | } | 615 | } |
| 617 | | 616 | |
| 618 | /// Checks whether the type is "composite int", an integer consisting of multiple native integers. These are represented by | | |
| 619 | /// arrays of largestSupportedIntBits(). | | |
| 620 | /// Asserts `ty` is an integer. | | |
| 621 | fn isCompositeInt(self: *NavGen, ty: Type) bool { | | |
| 622 | return self.backingIntBits(ty) == null; | | |
| 623 | } | | |
| 624 | | | |
| 625 | /// Checks whether the type can be directly translated to SPIR-V vectors | | |
| 626 | fn isSpvVector(self: *NavGen, ty: Type) bool { | | |
| 627 | const zcu = self.pt.zcu; | | |
| 628 | if (ty.zigTypeTag(zcu) != .vector) return false; | | |
| 629 | | | |
| 630 | // TODO: This check must be expanded for types that can be represented | | |
| 631 | // as integers (enums / packed structs?) and types that are represented | | |
| 632 | // by multiple SPIR-V values. | | |
| 633 | const scalar_ty = ty.scalarType(zcu); | | |
| 634 | switch (scalar_ty.zigTypeTag(zcu)) { | | |
| 635 | .bool, | | |
| 636 | .int, | | |
| 637 | .float, | | |
| 638 | => {}, | | |
| 639 | else => return false, | | |
| 640 | } | | |
| 641 | | | |
| 642 | const elem_ty = ty.childType(zcu); | | |
| 643 | const len = ty.vectorLen(zcu); | | |
| 644 | | | |
| 645 | if (elem_ty.isNumeric(zcu) or elem_ty.toIntern() == .bool_type) { | | |
| 646 | if (len > 1 and len <= 4) return true; | | |
| 647 | if (self.spv.hasFeature(.vector16)) return (len == 8 or len == 16); | | |
| 648 | } | | |
| 649 | | | |
| 650 | return false; | | |
| 651 | } | | |
| 652 | | | |
| 653 | fn arithmeticTypeInfo(self: *NavGen, ty: Type) ArithmeticTypeInfo { | 617 | fn arithmeticTypeInfo(self: *NavGen, ty: Type) ArithmeticTypeInfo { |
| 654 | const zcu = self.pt.zcu; | 618 | const zcu = self.pt.zcu; |
| 655 | const target = self.spv.target; | 619 | const target = self.spv.target; |
| ... | @@ -659,14 +623,14 @@ const NavGen = struct { | ... | @@ -659,14 +623,14 @@ const NavGen = struct { |
| 659 | } | 623 | } |
| 660 | const vector_len = if (ty.isVector(zcu)) ty.vectorLen(zcu) else null; | 624 | const vector_len = if (ty.isVector(zcu)) ty.vectorLen(zcu) else null; |
| 661 | return switch (scalar_ty.zigTypeTag(zcu)) { | 625 | return switch (scalar_ty.zigTypeTag(zcu)) { |
| 662 | .bool => ArithmeticTypeInfo{ | 626 | .bool => .{ |
| 663 | .bits = 1, // Doesn't matter for this class. | 627 | .bits = 1, // Doesn't matter for this class. |
| 664 | .backing_bits = self.backingIntBits(1).?, | 628 | .backing_bits = self.backingIntBits(1).?, |
| 665 | .vector_len = vector_len, | 629 | .vector_len = vector_len, |
| 666 | .signedness = .unsigned, // Technically, but doesn't matter for this class. | 630 | .signedness = .unsigned, // Technically, but doesn't matter for this class. |
| 667 | .class = .bool, | 631 | .class = .bool, |
| 668 | }, | 632 | }, |
| 669 | .float => ArithmeticTypeInfo{ | 633 | .float => .{ |
| 670 | .bits = scalar_ty.floatBits(target), | 634 | .bits = scalar_ty.floatBits(target), |
| 671 | .backing_bits = scalar_ty.floatBits(target), // TODO: F80? | 635 | .backing_bits = scalar_ty.floatBits(target), // TODO: F80? |
| 672 | .vector_len = vector_len, | 636 | .vector_len = vector_len, |
| ... | @@ -677,16 +641,16 @@ const NavGen = struct { | ... | @@ -677,16 +641,16 @@ const NavGen = struct { |
| 677 | const int_info = scalar_ty.intInfo(zcu); | 641 | const int_info = scalar_ty.intInfo(zcu); |
| 678 | // TODO: Maybe it's useful to also return this value. | 642 | // TODO: Maybe it's useful to also return this value. |
| 679 | const maybe_backing_bits = self.backingIntBits(int_info.bits); | 643 | const maybe_backing_bits = self.backingIntBits(int_info.bits); |
| 680 | break :blk ArithmeticTypeInfo{ | 644 | break :blk .{ |
| 681 | .bits = int_info.bits, | 645 | .bits = int_info.bits, |
| 682 | .backing_bits = maybe_backing_bits orelse 0, | 646 | .backing_bits = maybe_backing_bits orelse 0, |
| 683 | .vector_len = vector_len, | 647 | .vector_len = vector_len, |
| 684 | .signedness = int_info.signedness, | 648 | .signedness = int_info.signedness, |
| 685 | .class = if (maybe_backing_bits) |backing_bits| | 649 | .class = if (maybe_backing_bits) |backing_bits| |
| 686 | if (backing_bits == int_info.bits) | 650 | if (backing_bits == int_info.bits) |
| 687 | ArithmeticTypeInfo.Class.integer | 651 | .integer |
| 688 | else | 652 | else |
| 689 | ArithmeticTypeInfo.Class.strange_integer | 653 | .strange_integer |
| 690 | else | 654 | else |
| 691 | .composite_integer, | 655 | .composite_integer, |
| 692 | }; | 656 | }; |
| ... | @@ -1338,19 +1302,6 @@ const NavGen = struct { | ... | @@ -1338,19 +1302,6 @@ const NavGen = struct { |
| 1338 | return self.spv.functionType(return_ty_id, param_ids); | 1302 | return self.spv.functionType(return_ty_id, param_ids); |
| 1339 | } | 1303 | } |
| 1340 | | 1304 | |
| 1341 | fn zigScalarOrVectorTypeLike(self: *NavGen, new_ty: Type, base_ty: Type) !Type { | | |
| 1342 | const pt = self.pt; | | |
| 1343 | const new_scalar_ty = new_ty.scalarType(pt.zcu); | | |
| 1344 | if (!base_ty.isVector(pt.zcu)) { | | |
| 1345 | return new_scalar_ty; | | |
| 1346 | } | | |
| 1347 | | | |
| 1348 | return try pt.vectorType(.{ | | |
| 1349 | .len = base_ty.vectorLen(pt.zcu), | | |
| 1350 | .child = new_scalar_ty.toIntern(), | | |
| 1351 | }); | | |
| 1352 | } | | |
| 1353 | | | |
| 1354 | /// Generate a union type. Union types are always generated with the | 1305 | /// Generate a union type. Union types are always generated with the |
| 1355 | /// most aligned field active. If the tag alignment is greater | 1306 | /// most aligned field active. If the tag alignment is greater |
| 1356 | /// than that of the payload, a regular union (non-packed, with both tag and | 1307 | /// than that of the payload, a regular union (non-packed, with both tag and |
| ... | @@ -1632,12 +1583,7 @@ const NavGen = struct { | ... | @@ -1632,12 +1583,7 @@ const NavGen = struct { |
| 1632 | const elem_ty = ty.childType(zcu); | 1583 | const elem_ty = ty.childType(zcu); |
| 1633 | const elem_ty_id = try self.resolveType(elem_ty, repr); | 1584 | const elem_ty_id = try self.resolveType(elem_ty, repr); |
| 1634 | const len = ty.vectorLen(zcu); | 1585 | const len = ty.vectorLen(zcu); |
| 1635 | | 1586 | return self.arrayType(len, elem_ty_id); |
| 1636 | if (self.isSpvVector(ty)) { | | |
| 1637 | return try self.spv.vectorType(len, elem_ty_id); | | |
| 1638 | } else { | | |
| 1639 | return try self.arrayType(len, elem_ty_id); | | |
| 1640 | } | | |
| 1641 | }, | 1587 | }, |
| 1642 | .@"struct" => { | 1588 | .@"struct" => { |
| 1643 | const struct_type = switch (ip.indexToKey(ty.toIntern())) { | 1589 | const struct_type = switch (ip.indexToKey(ty.toIntern())) { |
| ... | @@ -2035,69 +1981,32 @@ const NavGen = struct { | ... | @@ -2035,69 +1981,32 @@ const NavGen = struct { |
| 2035 | const Vectorization = union(enum) { | 1981 | const Vectorization = union(enum) { |
| 2036 | /// This is an operation between scalars. | 1982 | /// This is an operation between scalars. |
| 2037 | scalar, | 1983 | scalar, |
| 2038 | /// This is an operation between SPIR-V vectors. | | |
| 2039 | /// Value is number of components. | | |
| 2040 | spv_vectorized: u32, | | |
| 2041 | /// This operation is unrolled into separate operations. | 1984 | /// This operation is unrolled into separate operations. |
| 2042 | /// Inputs may still be SPIR-V vectors, for example, | 1985 | /// Inputs may still be SPIR-V vectors, for example, |
| 2043 | /// when the operation can't be vectorized in SPIR-V. | 1986 | /// when the operation can't be vectorized in SPIR-V. |
| 2044 | /// Value is number of components. | 1987 | /// Value is number of components. |
| 2045 | unrolled: u32, | 1988 | unrolled: u32, |
| 2046 | | 1989 | |
| 2047 | /// Derive a vectorization from a particular type. This usually | 1990 | /// Derive a vectorization from a particular type |
| 2048 | /// only checks the size, but the source-of-truth is implemented | | |
| 2049 | /// by `isSpvVector()`. | | |
| 2050 | fn fromType(ty: Type, ng: *NavGen) Vectorization { | 1991 | fn fromType(ty: Type, ng: *NavGen) Vectorization { |
| 2051 | const zcu = ng.pt.zcu; | 1992 | const zcu = ng.pt.zcu; |
| 2052 | if (!ty.isVector(zcu)) { | 1993 | if (!ty.isVector(zcu)) return .scalar; |
| 2053 | return .scalar; | 1994 | return .{ .unrolled = ty.vectorLen(zcu) }; |
| 2054 | } else if (ng.isSpvVector(ty)) { | | |
| 2055 | return .{ .spv_vectorized = ty.vectorLen(zcu) }; | | |
| 2056 | } else { | | |
| 2057 | return .{ .unrolled = ty.vectorLen(zcu) }; | | |
| 2058 | } | | |
| 2059 | } | 1995 | } |
| 2060 | | 1996 | |
| 2061 | /// Given two vectorization methods, compute a "unification": a fallback | 1997 | /// Given two vectorization methods, compute a "unification": a fallback |
| 2062 | /// that works for both, according to the following rules: | 1998 | /// that works for both, according to the following rules: |
| 2063 | /// - Scalars may broadcast | 1999 | /// - Scalars may broadcast |
| 2064 | /// - SPIR-V vectorized operations may unroll | 2000 | /// - SPIR-V vectorized operations will unroll |
| 2065 | /// - Prefer scalar > SPIR-V vectorized > unrolled | 2001 | /// - Prefer scalar > unrolled |
| 2066 | fn unify(a: Vectorization, b: Vectorization) Vectorization { | 2002 | fn unify(a: Vectorization, b: Vectorization) Vectorization { |
| 2067 | if (a == .scalar and b == .scalar) { | 2003 | if (a == .scalar and b == .scalar) return .scalar; |
| 2068 | return .scalar; | 2004 | if (a == .unrolled or b == .unrolled) { |
| 2069 | } else if (a == .spv_vectorized and b == .spv_vectorized) { | 2005 | if (a == .unrolled and b == .unrolled) assert(a.components() == b.components()); |
| 2070 | assert(a.components() == b.components()); | 2006 | if (a == .unrolled) return .{ .unrolled = a.components() }; |
| 2071 | return .{ .spv_vectorized = a.components() }; | 2007 | return .{ .unrolled = b.components() }; |
| 2072 | } else if (a == .unrolled or b == .unrolled) { | | |
| 2073 | if (a == .unrolled and b == .unrolled) { | | |
| 2074 | assert(a.components() == b.components()); | | |
| 2075 | return .{ .unrolled = a.components() }; | | |
| 2076 | } else if (a == .unrolled) { | | |
| 2077 | return .{ .unrolled = a.components() }; | | |
| 2078 | } else if (b == .unrolled) { | | |
| 2079 | return .{ .unrolled = b.components() }; | | |
| 2080 | } else { | | |
| 2081 | unreachable; | | |
| 2082 | } | | |
| 2083 | } else { | | |
| 2084 | if (a == .spv_vectorized) { | | |
| 2085 | return .{ .spv_vectorized = a.components() }; | | |
| 2086 | } else if (b == .spv_vectorized) { | | |
| 2087 | return .{ .spv_vectorized = b.components() }; | | |
| 2088 | } else { | | |
| 2089 | unreachable; | | |
| 2090 | } | | |
| 2091 | } | 2008 | } |
| 2092 | } | 2009 | unreachable; |
| 2093 | | | |
| 2094 | /// Force this vectorization to be unrolled, if its | | |
| 2095 | /// an operation involving vectors. | | |
| 2096 | fn unroll(self: Vectorization) Vectorization { | | |
| 2097 | return switch (self) { | | |
| 2098 | .scalar, .unrolled => self, | | |
| 2099 | .spv_vectorized => |n| .{ .unrolled = n }, | | |
| 2100 | }; | | |
| 2101 | } | 2010 | } |
| 2102 | | 2011 | |
| 2103 | /// Query the number of components that inputs of this operation have. | 2012 | /// Query the number of components that inputs of this operation have. |
| ... | @@ -2106,35 +2015,10 @@ const NavGen = struct { | ... | @@ -2106,35 +2015,10 @@ const NavGen = struct { |
| 2106 | fn components(self: Vectorization) u32 { | 2015 | fn components(self: Vectorization) u32 { |
| 2107 | return switch (self) { | 2016 | return switch (self) { |
| 2108 | .scalar => 1, | 2017 | .scalar => 1, |
| 2109 | .spv_vectorized => |n| n, | | |
| 2110 | .unrolled => |n| n, | 2018 | .unrolled => |n| n, |
| 2111 | }; | 2019 | }; |
| 2112 | } | 2020 | } |
| 2113 | | 2021 | |
| 2114 | /// Query the number of operations involving this vectorization. | | |
| 2115 | /// This is basically the number of components, except that SPIR-V vectorized | | |
| 2116 | /// operations only need a single SPIR-V instruction. | | |
| 2117 | fn operations(self: Vectorization) u32 { | | |
| 2118 | return switch (self) { | | |
| 2119 | .scalar, .spv_vectorized => 1, | | |
| 2120 | .unrolled => |n| n, | | |
| 2121 | }; | | |
| 2122 | } | | |
| 2123 | | | |
| 2124 | /// Turns `ty` into the result-type of an individual vector operation. | | |
| 2125 | /// `ty` may be a scalar or vector, it doesn't matter. | | |
| 2126 | fn operationType(self: Vectorization, ng: *NavGen, ty: Type) !Type { | | |
| 2127 | const pt = ng.pt; | | |
| 2128 | const scalar_ty = ty.scalarType(pt.zcu); | | |
| 2129 | return switch (self) { | | |
| 2130 | .scalar, .unrolled => scalar_ty, | | |
| 2131 | .spv_vectorized => |n| try pt.vectorType(.{ | | |
| 2132 | .len = n, | | |
| 2133 | .child = scalar_ty.toIntern(), | | |
| 2134 | }), | | |
| 2135 | }; | | |
| 2136 | } | | |
| 2137 | | | |
| 2138 | /// Turns `ty` into the result-type of the entire operation. | 2022 | /// Turns `ty` into the result-type of the entire operation. |
| 2139 | /// `ty` may be a scalar or vector, it doesn't matter. | 2023 | /// `ty` may be a scalar or vector, it doesn't matter. |
| 2140 | fn resultType(self: Vectorization, ng: *NavGen, ty: Type) !Type { | 2024 | fn resultType(self: Vectorization, ng: *NavGen, ty: Type) !Type { |
| ... | @@ -2142,10 +2026,7 @@ const NavGen = struct { | ... | @@ -2142,10 +2026,7 @@ const NavGen = struct { |
| 2142 | const scalar_ty = ty.scalarType(pt.zcu); | 2026 | const scalar_ty = ty.scalarType(pt.zcu); |
| 2143 | return switch (self) { | 2027 | return switch (self) { |
| 2144 | .scalar => scalar_ty, | 2028 | .scalar => scalar_ty, |
| 2145 | .unrolled, .spv_vectorized => |n| try pt.vectorType(.{ | 2029 | .unrolled => |n| try pt.vectorType(.{ .len = n, .child = scalar_ty.toIntern() }), |
| 2146 | .len = n, | | |
| 2147 | .child = scalar_ty.toIntern(), | | |
| 2148 | }), | | |
| 2149 | }; | 2030 | }; |
| 2150 | } | 2031 | } |
| 2151 | | 2032 | |
| ... | @@ -2155,51 +2036,19 @@ const NavGen = struct { | ... | @@ -2155,51 +2036,19 @@ const NavGen = struct { |
| 2155 | fn prepare(self: Vectorization, ng: *NavGen, tmp: Temporary) !PreparedOperand { | 2036 | fn prepare(self: Vectorization, ng: *NavGen, tmp: Temporary) !PreparedOperand { |
| 2156 | const pt = ng.pt; | 2037 | const pt = ng.pt; |
| 2157 | const is_vector = tmp.ty.isVector(pt.zcu); | 2038 | const is_vector = tmp.ty.isVector(pt.zcu); |
| 2158 | const is_spv_vector = ng.isSpvVector(tmp.ty); | | |
| 2159 | const value: PreparedOperand.Value = switch (tmp.value) { | 2039 | const value: PreparedOperand.Value = switch (tmp.value) { |
| 2160 | .singleton => |id| switch (self) { | 2040 | .singleton => |id| switch (self) { |
| 2161 | .scalar => blk: { | 2041 | .scalar => blk: { |
| 2162 | assert(!is_vector); | 2042 | assert(!is_vector); |
| 2163 | break :blk .{ .scalar = id }; | 2043 | break :blk .{ .scalar = id }; |
| 2164 | }, | 2044 | }, |
| 2165 | .spv_vectorized => blk: { | | |
| 2166 | if (is_vector) { | | |
| 2167 | assert(is_spv_vector); | | |
| 2168 | break :blk .{ .spv_vectorwise = id }; | | |
| 2169 | } | | |
| 2170 | | | |
| 2171 | // Broadcast scalar into vector. | | |
| 2172 | const vector_ty = try pt.vectorType(.{ | | |
| 2173 | .len = self.components(), | | |
| 2174 | .child = tmp.ty.toIntern(), | | |
| 2175 | }); | | |
| 2176 | | | |
| 2177 | const vector = try ng.constructCompositeSplat(vector_ty, id); | | |
| 2178 | return .{ | | |
| 2179 | .ty = vector_ty, | | |
| 2180 | .value = .{ .spv_vectorwise = vector }, | | |
| 2181 | }; | | |
| 2182 | }, | | |
| 2183 | .unrolled => blk: { | 2045 | .unrolled => blk: { |
| 2184 | if (is_vector) { | 2046 | if (is_vector) break :blk .{ .vector_exploded = try tmp.explode(ng) }; |
| 2185 | break :blk .{ .vector_exploded = try tmp.explode(ng) }; | 2047 | break :blk .{ .scalar_broadcast = id }; |
| 2186 | } else { | | |
| 2187 | break :blk .{ .scalar_broadcast = id }; | | |
| 2188 | } | | |
| 2189 | }, | 2048 | }, |
| 2190 | }, | 2049 | }, |
| 2191 | .exploded_vector => |range| switch (self) { | 2050 | .exploded_vector => |range| switch (self) { |
| 2192 | .scalar => unreachable, | 2051 | .scalar => unreachable, |
| 2193 | .spv_vectorized => |n| blk: { | | |
| 2194 | // We can vectorize this operation, but we have an exploded vector. This can happen | | |
| 2195 | // when a vectorizable operation succeeds a non-vectorizable operation. In this case, | | |
| 2196 | // pack up the IDs into a SPIR-V vector. This path should not be able to be hit with | | |
| 2197 | // a type that cannot do that. | | |
| 2198 | assert(is_spv_vector); | | |
| 2199 | assert(range.len == n); | | |
| 2200 | const vec = try tmp.materialize(ng); | | |
| 2201 | break :blk .{ .spv_vectorwise = vec }; | | |
| 2202 | }, | | |
| 2203 | .unrolled => |n| blk: { | 2052 | .unrolled => |n| blk: { |
| 2204 | assert(range.len == n); | 2053 | assert(range.len == n); |
| 2205 | break :blk .{ .vector_exploded = range }; | 2054 | break :blk .{ .vector_exploded = range }; |
| ... | @@ -2216,17 +2065,14 @@ const NavGen = struct { | ... | @@ -2216,17 +2065,14 @@ const NavGen = struct { |
| 2216 | /// Finalize the results of an operation back into a temporary. `results` is | 2065 | /// Finalize the results of an operation back into a temporary. `results` is |
| 2217 | /// a list of result-ids of the operation. | 2066 | /// a list of result-ids of the operation. |
| 2218 | fn finalize(self: Vectorization, ty: Type, results: IdRange) Temporary { | 2067 | fn finalize(self: Vectorization, ty: Type, results: IdRange) Temporary { |
| 2219 | assert(self.operations() == results.len); | 2068 | assert(self.components() == results.len); |
| 2220 | const value: Temporary.Value = switch (self) { | 2069 | return .{ |
| 2221 | .scalar, .spv_vectorized => blk: { | 2070 | .ty = ty, |
| 2222 | break :blk .{ .singleton = results.at(0) }; | 2071 | .value = switch (self) { |
| 2223 | }, | 2072 | .scalar => .{ .singleton = results.at(0) }, |
| 2224 | .unrolled => blk: { | 2073 | .unrolled => .{ .exploded_vector = results }, |
| 2225 | break :blk .{ .exploded_vector = results }; | | |
| 2226 | }, | 2074 | }, |
| 2227 | }; | 2075 | }; |
| 2228 | | | |
| 2229 | return .{ .ty = ty, .value = value }; | | |
| 2230 | } | 2076 | } |
| 2231 | | 2077 | |
| 2232 | /// This struct represents an operand that has gone through some setup, and is | 2078 | /// This struct represents an operand that has gone through some setup, and is |
| ... | @@ -2242,32 +2088,20 @@ const NavGen = struct { | ... | @@ -2242,32 +2088,20 @@ const NavGen = struct { |
| 2242 | scalar: IdResult, | 2088 | scalar: IdResult, |
| 2243 | /// A single scalar that is broadcasted in an unrolled operation. | 2089 | /// A single scalar that is broadcasted in an unrolled operation. |
| 2244 | scalar_broadcast: IdResult, | 2090 | scalar_broadcast: IdResult, |
| 2245 | /// A SPIR-V vector that is used in SPIR-V vectorize operation. | | |
| 2246 | spv_vectorwise: IdResult, | | |
| 2247 | /// A vector represented by a consecutive list of IDs that is used in an unrolled operation. | 2091 | /// A vector represented by a consecutive list of IDs that is used in an unrolled operation. |
| 2248 | vector_exploded: IdRange, | 2092 | vector_exploded: IdRange, |
| 2249 | }; | 2093 | }; |
| 2250 | | 2094 | |
| 2251 | /// Query the value at a particular index of the operation. Note that | 2095 | /// Query the value at a particular index of the operation. Note that |
| 2252 | /// the index is *not* the component/lane, but the index of the *operation*. When | 2096 | /// the index is *not* the component/lane, but the index of the *operation*. |
| 2253 | /// this operation is vectorized, the return value of this function is a SPIR-V vector. | | |
| 2254 | /// See also `Vectorization.operations()`. | | |
| 2255 | fn at(self: PreparedOperand, i: usize) IdResult { | 2097 | fn at(self: PreparedOperand, i: usize) IdResult { |
| 2256 | switch (self.value) { | 2098 | switch (self.value) { |
| 2257 | .scalar => |id| { | 2099 | .scalar => |id| { |
| 2258 | assert(i == 0); | 2100 | assert(i == 0); |
| 2259 | return id; | 2101 | return id; |
| 2260 | }, | 2102 | }, |
| 2261 | .scalar_broadcast => |id| { | 2103 | .scalar_broadcast => |id| return id, |
| 2262 | return id; | 2104 | .vector_exploded => |range| return range.at(i), |
| 2263 | }, | | |
| 2264 | .spv_vectorwise => |id| { | | |
| 2265 | assert(i == 0); | | |
| 2266 | return id; | | |
| 2267 | }, | | |
| 2268 | .vector_exploded => |range| { | | |
| 2269 | return range.at(i); | | |
| 2270 | }, | | |
| 2271 | } | 2105 | } |
| 2272 | } | 2106 | } |
| 2273 | }; | 2107 | }; |
| ... | @@ -2299,7 +2133,7 @@ const NavGen = struct { | ... | @@ -2299,7 +2133,7 @@ const NavGen = struct { |
| 2299 | | 2133 | |
| 2300 | /// This function builds an OpSConvert of OpUConvert depending on the | 2134 | /// This function builds an OpSConvert of OpUConvert depending on the |
| 2301 | /// signedness of the types. | 2135 | /// signedness of the types. |
| 2302 | fn buildIntConvert(self: *NavGen, dst_ty: Type, src: Temporary) !Temporary { | 2136 | fn buildConvert(self: *NavGen, dst_ty: Type, src: Temporary) !Temporary { |
| 2303 | const zcu = self.pt.zcu; | 2137 | const zcu = self.pt.zcu; |
| 2304 | | 2138 | |
| 2305 | const dst_ty_id = try self.resolveType(dst_ty.scalarType(zcu), .direct); | 2139 | const dst_ty_id = try self.resolveType(dst_ty.scalarType(zcu), .direct); |
| ... | @@ -2318,13 +2152,17 @@ const NavGen = struct { | ... | @@ -2318,13 +2152,17 @@ const NavGen = struct { |
| 2318 | return src.pun(result_ty); | 2152 | return src.pun(result_ty); |
| 2319 | } | 2153 | } |
| 2320 | | 2154 | |
| 2321 | const ops = v.operations(); | 2155 | const ops = v.components(); |
| 2322 | const results = self.spv.allocIds(ops); | 2156 | const results = self.spv.allocIds(ops); |
| 2323 | | 2157 | |
| 2324 | const op_result_ty = try v.operationType(self, dst_ty); | 2158 | const op_result_ty = dst_ty.scalarType(zcu); |
| 2325 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); | 2159 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); |
| 2326 | | 2160 | |
| 2327 | const opcode: Opcode = if (dst_ty.isSignedInt(zcu)) .OpSConvert else .OpUConvert; | 2161 | const opcode: Opcode = blk: { |
| | 2162 | if (dst_ty.scalarType(zcu).isAnyFloat()) break :blk .OpFConvert; |
| | 2163 | if (dst_ty.scalarType(zcu).isSignedInt(zcu)) break :blk .OpSConvert; |
| | 2164 | break :blk .OpUConvert; |
| | 2165 | }; |
| 2328 | | 2166 | |
| 2329 | const op_src = try v.prepare(self, src); | 2167 | const op_src = try v.prepare(self, src); |
| 2330 | | 2168 | |
| ... | @@ -2339,13 +2177,14 @@ const NavGen = struct { | ... | @@ -2339,13 +2177,14 @@ const NavGen = struct { |
| 2339 | } | 2177 | } |
| 2340 | | 2178 | |
| 2341 | fn buildFma(self: *NavGen, a: Temporary, b: Temporary, c: Temporary) !Temporary { | 2179 | fn buildFma(self: *NavGen, a: Temporary, b: Temporary, c: Temporary) !Temporary { |
| | 2180 | const zcu = self.pt.zcu; |
| 2342 | const target = self.spv.target; | 2181 | const target = self.spv.target; |
| 2343 | | 2182 | |
| 2344 | const v = self.vectorization(.{ a, b, c }); | 2183 | const v = self.vectorization(.{ a, b, c }); |
| 2345 | const ops = v.operations(); | 2184 | const ops = v.components(); |
| 2346 | const results = self.spv.allocIds(ops); | 2185 | const results = self.spv.allocIds(ops); |
| 2347 | | 2186 | |
| 2348 | const op_result_ty = try v.operationType(self, a.ty); | 2187 | const op_result_ty = a.ty.scalarType(zcu); |
| 2349 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); | 2188 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); |
| 2350 | const result_ty = try v.resultType(self, a.ty); | 2189 | const result_ty = try v.resultType(self, a.ty); |
| 2351 | | 2190 | |
| ... | @@ -2382,10 +2221,10 @@ const NavGen = struct { | ... | @@ -2382,10 +2221,10 @@ const NavGen = struct { |
| 2382 | const zcu = self.pt.zcu; | 2221 | const zcu = self.pt.zcu; |
| 2383 | | 2222 | |
| 2384 | const v = self.vectorization(.{ condition, lhs, rhs }); | 2223 | const v = self.vectorization(.{ condition, lhs, rhs }); |
| 2385 | const ops = v.operations(); | 2224 | const ops = v.components(); |
| 2386 | const results = self.spv.allocIds(ops); | 2225 | const results = self.spv.allocIds(ops); |
| 2387 | | 2226 | |
| 2388 | const op_result_ty = try v.operationType(self, lhs.ty); | 2227 | const op_result_ty = lhs.ty.scalarType(zcu); |
| 2389 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); | 2228 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); |
| 2390 | const result_ty = try v.resultType(self, lhs.ty); | 2229 | const result_ty = try v.resultType(self, lhs.ty); |
| 2391 | | 2230 | |
| ... | @@ -2431,10 +2270,10 @@ const NavGen = struct { | ... | @@ -2431,10 +2270,10 @@ const NavGen = struct { |
| 2431 | | 2270 | |
| 2432 | fn buildCmp(self: *NavGen, pred: CmpPredicate, lhs: Temporary, rhs: Temporary) !Temporary { | 2271 | fn buildCmp(self: *NavGen, pred: CmpPredicate, lhs: Temporary, rhs: Temporary) !Temporary { |
| 2433 | const v = self.vectorization(.{ lhs, rhs }); | 2272 | const v = self.vectorization(.{ lhs, rhs }); |
| 2434 | const ops = v.operations(); | 2273 | const ops = v.components(); |
| 2435 | const results = self.spv.allocIds(ops); | 2274 | const results = self.spv.allocIds(ops); |
| 2436 | | 2275 | |
| 2437 | const op_result_ty = try v.operationType(self, Type.bool); | 2276 | const op_result_ty: Type = .bool; |
| 2438 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); | 2277 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); |
| 2439 | const result_ty = try v.resultType(self, Type.bool); | 2278 | const result_ty = try v.resultType(self, Type.bool); |
| 2440 | | 2279 | |
| ... | @@ -2498,22 +2337,12 @@ const NavGen = struct { | ... | @@ -2498,22 +2337,12 @@ const NavGen = struct { |
| 2498 | }; | 2337 | }; |
| 2499 | | 2338 | |
| 2500 | fn buildUnary(self: *NavGen, op: UnaryOp, operand: Temporary) !Temporary { | 2339 | fn buildUnary(self: *NavGen, op: UnaryOp, operand: Temporary) !Temporary { |
| | 2340 | const zcu = self.pt.zcu; |
| 2501 | const target = self.spv.target; | 2341 | const target = self.spv.target; |
| 2502 | const v = blk: { | 2342 | const v = self.vectorization(.{operand}); |
| 2503 | const v = self.vectorization(.{operand}); | 2343 | const ops = v.components(); |
| 2504 | break :blk switch (op) { | | |
| 2505 | // TODO: These instructions don't seem to be working | | |
| 2506 | // properly for LLVM-based backends on OpenCL for 8- and | | |
| 2507 | // 16-component vectors. | | |
| 2508 | .i_abs => if (self.spv.hasFeature(.vector16) and v.components() >= 8) v.unroll() else v, | | |
| 2509 | else => v, | | |
| 2510 | }; | | |
| 2511 | }; | | |
| 2512 | | | |
| 2513 | const ops = v.operations(); | | |
| 2514 | const results = self.spv.allocIds(ops); | 2344 | const results = self.spv.allocIds(ops); |
| 2515 | | 2345 | const op_result_ty = operand.ty.scalarType(zcu); |
| 2516 | const op_result_ty = try v.operationType(self, operand.ty); | | |
| 2517 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); | 2346 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); |
| 2518 | const result_ty = try v.resultType(self, operand.ty); | 2347 | const result_ty = try v.resultType(self, operand.ty); |
| 2519 | | 2348 | |
| ... | @@ -2628,13 +2457,14 @@ const NavGen = struct { | ... | @@ -2628,13 +2457,14 @@ const NavGen = struct { |
| 2628 | }; | 2457 | }; |
| 2629 | | 2458 | |
| 2630 | fn buildBinary(self: *NavGen, op: BinaryOp, lhs: Temporary, rhs: Temporary) !Temporary { | 2459 | fn buildBinary(self: *NavGen, op: BinaryOp, lhs: Temporary, rhs: Temporary) !Temporary { |
| | 2460 | const zcu = self.pt.zcu; |
| 2631 | const target = self.spv.target; | 2461 | const target = self.spv.target; |
| 2632 | | 2462 | |
| 2633 | const v = self.vectorization(.{ lhs, rhs }); | 2463 | const v = self.vectorization(.{ lhs, rhs }); |
| 2634 | const ops = v.operations(); | 2464 | const ops = v.components(); |
| 2635 | const results = self.spv.allocIds(ops); | 2465 | const results = self.spv.allocIds(ops); |
| 2636 | | 2466 | |
| 2637 | const op_result_ty = try v.operationType(self, lhs.ty); | 2467 | const op_result_ty = lhs.ty.scalarType(zcu); |
| 2638 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); | 2468 | const op_result_ty_id = try self.resolveType(op_result_ty, .direct); |
| 2639 | const result_ty = try v.resultType(self, lhs.ty); | 2469 | const result_ty = try v.resultType(self, lhs.ty); |
| 2640 | | 2470 | |
| ... | @@ -2730,9 +2560,9 @@ const NavGen = struct { | ... | @@ -2730,9 +2560,9 @@ const NavGen = struct { |
| 2730 | const ip = &zcu.intern_pool; | 2560 | const ip = &zcu.intern_pool; |
| 2731 | | 2561 | |
| 2732 | const v = lhs.vectorization(self).unify(rhs.vectorization(self)); | 2562 | const v = lhs.vectorization(self).unify(rhs.vectorization(self)); |
| 2733 | const ops = v.operations(); | 2563 | const ops = v.components(); |
| 2734 | | 2564 | |
| 2735 | const arith_op_ty = try v.operationType(self, lhs.ty); | 2565 | const arith_op_ty = lhs.ty.scalarType(zcu); |
| 2736 | const arith_op_ty_id = try self.resolveType(arith_op_ty, .direct); | 2566 | const arith_op_ty_id = try self.resolveType(arith_op_ty, .direct); |
| 2737 | | 2567 | |
| 2738 | const lhs_op = try v.prepare(self, lhs); | 2568 | const lhs_op = try v.prepare(self, lhs); |
| ... | @@ -3175,17 +3005,18 @@ const NavGen = struct { | ... | @@ -3175,17 +3005,18 @@ const NavGen = struct { |
| 3175 | /// Convert representation from indirect (in memory) to direct (in 'register') | 3005 | /// Convert representation from indirect (in memory) to direct (in 'register') |
| 3176 | /// This converts the argument type from resolveType(ty, .indirect) to resolveType(ty, .direct). | 3006 | /// This converts the argument type from resolveType(ty, .indirect) to resolveType(ty, .direct). |
| 3177 | fn convertToDirect(self: *NavGen, ty: Type, operand_id: IdRef) !IdRef { | 3007 | fn convertToDirect(self: *NavGen, ty: Type, operand_id: IdRef) !IdRef { |
| 3178 | const zcu = self.pt.zcu; | 3008 | const pt = self.pt; |
| | 3009 | const zcu = pt.zcu; |
| 3179 | switch (ty.scalarType(zcu).zigTypeTag(zcu)) { | 3010 | switch (ty.scalarType(zcu).zigTypeTag(zcu)) { |
| 3180 | .bool => { | 3011 | .bool => { |
| 3181 | const false_id = try self.constBool(false, .indirect); | 3012 | const false_id = try self.constBool(false, .indirect); |
| 3182 | // The operation below requires inputs in direct representation, but the operand | 3013 | const operand_ty = blk: { |
| 3183 | // is actually in indirect representation. | 3014 | if (!ty.isVector(pt.zcu)) break :blk Type.u1; |
| 3184 | // Cheekily swap out the type to the direct equivalent of the indirect type here, they have the | 3015 | break :blk try pt.vectorType(.{ |
| 3185 | // same representation when converted to SPIR-V. | 3016 | .len = ty.vectorLen(pt.zcu), |
| 3186 | const operand_ty = try self.zigScalarOrVectorTypeLike(Type.u1, ty); | 3017 | .child = Type.u1.toIntern(), |
| 3187 | // Note: We can guarantee that these are the same ID due to the SPIR-V Module's `vector_types` cache! | 3018 | }); |
| 3188 | assert(try self.resolveType(operand_ty, .direct) == try self.resolveType(ty, .indirect)); | 3019 | }; |
| 3189 | | 3020 | |
| 3190 | const result = try self.buildCmp( | 3021 | const result = try self.buildCmp( |
| 3191 | .i_ne, | 3022 | .i_ne, |
| ... | @@ -3226,7 +3057,6 @@ const NavGen = struct { | ... | @@ -3226,7 +3057,6 @@ const NavGen = struct { |
| 3226 | } | 3057 | } |
| 3227 | | 3058 | |
| 3228 | fn extractVectorComponent(self: *NavGen, result_ty: Type, vector_id: IdRef, field: u32) !IdRef { | 3059 | fn extractVectorComponent(self: *NavGen, result_ty: Type, vector_id: IdRef, field: u32) !IdRef { |
| 3229 | // Whether this is an OpTypeVector or OpTypeArray, we need to emit the same instruction regardless. | | |
| 3230 | const result_ty_id = try self.resolveType(result_ty, .direct); | 3060 | const result_ty_id = try self.resolveType(result_ty, .direct); |
| 3231 | const result_id = self.spv.allocId(); | 3061 | const result_id = self.spv.allocId(); |
| 3232 | const indexes = [_]u32{field}; | 3062 | const indexes = [_]u32{field}; |
| ... | @@ -3485,7 +3315,7 @@ const NavGen = struct { | ... | @@ -3485,7 +3315,7 @@ const NavGen = struct { |
| 3485 | // Note: The sign may differ here between the shift and the base type, in case | 3315 | // Note: The sign may differ here between the shift and the base type, in case |
| 3486 | // of an arithmetic right shift. SPIR-V still expects the same type, | 3316 | // of an arithmetic right shift. SPIR-V still expects the same type, |
| 3487 | // so in that case we have to cast convert to signed. | 3317 | // so in that case we have to cast convert to signed. |
| 3488 | const casted_shift = try self.buildIntConvert(base.ty.scalarType(zcu), shift); | 3318 | const casted_shift = try self.buildConvert(base.ty.scalarType(zcu), shift); |
| 3489 | | 3319 | |
| 3490 | const shifted = switch (info.signedness) { | 3320 | const shifted = switch (info.signedness) { |
| 3491 | .unsigned => try self.buildBinary(unsigned, base, casted_shift), | 3321 | .unsigned => try self.buildBinary(unsigned, base, casted_shift), |
| ... | @@ -3815,12 +3645,12 @@ const NavGen = struct { | ... | @@ -3815,12 +3645,12 @@ const NavGen = struct { |
| 3815 | .unsigned => blk: { | 3645 | .unsigned => blk: { |
| 3816 | if (maybe_op_ty_bits) |op_ty_bits| { | 3646 | if (maybe_op_ty_bits) |op_ty_bits| { |
| 3817 | const op_ty = try pt.intType(.unsigned, op_ty_bits); | 3647 | const op_ty = try pt.intType(.unsigned, op_ty_bits); |
| 3818 | const casted_lhs = try self.buildIntConvert(op_ty, lhs); | 3648 | const casted_lhs = try self.buildConvert(op_ty, lhs); |
| 3819 | const casted_rhs = try self.buildIntConvert(op_ty, rhs); | 3649 | const casted_rhs = try self.buildConvert(op_ty, rhs); |
| 3820 | | 3650 | |
| 3821 | const full_result = try self.buildBinary(.i_mul, casted_lhs, casted_rhs); | 3651 | const full_result = try self.buildBinary(.i_mul, casted_lhs, casted_rhs); |
| 3822 | | 3652 | |
| 3823 | const low_bits = try self.buildIntConvert(lhs.ty, full_result); | 3653 | const low_bits = try self.buildConvert(lhs.ty, full_result); |
| 3824 | const result = try self.normalize(low_bits, info); | 3654 | const result = try self.normalize(low_bits, info); |
| 3825 | | 3655 | |
| 3826 | // Shift the result bits away to get the overflow bits. | 3656 | // Shift the result bits away to get the overflow bits. |
| ... | @@ -3846,9 +3676,7 @@ const NavGen = struct { | ... | @@ -3846,9 +3676,7 @@ const NavGen = struct { |
| 3846 | const high_overflowed = try self.buildCmp(.i_ne, zero, high_bits); | 3676 | const high_overflowed = try self.buildCmp(.i_ne, zero, high_bits); |
| 3847 | | 3677 | |
| 3848 | // If no overflow bits in low_bits, no extra work needs to be done. | 3678 | // If no overflow bits in low_bits, no extra work needs to be done. |
| 3849 | if (info.backing_bits == info.bits) { | 3679 | if (info.backing_bits == info.bits) break :blk .{ result, high_overflowed }; |
| 3850 | break :blk .{ result, high_overflowed }; | | |
| 3851 | } | | |
| 3852 | | 3680 | |
| 3853 | // Shift the result bits away to get the overflow bits. | 3681 | // Shift the result bits away to get the overflow bits. |
| 3854 | const shift = Temporary.init(lhs.ty, try self.constInt(lhs.ty, info.bits)); | 3682 | const shift = Temporary.init(lhs.ty, try self.constInt(lhs.ty, info.bits)); |
| ... | @@ -3886,13 +3714,13 @@ const NavGen = struct { | ... | @@ -3886,13 +3714,13 @@ const NavGen = struct { |
| 3886 | if (maybe_op_ty_bits) |op_ty_bits| { | 3714 | if (maybe_op_ty_bits) |op_ty_bits| { |
| 3887 | const op_ty = try pt.intType(.signed, op_ty_bits); | 3715 | const op_ty = try pt.intType(.signed, op_ty_bits); |
| 3888 | // Assume normalized; sign bit is set. We want a sign extend. | 3716 | // Assume normalized; sign bit is set. We want a sign extend. |
| 3889 | const casted_lhs = try self.buildIntConvert(op_ty, lhs); | 3717 | const casted_lhs = try self.buildConvert(op_ty, lhs); |
| 3890 | const casted_rhs = try self.buildIntConvert(op_ty, rhs); | 3718 | const casted_rhs = try self.buildConvert(op_ty, rhs); |
| 3891 | | 3719 | |
| 3892 | const full_result = try self.buildBinary(.i_mul, casted_lhs, casted_rhs); | 3720 | const full_result = try self.buildBinary(.i_mul, casted_lhs, casted_rhs); |
| 3893 | | 3721 | |
| 3894 | // Truncate to the result type. | 3722 | // Truncate to the result type. |
| 3895 | const low_bits = try self.buildIntConvert(lhs.ty, full_result); | 3723 | const low_bits = try self.buildConvert(lhs.ty, full_result); |
| 3896 | const result = try self.normalize(low_bits, info); | 3724 | const result = try self.normalize(low_bits, info); |
| 3897 | | 3725 | |
| 3898 | // Now, we need to check the overflow bits AND the sign | 3726 | // Now, we need to check the overflow bits AND the sign |
| ... | @@ -3929,9 +3757,7 @@ const NavGen = struct { | ... | @@ -3929,9 +3757,7 @@ const NavGen = struct { |
| 3929 | // If no overflow bits in low_bits, no extra work needs to be done. | 3757 | // If no overflow bits in low_bits, no extra work needs to be done. |
| 3930 | // Careful, we still have to check the sign bit, so this branch | 3758 | // Careful, we still have to check the sign bit, so this branch |
| 3931 | // only goes for i33 and such. | 3759 | // only goes for i33 and such. |
| 3932 | if (info.backing_bits == info.bits + 1) { | 3760 | if (info.backing_bits == info.bits + 1) break :blk .{ result, high_overflowed }; |
| 3933 | break :blk .{ result, high_overflowed }; | | |
| 3934 | } | | |
| 3935 | | 3761 | |
| 3936 | // Shift the result bits away to get the overflow bits. | 3762 | // Shift the result bits away to get the overflow bits. |
| 3937 | const shift = Temporary.init(lhs.ty, try self.constInt(lhs.ty, info.bits - 1)); | 3763 | const shift = Temporary.init(lhs.ty, try self.constInt(lhs.ty, info.bits - 1)); |
| ... | @@ -3972,7 +3798,7 @@ const NavGen = struct { | ... | @@ -3972,7 +3798,7 @@ const NavGen = struct { |
| 3972 | | 3798 | |
| 3973 | // Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that, | 3799 | // Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that, |
| 3974 | // so just manually upcast it if required. | 3800 | // so just manually upcast it if required. |
| 3975 | const casted_shift = try self.buildIntConvert(base.ty.scalarType(zcu), shift); | 3801 | const casted_shift = try self.buildConvert(base.ty.scalarType(zcu), shift); |
| 3976 | | 3802 | |
| 3977 | const left = try self.buildBinary(.sll, base, casted_shift); | 3803 | const left = try self.buildBinary(.sll, base, casted_shift); |
| 3978 | const result = try self.normalize(left, info); | 3804 | const result = try self.normalize(left, info); |
| ... | @@ -4026,7 +3852,7 @@ const NavGen = struct { | ... | @@ -4026,7 +3852,7 @@ const NavGen = struct { |
| 4026 | // Result of OpenCL ctz/clz returns operand.ty, and we want result_ty. | 3852 | // Result of OpenCL ctz/clz returns operand.ty, and we want result_ty. |
| 4027 | // result_ty is always large enough to hold the result, so we might have to down | 3853 | // result_ty is always large enough to hold the result, so we might have to down |
| 4028 | // cast it. | 3854 | // cast it. |
| 4029 | const result = try self.buildIntConvert(scalar_result_ty, count); | 3855 | const result = try self.buildConvert(scalar_result_ty, count); |
| 4030 | return try result.materialize(self); | 3856 | return try result.materialize(self); |
| 4031 | } | 3857 | } |
| 4032 | | 3858 | |
| ... | @@ -4057,11 +3883,8 @@ const NavGen = struct { | ... | @@ -4057,11 +3883,8 @@ const NavGen = struct { |
| 4057 | const operand_ty = self.typeOf(reduce.operand); | 3883 | const operand_ty = self.typeOf(reduce.operand); |
| 4058 | const scalar_ty = operand_ty.scalarType(zcu); | 3884 | const scalar_ty = operand_ty.scalarType(zcu); |
| 4059 | const scalar_ty_id = try self.resolveType(scalar_ty, .direct); | 3885 | const scalar_ty_id = try self.resolveType(scalar_ty, .direct); |
| 4060 | | | |
| 4061 | const info = self.arithmeticTypeInfo(operand_ty); | 3886 | const info = self.arithmeticTypeInfo(operand_ty); |
| 4062 | | | |
| 4063 | const len = operand_ty.vectorLen(zcu); | 3887 | const len = operand_ty.vectorLen(zcu); |
| 4064 | | | |
| 4065 | const first = try self.extractVectorComponent(scalar_ty, operand, 0); | 3888 | const first = try self.extractVectorComponent(scalar_ty, operand, 0); |
| 4066 | | 3889 | |
| 4067 | switch (reduce.operation) { | 3890 | switch (reduce.operation) { |
| ... | @@ -4136,51 +3959,9 @@ const NavGen = struct { | ... | @@ -4136,51 +3959,9 @@ const NavGen = struct { |
| 4136 | | 3959 | |
| 4137 | // Note: number of components in the result, a, and b may differ. | 3960 | // Note: number of components in the result, a, and b may differ. |
| 4138 | const result_ty = self.typeOfIndex(inst); | 3961 | const result_ty = self.typeOfIndex(inst); |
| 4139 | const a_ty = self.typeOf(extra.a); | | |
| 4140 | const b_ty = self.typeOf(extra.b); | | |
| 4141 | | | |
| 4142 | const scalar_ty = result_ty.scalarType(zcu); | 3962 | const scalar_ty = result_ty.scalarType(zcu); |
| 4143 | const scalar_ty_id = try self.resolveType(scalar_ty, .direct); | 3963 | const scalar_ty_id = try self.resolveType(scalar_ty, .direct); |
| 4144 | | 3964 | |
| 4145 | // If all of the types are SPIR-V vectors, we can use OpVectorShuffle. | | |
| 4146 | if (self.isSpvVector(result_ty) and self.isSpvVector(a_ty) and self.isSpvVector(b_ty)) { | | |
| 4147 | // The SPIR-V shuffle instruction is similar to the Air instruction, except that the elements are | | |
| 4148 | // numbered consecutively instead of using negatives. | | |
| 4149 | | | |
| 4150 | const components = try self.gpa.alloc(Word, result_ty.vectorLen(zcu)); | | |
| 4151 | defer self.gpa.free(components); | | |
| 4152 | | | |
| 4153 | const a_len = a_ty.vectorLen(zcu); | | |
| 4154 | | | |
| 4155 | for (components, 0..) |*component, i| { | | |
| 4156 | const elem = try mask.elemValue(pt, i); | | |
| 4157 | if (elem.isUndef(zcu)) { | | |
| 4158 | // This is explicitly valid for OpVectorShuffle, it indicates undefined. | | |
| 4159 | component.* = 0xFFFF_FFFF; | | |
| 4160 | continue; | | |
| 4161 | } | | |
| 4162 | | | |
| 4163 | const index = elem.toSignedInt(zcu); | | |
| 4164 | if (index >= 0) { | | |
| 4165 | component.* = @intCast(index); | | |
| 4166 | } else { | | |
| 4167 | component.* = @intCast(~index + a_len); | | |
| 4168 | } | | |
| 4169 | } | | |
| 4170 | | | |
| 4171 | const result_id = self.spv.allocId(); | | |
| 4172 | try self.func.body.emit(self.spv.gpa, .OpVectorShuffle, .{ | | |
| 4173 | .id_result_type = try self.resolveType(result_ty, .direct), | | |
| 4174 | .id_result = result_id, | | |
| 4175 | .vector_1 = a, | | |
| 4176 | .vector_2 = b, | | |
| 4177 | .components = components, | | |
| 4178 | }); | | |
| 4179 | return result_id; | | |
| 4180 | } | | |
| 4181 | | | |
| 4182 | // Fall back to manually extracting and inserting components. | | |
| 4183 | | | |
| 4184 | const constituents = try self.gpa.alloc(IdRef, result_ty.vectorLen(zcu)); | 3965 | const constituents = try self.gpa.alloc(IdRef, result_ty.vectorLen(zcu)); |
| 4185 | defer self.gpa.free(constituents); | 3966 | defer self.gpa.free(constituents); |
| 4186 | | 3967 | |
| ... | @@ -4535,9 +4316,7 @@ const NavGen = struct { | ... | @@ -4535,9 +4316,7 @@ const NavGen = struct { |
| 4535 | const dst_ty_id = try self.resolveType(dst_ty, .direct); | 4316 | const dst_ty_id = try self.resolveType(dst_ty, .direct); |
| 4536 | | 4317 | |
| 4537 | const result_id = blk: { | 4318 | const result_id = blk: { |
| 4538 | if (src_ty_id == dst_ty_id) { | 4319 | if (src_ty_id == dst_ty_id) break :blk src_id; |
| 4539 | break :blk src_id; | | |
| 4540 | } | | |
| 4541 | | 4320 | |
| 4542 | // TODO: Some more cases are missing here | 4321 | // TODO: Some more cases are missing here |
| 4543 | // See fn bitCast in llvm.zig | 4322 | // See fn bitCast in llvm.zig |
| ... | @@ -4618,7 +4397,7 @@ const NavGen = struct { | ... | @@ -4618,7 +4397,7 @@ const NavGen = struct { |
| 4618 | return try src.materialize(self); | 4397 | return try src.materialize(self); |
| 4619 | } | 4398 | } |
| 4620 | | 4399 | |
| 4621 | const converted = try self.buildIntConvert(dst_ty, src); | 4400 | const converted = try self.buildConvert(dst_ty, src); |
| 4622 | | 4401 | |
| 4623 | // Make sure to normalize the result if shrinking. | 4402 | // Make sure to normalize the result if shrinking. |
| 4624 | // Because strange ints are sign extended in their backing | 4403 | // Because strange ints are sign extended in their backing |
| ... | @@ -4698,17 +4477,10 @@ const NavGen = struct { | ... | @@ -4698,17 +4477,10 @@ const NavGen = struct { |
| 4698 | | 4477 | |
| 4699 | fn airFloatCast(self: *NavGen, inst: Air.Inst.Index) !?IdRef { | 4478 | fn airFloatCast(self: *NavGen, inst: Air.Inst.Index) !?IdRef { |
| 4700 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | 4479 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 4701 | const operand_id = try self.resolve(ty_op.operand); | 4480 | const operand = try self.temporary(ty_op.operand); |
| 4702 | const dest_ty = self.typeOfIndex(inst); | 4481 | const dest_ty = self.typeOfIndex(inst); |
| 4703 | const dest_ty_id = try self.resolveType(dest_ty, .direct); | 4482 | const result = try self.buildConvert(dest_ty, operand); |
| 4704 | | 4483 | return try result.materialize(self); |
| 4705 | const result_id = self.spv.allocId(); | | |
| 4706 | try self.func.body.emit(self.spv.gpa, .OpFConvert, .{ | | |
| 4707 | .id_result_type = dest_ty_id, | | |
| 4708 | .id_result = result_id, | | |
| 4709 | .float_value = operand_id, | | |
| 4710 | }); | | |
| 4711 | return result_id; | | |
| 4712 | } | 4484 | } |
| 4713 | | 4485 | |
| 4714 | fn airNot(self: *NavGen, inst: Air.Inst.Index) !?IdRef { | 4486 | fn airNot(self: *NavGen, inst: Air.Inst.Index) !?IdRef { |
| ... | @@ -4796,7 +4568,7 @@ const NavGen = struct { | ... | @@ -4796,7 +4568,7 @@ const NavGen = struct { |
| 4796 | break :blk try self.bitCast(field_int_ty, field_ty, field_id); | 4568 | break :blk try self.bitCast(field_int_ty, field_ty, field_id); |
| 4797 | }; | 4569 | }; |
| 4798 | const shift_rhs = try self.constInt(backing_int_ty, running_bits); | 4570 | const shift_rhs = try self.constInt(backing_int_ty, running_bits); |
| 4799 | const extended_int_conv = try self.buildIntConvert(backing_int_ty, .{ | 4571 | const extended_int_conv = try self.buildConvert(backing_int_ty, .{ |
| 4800 | .ty = field_int_ty, | 4572 | .ty = field_int_ty, |
| 4801 | .value = .{ .singleton = field_int_id }, | 4573 | .value = .{ .singleton = field_int_id }, |
| 4802 | }); | 4574 | }); |
| ... | @@ -5016,17 +4788,6 @@ const NavGen = struct { | ... | @@ -5016,17 +4788,6 @@ const NavGen = struct { |
| 5016 | const array_id = try self.resolve(bin_op.lhs); | 4788 | const array_id = try self.resolve(bin_op.lhs); |
| 5017 | const index_id = try self.resolve(bin_op.rhs); | 4789 | const index_id = try self.resolve(bin_op.rhs); |
| 5018 | | 4790 | |
| 5019 | if (self.isSpvVector(array_ty)) { | | |
| 5020 | const result_id = self.spv.allocId(); | | |
| 5021 | try self.func.body.emit(self.spv.gpa, .OpVectorExtractDynamic, .{ | | |
| 5022 | .id_result_type = try self.resolveType(elem_ty, .direct), | | |
| 5023 | .id_result = result_id, | | |
| 5024 | .vector = array_id, | | |
| 5025 | .index = index_id, | | |
| 5026 | }); | | |
| 5027 | return result_id; | | |
| 5028 | } | | |
| 5029 | | | |
| 5030 | // SPIR-V doesn't have an array indexing function for some damn reason. | 4791 | // SPIR-V doesn't have an array indexing function for some damn reason. |
| 5031 | // For now, just generate a temporary and use that. | 4792 | // For now, just generate a temporary and use that. |
| 5032 | // TODO: This backend probably also should use isByRef from llvm... | 4793 | // TODO: This backend probably also should use isByRef from llvm... |
| ... | @@ -5173,7 +4934,7 @@ const NavGen = struct { | ... | @@ -5173,7 +4934,7 @@ const NavGen = struct { |
| 5173 | return self.bitCast(ty, payload_ty, payload.?); | 4934 | return self.bitCast(ty, payload_ty, payload.?); |
| 5174 | } | 4935 | } |
| 5175 | | 4936 | |
| 5176 | const trunc = try self.buildIntConvert(ty, .{ .ty = payload_ty, .value = .{ .singleton = payload.? } }); | 4937 | const trunc = try self.buildConvert(ty, .{ .ty = payload_ty, .value = .{ .singleton = payload.? } }); |
| 5177 | return try trunc.materialize(self); | 4938 | return try trunc.materialize(self); |
| 5178 | } | 4939 | } |
| 5179 | | 4940 | |
| ... | @@ -5182,7 +4943,7 @@ const NavGen = struct { | ... | @@ -5182,7 +4943,7 @@ const NavGen = struct { |
| 5182 | try self.convertToIndirect(payload_ty, payload.?) | 4943 | try self.convertToIndirect(payload_ty, payload.?) |
| 5183 | else | 4944 | else |
| 5184 | try self.bitCast(payload_int_ty, payload_ty, payload.?); | 4945 | try self.bitCast(payload_int_ty, payload_ty, payload.?); |
| 5185 | const trunc = try self.buildIntConvert(ty, .{ .ty = payload_int_ty, .value = .{ .singleton = payload_int } }); | 4946 | const trunc = try self.buildConvert(ty, .{ .ty = payload_int_ty, .value = .{ .singleton = payload_int } }); |
| 5186 | return try trunc.materialize(self); | 4947 | return try trunc.materialize(self); |
| 5187 | } | 4948 | } |
| 5188 | | 4949 | |
| ... | @@ -5273,7 +5034,7 @@ const NavGen = struct { | ... | @@ -5273,7 +5034,7 @@ const NavGen = struct { |
| 5273 | const result_id = blk: { | 5034 | const result_id = blk: { |
| 5274 | if (self.backingIntBits(field_bit_size).? == self.backingIntBits(@intCast(object_ty.bitSize(zcu))).?) | 5035 | if (self.backingIntBits(field_bit_size).? == self.backingIntBits(@intCast(object_ty.bitSize(zcu))).?) |
| 5275 | break :blk try self.bitCast(field_int_ty, object_ty, try masked.materialize(self)); | 5036 | break :blk try self.bitCast(field_int_ty, object_ty, try masked.materialize(self)); |
| 5276 | const trunc = try self.buildIntConvert(field_int_ty, masked); | 5037 | const trunc = try self.buildConvert(field_int_ty, masked); |
| 5277 | break :blk try trunc.materialize(self); | 5038 | break :blk try trunc.materialize(self); |
| 5278 | }; | 5039 | }; |
| 5279 | if (field_ty.ip_index == .bool_type) return try self.convertToDirect(.bool, result_id); | 5040 | if (field_ty.ip_index == .bool_type) return try self.convertToDirect(.bool, result_id); |
| ... | @@ -5297,7 +5058,7 @@ const NavGen = struct { | ... | @@ -5297,7 +5058,7 @@ const NavGen = struct { |
| 5297 | const result_id = blk: { | 5058 | const result_id = blk: { |
| 5298 | if (self.backingIntBits(field_bit_size).? == self.backingIntBits(@intCast(backing_int_ty.bitSize(zcu))).?) | 5059 | if (self.backingIntBits(field_bit_size).? == self.backingIntBits(@intCast(backing_int_ty.bitSize(zcu))).?) |
| 5299 | break :blk try self.bitCast(int_ty, backing_int_ty, try masked.materialize(self)); | 5060 | break :blk try self.bitCast(int_ty, backing_int_ty, try masked.materialize(self)); |
| 5300 | const trunc = try self.buildIntConvert(int_ty, masked); | 5061 | const trunc = try self.buildConvert(int_ty, masked); |
| 5301 | break :blk try trunc.materialize(self); | 5062 | break :blk try trunc.materialize(self); |
| 5302 | }; | 5063 | }; |
| 5303 | if (field_ty.ip_index == .bool_type) return try self.convertToDirect(.bool, result_id); | 5064 | if (field_ty.ip_index == .bool_type) return try self.convertToDirect(.bool, result_id); |
| ... | @@ -6752,7 +6513,7 @@ const NavGen = struct { | ... | @@ -6752,7 +6513,7 @@ const NavGen = struct { |
| 6752 | // TODO: Should we make these builtins return usize? | 6513 | // TODO: Should we make these builtins return usize? |
| 6753 | const result_id = try self.builtin3D(Type.u64, .LocalInvocationId, dimension, 0); | 6514 | const result_id = try self.builtin3D(Type.u64, .LocalInvocationId, dimension, 0); |
| 6754 | const tmp = Temporary.init(Type.u64, result_id); | 6515 | const tmp = Temporary.init(Type.u64, result_id); |
| 6755 | const result = try self.buildIntConvert(Type.u32, tmp); | 6516 | const result = try self.buildConvert(Type.u32, tmp); |
| 6756 | return try result.materialize(self); | 6517 | return try result.materialize(self); |
| 6757 | } | 6518 | } |
| 6758 | | 6519 | |
| ... | @@ -6763,7 +6524,7 @@ const NavGen = struct { | ... | @@ -6763,7 +6524,7 @@ const NavGen = struct { |
| 6763 | // TODO: Should we make these builtins return usize? | 6524 | // TODO: Should we make these builtins return usize? |
| 6764 | const result_id = try self.builtin3D(Type.u64, .WorkgroupSize, dimension, 0); | 6525 | const result_id = try self.builtin3D(Type.u64, .WorkgroupSize, dimension, 0); |
| 6765 | const tmp = Temporary.init(Type.u64, result_id); | 6526 | const tmp = Temporary.init(Type.u64, result_id); |
| 6766 | const result = try self.buildIntConvert(Type.u32, tmp); | 6527 | const result = try self.buildConvert(Type.u32, tmp); |
| 6767 | return try result.materialize(self); | 6528 | return try result.materialize(self); |
| 6768 | } | 6529 | } |
| 6769 | | 6530 | |
| ... | @@ -6774,7 +6535,7 @@ const NavGen = struct { | ... | @@ -6774,7 +6535,7 @@ const NavGen = struct { |
| 6774 | // TODO: Should we make these builtins return usize? | 6535 | // TODO: Should we make these builtins return usize? |
| 6775 | const result_id = try self.builtin3D(Type.u64, .WorkgroupId, dimension, 0); | 6536 | const result_id = try self.builtin3D(Type.u64, .WorkgroupId, dimension, 0); |
| 6776 | const tmp = Temporary.init(Type.u64, result_id); | 6537 | const tmp = Temporary.init(Type.u64, result_id); |
| 6777 | const result = try self.buildIntConvert(Type.u32, tmp); | 6538 | const result = try self.buildConvert(Type.u32, tmp); |
| 6778 | return try result.materialize(self); | 6539 | return try result.materialize(self); |
| 6779 | } | 6540 | } |
| 6780 | | 6541 | |
Ali Chraghi <alichraghi@proton.me> 2025-05-06 16:25:08+03:30
src