authorgravatar for alichraghi@proton.meAli Chraghi <alichraghi@proton.me> 2024-02-09 09:13:11+03:30
committergravatar for alichraghi@proton.meAli Chraghi <alichraghi@proton.me> 2024-02-09 16:40:34+03:30
log5ee96688a7456b9581fa605520fe90f6723bb95b
tree7d49c0006adc1bbc1221795f012e64c12c76cb14
parentddcea2cad486684607039aab45634e0e30e7312a

spirv: emit vectorized operations

1 files changed, 154 insertions(+), 114 deletions(-)

src/codegen/spirv.zig+154-114
......@@ -629,6 +629,16 @@ const DeclGen = struct {
629629 return self.backingIntBits(ty) == null;
630630 }
631631
632 /// Checks whether the type can be directly translated to SPIR-V vectors
633 fn isVector(self: *DeclGen, ty: Type) bool {
634 const mod = self.module;
635 if (ty.zigTypeTag(mod) != .Vector) return false;
636 const elem_ty = ty.childType(mod);
637 const len = ty.vectorLen(mod);
638 const is_scalar = elem_ty.isNumeric(mod) or elem_ty.toIntern() == .bool_type;
639 return is_scalar and len > 1 and len <= 4;
640 }
641
632642 fn arithmeticTypeInfo(self: *DeclGen, ty: Type) ArithmeticTypeInfo {
633643 const mod = self.module;
634644 const target = self.getTarget();
......@@ -694,6 +704,24 @@ const DeclGen = struct {
694704 /// This function, unlike SpvModule.constInt, takes care to bitcast
695705 /// the value to an unsigned int first for Kernels.
696706 fn constInt(self: *DeclGen, ty_ref: CacheRef, value: anytype) !IdRef {
707 switch (self.spv.cache.lookup(ty_ref)) {
708 .vector_type => |vec_type| {
709 const elem_ids = try self.gpa.alloc(IdRef, vec_type.component_count);
710 defer self.gpa.free(elem_ids);
711 const int_value = try self.constInt(vec_type.component_type, value);
712 @memset(elem_ids, int_value);
713
714 const constituents_id = self.spv.allocId();
715 try self.func.body.emit(self.spv.gpa, .OpCompositeConstruct, .{
716 .id_result_type = self.typeId(ty_ref),
717 .id_result = constituents_id,
718 .constituents = elem_ids,
719 });
720 return constituents_id;
721 },
722 else => {},
723 }
724
697725 if (value < 0) {
698726 const ty = self.spv.cache.lookup(ty_ref).int_type;
699727 // Manually truncate the value so that the resulting value
......@@ -711,6 +739,24 @@ const DeclGen = struct {
711739
712740 /// Emits a float constant
713741 fn constFloat(self: *DeclGen, ty_ref: CacheRef, value: f128) !IdRef {
742 switch (self.spv.cache.lookup(ty_ref)) {
743 .vector_type => |vec_type| {
744 const elem_ids = try self.gpa.alloc(IdRef, vec_type.component_count);
745 defer self.gpa.free(elem_ids);
746 const int_value = try self.constFloat(vec_type.component_type, value);
747 @memset(elem_ids, int_value);
748
749 const constituents_id = self.spv.allocId();
750 try self.func.body.emit(self.spv.gpa, .OpCompositeConstruct, .{
751 .id_result_type = self.typeId(ty_ref),
752 .id_result = constituents_id,
753 .constituents = elem_ids,
754 });
755 return constituents_id;
756 },
757 else => {},
758 }
759
714760 const ty = self.spv.cache.lookup(ty_ref).float_type;
715761 return switch (ty.bits) {
716762 16 => try self.spv.resolveId(.{ .float = .{ .ty = ty_ref, .value = .{ .float16 = @floatCast(value) } } }),
......@@ -726,9 +772,9 @@ const DeclGen = struct {
726772 /// if the parameters are in indirect representation, then the result is too.
727773 fn constructComposite(self: *DeclGen, ty: Type, constituents: []const IdRef) !IdRef {
728774 const constituents_id = self.spv.allocId();
729 const type_id = try self.resolveTypeId(ty);
775 const type_id = try self.resolveType(ty, .direct);
730776 try self.func.body.emit(self.spv.gpa, .OpCompositeConstruct, .{
731 .id_result_type = type_id,
777 .id_result_type = self.typeId(type_id),
732778 .id_result = constituents_id,
733779 .constituents = constituents,
734780 });
......@@ -901,19 +947,19 @@ const DeclGen = struct {
901947 .bytes => |bytes| {
902948 // TODO: This is really space inefficient, perhaps there is a better
903949 // way to do it?
904 for (bytes, 0..) |byte, i| {
905 constituents[i] = try self.constInt(elem_ty_ref, byte);
950 for (constituents, bytes) |*constituent, byte| {
951 constituent.* = try self.constInt(elem_ty_ref, byte);
906952 }
907953 },
908954 .elems => |elems| {
909 for (0..@as(usize, @intCast(array_type.len))) |i| {
910 constituents[i] = try self.constant(elem_ty, Value.fromInterned(elems[i]), .indirect);
955 for (constituents, elems) |*constituent, elem| {
956 constituent.* = try self.constant(elem_ty, Value.fromInterned(elem), .indirect);
911957 }
912958 },
913959 .repeated_elem => |elem| {
914960 const val_id = try self.constant(elem_ty, Value.fromInterned(elem), .indirect);
915 for (0..@as(usize, @intCast(array_type.len))) |i| {
916 constituents[i] = val_id;
961 for (constituents) |*constituent| {
962 constituent.* = val_id;
917963 }
918964 },
919965 }
......@@ -1448,12 +1494,11 @@ const DeclGen = struct {
14481494 const elem_ty = ty.childType(mod);
14491495 const elem_ty_ref = try self.resolveType(elem_ty, .indirect);
14501496 const len = ty.vectorLen(mod);
1451 const is_scalar = elem_ty.isNumeric(mod) or elem_ty.toIntern() == .bool_type;
14521497
1453 const ty_ref = if (is_scalar and len > 1 and len <= 4)
1454 try self.spv.vectorType(ty.vectorLen(mod), elem_ty_ref)
1498 const ty_ref = if (self.isVector(ty))
1499 try self.spv.vectorType(len, elem_ty_ref)
14551500 else
1456 try self.spv.arrayType(ty.vectorLen(mod), elem_ty_ref);
1501 try self.spv.arrayType(len, elem_ty_ref);
14571502
14581503 try self.type_map.put(self.gpa, ty.toIntern(), .{ .ty_ref = ty_ref });
14591504 return ty_ref;
......@@ -1752,18 +1797,16 @@ const DeclGen = struct {
17521797 }
17531798
17541799 /// This structure is used as helper for element-wise operations. It is intended
1755 /// to be used with both vectors and single elements.
1800 /// to be used with vectors, fake vectors (arrays) and single elements.
17561801 const WipElementWise = struct {
17571802 dg: *DeclGen,
17581803 result_ty: Type,
1804 ty: Type,
17591805 /// Always in direct representation.
1760 result_ty_ref: CacheRef,
1761 scalar_ty: Type,
1762 /// Always in direct representation.
1763 scalar_ty_ref: CacheRef,
1764 scalar_ty_id: IdRef,
1765 /// True if the input is actually a vector type.
1766 is_vector: bool,
1806 ty_ref: CacheRef,
1807 ty_id: IdRef,
1808 /// True if the input is an array type.
1809 is_array: bool,
17671810 /// The element-wise operation should fill these results before calling finalize().
17681811 /// These should all be in **direct** representation! `finalize()` will convert
17691812 /// them to indirect if required.
......@@ -1774,29 +1817,28 @@ const DeclGen = struct {
17741817 }
17751818
17761819 /// Utility function to extract the element at a particular index in an
1777 /// input vector. This type is expected to be a vector if `wip.is_vector`, and
1778 /// a scalar otherwise.
1820 /// input array. This type is expected to be a fake vector (array) if `wip.is_array`, and
1821 /// a vector or scalar otherwise.
17791822 fn elementAt(wip: WipElementWise, ty: Type, value: IdRef, index: usize) !IdRef {
17801823 const mod = wip.dg.module;
1781 if (wip.is_vector) {
1824 if (wip.is_array) {
17821825 assert(ty.isVector(mod));
17831826 return try wip.dg.extractField(ty.childType(mod), value, @intCast(index));
17841827 } else {
1785 assert(!ty.isVector(mod));
17861828 assert(index == 0);
17871829 return value;
17881830 }
17891831 }
17901832
1791 /// Turns the results of this WipElementWise into a result. This can either
1792 /// be a vector or single element, depending on `result_ty`.
1833 /// Turns the results of this WipElementWise into a result. This can be
1834 /// vectors, fake vectors (arrays) and single elements, depending on `result_ty`.
17931835 /// After calling this function, this WIP is no longer usable.
17941836 /// Results is in `direct` representation.
17951837 fn finalize(wip: *WipElementWise) !IdRef {
1796 if (wip.is_vector) {
1838 if (wip.is_array) {
17971839 // Convert all the constituents to indirect, as required for the array.
17981840 for (wip.results) |*result| {
1799 result.* = try wip.dg.convertToIndirect(wip.scalar_ty, result.*);
1841 result.* = try wip.dg.convertToIndirect(wip.ty, result.*);
18001842 }
18011843 return try wip.dg.constructComposite(wip.result_ty, wip.results);
18021844 } else {
......@@ -1806,33 +1848,30 @@ const DeclGen = struct {
18061848
18071849 /// Allocate a result id at a particular index, and return it.
18081850 fn allocId(wip: *WipElementWise, index: usize) IdRef {
1809 assert(wip.is_vector or index == 0);
1851 assert(wip.is_array or index == 0);
18101852 wip.results[index] = wip.dg.spv.allocId();
18111853 return wip.results[index];
18121854 }
18131855 };
18141856
18151857 /// Create a new element-wise operation.
1816 fn elementWise(self: *DeclGen, result_ty: Type) !WipElementWise {
1858 fn elementWise(self: *DeclGen, result_ty: Type, force_element_wise: bool) !WipElementWise {
18171859 const mod = self.module;
1818 // For now, this operation also reasons in terms of `.direct` representation.
1819 const result_ty_ref = try self.resolveType(result_ty, .direct);
1820 const is_vector = result_ty.isVector(mod);
1821 const num_results = if (is_vector) result_ty.vectorLen(mod) else 1;
1860 const is_array = result_ty.isVector(mod) and (!self.isVector(result_ty) or force_element_wise);
1861 const num_results = if (is_array) result_ty.vectorLen(mod) else 1;
18221862 const results = try self.gpa.alloc(IdRef, num_results);
1823 for (results) |*result| result.* = undefined;
1863 @memset(results, undefined);
18241864
1825 const scalar_ty = result_ty.scalarType(mod);
1826 const scalar_ty_ref = try self.resolveType(scalar_ty, .direct);
1865 const ty = if (is_array) result_ty.scalarType(mod) else result_ty;
1866 const ty_ref = try self.resolveType(ty, .direct);
18271867
18281868 return .{
18291869 .dg = self,
18301870 .result_ty = result_ty,
1831 .result_ty_ref = result_ty_ref,
1832 .scalar_ty = scalar_ty,
1833 .scalar_ty_ref = scalar_ty_ref,
1834 .scalar_ty_id = self.typeId(scalar_ty_ref),
1835 .is_vector = is_vector,
1871 .ty = ty,
1872 .ty_ref = ty_ref,
1873 .ty_id = self.typeId(ty_ref),
1874 .is_array = is_array,
18361875 .results = results,
18371876 };
18381877 }
......@@ -2312,11 +2351,11 @@ const DeclGen = struct {
23122351 }
23132352
23142353 fn binOpSimple(self: *DeclGen, ty: Type, lhs_id: IdRef, rhs_id: IdRef, comptime opcode: Opcode) !IdRef {
2315 var wip = try self.elementWise(ty);
2354 var wip = try self.elementWise(ty, false);
23162355 defer wip.deinit();
23172356 for (0..wip.results.len) |i| {
23182357 try self.func.body.emit(self.spv.gpa, opcode, .{
2319 .id_result_type = wip.scalar_ty_id,
2358 .id_result_type = wip.ty_id,
23202359 .id_result = wip.allocId(i),
23212360 .operand_1 = try wip.elementAt(ty, lhs_id, i),
23222361 .operand_2 = try wip.elementAt(ty, rhs_id, i),
......@@ -2345,7 +2384,7 @@ const DeclGen = struct {
23452384
23462385 const result_ty = self.typeOfIndex(inst);
23472386 const shift_ty = self.typeOf(bin_op.rhs);
2348 const scalar_shift_ty_ref = try self.resolveType(shift_ty.scalarType(mod), .direct);
2387 const shift_ty_ref = try self.resolveType(shift_ty, .direct);
23492388
23502389 const info = self.arithmeticTypeInfo(result_ty);
23512390 switch (info.class) {
......@@ -2354,7 +2393,7 @@ const DeclGen = struct {
23542393 .float, .bool => unreachable,
23552394 }
23562395
2357 var wip = try self.elementWise(result_ty);
2396 var wip = try self.elementWise(result_ty, false);
23582397 defer wip.deinit();
23592398 for (wip.results, 0..) |*result_id, i| {
23602399 const lhs_elem_id = try wip.elementAt(result_ty, lhs_id, i);
......@@ -2362,10 +2401,10 @@ const DeclGen = struct {
23622401
23632402 // Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that,
23642403 // so just manually upcast it if required.
2365 const shift_id = if (scalar_shift_ty_ref != wip.scalar_ty_ref) blk: {
2404 const shift_id = if (shift_ty_ref != wip.ty_ref) blk: {
23662405 const shift_id = self.spv.allocId();
23672406 try self.func.body.emit(self.spv.gpa, .OpUConvert, .{
2368 .id_result_type = wip.scalar_ty_id,
2407 .id_result_type = wip.ty_id,
23692408 .id_result = shift_id,
23702409 .unsigned_value = rhs_elem_id,
23712410 });
......@@ -2374,7 +2413,7 @@ const DeclGen = struct {
23742413
23752414 const value_id = self.spv.allocId();
23762415 const args = .{
2377 .id_result_type = wip.scalar_ty_id,
2416 .id_result_type = wip.ty_id,
23782417 .id_result = value_id,
23792418 .base = lhs_elem_id,
23802419 .shift = shift_id,
......@@ -2386,7 +2425,7 @@ const DeclGen = struct {
23862425 try self.func.body.emit(self.spv.gpa, unsigned, args);
23872426 }
23882427
2389 result_id.* = try self.normalize(wip.scalar_ty_ref, value_id, info);
2428 result_id.* = try self.normalize(wip.ty_ref, value_id, info);
23902429 }
23912430 return try wip.finalize();
23922431 }
......@@ -2405,14 +2444,14 @@ const DeclGen = struct {
24052444 fn minMax(self: *DeclGen, result_ty: Type, op: std.math.CompareOperator, lhs_id: IdRef, rhs_id: IdRef) !IdRef {
24062445 const info = self.arithmeticTypeInfo(result_ty);
24072446
2408 var wip = try self.elementWise(result_ty);
2447 var wip = try self.elementWise(result_ty, true);
24092448 defer wip.deinit();
24102449 for (wip.results, 0..) |*result_id, i| {
24112450 const lhs_elem_id = try wip.elementAt(result_ty, lhs_id, i);
24122451 const rhs_elem_id = try wip.elementAt(result_ty, rhs_id, i);
24132452
24142453 // TODO: Use fmin for OpenCL
2415 const cmp_id = try self.cmp(op, Type.bool, wip.scalar_ty, lhs_elem_id, rhs_elem_id);
2454 const cmp_id = try self.cmp(op, Type.bool, wip.ty, lhs_elem_id, rhs_elem_id);
24162455 const selection_id = switch (info.class) {
24172456 .float => blk: {
24182457 // cmp uses OpFOrd. When we have 0 [<>] nan this returns false,
......@@ -2440,7 +2479,7 @@ const DeclGen = struct {
24402479
24412480 result_id.* = self.spv.allocId();
24422481 try self.func.body.emit(self.spv.gpa, .OpSelect, .{
2443 .id_result_type = wip.scalar_ty_id,
2482 .id_result_type = wip.ty_id,
24442483 .id_result = result_id.*,
24452484 .condition = selection_id,
24462485 .object_1 = lhs_elem_id,
......@@ -2545,7 +2584,7 @@ const DeclGen = struct {
25452584 .bool => unreachable,
25462585 };
25472586
2548 var wip = try self.elementWise(ty);
2587 var wip = try self.elementWise(ty, false);
25492588 defer wip.deinit();
25502589 for (wip.results, 0..) |*result_id, i| {
25512590 const lhs_elem_id = try wip.elementAt(ty, lhs_id, i);
......@@ -2553,7 +2592,7 @@ const DeclGen = struct {
25532592
25542593 const value_id = self.spv.allocId();
25552594 const operands = .{
2556 .id_result_type = wip.scalar_ty_id,
2595 .id_result_type = wip.ty_id,
25572596 .id_result = value_id,
25582597 .operand_1 = lhs_elem_id,
25592598 .operand_2 = rhs_elem_id,
......@@ -2568,7 +2607,7 @@ const DeclGen = struct {
25682607
25692608 // TODO: Trap on overflow? Probably going to be annoying.
25702609 // TODO: Look into SPV_KHR_no_integer_wrap_decoration which provides NoSignedWrap/NoUnsignedWrap.
2571 result_id.* = try self.normalize(wip.scalar_ty_ref, value_id, info);
2610 result_id.* = try self.normalize(wip.ty_ref, value_id, info);
25722611 }
25732612
25742613 return try wip.finalize();
......@@ -2582,12 +2621,12 @@ const DeclGen = struct {
25822621 const operand_id = try self.resolve(ty_op.operand);
25832622 // Note: operand_ty may be signed, while ty is always unsigned!
25842623 const operand_ty = self.typeOf(ty_op.operand);
2585 const ty = self.typeOfIndex(inst);
2586 const info = self.arithmeticTypeInfo(ty);
2624 const result_ty = self.typeOfIndex(inst);
2625 const info = self.arithmeticTypeInfo(result_ty);
25872626 const operand_scalar_ty = operand_ty.scalarType(mod);
25882627 const operand_scalar_ty_ref = try self.resolveType(operand_scalar_ty, .direct);
25892628
2590 var wip = try self.elementWise(ty);
2629 var wip = try self.elementWise(result_ty, true);
25912630 defer wip.deinit();
25922631
25932632 const zero_id = switch (info.class) {
......@@ -2615,7 +2654,7 @@ const DeclGen = struct {
26152654 .composite_integer => unreachable, // TODO
26162655 .bool => unreachable,
26172656 }
2618 const neg_norm_id = try self.normalize(wip.scalar_ty_ref, neg_id, info);
2657 const neg_norm_id = try self.normalize(wip.ty_ref, neg_id, info);
26192658
26202659 const gt_zero_id = try self.cmp(.gt, Type.bool, operand_scalar_ty, elem_id, zero_id);
26212660 const abs_id = self.spv.allocId();
......@@ -2627,7 +2666,7 @@ const DeclGen = struct {
26272666 .object_2 = neg_norm_id,
26282667 });
26292668 // For Shader, we may need to cast from signed to unsigned here.
2630 result_id.* = try self.bitCast(wip.scalar_ty, operand_scalar_ty, abs_id);
2669 result_id.* = try self.bitCast(wip.ty, operand_scalar_ty, abs_id);
26312670 }
26322671 return try wip.finalize();
26332672 }
......@@ -2641,6 +2680,7 @@ const DeclGen = struct {
26412680 ) !?IdRef {
26422681 if (self.liveness.isUnused(inst)) return null;
26432682
2683 const mod = self.module;
26442684 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
26452685 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
26462686 const lhs = try self.resolve(extra.lhs);
......@@ -2651,6 +2691,10 @@ const DeclGen = struct {
26512691 const ov_ty = result_ty.structFieldType(1, self.module);
26522692
26532693 const bool_ty_ref = try self.resolveType(Type.bool, .direct);
2694 const cmp_ty_ref = if (self.isVector(operand_ty))
2695 try self.spv.vectorType(operand_ty.vectorLen(mod), bool_ty_ref)
2696 else
2697 bool_ty_ref;
26542698
26552699 const info = self.arithmeticTypeInfo(operand_ty);
26562700 switch (info.class) {
......@@ -2659,9 +2703,9 @@ const DeclGen = struct {
26592703 .float, .bool => unreachable,
26602704 }
26612705
2662 var wip_result = try self.elementWise(operand_ty);
2706 var wip_result = try self.elementWise(operand_ty, false);
26632707 defer wip_result.deinit();
2664 var wip_ov = try self.elementWise(ov_ty);
2708 var wip_ov = try self.elementWise(ov_ty, false);
26652709 defer wip_ov.deinit();
26662710 for (wip_result.results, wip_ov.results, 0..) |*result_id, *ov_id, i| {
26672711 const lhs_elem_id = try wip_result.elementAt(operand_ty, lhs, i);
......@@ -2671,14 +2715,14 @@ const DeclGen = struct {
26712715 const value_id = self.spv.allocId();
26722716
26732717 try self.func.body.emit(self.spv.gpa, add, .{
2674 .id_result_type = wip_result.scalar_ty_id,
2718 .id_result_type = wip_result.ty_id,
26752719 .id_result = value_id,
26762720 .operand_1 = lhs_elem_id,
26772721 .operand_2 = rhs_elem_id,
26782722 });
26792723
26802724 // Normalize the result so that the comparisons go well
2681 result_id.* = try self.normalize(wip_result.scalar_ty_ref, value_id, info);
2725 result_id.* = try self.normalize(wip_result.ty_ref, value_id, info);
26822726
26832727 const overflowed_id = switch (info.signedness) {
26842728 .unsigned => blk: {
......@@ -2686,7 +2730,7 @@ const DeclGen = struct {
26862730 // For subtraction the conditions need to be swapped.
26872731 const overflowed_id = self.spv.allocId();
26882732 try self.func.body.emit(self.spv.gpa, ucmp, .{
2689 .id_result_type = self.typeId(bool_ty_ref),
2733 .id_result_type = self.typeId(cmp_ty_ref),
26902734 .id_result = overflowed_id,
26912735 .operand_1 = result_id.*,
26922736 .operand_2 = lhs_elem_id,
......@@ -2712,9 +2756,9 @@ const DeclGen = struct {
27122756 // = (rhs < 0) == (lhs > value)
27132757
27142758 const rhs_lt_zero_id = self.spv.allocId();
2715 const zero_id = try self.constInt(wip_result.scalar_ty_ref, 0);
2759 const zero_id = try self.constInt(wip_result.ty_ref, 0);
27162760 try self.func.body.emit(self.spv.gpa, .OpSLessThan, .{
2717 .id_result_type = self.typeId(bool_ty_ref),
2761 .id_result_type = self.typeId(cmp_ty_ref),
27182762 .id_result = rhs_lt_zero_id,
27192763 .operand_1 = rhs_elem_id,
27202764 .operand_2 = zero_id,
......@@ -2722,7 +2766,7 @@ const DeclGen = struct {
27222766
27232767 const value_gt_lhs_id = self.spv.allocId();
27242768 try self.func.body.emit(self.spv.gpa, scmp, .{
2725 .id_result_type = self.typeId(bool_ty_ref),
2769 .id_result_type = self.typeId(cmp_ty_ref),
27262770 .id_result = value_gt_lhs_id,
27272771 .operand_1 = lhs_elem_id,
27282772 .operand_2 = result_id.*,
......@@ -2730,7 +2774,7 @@ const DeclGen = struct {
27302774
27312775 const overflowed_id = self.spv.allocId();
27322776 try self.func.body.emit(self.spv.gpa, .OpLogicalEqual, .{
2733 .id_result_type = self.typeId(bool_ty_ref),
2777 .id_result_type = self.typeId(cmp_ty_ref),
27342778 .id_result = overflowed_id,
27352779 .operand_1 = rhs_lt_zero_id,
27362780 .operand_2 = value_gt_lhs_id,
......@@ -2739,7 +2783,7 @@ const DeclGen = struct {
27392783 },
27402784 };
27412785
2742 ov_id.* = try self.intFromBool(wip_ov.scalar_ty_ref, overflowed_id);
2786 ov_id.* = try self.intFromBool(wip_ov.ty_ref, overflowed_id);
27432787 }
27442788
27452789 return try self.constructComposite(
......@@ -2759,11 +2803,15 @@ const DeclGen = struct {
27592803 const result_ty = self.typeOfIndex(inst);
27602804 const operand_ty = self.typeOf(extra.lhs);
27612805 const shift_ty = self.typeOf(extra.rhs);
2762 const scalar_shift_ty_ref = try self.resolveType(shift_ty.scalarType(mod), .direct);
2806 const shift_ty_ref = try self.resolveType(shift_ty, .direct);
27632807
27642808 const ov_ty = result_ty.structFieldType(1, self.module);
27652809
27662810 const bool_ty_ref = try self.resolveType(Type.bool, .direct);
2811 const cmp_ty_ref = if (self.isVector(operand_ty))
2812 try self.spv.vectorType(operand_ty.vectorLen(mod), bool_ty_ref)
2813 else
2814 bool_ty_ref;
27672815
27682816 const info = self.arithmeticTypeInfo(operand_ty);
27692817 switch (info.class) {
......@@ -2772,9 +2820,9 @@ const DeclGen = struct {
27722820 .float, .bool => unreachable,
27732821 }
27742822
2775 var wip_result = try self.elementWise(operand_ty);
2823 var wip_result = try self.elementWise(operand_ty, false);
27762824 defer wip_result.deinit();
2777 var wip_ov = try self.elementWise(ov_ty);
2825 var wip_ov = try self.elementWise(ov_ty, false);
27782826 defer wip_ov.deinit();
27792827 for (wip_result.results, wip_ov.results, 0..) |*result_id, *ov_id, i| {
27802828 const lhs_elem_id = try wip_result.elementAt(operand_ty, lhs, i);
......@@ -2782,10 +2830,10 @@ const DeclGen = struct {
27822830
27832831 // Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that,
27842832 // so just manually upcast it if required.
2785 const shift_id = if (scalar_shift_ty_ref != wip_result.scalar_ty_ref) blk: {
2833 const shift_id = if (shift_ty_ref != wip_result.ty_ref) blk: {
27862834 const shift_id = self.spv.allocId();
27872835 try self.func.body.emit(self.spv.gpa, .OpUConvert, .{
2788 .id_result_type = wip_result.scalar_ty_id,
2836 .id_result_type = wip_result.ty_id,
27892837 .id_result = shift_id,
27902838 .unsigned_value = rhs_elem_id,
27912839 });
......@@ -2794,18 +2842,18 @@ const DeclGen = struct {
27942842
27952843 const value_id = self.spv.allocId();
27962844 try self.func.body.emit(self.spv.gpa, .OpShiftLeftLogical, .{
2797 .id_result_type = wip_result.scalar_ty_id,
2845 .id_result_type = wip_result.ty_id,
27982846 .id_result = value_id,
27992847 .base = lhs_elem_id,
28002848 .shift = shift_id,
28012849 });
2802 result_id.* = try self.normalize(wip_result.scalar_ty_ref, value_id, info);
2850 result_id.* = try self.normalize(wip_result.ty_ref, value_id, info);
28032851
28042852 const right_shift_id = self.spv.allocId();
28052853 switch (info.signedness) {
28062854 .signed => {
28072855 try self.func.body.emit(self.spv.gpa, .OpShiftRightArithmetic, .{
2808 .id_result_type = wip_result.scalar_ty_id,
2856 .id_result_type = wip_result.ty_id,
28092857 .id_result = right_shift_id,
28102858 .base = result_id.*,
28112859 .shift = shift_id,
......@@ -2813,7 +2861,7 @@ const DeclGen = struct {
28132861 },
28142862 .unsigned => {
28152863 try self.func.body.emit(self.spv.gpa, .OpShiftRightLogical, .{
2816 .id_result_type = wip_result.scalar_ty_id,
2864 .id_result_type = wip_result.ty_id,
28172865 .id_result = right_shift_id,
28182866 .base = result_id.*,
28192867 .shift = shift_id,
......@@ -2823,13 +2871,13 @@ const DeclGen = struct {
28232871
28242872 const overflowed_id = self.spv.allocId();
28252873 try self.func.body.emit(self.spv.gpa, .OpINotEqual, .{
2826 .id_result_type = self.typeId(bool_ty_ref),
2874 .id_result_type = self.typeId(cmp_ty_ref),
28272875 .id_result = overflowed_id,
28282876 .operand_1 = lhs_elem_id,
28292877 .operand_2 = right_shift_id,
28302878 });
28312879
2832 ov_id.* = try self.intFromBool(wip_ov.scalar_ty_ref, overflowed_id);
2880 ov_id.* = try self.intFromBool(wip_ov.ty_ref, overflowed_id);
28332881 }
28342882
28352883 return try self.constructComposite(
......@@ -2853,19 +2901,19 @@ const DeclGen = struct {
28532901 const info = self.arithmeticTypeInfo(ty);
28542902 assert(info.class == .float); // .mul_add is only emitted for floats
28552903
2856 var wip = try self.elementWise(ty);
2904 var wip = try self.elementWise(ty, false);
28572905 defer wip.deinit();
28582906 for (0..wip.results.len) |i| {
28592907 const mul_result = self.spv.allocId();
28602908 try self.func.body.emit(self.spv.gpa, .OpFMul, .{
2861 .id_result_type = wip.scalar_ty_id,
2909 .id_result_type = wip.ty_id,
28622910 .id_result = mul_result,
28632911 .operand_1 = try wip.elementAt(ty, mulend1, i),
28642912 .operand_2 = try wip.elementAt(ty, mulend2, i),
28652913 });
28662914
28672915 try self.func.body.emit(self.spv.gpa, .OpFAdd, .{
2868 .id_result_type = wip.scalar_ty_id,
2916 .id_result_type = wip.ty_id,
28692917 .id_result = wip.allocId(i),
28702918 .operand_1 = mul_result,
28712919 .operand_2 = try wip.elementAt(ty, addend, i),
......@@ -2879,11 +2927,9 @@ const DeclGen = struct {
28792927 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
28802928 const operand_id = try self.resolve(ty_op.operand);
28812929 const result_ty = self.typeOfIndex(inst);
2882 var wip = try self.elementWise(result_ty);
2930 var wip = try self.elementWise(result_ty, true);
28832931 defer wip.deinit();
2884 for (wip.results) |*result_id| {
2885 result_id.* = operand_id;
2886 }
2932 @memset(wip.results, operand_id);
28872933 return try wip.finalize();
28882934 }
28892935
......@@ -2965,20 +3011,20 @@ const DeclGen = struct {
29653011
29663012 const ty = self.typeOfIndex(inst);
29673013
2968 var wip = try self.elementWise(ty);
3014 var wip = try self.elementWise(ty, true);
29693015 defer wip.deinit();
29703016 for (wip.results, 0..) |*result_id, i| {
29713017 const elem = try mask.elemValue(mod, i);
29723018 if (elem.isUndef(mod)) {
2973 result_id.* = try self.spv.constUndef(wip.scalar_ty_ref);
3019 result_id.* = try self.spv.constUndef(wip.ty_ref);
29743020 continue;
29753021 }
29763022
29773023 const index = elem.toSignedInt(mod);
29783024 if (index >= 0) {
2979 result_id.* = try self.extractField(wip.scalar_ty, a, @intCast(index));
3025 result_id.* = try self.extractField(wip.ty, a, @intCast(index));
29803026 } else {
2981 result_id.* = try self.extractField(wip.scalar_ty, b, @intCast(~index));
3027 result_id.* = try self.extractField(wip.ty, b, @intCast(~index));
29823028 }
29833029 }
29843030 return try wip.finalize();
......@@ -3188,7 +3234,7 @@ const DeclGen = struct {
31883234 return result_id;
31893235 },
31903236 .Vector => {
3191 var wip = try self.elementWise(result_ty);
3237 var wip = try self.elementWise(result_ty, true);
31923238 defer wip.deinit();
31933239 const scalar_ty = ty.scalarType(mod);
31943240 for (wip.results, 0..) |*result_id, i| {
......@@ -3374,19 +3420,19 @@ const DeclGen = struct {
33743420 return operand_id;
33753421 }
33763422
3377 var wip = try self.elementWise(dst_ty);
3423 var wip = try self.elementWise(dst_ty, false);
33783424 defer wip.deinit();
33793425 for (wip.results, 0..) |*result_id, i| {
33803426 const elem_id = try wip.elementAt(src_ty, operand_id, i);
33813427 const value_id = self.spv.allocId();
33823428 switch (dst_info.signedness) {
33833429 .signed => try self.func.body.emit(self.spv.gpa, .OpSConvert, .{
3384 .id_result_type = wip.scalar_ty_id,
3430 .id_result_type = wip.ty_id,
33853431 .id_result = value_id,
33863432 .signed_value = elem_id,
33873433 }),
33883434 .unsigned => try self.func.body.emit(self.spv.gpa, .OpUConvert, .{
3389 .id_result_type = wip.scalar_ty_id,
3435 .id_result_type = wip.ty_id,
33903436 .id_result = value_id,
33913437 .unsigned_value = elem_id,
33923438 }),
......@@ -3397,7 +3443,7 @@ const DeclGen = struct {
33973443 // type, we don't need to normalize when growing the type. The
33983444 // representation is already the same.
33993445 if (dst_info.bits < src_info.bits) {
3400 result_id.* = try self.normalize(wip.scalar_ty_ref, value_id, dst_info);
3446 result_id.* = try self.normalize(wip.ty_ref, value_id, dst_info);
34013447 } else {
34023448 result_id.* = value_id;
34033449 }
......@@ -3482,11 +3528,11 @@ const DeclGen = struct {
34823528 const operand_id = try self.resolve(un_op);
34833529 const result_ty = self.typeOfIndex(inst);
34843530
3485 var wip = try self.elementWise(result_ty);
3531 var wip = try self.elementWise(result_ty, false);
34863532 defer wip.deinit();
34873533 for (wip.results, 0..) |*result_id, i| {
34883534 const elem_id = try wip.elementAt(Type.bool, operand_id, i);
3489 result_id.* = try self.intFromBool(wip.scalar_ty_ref, elem_id);
3535 result_id.* = try self.intFromBool(wip.ty_ref, elem_id);
34903536 }
34913537 return try wip.finalize();
34923538 }
......@@ -3515,12 +3561,12 @@ const DeclGen = struct {
35153561 const result_ty = self.typeOfIndex(inst);
35163562 const info = self.arithmeticTypeInfo(result_ty);
35173563
3518 var wip = try self.elementWise(result_ty);
3564 var wip = try self.elementWise(result_ty, false);
35193565 defer wip.deinit();
35203566
35213567 for (0..wip.results.len) |i| {
35223568 const args = .{
3523 .id_result_type = wip.scalar_ty_id,
3569 .id_result_type = wip.ty_id,
35243570 .id_result = wip.allocId(i),
35253571 .operand = try wip.elementAt(result_ty, operand_id, i),
35263572 };
......@@ -3563,10 +3609,7 @@ const DeclGen = struct {
35633609 // Convert the pointer-to-array to a pointer to the first element.
35643610 try self.accessChain(elem_ptr_ty_ref, array_ptr_id, &.{0});
35653611
3566 return try self.constructComposite(
3567 slice_ty,
3568 &.{ elem_ptr_id, len_id },
3569 );
3612 return try self.constructComposite(slice_ty, &.{ elem_ptr_id, len_id });
35703613 }
35713614
35723615 fn airSlice(self: *DeclGen, inst: Air.Inst.Index) !?IdRef {
......@@ -3580,10 +3623,7 @@ const DeclGen = struct {
35803623
35813624 // Note: Types should not need to be converted to direct, these types
35823625 // dont need to be converted.
3583 return try self.constructComposite(
3584 slice_ty,
3585 &.{ ptr_id, len_id },
3586 );
3626 return try self.constructComposite(slice_ty, &.{ ptr_id, len_id });
35873627 }
35883628
35893629 fn airAggregateInit(self: *DeclGen, inst: Air.Inst.Index) !?IdRef {
......@@ -3641,9 +3681,9 @@ const DeclGen = struct {
36413681 const elem_ids = try self.gpa.alloc(IdRef, n_elems);
36423682 defer self.gpa.free(elem_ids);
36433683
3644 for (elements, 0..) |element, i| {
3684 for (elements, elem_ids) |element, *elem_id| {
36453685 const id = try self.resolve(element);
3646 elem_ids[i] = try self.convertToIndirect(result_ty.childType(mod), id);
3686 elem_id.* = try self.convertToIndirect(result_ty.childType(mod), id);
36473687 }
36483688
36493689 return try self.constructComposite(result_ty, elem_ids);
......@@ -3654,9 +3694,9 @@ const DeclGen = struct {
36543694 const elem_ids = try self.gpa.alloc(IdRef, n_elems);
36553695 defer self.gpa.free(elem_ids);
36563696
3657 for (elements, 0..) |element, i| {
3697 for (elements, elem_ids) |element, *elem_id| {
36583698 const id = try self.resolve(element);
3659 elem_ids[i] = try self.convertToIndirect(array_info.elem_type, id);
3699 elem_id.* = try self.convertToIndirect(array_info.elem_type, id);
36603700 }
36613701
36623702 if (array_info.sentinel) |sentinel_val| {