authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-05-26 05:07:13+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-06-01 08:24:01+01:00
logadd2976a9ba76ec661ae5668eb2a8dca2ccfad42
tree54ea8377660395007c1ac5188863f5cbe3a3310e
parentb48d6ff619de424e664cf11b43e2a03fecbca6ce
signaturelock-open Commit is signed but in an unrecognized format.

compiler: implement better shuffle AIR

Runtime `@shuffle` has two cases which backends generally want to handle differently for efficiency: * One runtime vector operand; some result elements may be comptime-known * Two runtime vector operands; some result elements may be undefined The latter case happens if both vectors given to `@shuffle` are runtime-known and they are both used (i.e. the mask refers to them). Otherwise, if the result is not entirely comptime-known, we are in the former case. `Sema` now diffentiates these two cases in the AIR so that backends can easily handle them however they want to. Note that this *doesn't* really involve Sema doing any more work than it would otherwise need to, so there's not really a negative here! Most existing backends have their lowerings for `@shuffle` migrated in this commit. The LLVM backend uses new lowerings suggested by Jacob as ones which it will handle effectively. The x86_64 backend has not yet been migrated; for now there's a panic in there. Jacob will implement that before this is merged anywhere.

18 files changed, 755 insertions(+), 321 deletions(-)

src/Air.zig+119-12
...@@ -699,9 +699,21 @@ pub const Inst = struct {...@@ -699,9 +699,21 @@ pub const Inst = struct {
699 /// equal to the scalar value.699 /// equal to the scalar value.
700 /// Uses the `ty_op` field.700 /// Uses the `ty_op` field.
701 splat,701 splat,
702 /// Constructs a vector by selecting elements from `a` and `b` based on `mask`.702 /// Constructs a vector by selecting elements from a single vector based on a mask. Each
703 /// Uses the `ty_pl` field with payload `Shuffle`.703 /// mask element is either an index into the vector, or a comptime-known value, or "undef".
704 shuffle,704 /// Uses the `ty_pl` field, where the payload index points to:
705 /// 1. mask_elem: ShuffleOneMask // for each `mask_len`, which comes from `ty_pl.ty`
706 /// 2. operand: Ref // guaranteed not to be an interned value
707 /// See `unwrapShufleOne`.
708 shuffle_one,
709 /// Constructs a vector by selecting elements from two vectors based on a mask. Each mask
710 /// element is either an index into one of the vectors, or "undef".
711 /// Uses the `ty_pl` field, where the payload index points to:
712 /// 1. mask_elem: ShuffleOneMask // for each `mask_len`, which comes from `ty_pl.ty`
713 /// 2. operand_a: Ref // guaranteed not to be an interned value
714 /// 3. operand_b: Ref // guaranteed not to be an interned value
715 /// See `unwrapShufleTwo`.
716 shuffle_two,
705 /// Constructs a vector element-wise from `a` or `b` based on `pred`.717 /// Constructs a vector element-wise from `a` or `b` based on `pred`.
706 /// Uses the `pl_op` field with `pred` as operand, and payload `Bin`.718 /// Uses the `pl_op` field with `pred` as operand, and payload `Bin`.
707 select,719 select,
...@@ -1299,13 +1311,6 @@ pub const FieldParentPtr = struct {...@@ -1299,13 +1311,6 @@ pub const FieldParentPtr = struct {
1299 field_index: u32,1311 field_index: u32,
1300};1312};
13011313
1302pub const Shuffle = struct {
1303 a: Inst.Ref,
1304 b: Inst.Ref,
1305 mask: InternPool.Index,
1306 mask_len: u32,
1307};
1308
1309pub const VectorCmp = struct {1314pub const VectorCmp = struct {
1310 lhs: Inst.Ref,1315 lhs: Inst.Ref,
1311 rhs: Inst.Ref,1316 rhs: Inst.Ref,
...@@ -1320,6 +1325,64 @@ pub const VectorCmp = struct {...@@ -1320,6 +1325,64 @@ pub const VectorCmp = struct {
1320 }1325 }
1321};1326};
13221327
1328/// Used by `Inst.Tag.shuffle_one`. Represents a mask element which either indexes into a
1329/// runtime-known vector, or is a comptime-known value.
1330pub const ShuffleOneMask = packed struct(u32) {
1331 index: u31,
1332 kind: enum(u1) { elem, value },
1333 pub fn elem(idx: u32) ShuffleOneMask {
1334 return .{ .index = @intCast(idx), .kind = .elem };
1335 }
1336 pub fn value(val: Value) ShuffleOneMask {
1337 return .{ .index = @intCast(@intFromEnum(val.toIntern())), .kind = .value };
1338 }
1339 pub const Unwrapped = union(enum) {
1340 /// The resulting element is this index into the runtime vector.
1341 elem: u32,
1342 /// The resulting element is this comptime-known value.
1343 /// It is correctly typed. It might be `undefined`.
1344 value: InternPool.Index,
1345 };
1346 pub fn unwrap(raw: ShuffleOneMask) Unwrapped {
1347 return switch (raw.kind) {
1348 .elem => .{ .elem = raw.index },
1349 .value => .{ .value = @enumFromInt(raw.index) },
1350 };
1351 }
1352};
1353
1354/// Used by `Inst.Tag.shuffle_two`. Represents a mask element which either indexes into one
1355/// of two runtime-known vectors, or is undefined.
1356pub const ShuffleTwoMask = enum(u32) {
1357 undef = std.math.maxInt(u32),
1358 _,
1359 pub fn aElem(idx: u32) ShuffleTwoMask {
1360 return @enumFromInt(idx << 1);
1361 }
1362 pub fn bElem(idx: u32) ShuffleTwoMask {
1363 return @enumFromInt(idx << 1 | 1);
1364 }
1365 pub const Unwrapped = union(enum) {
1366 /// The resulting element is this index into the first runtime vector.
1367 a_elem: u32,
1368 /// The resulting element is this index into the second runtime vector.
1369 b_elem: u32,
1370 /// The resulting element is `undefined`.
1371 undef,
1372 };
1373 pub fn unwrap(raw: ShuffleTwoMask) Unwrapped {
1374 switch (raw) {
1375 .undef => return .undef,
1376 _ => {},
1377 }
1378 const x = @intFromEnum(raw);
1379 return switch (@as(u1, @truncate(x))) {
1380 0 => .{ .a_elem = x >> 1 },
1381 1 => .{ .b_elem = x >> 1 },
1382 };
1383 }
1384};
1385
1323/// Trailing:1386/// Trailing:
1324/// 0. `Inst.Ref` for every outputs_len1387/// 0. `Inst.Ref` for every outputs_len
1325/// 1. `Inst.Ref` for every inputs_len1388/// 1. `Inst.Ref` for every inputs_len
...@@ -1503,7 +1566,6 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)...@@ -1503,7 +1566,6 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
1503 .cmpxchg_weak,1566 .cmpxchg_weak,
1504 .cmpxchg_strong,1567 .cmpxchg_strong,
1505 .slice,1568 .slice,
1506 .shuffle,
1507 .aggregate_init,1569 .aggregate_init,
1508 .union_init,1570 .union_init,
1509 .field_parent_ptr,1571 .field_parent_ptr,
...@@ -1517,6 +1579,8 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)...@@ -1517,6 +1579,8 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
1517 .ptr_sub,1579 .ptr_sub,
1518 .try_ptr,1580 .try_ptr,
1519 .try_ptr_cold,1581 .try_ptr_cold,
1582 .shuffle_one,
1583 .shuffle_two,
1520 => return datas[@intFromEnum(inst)].ty_pl.ty.toType(),1584 => return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
15211585
1522 .not,1586 .not,
...@@ -1903,7 +1967,8 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -1903,7 +1967,8 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1903 .reduce,1967 .reduce,
1904 .reduce_optimized,1968 .reduce_optimized,
1905 .splat,1969 .splat,
1906 .shuffle,1970 .shuffle_one,
1971 .shuffle_two,
1907 .select,1972 .select,
1908 .is_named_enum_value,1973 .is_named_enum_value,
1909 .tag_name,1974 .tag_name,
...@@ -2030,6 +2095,48 @@ pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch {...@@ -2030,6 +2095,48 @@ pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch {
2030 };2095 };
2031}2096}
20322097
2098pub fn unwrapShuffleOne(air: *const Air, zcu: *const Zcu, inst_index: Inst.Index) struct {
2099 result_ty: Type,
2100 operand: Inst.Ref,
2101 mask: []const ShuffleOneMask,
2102} {
2103 const inst = air.instructions.get(@intFromEnum(inst_index));
2104 switch (inst.tag) {
2105 .shuffle_one => {},
2106 else => unreachable, // assertion failure
2107 }
2108 const result_ty: Type = .fromInterned(inst.data.ty_pl.ty.toInterned().?);
2109 const mask_len: u32 = result_ty.vectorLen(zcu);
2110 const extra_idx = inst.data.ty_pl.payload;
2111 return .{
2112 .result_ty = result_ty,
2113 .operand = @enumFromInt(air.extra.items[extra_idx + mask_len]),
2114 .mask = @ptrCast(air.extra.items[extra_idx..][0..mask_len]),
2115 };
2116}
2117
2118pub fn unwrapShuffleTwo(air: *const Air, zcu: *const Zcu, inst_index: Inst.Index) struct {
2119 result_ty: Type,
2120 operand_a: Inst.Ref,
2121 operand_b: Inst.Ref,
2122 mask: []const ShuffleTwoMask,
2123} {
2124 const inst = air.instructions.get(@intFromEnum(inst_index));
2125 switch (inst.tag) {
2126 .shuffle_two => {},
2127 else => unreachable, // assertion failure
2128 }
2129 const result_ty: Type = .fromInterned(inst.data.ty_pl.ty.toInterned().?);
2130 const mask_len: u32 = result_ty.vectorLen(zcu);
2131 const extra_idx = inst.data.ty_pl.payload;
2132 return .{
2133 .result_ty = result_ty,
2134 .operand_a = @enumFromInt(air.extra.items[extra_idx + mask_len + 0]),
2135 .operand_b = @enumFromInt(air.extra.items[extra_idx + mask_len + 1]),
2136 .mask = @ptrCast(air.extra.items[extra_idx..][0..mask_len]),
2137 };
2138}
2139
2033pub const typesFullyResolved = types_resolved.typesFullyResolved;2140pub const typesFullyResolved = types_resolved.typesFullyResolved;
2034pub const typeFullyResolved = types_resolved.checkType;2141pub const typeFullyResolved = types_resolved.checkType;
2035pub const valFullyResolved = types_resolved.checkVal;2142pub const valFullyResolved = types_resolved.checkVal;
src/Air/Legalize.zig+2-1
...@@ -521,7 +521,8 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -521,7 +521,8 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
521 }521 }
522 },522 },
523 .splat,523 .splat,
524 .shuffle,524 .shuffle_one,
525 .shuffle_two,
525 => {},526 => {},
526 .select,527 .select,
527 => if (l.features.contains(.scalarize_select)) continue :inst try l.scalarize(inst, .select_pl_op_bin),528 => if (l.features.contains(.scalarize_select)) continue :inst try l.scalarize(inst, .select_pl_op_bin),
src/Air/Liveness.zig+24-9
...@@ -15,6 +15,7 @@ const Liveness = @This();...@@ -15,6 +15,7 @@ const Liveness = @This();
15const trace = @import("../tracy.zig").trace;15const trace = @import("../tracy.zig").trace;
16const Air = @import("../Air.zig");16const Air = @import("../Air.zig");
17const InternPool = @import("../InternPool.zig");17const InternPool = @import("../InternPool.zig");
18const Zcu = @import("../Zcu.zig");
1819
19pub const Verify = @import("Liveness/Verify.zig");20pub const Verify = @import("Liveness/Verify.zig");
2021
...@@ -136,12 +137,15 @@ fn LivenessPassData(comptime pass: LivenessPass) type {...@@ -136,12 +137,15 @@ fn LivenessPassData(comptime pass: LivenessPass) type {
136 };137 };
137}138}
138139
139pub fn analyze(gpa: Allocator, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness {140pub fn analyze(zcu: *Zcu, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness {
140 const tracy = trace(@src());141 const tracy = trace(@src());
141 defer tracy.end();142 defer tracy.end();
142143
144 const gpa = zcu.gpa;
145
143 var a: Analysis = .{146 var a: Analysis = .{
144 .gpa = gpa,147 .gpa = gpa,
148 .zcu = zcu,
145 .air = air,149 .air = air,
146 .tomb_bits = try gpa.alloc(150 .tomb_bits = try gpa.alloc(
147 usize,151 usize,
...@@ -220,6 +224,7 @@ const OperandCategory = enum {...@@ -220,6 +224,7 @@ const OperandCategory = enum {
220pub fn categorizeOperand(224pub fn categorizeOperand(
221 l: Liveness,225 l: Liveness,
222 air: Air,226 air: Air,
227 zcu: *Zcu,
223 inst: Air.Inst.Index,228 inst: Air.Inst.Index,
224 operand: Air.Inst.Index,229 operand: Air.Inst.Index,
225 ip: *const InternPool,230 ip: *const InternPool,
...@@ -511,10 +516,15 @@ pub fn categorizeOperand(...@@ -511,10 +516,15 @@ pub fn categorizeOperand(
511 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);516 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
512 return .none;517 return .none;
513 },518 },
514 .shuffle => {519 .shuffle_one => {
515 const extra = air.extraData(Air.Shuffle, air_datas[@intFromEnum(inst)].ty_pl.payload).data;520 const unwrapped = air.unwrapShuffleOne(zcu, inst);
516 if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);521 if (unwrapped.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
517 if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);522 return .none;
523 },
524 .shuffle_two => {
525 const unwrapped = air.unwrapShuffleTwo(zcu, inst);
526 if (unwrapped.operand_a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
527 if (unwrapped.operand_b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
518 return .none;528 return .none;
519 },529 },
520 .reduce, .reduce_optimized => {530 .reduce, .reduce_optimized => {
...@@ -639,7 +649,7 @@ pub fn categorizeOperand(...@@ -639,7 +649,7 @@ pub fn categorizeOperand(
639649
640 var operand_live: bool = true;650 var operand_live: bool = true;
641 for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| {651 for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| {
642 if (l.categorizeOperand(air, cond_inst, operand, ip) == .tomb)652 if (l.categorizeOperand(air, zcu, cond_inst, operand, ip) == .tomb)
643 operand_live = false;653 operand_live = false;
644654
645 switch (air_tags[@intFromEnum(cond_inst)]) {655 switch (air_tags[@intFromEnum(cond_inst)]) {
...@@ -824,6 +834,7 @@ pub const BigTomb = struct {...@@ -824,6 +834,7 @@ pub const BigTomb = struct {
824/// In-progress data; on successful analysis converted into `Liveness`.834/// In-progress data; on successful analysis converted into `Liveness`.
825const Analysis = struct {835const Analysis = struct {
826 gpa: Allocator,836 gpa: Allocator,
837 zcu: *Zcu,
827 air: Air,838 air: Air,
828 intern_pool: *InternPool,839 intern_pool: *InternPool,
829 tomb_bits: []usize,840 tomb_bits: []usize,
...@@ -1119,9 +1130,13 @@ fn analyzeInst(...@@ -1119,9 +1130,13 @@ fn analyzeInst(
1119 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;1130 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1120 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs });1131 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs });
1121 },1132 },
1122 .shuffle => {1133 .shuffle_one => {
1123 const extra = a.air.extraData(Air.Shuffle, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;1134 const unwrapped = a.air.unwrapShuffleOne(a.zcu, inst);
1124 return analyzeOperands(a, pass, data, inst, .{ extra.a, extra.b, .none });1135 return analyzeOperands(a, pass, data, inst, .{ unwrapped.operand, .none, .none });
1136 },
1137 .shuffle_two => {
1138 const unwrapped = a.air.unwrapShuffleTwo(a.zcu, inst);
1139 return analyzeOperands(a, pass, data, inst, .{ unwrapped.operand_a, unwrapped.operand_b, .none });
1125 },1140 },
1126 .reduce, .reduce_optimized => {1141 .reduce, .reduce_optimized => {
1127 const reduce = inst_datas[@intFromEnum(inst)].reduce;1142 const reduce = inst_datas[@intFromEnum(inst)].reduce;
src/Air/Liveness/Verify.zig+9-4
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1//! Verifies that Liveness information is valid.1//! Verifies that Liveness information is valid.
22
3gpa: std.mem.Allocator,3gpa: std.mem.Allocator,
4zcu: *Zcu,
4air: Air,5air: Air,
5liveness: Liveness,6liveness: Liveness,
6live: LiveMap = .{},7live: LiveMap = .{},
...@@ -287,10 +288,13 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {...@@ -287,10 +288,13 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
287 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;288 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
288 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });289 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
289 },290 },
290 .shuffle => {291 .shuffle_one => {
291 const ty_pl = data[@intFromEnum(inst)].ty_pl;292 const unwrapped = self.air.unwrapShuffleOne(self.zcu, inst);
292 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;293 try self.verifyInstOperands(inst, .{ unwrapped.operand, .none, .none });
293 try self.verifyInstOperands(inst, .{ extra.a, extra.b, .none });294 },
295 .shuffle_two => {
296 const unwrapped = self.air.unwrapShuffleTwo(self.zcu, inst);
297 try self.verifyInstOperands(inst, .{ unwrapped.operand_a, unwrapped.operand_b, .none });
294 },298 },
295 .cmp_vector,299 .cmp_vector,
296 .cmp_vector_optimized,300 .cmp_vector_optimized,
...@@ -639,4 +643,5 @@ const log = std.log.scoped(.liveness_verify);...@@ -639,4 +643,5 @@ const log = std.log.scoped(.liveness_verify);
639const Air = @import("../../Air.zig");643const Air = @import("../../Air.zig");
640const Liveness = @import("../Liveness.zig");644const Liveness = @import("../Liveness.zig");
641const InternPool = @import("../../InternPool.zig");645const InternPool = @import("../../InternPool.zig");
646const Zcu = @import("../../Zcu.zig");
642const Verify = @This();647const Verify = @This();
src/Air/types_resolved.zig+16-6
...@@ -249,12 +249,22 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {...@@ -249,12 +249,22 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
249 if (!checkRef(extra.struct_operand, zcu)) return false;249 if (!checkRef(extra.struct_operand, zcu)) return false;
250 },250 },
251251
252 .shuffle => {252 .shuffle_one => {
253 const extra = air.extraData(Air.Shuffle, data.ty_pl.payload).data;253 const unwrapped = air.unwrapShuffleOne(zcu, inst);
254 if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;254 if (!checkType(unwrapped.result_ty, zcu)) return false;
255 if (!checkRef(extra.a, zcu)) return false;255 if (!checkRef(unwrapped.operand, zcu)) return false;
256 if (!checkRef(extra.b, zcu)) return false;256 for (unwrapped.mask) |m| switch (m.unwrap()) {
257 if (!checkVal(Value.fromInterned(extra.mask), zcu)) return false;257 .elem => {},
258 .value => |val| if (!checkVal(.fromInterned(val), zcu)) return false,
259 };
260 },
261
262 .shuffle_two => {
263 const unwrapped = air.unwrapShuffleTwo(zcu, inst);
264 if (!checkType(unwrapped.result_ty, zcu)) return false;
265 if (!checkRef(unwrapped.operand_a, zcu)) return false;
266 if (!checkRef(unwrapped.operand_b, zcu)) return false;
267 // No values to check because there are no comptime-known values other than undef
258 },268 },
259269
260 .cmpxchg_weak,270 .cmpxchg_weak,
src/Sema.zig+136-132
...@@ -24256,8 +24256,8 @@ fn analyzeShuffle(...@@ -24256,8 +24256,8 @@ fn analyzeShuffle(
24256 block: *Block,24256 block: *Block,
24257 src_node: std.zig.Ast.Node.Offset,24257 src_node: std.zig.Ast.Node.Offset,
24258 elem_ty: Type,24258 elem_ty: Type,
24259 a_arg: Air.Inst.Ref,24259 a_uncoerced: Air.Inst.Ref,
24260 b_arg: Air.Inst.Ref,24260 b_uncoerced: Air.Inst.Ref,
24261 mask: Value,24261 mask: Value,
24262 mask_len: u32,24262 mask_len: u32,
24263) CompileError!Air.Inst.Ref {24263) CompileError!Air.Inst.Ref {
...@@ -24266,150 +24266,154 @@ fn analyzeShuffle(...@@ -24266,150 +24266,154 @@ fn analyzeShuffle(
24266 const a_src = block.builtinCallArgSrc(src_node, 1);24266 const a_src = block.builtinCallArgSrc(src_node, 1);
24267 const b_src = block.builtinCallArgSrc(src_node, 2);24267 const b_src = block.builtinCallArgSrc(src_node, 2);
24268 const mask_src = block.builtinCallArgSrc(src_node, 3);24268 const mask_src = block.builtinCallArgSrc(src_node, 3);
24269 var a = a_arg;
24270 var b = b_arg;
2427124269
24272 const res_ty = try pt.vectorType(.{24270 // If the type of `a` is `@Type(.undefined)`, i.e. the argument is untyped, this is 0, because it is an error to index into this vector.
24273 .len = mask_len,24271 const a_len: u32 = switch (sema.typeOf(a_uncoerced).zigTypeTag(zcu)) {
24274 .child = elem_ty.toIntern(),24272 .array, .vector => @intCast(sema.typeOf(a_uncoerced).arrayLen(zcu)),
24275 });24273 .undefined => 0,
2427624274 else => return sema.fail(block, a_src, "expected vector of '{}', found '{}'", .{ elem_ty.fmt(pt), sema.typeOf(a_uncoerced).fmt(pt) }),
24277 const maybe_a_len = switch (sema.typeOf(a).zigTypeTag(zcu)) {24275 };
24278 .array, .vector => sema.typeOf(a).arrayLen(zcu),24276 const a_ty = try pt.vectorType(.{ .len = a_len, .child = elem_ty.toIntern() });
24279 .undefined => null,24277 const a_coerced = try sema.coerce(block, a_ty, a_uncoerced, a_src);
24280 else => return sema.fail(block, a_src, "expected vector or array with element type '{}', found '{}'", .{24278
24281 elem_ty.fmt(pt),24279 // If the type of `b` is `@Type(.undefined)`, i.e. the argument is untyped, this is 0, because it is an error to index into this vector.
24282 sema.typeOf(a).fmt(pt),24280 const b_len: u32 = switch (sema.typeOf(b_uncoerced).zigTypeTag(zcu)) {
24283 }),24281 .array, .vector => @intCast(sema.typeOf(b_uncoerced).arrayLen(zcu)),
24284 };24282 .undefined => 0,
24285 const maybe_b_len = switch (sema.typeOf(b).zigTypeTag(zcu)) {24283 else => return sema.fail(block, b_src, "expected vector of '{}', found '{}'", .{ elem_ty.fmt(pt), sema.typeOf(b_uncoerced).fmt(pt) }),
24286 .array, .vector => sema.typeOf(b).arrayLen(zcu),24284 };
24287 .undefined => null,24285 const b_ty = try pt.vectorType(.{ .len = b_len, .child = elem_ty.toIntern() });
24288 else => return sema.fail(block, b_src, "expected vector or array with element type '{}', found '{}'", .{24286 const b_coerced = try sema.coerce(block, b_ty, b_uncoerced, b_src);
24289 elem_ty.fmt(pt),24287
24290 sema.typeOf(b).fmt(pt),24288 const result_ty = try pt.vectorType(.{ .len = mask_len, .child = elem_ty.toIntern() });
24291 }),24289
24292 };24290 // We're going to pre-emptively reserve space in `sema.air_extra`. The reason for this is we need
24293 if (maybe_a_len == null and maybe_b_len == null) {24291 // a `u32` buffer of length `mask_len` anyway, and putting it in `sema.air_extra` avoids a copy
24294 return pt.undefRef(res_ty);24292 // in the runtime case. If the result is comptime-known, we'll shrink `air_extra` back.
24295 }24293 const air_extra_idx: u32 = @intCast(sema.air_extra.items.len);
24296 const a_len: u32 = @intCast(maybe_a_len orelse maybe_b_len.?);24294 const air_mask_buf = try sema.air_extra.addManyAsSlice(sema.gpa, mask_len);
24297 const b_len: u32 = @intCast(maybe_b_len orelse a_len);24295
2429824296 // We want to interpret that buffer in `air_extra` in a few ways. Initially, we'll consider its
24299 const a_ty = try pt.vectorType(.{24297 // elements as `Air.Inst.ShuffleTwoMask`, essentially representing the raw mask values; then, we'll
24300 .len = a_len,24298 // convert it to `InternPool.Index` or `Air.Inst.ShuffleOneMask` if there are comptime-known operands.
24301 .child = elem_ty.toIntern(),24299 const mask_ip_index: []InternPool.Index = @ptrCast(air_mask_buf);
24302 });24300 const mask_shuffle_one: []Air.ShuffleOneMask = @ptrCast(air_mask_buf);
24303 const b_ty = try pt.vectorType(.{24301 const mask_shuffle_two: []Air.ShuffleTwoMask = @ptrCast(air_mask_buf);
24304 .len = b_len,24302
24305 .child = elem_ty.toIntern(),24303 // Initial loop: check mask elements, populate `mask_shuffle_two`.
24306 });24304 var a_used = false;
2430724305 var b_used = false;
24308 if (maybe_a_len == null) a = try pt.undefRef(a_ty) else a = try sema.coerce(block, a_ty, a, a_src);24306 for (mask_shuffle_two, 0..mask_len) |*out, mask_idx| {
24309 if (maybe_b_len == null) b = try pt.undefRef(b_ty) else b = try sema.coerce(block, b_ty, b, b_src);24307 const mask_val = try mask.elemValue(pt, mask_idx);
2431024308 if (mask_val.isUndef(zcu)) {
24311 const operand_info = [2]std.meta.Tuple(&.{ u64, LazySrcLoc, Type }){24309 out.* = .undef;
24312 .{ a_len, a_src, a_ty },24310 continue;
24313 .{ b_len, b_src, b_ty },
24314 };
24315
24316 for (0..@intCast(mask_len)) |i| {
24317 const elem = try mask.elemValue(pt, i);
24318 if (elem.isUndef(zcu)) continue;
24319 const elem_resolved = try sema.resolveLazyValue(elem);
24320 const int = elem_resolved.toSignedInt(zcu);
24321 var unsigned: u32 = undefined;
24322 var chosen: u32 = undefined;
24323 if (int >= 0) {
24324 unsigned = @intCast(int);
24325 chosen = 0;
24326 } else {
24327 unsigned = @intCast(~int);
24328 chosen = 1;
24329 }24311 }
24330 if (unsigned >= operand_info[chosen][0]) {24312 // Safe because mask elements are `i32` and we already checked for undef:
24331 const msg = msg: {24313 const raw = (try sema.resolveLazyValue(mask_val)).toSignedInt(zcu);
24332 const msg = try sema.errMsg(mask_src, "mask index '{d}' has out-of-bounds selection", .{i});24314 if (raw >= 0) {
24315 const idx: u32 = @intCast(raw);
24316 a_used = true;
24317 out.* = .aElem(idx);
24318 if (idx >= a_len) return sema.failWithOwnedErrorMsg(block, msg: {
24319 const msg = try sema.errMsg(mask_src, "mask element at index '{d}' selects out-of-bounds index", .{mask_idx});
24333 errdefer msg.destroy(sema.gpa);24320 errdefer msg.destroy(sema.gpa);
2433424321 try sema.errNote(a_src, msg, "index '{d}' exceeds bounds of '{}' given here", .{ idx, a_ty.fmt(pt) });
24335 try sema.errNote(operand_info[chosen][1], msg, "selected index '{d}' out of bounds of '{}'", .{24322 if (idx < b_len) {
24336 unsigned,24323 try sema.errNote(b_src, msg, "use '~@as(u32, {d})' to index into second vector given here", .{idx});
24337 operand_info[chosen][2].fmt(pt),
24338 });
24339
24340 if (chosen == 0) {
24341 try sema.errNote(b_src, msg, "selections from the second vector are specified with negative numbers", .{});
24342 }24324 }
24343
24344 break :msg msg;24325 break :msg msg;
24345 };24326 });
24346 return sema.failWithOwnedErrorMsg(block, msg);24327 } else {
24328 const idx: u32 = @intCast(~raw);
24329 b_used = true;
24330 out.* = .bElem(idx);
24331 if (idx >= b_len) return sema.failWithOwnedErrorMsg(block, msg: {
24332 const msg = try sema.errMsg(mask_src, "mask element at index '{d}' selects out-of-bounds index", .{mask_idx});
24333 errdefer msg.destroy(sema.gpa);
24334 try sema.errNote(b_src, msg, "index '{d}' exceeds bounds of '{}' given here", .{ idx, b_ty.fmt(pt) });
24335 break :msg msg;
24336 });
24347 }24337 }
24348 }24338 }
2434924339
24350 if (try sema.resolveValue(a)) |a_val| {24340 const maybe_a_val = try sema.resolveValue(a_coerced);
24351 if (try sema.resolveValue(b)) |b_val| {24341 const maybe_b_val = try sema.resolveValue(b_coerced);
24352 const values = try sema.arena.alloc(InternPool.Index, mask_len);
24353 for (values, 0..) |*value, i| {
24354 const mask_elem_val = try mask.elemValue(pt, i);
24355 if (mask_elem_val.isUndef(zcu)) {
24356 value.* = try pt.intern(.{ .undef = elem_ty.toIntern() });
24357 continue;
24358 }
24359 const int = mask_elem_val.toSignedInt(zcu);
24360 const unsigned: u32 = @intCast(if (int >= 0) int else ~int);
24361 values[i] = (try (if (int >= 0) a_val else b_val).elemValue(pt, unsigned)).toIntern();
24362 }
24363 return Air.internedToRef((try pt.intern(.{ .aggregate = .{
24364 .ty = res_ty.toIntern(),
24365 .storage = .{ .elems = values },
24366 } })));
24367 }
24368 }
2436924342
24370 // All static analysis passed, and not comptime.24343 const a_rt = a_used and maybe_a_val == null;
24371 // For runtime codegen, vectors a and b must be the same length. Here we24344 const b_rt = b_used and maybe_b_val == null;
24372 // recursively @shuffle the smaller vector to append undefined elements
24373 // to it up to the length of the longer vector. This recursion terminates
24374 // in 1 call because these calls to analyzeShuffle guarantee a_len == b_len.
24375 if (a_len != b_len) {
24376 const min_len = @min(a_len, b_len);
24377 const max_src = if (a_len > b_len) a_src else b_src;
24378 const max_len = try sema.usizeCast(block, max_src, @max(a_len, b_len));
2437924345
24380 const expand_mask_values = try sema.arena.alloc(InternPool.Index, max_len);24346 if (a_rt and b_rt) {
24381 for (@intCast(0)..@intCast(min_len)) |i| {24347 // Both operands are needed and runtime-known. We need a `[]ShuffleTwomask`... which is
24382 expand_mask_values[i] = (try pt.intValue(.comptime_int, i)).toIntern();24348 // exactly what we already have in `mask_shuffle_two`! So, we're basically done already.
24349 // We just need to append the two operands.
24350 try sema.air_extra.ensureUnusedCapacity(sema.gpa, 2);
24351 sema.appendRefsAssumeCapacity(&.{ a_coerced, b_coerced });
24352 return block.addInst(.{
24353 .tag = .shuffle_two,
24354 .data = .{ .ty_pl = .{
24355 .ty = Air.internedToRef(result_ty.toIntern()),
24356 .payload = air_extra_idx,
24357 } },
24358 });
24359 } else if (a_rt) {
24360 // We need to convert the `ShuffleTwoMask` values to `ShuffleOneMask`.
24361 for (mask_shuffle_two, mask_shuffle_one) |in, *out| {
24362 out.* = switch (in.unwrap()) {
24363 .undef => .value(try pt.undefValue(elem_ty)),
24364 .a_elem => |idx| .elem(idx),
24365 .b_elem => |idx| .value(try maybe_b_val.?.elemValue(pt, idx)),
24366 };
24383 }24367 }
24384 for (@intCast(min_len)..@intCast(max_len)) |i| {24368 // Now just append our single runtime operand, and we're done.
24385 expand_mask_values[i] = .negative_one;24369 try sema.air_extra.ensureUnusedCapacity(sema.gpa, 1);
24370 sema.appendRefsAssumeCapacity(&.{a_coerced});
24371 return block.addInst(.{
24372 .tag = .shuffle_one,
24373 .data = .{ .ty_pl = .{
24374 .ty = Air.internedToRef(result_ty.toIntern()),
24375 .payload = air_extra_idx,
24376 } },
24377 });
24378 } else if (b_rt) {
24379 // We need to convert the `ShuffleTwoMask` values to `ShuffleOneMask`.
24380 for (mask_shuffle_two, mask_shuffle_one) |in, *out| {
24381 out.* = switch (in.unwrap()) {
24382 .undef => .value(try pt.undefValue(elem_ty)),
24383 .a_elem => |idx| .value(try maybe_a_val.?.elemValue(pt, idx)),
24384 .b_elem => |idx| .elem(idx),
24385 };
24386 }24386 }
24387 const expand_mask = try pt.intern(.{ .aggregate = .{24387 // Now just append our single runtime operand, and we're done.
24388 .ty = (try pt.vectorType(.{ .len = @intCast(max_len), .child = .comptime_int_type })).toIntern(),24388 try sema.air_extra.ensureUnusedCapacity(sema.gpa, 1);
24389 .storage = .{ .elems = expand_mask_values },24389 sema.appendRefsAssumeCapacity(&.{b_coerced});
24390 } });24390 return block.addInst(.{
2439124391 .tag = .shuffle_one,
24392 if (a_len < b_len) {24392 .data = .{ .ty_pl = .{
24393 const undef = try pt.undefRef(a_ty);24393 .ty = Air.internedToRef(result_ty.toIntern()),
24394 a = try sema.analyzeShuffle(block, src_node, elem_ty, a, undef, Value.fromInterned(expand_mask), @intCast(max_len));24394 .payload = air_extra_idx,
24395 } else {24395 } },
24396 const undef = try pt.undefRef(b_ty);24396 });
24397 b = try sema.analyzeShuffle(block, src_node, elem_ty, b, undef, Value.fromInterned(expand_mask), @intCast(max_len));24397 } else {
24398 // The result will be comptime-known. We must convert the `ShuffleTwoMask` values to
24399 // `InternPool.Index` values using the known operands.
24400 for (mask_shuffle_two, mask_ip_index) |in, *out| {
24401 const val: Value = switch (in.unwrap()) {
24402 .undef => try pt.undefValue(elem_ty),
24403 .a_elem => |idx| try maybe_a_val.?.elemValue(pt, idx),
24404 .b_elem => |idx| try maybe_b_val.?.elemValue(pt, idx),
24405 };
24406 out.* = val.toIntern();
24398 }24407 }
24408 const res = try pt.intern(.{ .aggregate = .{
24409 .ty = result_ty.toIntern(),
24410 .storage = .{ .elems = mask_ip_index },
24411 } });
24412 // We have a comptime-known result, so didn't need `air_mask_buf` -- remove it from `sema.air_extra`.
24413 assert(sema.air_extra.items.len == air_extra_idx + air_mask_buf.len);
24414 sema.air_extra.shrinkRetainingCapacity(air_extra_idx);
24415 return Air.internedToRef(res);
24399 }24416 }
24400
24401 return block.addInst(.{
24402 .tag = .shuffle,
24403 .data = .{ .ty_pl = .{
24404 .ty = Air.internedToRef(res_ty.toIntern()),
24405 .payload = try block.sema.addExtra(Air.Shuffle{
24406 .a = a,
24407 .b = b,
24408 .mask = mask.toIntern(),
24409 .mask_len = mask_len,
24410 }),
24411 } },
24412 });
24413}24417}
2441424418
24415fn zirSelect(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {24419fn zirSelect(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {
src/Zcu/PerThread.zig+2-1
...@@ -1745,7 +1745,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *A...@@ -1745,7 +1745,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *A
1745 try air.legalize(pt, @import("../codegen.zig").legalizeFeatures(pt, nav_index) orelse break :legalize);1745 try air.legalize(pt, @import("../codegen.zig").legalizeFeatures(pt, nav_index) orelse break :legalize);
1746 }1746 }
17471747
1748 var liveness = try Air.Liveness.analyze(gpa, air.*, ip);1748 var liveness = try Air.Liveness.analyze(zcu, air.*, ip);
1749 defer liveness.deinit(gpa);1749 defer liveness.deinit(gpa);
17501750
1751 if (build_options.enable_debug_extensions and comp.verbose_air) {1751 if (build_options.enable_debug_extensions and comp.verbose_air) {
...@@ -1757,6 +1757,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *A...@@ -1757,6 +1757,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *A
1757 if (std.debug.runtime_safety) {1757 if (std.debug.runtime_safety) {
1758 var verify: Air.Liveness.Verify = .{1758 var verify: Air.Liveness.Verify = .{
1759 .gpa = gpa,1759 .gpa = gpa,
1760 .zcu = zcu,
1760 .air = air.*,1761 .air = air.*,
1761 .liveness = liveness,1762 .liveness = liveness,
1762 .intern_pool = ip,1763 .intern_pool = ip,
src/arch/aarch64/CodeGen.zig+10-6
...@@ -778,7 +778,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -778,7 +778,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
778 .error_name => try self.airErrorName(inst),778 .error_name => try self.airErrorName(inst),
779 .splat => try self.airSplat(inst),779 .splat => try self.airSplat(inst),
780 .select => try self.airSelect(inst),780 .select => try self.airSelect(inst),
781 .shuffle => try self.airShuffle(inst),781 .shuffle_one => try self.airShuffleOne(inst),
782 .shuffle_two => try self.airShuffleTwo(inst),
782 .reduce => try self.airReduce(inst),783 .reduce => try self.airReduce(inst),
783 .aggregate_init => try self.airAggregateInit(inst),784 .aggregate_init => try self.airAggregateInit(inst),
784 .union_init => try self.airUnionInit(inst),785 .union_init => try self.airUnionInit(inst),
...@@ -6049,11 +6050,14 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) InnerError!void {...@@ -6049,11 +6050,14 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) InnerError!void {
6049 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });6050 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
6050}6051}
60516052
6052fn airShuffle(self: *Self, inst: Air.Inst.Index) InnerError!void {6053fn airShuffleOne(self: *Self, inst: Air.Inst.Index) InnerError!void {
6053 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;6054 _ = inst;
6054 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;6055 return self.fail("TODO implement airShuffleOne for {}", .{self.target.cpu.arch});
6055 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airShuffle for {}", .{self.target.cpu.arch});6056}
6056 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });6057
6058fn airShuffleTwo(self: *Self, inst: Air.Inst.Index) InnerError!void {
6059 _ = inst;
6060 return self.fail("TODO implement airShuffleTwo for {}", .{self.target.cpu.arch});
6057}6061}
60586062
6059fn airReduce(self: *Self, inst: Air.Inst.Index) InnerError!void {6063fn airReduce(self: *Self, inst: Air.Inst.Index) InnerError!void {
src/arch/arm/CodeGen.zig+10-5
...@@ -767,7 +767,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -767,7 +767,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
767 .error_name => try self.airErrorName(inst),767 .error_name => try self.airErrorName(inst),
768 .splat => try self.airSplat(inst),768 .splat => try self.airSplat(inst),
769 .select => try self.airSelect(inst),769 .select => try self.airSelect(inst),
770 .shuffle => try self.airShuffle(inst),770 .shuffle_one => try self.airShuffleOne(inst),
771 .shuffle_two => try self.airShuffleTwo(inst),
771 .reduce => try self.airReduce(inst),772 .reduce => try self.airReduce(inst),
772 .aggregate_init => try self.airAggregateInit(inst),773 .aggregate_init => try self.airAggregateInit(inst),
773 .union_init => try self.airUnionInit(inst),774 .union_init => try self.airUnionInit(inst),
...@@ -6021,10 +6022,14 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -6021,10 +6022,14 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
6021 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });6022 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
6022}6023}
60236024
6024fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {6025fn airShuffleOne(self: *Self, inst: Air.Inst.Index) !void {
6025 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6026 _ = inst;
6026 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airShuffle for arm", .{});6027 return self.fail("TODO implement airShuffleOne for arm", .{});
6027 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });6028}
6029
6030fn airShuffleTwo(self: *Self, inst: Air.Inst.Index) !void {
6031 _ = inst;
6032 return self.fail("TODO implement airShuffleTwo for arm", .{});
6028}6033}
60296034
6030fn airReduce(self: *Self, inst: Air.Inst.Index) !void {6035fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
src/arch/riscv64/CodeGen.zig+10-5
...@@ -1586,7 +1586,8 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1586,7 +1586,8 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1586 .error_name => try func.airErrorName(inst),1586 .error_name => try func.airErrorName(inst),
1587 .splat => try func.airSplat(inst),1587 .splat => try func.airSplat(inst),
1588 .select => try func.airSelect(inst),1588 .select => try func.airSelect(inst),
1589 .shuffle => try func.airShuffle(inst),1589 .shuffle_one => try func.airShuffleOne(inst),
1590 .shuffle_two => try func.airShuffleTwo(inst),
1590 .reduce => try func.airReduce(inst),1591 .reduce => try func.airReduce(inst),
1591 .aggregate_init => try func.airAggregateInit(inst),1592 .aggregate_init => try func.airAggregateInit(inst),
1592 .union_init => try func.airUnionInit(inst),1593 .union_init => try func.airUnionInit(inst),
...@@ -8030,10 +8031,14 @@ fn airSelect(func: *Func, inst: Air.Inst.Index) !void {...@@ -8030,10 +8031,14 @@ fn airSelect(func: *Func, inst: Air.Inst.Index) !void {
8030 return func.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });8031 return func.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
8031}8032}
80328033
8033fn airShuffle(func: *Func, inst: Air.Inst.Index) !void {8034fn airShuffleOne(func: *Func, inst: Air.Inst.Index) !void {
8034 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;8035 _ = inst;
8035 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airShuffle for riscv64", .{});8036 return func.fail("TODO implement airShuffleOne for riscv64", .{});
8036 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });8037}
8038
8039fn airShuffleTwo(func: *Func, inst: Air.Inst.Index) !void {
8040 _ = inst;
8041 return func.fail("TODO implement airShuffleTwo for riscv64", .{});
8037}8042}
80388043
8039fn airReduce(func: *Func, inst: Air.Inst.Index) !void {8044fn airReduce(func: *Func, inst: Air.Inst.Index) !void {
src/arch/sparc64/CodeGen.zig+2-1
...@@ -621,7 +621,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -621,7 +621,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
621 .error_name => try self.airErrorName(inst),621 .error_name => try self.airErrorName(inst),
622 .splat => try self.airSplat(inst),622 .splat => try self.airSplat(inst),
623 .select => @panic("TODO try self.airSelect(inst)"),623 .select => @panic("TODO try self.airSelect(inst)"),
624 .shuffle => @panic("TODO try self.airShuffle(inst)"),624 .shuffle_one => @panic("TODO try self.airShuffleOne(inst)"),
625 .shuffle_two => @panic("TODO try self.airShuffleTwo(inst)"),
625 .reduce => @panic("TODO try self.airReduce(inst)"),626 .reduce => @panic("TODO try self.airReduce(inst)"),
626 .aggregate_init => try self.airAggregateInit(inst),627 .aggregate_init => try self.airAggregateInit(inst),
627 .union_init => try self.airUnionInit(inst),628 .union_init => try self.airUnionInit(inst),
src/arch/wasm/CodeGen.zig+79-44
...@@ -2004,7 +2004,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2004,7 +2004,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2004 .ret_load => cg.airRetLoad(inst),2004 .ret_load => cg.airRetLoad(inst),
2005 .splat => cg.airSplat(inst),2005 .splat => cg.airSplat(inst),
2006 .select => cg.airSelect(inst),2006 .select => cg.airSelect(inst),
2007 .shuffle => cg.airShuffle(inst),2007 .shuffle_one => cg.airShuffleOne(inst),
2008 .shuffle_two => cg.airShuffleTwo(inst),
2008 .reduce => cg.airReduce(inst),2009 .reduce => cg.airReduce(inst),
2009 .aggregate_init => cg.airAggregateInit(inst),2010 .aggregate_init => cg.airAggregateInit(inst),
2010 .union_init => cg.airUnionInit(inst),2011 .union_init => cg.airUnionInit(inst),
...@@ -5177,66 +5178,100 @@ fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5177,66 +5178,100 @@ fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5177 return cg.fail("TODO: Implement wasm airSelect", .{});5178 return cg.fail("TODO: Implement wasm airSelect", .{});
5178}5179}
51795180
5180fn airShuffle(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5181fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5181 const pt = cg.pt;5182 const pt = cg.pt;
5182 const zcu = pt.zcu;5183 const zcu = pt.zcu;
5183 const inst_ty = cg.typeOfIndex(inst);
5184 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5185 const extra = cg.air.extraData(Air.Shuffle, ty_pl.payload).data;
5186
5187 const a = try cg.resolveInst(extra.a);
5188 const b = try cg.resolveInst(extra.b);
5189 const mask = Value.fromInterned(extra.mask);
5190 const mask_len = extra.mask_len;
51915184
5192 const child_ty = inst_ty.childType(zcu);5185 const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);
5193 const elem_size = child_ty.abiSize(zcu);5186 const result_ty = unwrapped.result_ty;
5187 const mask = unwrapped.mask;
5188 const operand = try cg.resolveInst(unwrapped.operand);
51945189
5195 // TODO: One of them could be by ref; handle in loop5190 const elem_ty = result_ty.childType(zcu);
5196 if (isByRef(cg.typeOf(extra.a), zcu, cg.target) or isByRef(inst_ty, zcu, cg.target)) {5191 const elem_size = elem_ty.abiSize(zcu);
5197 const result = try cg.allocStack(inst_ty);
51985192
5199 for (0..mask_len) |index| {5193 // TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were
5200 const value = (try mask.elemValue(pt, index)).toSignedInt(zcu);5194 // to lower the comptime-known operands to a non-by-ref vector value.
52015195
5202 try cg.emitWValue(result);5196 // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
5197 // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
52035198
5204 const loaded = if (value >= 0)5199 const dest_alloc = try cg.allocStack(result_ty);
5205 try cg.load(a, child_ty, @as(u32, @intCast(@as(i64, @intCast(elem_size)) * value)))5200 for (mask, 0..) |mask_elem, out_idx| {
5206 else5201 try cg.emitWValue(dest_alloc);
5207 try cg.load(b, child_ty, @as(u32, @intCast(@as(i64, @intCast(elem_size)) * ~value)));5202 const elem_val = switch (mask_elem.unwrap()) {
5203 .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
5204 .value => |val| try cg.lowerConstant(.fromInterned(val), elem_ty),
5205 };
5206 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
5207 }
5208 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
5209}
52085210
5209 try cg.store(.stack, loaded, child_ty, result.stack_offset.value + @as(u32, @intCast(elem_size)) * @as(u32, @intCast(index)));5211fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5210 }5212 const pt = cg.pt;
5213 const zcu = pt.zcu;
52115214
5212 return cg.finishAir(inst, result, &.{ extra.a, extra.b });5215 const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);
5213 } else {5216 const result_ty = unwrapped.result_ty;
5214 var operands = [_]u32{5217 const mask = unwrapped.mask;
5215 @intFromEnum(std.wasm.SimdOpcode.i8x16_shuffle),5218 const operand_a = try cg.resolveInst(unwrapped.operand_a);
5216 } ++ [1]u32{undefined} ** 4;5219 const operand_b = try cg.resolveInst(unwrapped.operand_b);
52175220
5218 var lanes = mem.asBytes(operands[1..]);5221 const a_ty = cg.typeOf(unwrapped.operand_a);
5219 for (0..@as(usize, @intCast(mask_len))) |index| {5222 const b_ty = cg.typeOf(unwrapped.operand_b);
5220 const mask_elem = (try mask.elemValue(pt, index)).toSignedInt(zcu);5223 const elem_ty = result_ty.childType(zcu);
5221 const base_index = if (mask_elem >= 0)5224 const elem_size = elem_ty.abiSize(zcu);
5222 @as(u8, @intCast(@as(i64, @intCast(elem_size)) * mask_elem))
5223 else
5224 16 + @as(u8, @intCast(@as(i64, @intCast(elem_size)) * ~mask_elem));
52255225
5226 for (0..@as(usize, @intCast(elem_size))) |byte_offset| {5226 // WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 8
5227 lanes[index * @as(usize, @intCast(elem_size)) + byte_offset] = base_index + @as(u8, @intCast(byte_offset));5227 // and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise,
5228 // we fall back to a naive loop lowering.
5229 if (!isByRef(a_ty, zcu, cg.target) and
5230 !isByRef(b_ty, zcu, cg.target) and
5231 !isByRef(result_ty, zcu, cg.target) and
5232 elem_ty.bitSize(zcu) % 8 == 0)
5233 {
5234 var lane_map: [16]u8 align(4) = undefined;
5235 const lanes_per_elem = elem_ty.bitSize(zcu) / 8;
5236 for (mask, 0..) |mask_elem, out_idx| {
5237 const out_first_lane = out_idx * lanes_per_elem;
5238 const in_first_lane = switch (mask_elem.unwrap()) {
5239 .a_elem => |i| i * lanes_per_elem,
5240 .b_elem => |i| i * lanes_per_elem + 16,
5241 .undef => 0, // doesn't matter
5242 };
5243 for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| {
5244 out.* = @intCast(in);
5228 }5245 }
5229 }5246 }
52305247 try cg.emitWValue(operand_a);
5231 try cg.emitWValue(a);5248 try cg.emitWValue(operand_b);
5232 try cg.emitWValue(b);
5233
5234 const extra_index = cg.extraLen();5249 const extra_index = cg.extraLen();
5235 try cg.mir_extra.appendSlice(cg.gpa, &operands);5250 try cg.mir_extra.appendSlice(cg.gpa, &.{
5251 @intFromEnum(std.wasm.SimdOpcode.i8x16_shuffle),
5252 @bitCast(lane_map[0..4].*),
5253 @bitCast(lane_map[4..8].*),
5254 @bitCast(lane_map[8..12].*),
5255 @bitCast(lane_map[12..].*),
5256 });
5236 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });5257 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
5258 return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b });
5259 }
5260
5261 // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
5262 // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
52375263
5238 return cg.finishAir(inst, .stack, &.{ extra.a, extra.b });5264 const dest_alloc = try cg.allocStack(result_ty);
5265 for (mask, 0..) |mask_elem, out_idx| {
5266 try cg.emitWValue(dest_alloc);
5267 const elem_val = switch (mask_elem.unwrap()) {
5268 .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),
5269 .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),
5270 .undef => try cg.emitUndefined(elem_ty),
5271 };
5272 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
5239 }5273 }
5274 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
5240}5275}
52415276
5242fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5277fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
src/arch/x86_64/CodeGen.zig+1-1
...@@ -2490,7 +2490,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2490,7 +2490,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2490 switch (air_tags[@intFromEnum(inst)]) {2490 switch (air_tags[@intFromEnum(inst)]) {
2491 // zig fmt: off2491 // zig fmt: off
2492 .select => try cg.airSelect(inst),2492 .select => try cg.airSelect(inst),
2493 .shuffle => try cg.airShuffle(inst),2493 .shuffle_one, .shuffle_two => @panic("x86_64 TODO: shuffle_one/shuffle_two"),
2494 // zig fmt: on2494 // zig fmt: on
24952495
2496 .arg => if (cg.debug_output != .none) {2496 .arg => if (cg.debug_output != .none) {
src/codegen/c.zig+57-17
...@@ -3374,7 +3374,8 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,...@@ -3374,7 +3374,8 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
3374 .error_name => try airErrorName(f, inst),3374 .error_name => try airErrorName(f, inst),
3375 .splat => try airSplat(f, inst),3375 .splat => try airSplat(f, inst),
3376 .select => try airSelect(f, inst),3376 .select => try airSelect(f, inst),
3377 .shuffle => try airShuffle(f, inst),3377 .shuffle_one => try airShuffleOne(f, inst),
3378 .shuffle_two => try airShuffleTwo(f, inst),
3378 .reduce => try airReduce(f, inst),3379 .reduce => try airReduce(f, inst),
3379 .aggregate_init => try airAggregateInit(f, inst),3380 .aggregate_init => try airAggregateInit(f, inst),
3380 .union_init => try airUnionInit(f, inst),3381 .union_init => try airUnionInit(f, inst),
...@@ -7163,34 +7164,73 @@ fn airSelect(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -7163,34 +7164,73 @@ fn airSelect(f: *Function, inst: Air.Inst.Index) !CValue {
7163 return local;7164 return local;
7164}7165}
71657166
7166fn airShuffle(f: *Function, inst: Air.Inst.Index) !CValue {7167fn airShuffleOne(f: *Function, inst: Air.Inst.Index) !CValue {
7167 const pt = f.object.dg.pt;7168 const pt = f.object.dg.pt;
7168 const zcu = pt.zcu;7169 const zcu = pt.zcu;
7169 const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7170 const extra = f.air.extraData(Air.Shuffle, ty_pl.payload).data;
7171
7172 const mask = Value.fromInterned(extra.mask);
7173 const lhs = try f.resolveInst(extra.a);
7174 const rhs = try f.resolveInst(extra.b);
71757170
7176 const inst_ty = f.typeOfIndex(inst);7171 const unwrapped = f.air.unwrapShuffleOne(zcu, inst);
7172 const mask = unwrapped.mask;
7173 const operand = try f.resolveInst(unwrapped.operand);
7174 const inst_ty = unwrapped.result_ty;
71777175
7178 const writer = f.object.writer();7176 const writer = f.object.writer();
7179 const local = try f.allocLocal(inst, inst_ty);7177 const local = try f.allocLocal(inst, inst_ty);
7180 try reap(f, inst, &.{ extra.a, extra.b }); // local cannot alias operands7178 try reap(f, inst, &.{unwrapped.operand}); // local cannot alias operand
7181 for (0..extra.mask_len) |index| {7179 for (mask, 0..) |mask_elem, out_idx| {
7182 try f.writeCValue(writer, local, .Other);7180 try f.writeCValue(writer, local, .Other);
7183 try writer.writeByte('[');7181 try writer.writeByte('[');
7184 try f.object.dg.renderValue(writer, try pt.intValue(.usize, index), .Other);7182 try f.object.dg.renderValue(writer, try pt.intValue(.usize, out_idx), .Other);
7185 try writer.writeAll("] = ");7183 try writer.writeAll("] = ");
7184 switch (mask_elem.unwrap()) {
7185 .elem => |src_idx| {
7186 try f.writeCValue(writer, operand, .Other);
7187 try writer.writeByte('[');
7188 try f.object.dg.renderValue(writer, try pt.intValue(.usize, src_idx), .Other);
7189 try writer.writeByte(']');
7190 },
7191 .value => |val| try f.object.dg.renderValue(writer, .fromInterned(val), .Other),
7192 }
7193 try writer.writeAll(";\n");
7194 }
71867195
7187 const mask_elem = (try mask.elemValue(pt, index)).toSignedInt(zcu);7196 return local;
7188 const src_val = try pt.intValue(.usize, @as(u64, @intCast(mask_elem ^ mask_elem >> 63)));7197}
71897198
7190 try f.writeCValue(writer, if (mask_elem >= 0) lhs else rhs, .Other);7199fn airShuffleTwo(f: *Function, inst: Air.Inst.Index) !CValue {
7200 const pt = f.object.dg.pt;
7201 const zcu = pt.zcu;
7202
7203 const unwrapped = f.air.unwrapShuffleTwo(zcu, inst);
7204 const mask = unwrapped.mask;
7205 const operand_a = try f.resolveInst(unwrapped.operand_a);
7206 const operand_b = try f.resolveInst(unwrapped.operand_b);
7207 const inst_ty = unwrapped.result_ty;
7208 const elem_ty = inst_ty.childType(zcu);
7209
7210 const writer = f.object.writer();
7211 const local = try f.allocLocal(inst, inst_ty);
7212 try reap(f, inst, &.{ unwrapped.operand_a, unwrapped.operand_b }); // local cannot alias operands
7213 for (mask, 0..) |mask_elem, out_idx| {
7214 try f.writeCValue(writer, local, .Other);
7191 try writer.writeByte('[');7215 try writer.writeByte('[');
7192 try f.object.dg.renderValue(writer, src_val, .Other);7216 try f.object.dg.renderValue(writer, try pt.intValue(.usize, out_idx), .Other);
7193 try writer.writeAll("];\n");7217 try writer.writeAll("] = ");
7218 switch (mask_elem.unwrap()) {
7219 .a_elem => |src_idx| {
7220 try f.writeCValue(writer, operand_a, .Other);
7221 try writer.writeByte('[');
7222 try f.object.dg.renderValue(writer, try pt.intValue(.usize, src_idx), .Other);
7223 try writer.writeByte(']');
7224 },
7225 .b_elem => |src_idx| {
7226 try f.writeCValue(writer, operand_b, .Other);
7227 try writer.writeByte('[');
7228 try f.object.dg.renderValue(writer, try pt.intValue(.usize, src_idx), .Other);
7229 try writer.writeByte(']');
7230 },
7231 .undef => try f.object.dg.renderUndefValue(writer, elem_ty, .Other),
7232 }
7233 try writer.writeAll(";\n");
7194 }7234 }
71957235
7196 return local;7236 return local;
src/codegen/llvm.zig+183-32
...@@ -4969,7 +4969,8 @@ pub const FuncGen = struct {...@@ -4969,7 +4969,8 @@ pub const FuncGen = struct {
4969 .error_name => try self.airErrorName(inst),4969 .error_name => try self.airErrorName(inst),
4970 .splat => try self.airSplat(inst),4970 .splat => try self.airSplat(inst),
4971 .select => try self.airSelect(inst),4971 .select => try self.airSelect(inst),
4972 .shuffle => try self.airShuffle(inst),4972 .shuffle_one => try self.airShuffleOne(inst),
4973 .shuffle_two => try self.airShuffleTwo(inst),
4973 .aggregate_init => try self.airAggregateInit(inst),4974 .aggregate_init => try self.airAggregateInit(inst),
4974 .union_init => try self.airUnionInit(inst),4975 .union_init => try self.airUnionInit(inst),
4975 .prefetch => try self.airPrefetch(inst),4976 .prefetch => try self.airPrefetch(inst),
...@@ -9666,7 +9667,7 @@ pub const FuncGen = struct {...@@ -9666,7 +9667,7 @@ pub const FuncGen = struct {
9666 const zcu = o.pt.zcu;9667 const zcu = o.pt.zcu;
9667 const ip = &zcu.intern_pool;9668 const ip = &zcu.intern_pool;
9668 for (body_tail[1..]) |body_inst| {9669 for (body_tail[1..]) |body_inst| {
9669 switch (fg.liveness.categorizeOperand(fg.air, body_inst, body_tail[0], ip)) {9670 switch (fg.liveness.categorizeOperand(fg.air, zcu, body_inst, body_tail[0], ip)) {
9670 .none => continue,9671 .none => continue,
9671 .write, .noret, .complex => return false,9672 .write, .noret, .complex => return false,
9672 .tomb => return true,9673 .tomb => return true,
...@@ -10421,42 +10422,192 @@ pub const FuncGen = struct {...@@ -10421,42 +10422,192 @@ pub const FuncGen = struct {
10421 return self.wip.select(.normal, pred, a, b, "");10422 return self.wip.select(.normal, pred, a, b, "");
10422 }10423 }
1042310424
10424 fn airShuffle(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {10425 fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10425 const o = self.ng.object;10426 const o = fg.ng.object;
10426 const pt = o.pt;10427 const pt = o.pt;
10427 const zcu = pt.zcu;10428 const zcu = pt.zcu;
10428 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;10429 const gpa = zcu.gpa;
10429 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;10430
10430 const a = try self.resolveInst(extra.a);10431 const unwrapped = fg.air.unwrapShuffleOne(zcu, inst);
10431 const b = try self.resolveInst(extra.b);10432
10432 const mask = Value.fromInterned(extra.mask);10433 const operand = try fg.resolveInst(unwrapped.operand);
10433 const mask_len = extra.mask_len;10434 const mask = unwrapped.mask;
10434 const a_len = self.typeOf(extra.a).vectorLen(zcu);10435 const operand_ty = fg.typeOf(unwrapped.operand);
1043510436 const llvm_operand_ty = try o.lowerType(operand_ty);
10436 // LLVM uses integers larger than the length of the first array to10437 const llvm_result_ty = try o.lowerType(unwrapped.result_ty);
10437 // index into the second array. This was deemed unnecessarily fragile10438 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
10438 // when changing code, so Zig uses negative numbers to index the10439 const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
10439 // second vector. These start at -1 and go down, and are easiest to use10440 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10440 // with the ~ operator. Here we convert between the two formats.10441 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10441 const values = try self.gpa.alloc(Builder.Constant, mask_len);10442
10442 defer self.gpa.free(values);10443 // LLVM requires that the two input vectors have the same length, so lowering isn't trivial.
1044310444 // And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at
10444 for (values, 0..) |*val, i| {10445 // least a bit". So, there are two cases here.
10445 const elem = try mask.elemValue(pt, i);10446 //
10446 if (elem.isUndef(zcu)) {10447 // If the operand length equals the mask length, we do just the one `shufflevector`, where
10447 val.* = try o.builder.undefConst(.i32);10448 // the second operand is a constant vector with comptime-known elements at the right indices
10448 } else {10449 // and poison values elsewhere (in the indices which won't be selected).
10449 const int = elem.toSignedInt(zcu);10450 //
10450 const unsigned: u32 = @intCast(if (int >= 0) int else ~int + a_len);10451 // Otherwise, we lower to *two* `shufflevector` instructions. The first shuffles the runtime
10451 val.* = try o.builder.intConst(.i32, unsigned);10452 // operand with an all-poison vector to extract and correctly position all of the runtime
10453 // elements. We also make a constant vector with all of the comptime elements correctly
10454 // positioned. Then, our second instruction selects elements from those "runtime-or-poison"
10455 // and "comptime-or-poison" vectors to compute the result.
10456
10457 // This buffer is used primarily for the mask constants.
10458 const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len);
10459 defer gpa.free(llvm_elem_buf);
10460
10461 // ...but first, we'll collect all of the comptime-known values.
10462 var any_defined_comptime_value = false;
10463 for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| {
10464 llvm_elem.* = switch (mask_elem.unwrap()) {
10465 .elem => llvm_poison_elem,
10466 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
10467 any_defined_comptime_value = true;
10468 break :elem try o.lowerValue(val);
10469 } else llvm_poison_elem,
10470 };
10471 }
10472 // This vector is like the result, but runtime elements are replaced with poison.
10473 const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: {
10474 break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf);
10475 } else try o.builder.poisonValue(llvm_result_ty);
10476
10477 if (operand_ty.vectorLen(zcu) == mask.len) {
10478 // input length equals mask/output length, so we lower to one instruction
10479 for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| {
10480 llvm_elem.* = switch (mask_elem.unwrap()) {
10481 .elem => |idx| try o.builder.intConst(.i32, idx),
10482 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10483 break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10484 } else llvm_poison_mask_elem,
10485 };
10452 }10486 }
10487 return fg.wip.shuffleVector(
10488 operand,
10489 comptime_and_poison,
10490 try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10491 "",
10492 );
10493 }
10494
10495 for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| {
10496 llvm_elem.* = switch (mask_elem.unwrap()) {
10497 .elem => |idx| try o.builder.intConst(.i32, idx),
10498 .value => llvm_poison_mask_elem,
10499 };
10500 }
10501 // This vector is like our result, but all comptime-known elements are poison.
10502 const runtime_and_poison = try fg.wip.shuffleVector(
10503 operand,
10504 try o.builder.poisonValue(llvm_operand_ty),
10505 try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10506 "",
10507 );
10508
10509 if (!any_defined_comptime_value) {
10510 // `comptime_and_poison` is just poison; a second shuffle would be a nop.
10511 return runtime_and_poison;
10512 }
10513
10514 // In this second shuffle, the inputs, the mask, and the output all have the same length.
10515 for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| {
10516 llvm_elem.* = switch (mask_elem.unwrap()) {
10517 .elem => try o.builder.intConst(.i32, elem_idx),
10518 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10519 break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10520 } else llvm_poison_mask_elem,
10521 };
10453 }10522 }
10523 // Merge the runtime and comptime elements with the mask we just built.
10524 return fg.wip.shuffleVector(
10525 runtime_and_poison,
10526 comptime_and_poison,
10527 try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10528 "",
10529 );
10530 }
10531
10532 fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10533 const o = fg.ng.object;
10534 const pt = o.pt;
10535 const zcu = pt.zcu;
10536 const gpa = zcu.gpa;
10537
10538 const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
10539
10540 const mask = unwrapped.mask;
10541 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
10542 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10543 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10544
10545 // This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the
10546 // length of the longer one with an initial `shufflevector` if necessary, and then do the
10547 // actual computation with a second `shufflevector`.
10548
10549 const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu);
10550 const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu);
10551 const operand_len: u32 = @max(operand_a_len, operand_b_len);
10552
10553 // If we need to extend an operand, this is the type that mask will have.
10554 const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32);
10555
10556 const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len));
10557 defer gpa.free(llvm_elem_buf);
1045410558
10455 const llvm_mask_value = try o.builder.vectorValue(10559 const operand_a: Builder.Value = extend: {
10456 try o.builder.vectorType(.normal, mask_len, .i32),10560 const raw = try fg.resolveInst(unwrapped.operand_a);
10457 values,10561 if (operand_a_len == operand_len) break :extend raw;
10562 // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10563 const mask_elems = llvm_elem_buf[0..operand_len];
10564 for (mask_elems[0..operand_a_len], 0..) |*llvm_elem, elem_idx| {
10565 llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10566 }
10567 @memset(mask_elems[operand_a_len..], llvm_poison_mask_elem);
10568 const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty);
10569 break :extend try fg.wip.shuffleVector(
10570 raw,
10571 try o.builder.poisonValue(llvm_this_operand_ty),
10572 try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10573 "",
10574 );
10575 };
10576 const operand_b: Builder.Value = extend: {
10577 const raw = try fg.resolveInst(unwrapped.operand_b);
10578 if (operand_b_len == operand_len) break :extend raw;
10579 // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10580 const mask_elems = llvm_elem_buf[0..operand_len];
10581 for (mask_elems[0..operand_b_len], 0..) |*llvm_elem, elem_idx| {
10582 llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10583 }
10584 @memset(mask_elems[operand_b_len..], llvm_poison_mask_elem);
10585 const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty);
10586 break :extend try fg.wip.shuffleVector(
10587 raw,
10588 try o.builder.poisonValue(llvm_this_operand_ty),
10589 try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10590 "",
10591 );
10592 };
10593
10594 // `operand_a` and `operand_b` now have the same length (we've extended the shorter one with
10595 // an initial shuffle if necessary). Now for the easy bit.
10596
10597 const mask_elems = llvm_elem_buf[0..mask.len];
10598 for (mask, mask_elems) |mask_elem, *llvm_mask_elem| {
10599 llvm_mask_elem.* = switch (mask_elem.unwrap()) {
10600 .a_elem => |idx| try o.builder.intConst(.i32, idx),
10601 .b_elem => |idx| try o.builder.intConst(.i32, operand_len + idx),
10602 .undef => llvm_poison_mask_elem,
10603 };
10604 }
10605 return fg.wip.shuffleVector(
10606 operand_a,
10607 operand_b,
10608 try o.builder.vectorValue(llvm_mask_ty, mask_elems),
10609 "",
10458 );10610 );
10459 return self.wip.shuffleVector(a, b, llvm_mask_value, "");
10460 }10611 }
1046110612
10462 /// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.10613 /// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
src/codegen/spirv.zig+46-28
...@@ -3252,7 +3252,8 @@ const NavGen = struct {...@@ -3252,7 +3252,8 @@ const NavGen = struct {
32523252
3253 .splat => try self.airSplat(inst),3253 .splat => try self.airSplat(inst),
3254 .reduce, .reduce_optimized => try self.airReduce(inst),3254 .reduce, .reduce_optimized => try self.airReduce(inst),
3255 .shuffle => try self.airShuffle(inst),3255 .shuffle_one => try self.airShuffleOne(inst),
3256 .shuffle_two => try self.airShuffleTwo(inst),
32563257
3257 .ptr_add => try self.airPtrAdd(inst),3258 .ptr_add => try self.airPtrAdd(inst),
3258 .ptr_sub => try self.airPtrSub(inst),3259 .ptr_sub => try self.airPtrSub(inst),
...@@ -4047,40 +4048,57 @@ const NavGen = struct {...@@ -4047,40 +4048,57 @@ const NavGen = struct {
4047 return result_id;4048 return result_id;
4048 }4049 }
40494050
4050 fn airShuffle(self: *NavGen, inst: Air.Inst.Index) !?IdRef {4051 fn airShuffleOne(ng: *NavGen, inst: Air.Inst.Index) !?IdRef {
4051 const pt = self.pt;4052 const pt = ng.pt;
4052 const zcu = pt.zcu;4053 const zcu = pt.zcu;
4053 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;4054 const gpa = zcu.gpa;
4054 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
4055 const a = try self.resolve(extra.a);
4056 const b = try self.resolve(extra.b);
4057 const mask = Value.fromInterned(extra.mask);
40584055
4059 // Note: number of components in the result, a, and b may differ.4056 const unwrapped = ng.air.unwrapShuffleOne(zcu, inst);
4060 const result_ty = self.typeOfIndex(inst);4057 const mask = unwrapped.mask;
4061 const scalar_ty = result_ty.scalarType(zcu);4058 const result_ty = unwrapped.result_ty;
4062 const scalar_ty_id = try self.resolveType(scalar_ty, .direct);4059 const elem_ty = result_ty.childType(zcu);
4060 const operand = try ng.resolve(unwrapped.operand);
40634061
4064 const constituents = try self.gpa.alloc(IdRef, result_ty.vectorLen(zcu));4062 const constituents = try gpa.alloc(IdRef, mask.len);
4065 defer self.gpa.free(constituents);4063 defer gpa.free(constituents);
40664064
4067 for (constituents, 0..) |*id, i| {4065 for (constituents, mask) |*id, mask_elem| {
4068 const elem = try mask.elemValue(pt, i);4066 id.* = switch (mask_elem.unwrap()) {
4069 if (elem.isUndef(zcu)) {4067 .elem => |idx| try ng.extractVectorComponent(elem_ty, operand, idx),
4070 id.* = try self.spv.constUndef(scalar_ty_id);4068 .value => |val| try ng.constant(elem_ty, .fromInterned(val), .direct),
4071 continue;4069 };
4072 }4070 }
40734071
4074 const index = elem.toSignedInt(zcu);4072 const result_ty_id = try ng.resolveType(result_ty, .direct);
4075 if (index >= 0) {4073 return try ng.constructComposite(result_ty_id, constituents);
4076 id.* = try self.extractVectorComponent(scalar_ty, a, @intCast(index));4074 }
4077 } else {4075
4078 id.* = try self.extractVectorComponent(scalar_ty, b, @intCast(~index));4076 fn airShuffleTwo(ng: *NavGen, inst: Air.Inst.Index) !?IdRef {
4079 }4077 const pt = ng.pt;
4078 const zcu = pt.zcu;
4079 const gpa = zcu.gpa;
4080
4081 const unwrapped = ng.air.unwrapShuffleTwo(zcu, inst);
4082 const mask = unwrapped.mask;
4083 const result_ty = unwrapped.result_ty;
4084 const elem_ty = result_ty.childType(zcu);
4085 const elem_ty_id = try ng.resolveType(elem_ty, .direct);
4086 const operand_a = try ng.resolve(unwrapped.operand_a);
4087 const operand_b = try ng.resolve(unwrapped.operand_b);
4088
4089 const constituents = try gpa.alloc(IdRef, mask.len);
4090 defer gpa.free(constituents);
4091
4092 for (constituents, mask) |*id, mask_elem| {
4093 id.* = switch (mask_elem.unwrap()) {
4094 .a_elem => |idx| try ng.extractVectorComponent(elem_ty, operand_a, idx),
4095 .b_elem => |idx| try ng.extractVectorComponent(elem_ty, operand_b, idx),
4096 .undef => try ng.spv.constUndef(elem_ty_id),
4097 };
4080 }4098 }
40814099
4082 const result_ty_id = try self.resolveType(result_ty, .direct);4100 const result_ty_id = try ng.resolveType(result_ty, .direct);
4083 return try self.constructComposite(result_ty_id, constituents);4101 return try ng.constructComposite(result_ty_id, constituents);
4084 }4102 }
40854103
4086 fn indicesToIds(self: *NavGen, indices: []const u32) ![]IdRef {4104 fn indicesToIds(self: *NavGen, indices: []const u32) ![]IdRef {
src/print_air.zig+33-7
...@@ -315,7 +315,8 @@ const Writer = struct {...@@ -315,7 +315,8 @@ const Writer = struct {
315 .wasm_memory_grow => try w.writeWasmMemoryGrow(s, inst),315 .wasm_memory_grow => try w.writeWasmMemoryGrow(s, inst),
316 .mul_add => try w.writeMulAdd(s, inst),316 .mul_add => try w.writeMulAdd(s, inst),
317 .select => try w.writeSelect(s, inst),317 .select => try w.writeSelect(s, inst),
318 .shuffle => try w.writeShuffle(s, inst),318 .shuffle_one => try w.writeShuffleOne(s, inst),
319 .shuffle_two => try w.writeShuffleTwo(s, inst),
319 .reduce, .reduce_optimized => try w.writeReduce(s, inst),320 .reduce, .reduce_optimized => try w.writeReduce(s, inst),
320 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),321 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),
321 .vector_store_elem => try w.writeVectorStoreElem(s, inst),322 .vector_store_elem => try w.writeVectorStoreElem(s, inst),
...@@ -499,14 +500,39 @@ const Writer = struct {...@@ -499,14 +500,39 @@ const Writer = struct {
499 try w.writeOperand(s, inst, 2, pl_op.operand);500 try w.writeOperand(s, inst, 2, pl_op.operand);
500 }501 }
501502
502 fn writeShuffle(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {503 fn writeShuffleOne(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
503 const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;504 const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);
504 const extra = w.air.extraData(Air.Shuffle, ty_pl.payload).data;505 try w.writeType(s, unwrapped.result_ty);
506 try s.writeAll(", ");
507 try w.writeOperand(s, inst, 0, unwrapped.operand);
508 try s.writeAll(", [");
509 for (unwrapped.mask, 0..) |mask_elem, mask_idx| {
510 if (mask_idx > 0) try s.writeAll(", ");
511 switch (mask_elem.unwrap()) {
512 .elem => |idx| try s.print("elem {d}", .{idx}),
513 .value => |val| try s.print("val {}", .{Value.fromInterned(val).fmtValue(w.pt)}),
514 }
515 }
516 try s.writeByte(']');
517 }
505518
506 try w.writeOperand(s, inst, 0, extra.a);519 fn writeShuffleTwo(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
520 const unwrapped = w.air.unwrapShuffleTwo(w.pt.zcu, inst);
521 try w.writeType(s, unwrapped.result_ty);
522 try s.writeAll(", ");
523 try w.writeOperand(s, inst, 0, unwrapped.operand_a);
507 try s.writeAll(", ");524 try s.writeAll(", ");
508 try w.writeOperand(s, inst, 1, extra.b);525 try w.writeOperand(s, inst, 1, unwrapped.operand_b);
509 try s.print(", mask {d}, len {d}", .{ extra.mask, extra.mask_len });526 try s.writeAll(", [");
527 for (unwrapped.mask, 0..) |mask_elem, mask_idx| {
528 if (mask_idx > 0) try s.writeAll(", ");
529 switch (mask_elem.unwrap()) {
530 .a_elem => |idx| try s.print("a_elem {d}", .{idx}),
531 .b_elem => |idx| try s.print("b_elem {d}", .{idx}),
532 .undef => try s.writeAll("undef"),
533 }
534 }
535 try s.writeByte(']');
510 }536 }
511537
512 fn writeSelect(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {538 fn writeSelect(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
test/cases/compile_errors/shuffle_with_selected_index_past_first_vector_length.zig+16-10
...@@ -1,14 +1,20 @@...@@ -1,14 +1,20 @@
1export fn entry() void {1export fn foo() void {
2 const v: @Vector(4, u32) = [4]u32{ 10, 11, 12, 13 };2 // Here, the bad index ('7') is not less than 'b.len', so the error shouldn't have a note suggesting a negative index.
3 const x: @Vector(4, u32) = [4]u32{ 14, 15, 16, 17 };3 const a: @Vector(4, u32) = .{ 10, 11, 12, 13 };
4 const z = @shuffle(u32, v, x, [8]i32{ 0, 1, 2, 3, 7, 6, 5, 4 });4 const b: @Vector(4, u32) = .{ 14, 15, 16, 17 };
5 _ = z;5 _ = @shuffle(u32, a, b, [8]i32{ 0, 1, 2, 3, 7, 6, 5, 4 });
6}
7export fn bar() void {
8 // Here, the bad index ('7') *is* less than 'b.len', so the error *should* have a note suggesting a negative index.
9 const a: @Vector(4, u32) = .{ 10, 11, 12, 13 };
10 const b: @Vector(9, u32) = .{ 14, 15, 16, 17, 18, 19, 20, 21, 22 };
11 _ = @shuffle(u32, a, b, [8]i32{ 0, 1, 2, 3, 7, 6, 5, 4 });
6}12}
713
8// error14// error
9// backend=stage2
10// target=native
11//15//
12// :4:41: error: mask index '4' has out-of-bounds selection16// :5:35: error: mask element at index '4' selects out-of-bounds index
13// :4:29: note: selected index '7' out of bounds of '@Vector(4, u32)'17// :5:23: note: index '7' exceeds bounds of '@Vector(4, u32)' given here
14// :4:32: note: selections from the second vector are specified with negative numbers18// :11:35: error: mask element at index '4' selects out-of-bounds index
19// :11:23: note: index '7' exceeds bounds of '@Vector(4, u32)' given here
20// :11:26: note: use '~@as(u32, 7)' to index into second vector given here