authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-14 06:05:01+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-25 11:22:10+01:00
log2fdc9e6ae8b6f1ec86050011e1170d639d8c9c2c
tree29ae1289a64ff0568e06da19a4ecaff1dad562ba
parentdefef3f1a15be3b8f4e2074d581d43deb8296349

x86_64: implement `@shuffle`

15 files changed, 443 insertions(+), 183 deletions(-)

lib/std/crypto/blake3.zig+2-1
...@@ -200,7 +200,8 @@ const CompressGeneric = struct {...@@ -200,7 +200,8 @@ const CompressGeneric = struct {
200 }200 }
201};201};
202202
203const compress = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64)203const compress = if (builtin.cpu.arch == .x86_64 and
204 (builtin.zig_backend != .stage2_x86_64 or std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)))
204 CompressVectorized.compress205 CompressVectorized.compress
205else206else
206 CompressGeneric.compress;207 CompressGeneric.compress;
lib/std/crypto/salsa20.zig+5-1
...@@ -302,7 +302,11 @@ fn SalsaNonVecImpl(comptime rounds: comptime_int) type {...@@ -302,7 +302,11 @@ fn SalsaNonVecImpl(comptime rounds: comptime_int) type {
302 };302 };
303}303}
304304
305const SalsaImpl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64) SalsaVecImpl else SalsaNonVecImpl;305const SalsaImpl = if (builtin.cpu.arch == .x86_64 and
306 (builtin.zig_backend != .stage2_x86_64 or std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)))
307 SalsaVecImpl
308else
309 SalsaNonVecImpl;
306310
307fn keyToWords(key: [32]u8) [8]u32 {311fn keyToWords(key: [32]u8) [8]u32 {
308 var k: [8]u32 = undefined;312 var k: [8]u32 = undefined;
lib/std/meta.zig+2-1
...@@ -1286,5 +1286,6 @@ test "hasUniqueRepresentation" {...@@ -1286,5 +1286,6 @@ test "hasUniqueRepresentation" {
1286 try testing.expect(!hasUniqueRepresentation([]u8));1286 try testing.expect(!hasUniqueRepresentation([]u8));
1287 try testing.expect(!hasUniqueRepresentation([]const u8));1287 try testing.expect(!hasUniqueRepresentation([]const u8));
12881288
1289 try testing.expect(hasUniqueRepresentation(@Vector(4, u16)));1289 try testing.expect(hasUniqueRepresentation(@Vector(std.simd.suggestVectorLength(u8) orelse 1, u8)));
1290 try testing.expect(@sizeOf(@Vector(3, u8)) == 3 or !hasUniqueRepresentation(@Vector(3, u8)));
1290}1291}
lib/std/unicode.zig+34-19
...@@ -239,18 +239,19 @@ pub fn utf8ValidateSlice(input: []const u8) bool {...@@ -239,18 +239,19 @@ pub fn utf8ValidateSlice(input: []const u8) bool {
239fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {239fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {
240 var remaining = input;240 var remaining = input;
241241
242 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;242 if (std.simd.suggestVectorLength(u8)) |chunk_len| {
243 const Chunk = @Vector(chunk_len, u8);243 const Chunk = @Vector(chunk_len, u8);
244244
245 // Fast path. Check for and skip ASCII characters at the start of the input.245 // Fast path. Check for and skip ASCII characters at the start of the input.
246 while (remaining.len >= chunk_len) {246 while (remaining.len >= chunk_len) {
247 const chunk: Chunk = remaining[0..chunk_len].*;247 const chunk: Chunk = remaining[0..chunk_len].*;
248 const mask: Chunk = @splat(0x80);248 const mask: Chunk = @splat(0x80);
249 if (@reduce(.Or, chunk & mask == mask)) {249 if (@reduce(.Or, chunk & mask == mask)) {
250 // found a non ASCII byte250 // found a non ASCII byte
251 break;251 break;
252 }
253 remaining = remaining[chunk_len..];
252 }254 }
253 remaining = remaining[chunk_len..];
254 }255 }
255256
256 // default lowest and highest continuation byte257 // default lowest and highest continuation byte
...@@ -937,8 +938,11 @@ fn utf16LeToUtf8ArrayListImpl(...@@ -937,8 +938,11 @@ fn utf16LeToUtf8ArrayListImpl(
937 try array_list.ensureTotalCapacityPrecise(utf16le.len);938 try array_list.ensureTotalCapacityPrecise(utf16le.len);
938939
939 var remaining = utf16le;940 var remaining = utf16le;
940 if (builtin.zig_backend != .stage2_x86_64) {941 if (builtin.zig_backend != .stage2_x86_64 or
941 const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;942 comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and
943 !std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx })))
944 vectorized: {
945 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
942 const Chunk = @Vector(chunk_len, u16);946 const Chunk = @Vector(chunk_len, u16);
943947
944 // Fast path. Check for and encode ASCII characters at the start of the input.948 // Fast path. Check for and encode ASCII characters at the start of the input.
...@@ -1029,8 +1033,11 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr...@@ -1029,8 +1033,11 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1029 var end_index: usize = 0;1033 var end_index: usize = 0;
10301034
1031 var remaining = utf16le;1035 var remaining = utf16le;
1032 if (builtin.zig_backend != .stage2_x86_64) {1036 if (builtin.zig_backend != .stage2_x86_64 or
1033 const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;1037 comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and
1038 !std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx })))
1039 vectorized: {
1040 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1034 const Chunk = @Vector(chunk_len, u16);1041 const Chunk = @Vector(chunk_len, u16);
10351042
1036 // Fast path. Check for and encode ASCII characters at the start of the input.1043 // Fast path. Check for and encode ASCII characters at the start of the input.
...@@ -1155,8 +1162,12 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,...@@ -1155,8 +1162,12 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,
11551162
1156 var remaining = utf8;1163 var remaining = utf8;
1157 // Need support for std.simd.interlace1164 // Need support for std.simd.interlace
1158 if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {1165 if ((builtin.zig_backend != .stage2_x86_64 or
1159 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;1166 comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and
1167 !std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx }))) and
1168 comptime !builtin.cpu.arch.isMIPS())
1169 vectorized: {
1170 const chunk_len = @divExact(std.simd.suggestVectorLength(u8) orelse break :vectorized, 2);
1160 const Chunk = @Vector(chunk_len, u8);1171 const Chunk = @Vector(chunk_len, u8);
11611172
1162 // Fast path. Check for and encode ASCII characters at the start of the input.1173 // Fast path. Check for and encode ASCII characters at the start of the input.
...@@ -1232,8 +1243,12 @@ pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates:...@@ -1232,8 +1243,12 @@ pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates:
12321243
1233 var remaining = utf8;1244 var remaining = utf8;
1234 // Need support for std.simd.interlace1245 // Need support for std.simd.interlace
1235 if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {1246 if ((builtin.zig_backend != .stage2_x86_64 or
1236 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;1247 comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and
1248 !std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx }))) and
1249 comptime !builtin.cpu.arch.isMIPS())
1250 vectorized: {
1251 const chunk_len = @divExact(std.simd.suggestVectorLength(u8) orelse break :vectorized, 2);
1237 const Chunk = @Vector(chunk_len, u8);1252 const Chunk = @Vector(chunk_len, u8);
12381253
1239 // Fast path. Check for and encode ASCII characters at the start of the input.1254 // Fast path. Check for and encode ASCII characters at the start of the input.
lib/std/zig/c_translation.zig+3-5
...@@ -308,14 +308,12 @@ test "promoteIntLiteral" {...@@ -308,14 +308,12 @@ test "promoteIntLiteral" {
308308
309/// Convert from clang __builtin_shufflevector index to Zig @shuffle index309/// Convert from clang __builtin_shufflevector index to Zig @shuffle index
310/// clang requires __builtin_shufflevector index arguments to be integer constants.310/// clang requires __builtin_shufflevector index arguments to be integer constants.
311/// negative values for `this_index` indicate "don't care" so we arbitrarily choose 0311/// negative values for `this_index` indicate "don't care".
312/// clang enforces that `this_index` is less than the total number of vector elements312/// clang enforces that `this_index` is less than the total number of vector elements
313/// See https://ziglang.org/documentation/master/#shuffle313/// See https://ziglang.org/documentation/master/#shuffle
314/// See https://clang.llvm.org/docs/LanguageExtensions.html#langext-builtin-shufflevector314/// See https://clang.llvm.org/docs/LanguageExtensions.html#langext-builtin-shufflevector
315pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len: usize) i32 {315pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len: usize) i32 {
316 if (this_index <= 0) return 0;316 const positive_index = std.math.cast(usize, this_index) orelse return undefined;
317
318 const positive_index = @as(usize, @intCast(this_index));
319 if (positive_index < source_vector_len) return @as(i32, @intCast(this_index));317 if (positive_index < source_vector_len) return @as(i32, @intCast(this_index));
320 const b_index = positive_index - source_vector_len;318 const b_index = positive_index - source_vector_len;
321 return ~@as(i32, @intCast(b_index));319 return ~@as(i32, @intCast(b_index));
...@@ -324,7 +322,7 @@ pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len...@@ -324,7 +322,7 @@ pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len
324test "shuffleVectorIndex" {322test "shuffleVectorIndex" {
325 const vector_len: usize = 4;323 const vector_len: usize = 4;
326324
327 try testing.expect(shuffleVectorIndex(-1, vector_len) == 0);325 _ = shuffleVectorIndex(-1, vector_len);
328326
329 try testing.expect(shuffleVectorIndex(0, vector_len) == 0);327 try testing.expect(shuffleVectorIndex(0, vector_len) == 0);
330 try testing.expect(shuffleVectorIndex(1, vector_len) == 1);328 try testing.expect(shuffleVectorIndex(1, vector_len) == 1);
src/InternPool.zig+1
...@@ -3587,6 +3587,7 @@ pub const Alignment = enum(u6) {...@@ -3587,6 +3587,7 @@ pub const Alignment = enum(u6) {
3587 @"8" = 3,3587 @"8" = 3,
3588 @"16" = 4,3588 @"16" = 4,
3589 @"32" = 5,3589 @"32" = 5,
3590 @"64" = 6,
3590 none = std.math.maxInt(u6),3591 none = std.math.maxInt(u6),
3591 _,3592 _,
35923593
src/arch/x86_64/CodeGen.zig+295-101
...@@ -2610,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -2610,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
26102610
2611 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;2611 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
2612 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =2612 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2613 if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);2613 if (opts.update_tracking)
2614 {} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
26142615
2615 var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(2616 var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
2616 stack.get(),2617 stack.get(),
...@@ -14116,30 +14117,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14116,30 +14117,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14116 else => {},14117 else => {},
14117 },14118 },
14118 .Int => switch (ty.childType(mod).intInfo(mod).bits) {14119 .Int => switch (ty.childType(mod).intInfo(mod).bits) {
14119 8 => switch (ty.vectorLen(mod)) {14120 1...8 => switch (ty.vectorLen(mod)) {
14120 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{14121 1...16 => return .{ .move = if (self.hasFeature(.avx))
14121 .insert = .{ .vp_b, .insr },
14122 .extract = .{ .vp_b, .extr },
14123 } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
14124 .insert = .{ .p_b, .insr },
14125 .extract = .{ .p_b, .extr },
14126 } },
14127 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
14128 .insert = .{ .vp_w, .insr },
14129 .extract = .{ .vp_w, .extr },
14130 } } else .{ .insert_extract = .{
14131 .insert = .{ .p_w, .insr },
14132 .extract = .{ .p_w, .extr },
14133 } },
14134 3...4 => return .{ .move = if (self.hasFeature(.avx))
14135 .{ .v_d, .mov }
14136 else
14137 .{ ._d, .mov } },
14138 5...8 => return .{ .move = if (self.hasFeature(.avx))
14139 .{ .v_q, .mov }
14140 else
14141 .{ ._q, .mov } },
14142 9...16 => return .{ .move = if (self.hasFeature(.avx))
14143 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14122 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14144 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14123 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14145 17...32 => if (self.hasFeature(.avx))14124 17...32 => if (self.hasFeature(.avx))
...@@ -14149,23 +14128,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14149,23 +14128,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14149 .{ .v_, .movdqu } },14128 .{ .v_, .movdqu } },
14150 else => {},14129 else => {},
14151 },14130 },
14152 16 => switch (ty.vectorLen(mod)) {14131 9...16 => switch (ty.vectorLen(mod)) {
14153 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{14132 1...8 => return .{ .move = if (self.hasFeature(.avx))
14154 .insert = .{ .vp_w, .insr },
14155 .extract = .{ .vp_w, .extr },
14156 } } else .{ .insert_extract = .{
14157 .insert = .{ .p_w, .insr },
14158 .extract = .{ .p_w, .extr },
14159 } },
14160 2 => return .{ .move = if (self.hasFeature(.avx))
14161 .{ .v_d, .mov }
14162 else
14163 .{ ._d, .mov } },
14164 3...4 => return .{ .move = if (self.hasFeature(.avx))
14165 .{ .v_q, .mov }
14166 else
14167 .{ ._q, .mov } },
14168 5...8 => return .{ .move = if (self.hasFeature(.avx))
14169 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14133 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14170 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14134 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14171 9...16 => if (self.hasFeature(.avx))14135 9...16 => if (self.hasFeature(.avx))
...@@ -14175,16 +14139,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14175,16 +14139,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14175 .{ .v_, .movdqu } },14139 .{ .v_, .movdqu } },
14176 else => {},14140 else => {},
14177 },14141 },
14178 32 => switch (ty.vectorLen(mod)) {14142 17...32 => switch (ty.vectorLen(mod)) {
14179 1 => return .{ .move = if (self.hasFeature(.avx))14143 1...4 => return .{ .move = if (self.hasFeature(.avx))
14180 .{ .v_d, .mov }
14181 else
14182 .{ ._d, .mov } },
14183 2 => return .{ .move = if (self.hasFeature(.avx))
14184 .{ .v_q, .mov }
14185 else
14186 .{ ._q, .mov } },
14187 3...4 => return .{ .move = if (self.hasFeature(.avx))
14188 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14144 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14189 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14145 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14190 5...8 => if (self.hasFeature(.avx))14146 5...8 => if (self.hasFeature(.avx))
...@@ -14194,12 +14150,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14194,12 +14150,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14194 .{ .v_, .movdqu } },14150 .{ .v_, .movdqu } },
14195 else => {},14151 else => {},
14196 },14152 },
14197 64 => switch (ty.vectorLen(mod)) {14153 33...64 => switch (ty.vectorLen(mod)) {
14198 1 => return .{ .move = if (self.hasFeature(.avx))14154 1...2 => return .{ .move = if (self.hasFeature(.avx))
14199 .{ .v_q, .mov }
14200 else
14201 .{ ._q, .mov } },
14202 2 => return .{ .move = if (self.hasFeature(.avx))
14203 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14155 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14204 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14156 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14205 3...4 => if (self.hasFeature(.avx))14157 3...4 => if (self.hasFeature(.avx))
...@@ -14209,7 +14161,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14209,7 +14161,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14209 .{ .v_, .movdqu } },14161 .{ .v_, .movdqu } },
14210 else => {},14162 else => {},
14211 },14163 },
14212 128 => switch (ty.vectorLen(mod)) {14164 65...128 => switch (ty.vectorLen(mod)) {
14213 1 => return .{ .move = if (self.hasFeature(.avx))14165 1 => return .{ .move = if (self.hasFeature(.avx))
14214 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14166 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14215 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14167 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
...@@ -14220,7 +14172,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14220,7 +14172,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14220 .{ .v_, .movdqu } },14172 .{ .v_, .movdqu } },
14221 else => {},14173 else => {},
14222 },14174 },
14223 256 => switch (ty.vectorLen(mod)) {14175 129...256 => switch (ty.vectorLen(mod)) {
14224 1 => if (self.hasFeature(.avx))14176 1 => if (self.hasFeature(.avx))
14225 return .{ .move = if (aligned)14177 return .{ .move = if (aligned)
14226 .{ .v_, .movdqa }14178 .{ .v_, .movdqa }
...@@ -14232,11 +14184,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14232,11 +14184,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14232 },14184 },
14233 .Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))14185 .Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))
14234 switch (ty.vectorLen(mod)) {14186 switch (ty.vectorLen(mod)) {
14235 1 => return .{ .move = if (self.hasFeature(.avx))14187 1...2 => return .{ .move = if (self.hasFeature(.avx))
14236 .{ .v_q, .mov }
14237 else
14238 .{ ._q, .mov } },
14239 2 => return .{ .move = if (self.hasFeature(.avx))
14240 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14188 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14241 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14189 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14242 3...4 => if (self.hasFeature(.avx))14190 3...4 => if (self.hasFeature(.avx))
...@@ -14250,22 +14198,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14250,22 +14198,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14250 unreachable,14198 unreachable,
14251 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {14199 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {
14252 16 => switch (ty.vectorLen(mod)) {14200 16 => switch (ty.vectorLen(mod)) {
14253 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{14201 1...8 => return .{ .move = if (self.hasFeature(.avx))
14254 .insert = .{ .vp_w, .insr },
14255 .extract = .{ .vp_w, .extr },
14256 } } else .{ .insert_extract = .{
14257 .insert = .{ .p_w, .insr },
14258 .extract = .{ .p_w, .extr },
14259 } },
14260 2 => return .{ .move = if (self.hasFeature(.avx))
14261 .{ .v_d, .mov }
14262 else
14263 .{ ._d, .mov } },
14264 3...4 => return .{ .move = if (self.hasFeature(.avx))
14265 .{ .v_q, .mov }
14266 else
14267 .{ ._q, .mov } },
14268 5...8 => return .{ .move = if (self.hasFeature(.avx))
14269 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14202 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14270 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14203 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14271 9...16 => if (self.hasFeature(.avx))14204 9...16 => if (self.hasFeature(.avx))
...@@ -14276,15 +14209,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14276,15 +14209,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14276 else => {},14209 else => {},
14277 },14210 },
14278 32 => switch (ty.vectorLen(mod)) {14211 32 => switch (ty.vectorLen(mod)) {
14279 1 => return .{ .move = if (self.hasFeature(.avx))14212 1...4 => return .{ .move = if (self.hasFeature(.avx))
14280 .{ .v_ss, .mov }
14281 else
14282 .{ ._ss, .mov } },
14283 2 => return .{ .move = if (self.hasFeature(.avx))
14284 .{ .v_sd, .mov }
14285 else
14286 .{ ._sd, .mov } },
14287 3...4 => return .{ .move = if (self.hasFeature(.avx))
14288 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }14213 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
14289 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },14214 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
14290 5...8 => if (self.hasFeature(.avx))14215 5...8 => if (self.hasFeature(.avx))
...@@ -14295,11 +14220,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14295,11 +14220,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14295 else => {},14220 else => {},
14296 },14221 },
14297 64 => switch (ty.vectorLen(mod)) {14222 64 => switch (ty.vectorLen(mod)) {
14298 1 => return .{ .move = if (self.hasFeature(.avx))14223 1...2 => return .{ .move = if (self.hasFeature(.avx))
14299 .{ .v_sd, .mov }
14300 else
14301 .{ ._sd, .mov } },
14302 2 => return .{ .move = if (self.hasFeature(.avx))
14303 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }14224 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
14304 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },14225 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
14305 3...4 => if (self.hasFeature(.avx))14226 3...4 => if (self.hasFeature(.avx))
...@@ -16551,7 +16472,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -16551,7 +16472,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
16551 const vec_len = ty.vectorLen(mod);16472 const vec_len = ty.vectorLen(mod);
16552 const elem_ty = ty.childType(mod);16473 const elem_ty = ty.childType(mod);
16553 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));16474 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
16554 const abi_size = elem_abi_size * vec_len;16475 const abi_size: u32 = @intCast(ty.abiSize(mod));
16555 const pred_ty = self.typeOf(pl_op.operand);16476 const pred_ty = self.typeOf(pl_op.operand);
1655616477
16557 const result = result: {16478 const result = result: {
...@@ -16882,10 +16803,283 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -16882,10 +16803,283 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
16882}16803}
1688316804
16884fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {16805fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
16806 const mod = self.bin_file.comp.module.?;
16885 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;16807 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
16886 _ = ty_pl;16808 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
16887 return self.fail("TODO implement airShuffle for x86_64", .{});16809
16888 //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });16810 const dst_ty = self.typeOfIndex(inst);
16811 const elem_ty = dst_ty.childType(mod);
16812 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
16813 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
16814 const lhs_ty = self.typeOf(extra.a);
16815 const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
16816 const rhs_ty = self.typeOf(extra.b);
16817 const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(mod));
16818 const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);
16819
16820 const ExpectedContents = [32]?i32;
16821 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
16822 std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
16823 const allocator = stack.get();
16824
16825 const mask_elems = try allocator.alloc(?i32, extra.mask_len);
16826 defer allocator.free(mask_elems);
16827 for (mask_elems, 0..) |*mask_elem, elem_index| {
16828 const mask_elem_val = Value
16829 .fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable;
16830 mask_elem.* = if (mask_elem_val.isUndef(mod))
16831 null
16832 else
16833 @intCast(mask_elem_val.toSignedInt(mod));
16834 }
16835
16836 const result = @as(?MCValue, result: {
16837 for (mask_elems) |mask_elem| {
16838 if (mask_elem) |_| break;
16839 } else break :result try self.allocRegOrMem(inst, true);
16840
16841 for (mask_elems, 0..) |mask_elem, elem_index| {
16842 if (mask_elem orelse continue != @as(i32, @intCast(elem_index))) break;
16843 } else {
16844 const lhs_mcv = try self.resolveInst(extra.a);
16845 if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;
16846 const dst_mcv = try self.allocRegOrMem(inst, true);
16847 try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});
16848 break :result dst_mcv;
16849 }
16850
16851 for (mask_elems, 0..) |mask_elem, elem_index| {
16852 if (mask_elem orelse continue != ~@as(i32, @intCast(elem_index))) break;
16853 } else {
16854 const rhs_mcv = try self.resolveInst(extra.b);
16855 if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;
16856 const dst_mcv = try self.allocRegOrMem(inst, true);
16857 try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});
16858 break :result dst_mcv;
16859 }
16860
16861 const has_avx = self.hasFeature(.avx);
16862 shufpd: {
16863 if (elem_abi_size != 8) break :shufpd;
16864 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
16865
16866 var control: u4 = 0b0_0_0_0;
16867 var sources = [1]?u1{null} ** 2;
16868 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
16869 const mask_elem = maybe_mask_elem orelse continue;
16870 const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
16871 if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;
16872
16873 const source = @intFromBool(mask_elem < 0);
16874 if (sources[elem_index & 0b01]) |prev_source| {
16875 if (source != prev_source) break :shufpd;
16876 } else sources[elem_index & 0b01] = source;
16877
16878 control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);
16879 }
16880 if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;
16881
16882 const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
16883 const operand_tys = [2]Type{ lhs_ty, rhs_ty };
16884 const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
16885 const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
16886
16887 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
16888 self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
16889 lhs_mcv
16890 else if (has_avx and lhs_mcv.isRegister())
16891 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
16892 else
16893 try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
16894 const dst_reg = dst_mcv.getReg().?;
16895 const dst_alias = registerAlias(dst_reg, max_abi_size);
16896
16897 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
16898 .{ .v_pd, .shuf },
16899 dst_alias,
16900 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
16901 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
16902 Immediate.u(control),
16903 ) else try self.asmRegisterRegisterRegisterImmediate(
16904 .{ .v_pd, .shuf },
16905 dst_alias,
16906 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
16907 registerAlias(if (rhs_mcv.isRegister())
16908 rhs_mcv.getReg().?
16909 else
16910 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
16911 Immediate.u(control),
16912 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
16913 .{ ._pd, .shuf },
16914 dst_alias,
16915 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
16916 Immediate.u(control),
16917 ) else try self.asmRegisterRegisterImmediate(
16918 .{ ._pd, .shuf },
16919 dst_alias,
16920 registerAlias(if (rhs_mcv.isRegister())
16921 rhs_mcv.getReg().?
16922 else
16923 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
16924 Immediate.u(control),
16925 );
16926 break :result dst_mcv;
16927 }
16928
16929 pshufb: {
16930 if (max_abi_size > 16) break :pshufb;
16931 if (!self.hasFeature(.ssse3)) break :pshufb;
16932
16933 const temp_regs =
16934 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
16935 const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
16936 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
16937
16938 const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
16939 try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
16940
16941 const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
16942 try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
16943
16944 var lhs_mask_elems: [16]InternPool.Index = undefined;
16945 for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| {
16946 const elem_index = byte_index / elem_abi_size;
16947 lhs_mask_elem.* = try mod.intern(.{ .int = .{
16948 .ty = .u8_type,
16949 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
16950 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
16951 if (mask_elem < 0) break :elem 0b1_00_00000;
16952 const mask_elem_index: u31 = @intCast(mask_elem);
16953 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
16954 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
16955 } },
16956 } });
16957 }
16958 const lhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
16959 const lhs_mask_mcv = try self.genTypedValue(.{
16960 .ty = lhs_mask_ty,
16961 .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
16962 .ty = lhs_mask_ty.toIntern(),
16963 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
16964 } })),
16965 });
16966 const lhs_mask_mem: Memory = .{
16967 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
16968 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
16969 };
16970 if (has_avx) try self.asmRegisterRegisterMemory(
16971 .{ .vp_b, .shuf },
16972 lhs_temp_alias,
16973 lhs_temp_alias,
16974 lhs_mask_mem,
16975 ) else try self.asmRegisterMemory(
16976 .{ .p_b, .shuf },
16977 lhs_temp_alias,
16978 lhs_mask_mem,
16979 );
16980
16981 var rhs_mask_elems: [16]InternPool.Index = undefined;
16982 for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| {
16983 const elem_index = byte_index / elem_abi_size;
16984 rhs_mask_elem.* = try mod.intern(.{ .int = .{
16985 .ty = .u8_type,
16986 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
16987 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
16988 if (mask_elem >= 0) break :elem 0b1_00_00000;
16989 const mask_elem_index: u31 = @intCast(~mask_elem);
16990 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
16991 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
16992 } },
16993 } });
16994 }
16995 const rhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
16996 const rhs_mask_mcv = try self.genTypedValue(.{
16997 .ty = rhs_mask_ty,
16998 .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
16999 .ty = rhs_mask_ty.toIntern(),
17000 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
17001 } })),
17002 });
17003 const rhs_mask_mem: Memory = .{
17004 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
17005 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
17006 };
17007 if (has_avx) try self.asmRegisterRegisterMemory(
17008 .{ .vp_b, .shuf },
17009 rhs_temp_alias,
17010 rhs_temp_alias,
17011 rhs_mask_mem,
17012 ) else try self.asmRegisterMemory(
17013 .{ .p_b, .shuf },
17014 rhs_temp_alias,
17015 rhs_mask_mem,
17016 );
17017
17018 if (has_avx) try self.asmRegisterRegisterRegister(
17019 .{ switch (elem_ty.zigTypeTag(mod)) {
17020 else => break :result null,
17021 .Int => .vp_,
17022 .Float => switch (elem_ty.floatBits(self.target.*)) {
17023 32 => .v_ps,
17024 64 => .v_pd,
17025 16, 80, 128 => break :result null,
17026 else => unreachable,
17027 },
17028 }, .@"or" },
17029 lhs_temp_alias,
17030 lhs_temp_alias,
17031 rhs_temp_alias,
17032 ) else try self.asmRegisterRegister(
17033 .{ switch (elem_ty.zigTypeTag(mod)) {
17034 else => break :result null,
17035 .Int => .p_,
17036 .Float => switch (elem_ty.floatBits(self.target.*)) {
17037 32 => ._ps,
17038 64 => ._pd,
17039 16, 80, 128 => break :result null,
17040 else => unreachable,
17041 },
17042 }, .@"or" },
17043 lhs_temp_alias,
17044 rhs_temp_alias,
17045 );
17046 break :result .{ .register = temp_regs[0] };
17047 }
17048
17049 if (max_abi_size <= 16) {
17050 const lhs_mcv = try self.resolveInst(extra.a);
17051 const lhs_reg = if (lhs_mcv.isRegister())
17052 lhs_mcv.getReg().?
17053 else
17054 try self.copyToTmpRegister(lhs_ty, lhs_mcv);
17055 const lhs_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);
17056 defer self.register_manager.unlockReg(lhs_lock);
17057
17058 const rhs_mcv = try self.resolveInst(extra.b);
17059 const rhs_reg = if (rhs_mcv.isRegister())
17060 rhs_mcv.getReg().?
17061 else
17062 try self.copyToTmpRegister(rhs_ty, rhs_mcv);
17063 const rhs_lock = self.register_manager.lockReg(rhs_reg);
17064 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
17065
17066 //const dst_mcv = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
17067 switch (elem_ty.zigTypeTag(mod)) {
17068 .Float => switch (elem_ty.floatBits(self.target.*)) {
17069 16, 32 => {},
17070 64 => unreachable, // fully handled by shufpd
17071 80, 128 => unreachable, // all possible masks already handled
17072 else => unreachable,
17073 },
17074 else => {},
17075 }
17076 }
17077
17078 break :result null;
17079 }) orelse return self.fail("TODO implement airShuffle from {} and {} to {}", .{
17080 lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod),
17081 });
17082 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
16889}17083}
1689017084
16891fn airReduce(self: *Self, inst: Air.Inst.Index) !void {17085fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
...@@ -17062,7 +17256,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -17062,7 +17256,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
17062 },17256 },
17063 .Array, .Vector => {17257 .Array, .Vector => {
17064 const elem_ty = result_ty.childType(mod);17258 const elem_ty = result_ty.childType(mod);
17065 if (result_ty.isVector(mod) and elem_ty.bitSize(mod) == 1) {17259 if (result_ty.isVector(mod) and elem_ty.toIntern() == .bool_type) {
17066 const result_size: u32 = @intCast(result_ty.abiSize(mod));17260 const result_size: u32 = @intCast(result_ty.abiSize(mod));
17067 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);17261 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
17068 try self.asmRegisterRegister(17262 try self.asmRegisterRegister(
...@@ -18112,7 +18306,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {...@@ -18112,7 +18306,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {
18112 else => unreachable,18306 else => unreachable,
18113 },18307 },
18114 .float => Type.f32,18308 .float => Type.f32,
18115 .float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),18309 .float_combine => try mod.arrayType(.{ .len = 2, .child = .f32_type }),
18116 .sse => Type.f64,18310 .sse => Type.f64,
18117 else => break,18311 else => break,
18118 };18312 };
src/arch/x86_64/Encoding.zig+2-2
...@@ -324,7 +324,7 @@ pub const Mnemonic = enum {...@@ -324,7 +324,7 @@ pub const Mnemonic = enum {
324 // SSE3324 // SSE3
325 movddup, movshdup, movsldup,325 movddup, movshdup, movsldup,
326 // SSSE3326 // SSSE3
327 pabsb, pabsd, pabsw, palignr,327 pabsb, pabsd, pabsw, palignr, pshufb,
328 // SSE4.1328 // SSE4.1
329 blendpd, blendps, blendvpd, blendvps,329 blendpd, blendps, blendvpd, blendvps,
330 extractps,330 extractps,
...@@ -389,7 +389,7 @@ pub const Mnemonic = enum {...@@ -389,7 +389,7 @@ pub const Mnemonic = enum {
389 vpmovmskb,389 vpmovmskb,
390 vpmulhw, vpmulld, vpmullw,390 vpmulhw, vpmulld, vpmullw,
391 vpor,391 vpor,
392 vpshufd, vpshufhw, vpshuflw,392 vpshufb, vpshufd, vpshufhw, vpshuflw,
393 vpslld, vpslldq, vpsllq, vpsllw,393 vpslld, vpslldq, vpsllq, vpsllw,
394 vpsrad, vpsraq, vpsraw,394 vpsrad, vpsraq, vpsraw,
395 vpsrld, vpsrldq, vpsrlq, vpsrlw,395 vpsrld, vpsrldq, vpsrlq, vpsrlw,
src/arch/x86_64/encodings.zig+5
...@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{...@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{
11851185
1186 .{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },1186 .{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },
11871187
1188 .{ .pshufb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 },
1189
1188 // SSE4.11190 // SSE4.1
1189 .{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },1191 .{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },
11901192
...@@ -1593,6 +1595,8 @@ pub const table = [_]Entry{...@@ -1593,6 +1595,8 @@ pub const table = [_]Entry{
15931595
1594 .{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },1596 .{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },
15951597
1598 .{ .vpshufb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx },
1599
1596 .{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },1600 .{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
15971601
1598 .{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },1602 .{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
...@@ -1820,6 +1824,7 @@ pub const table = [_]Entry{...@@ -1820,6 +1824,7 @@ pub const table = [_]Entry{
18201824
1821 .{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },1825 .{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },
18221826
1827 .{ .vpshufb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 },
1823 .{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },1828 .{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },
18241829
1825 .{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },1830 .{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },
src/codegen.zig+27-30
...@@ -405,7 +405,7 @@ pub fn generateSymbol(...@@ -405,7 +405,7 @@ pub fn generateSymbol(
405 .vector_type => |vector_type| {405 .vector_type => |vector_type| {
406 const abi_size = math.cast(usize, typed_value.ty.abiSize(mod)) orelse406 const abi_size = math.cast(usize, typed_value.ty.abiSize(mod)) orelse
407 return error.Overflow;407 return error.Overflow;
408 if (Type.fromInterned(vector_type.child).bitSize(mod) == 1) {408 if (vector_type.child == .bool_type) {
409 const bytes = try code.addManyAsSlice(abi_size);409 const bytes = try code.addManyAsSlice(abi_size);
410 @memset(bytes, 0xaa);410 @memset(bytes, 0xaa);
411 var index: usize = 0;411 var index: usize = 0;
...@@ -443,37 +443,34 @@ pub fn generateSymbol(...@@ -443,37 +443,34 @@ pub fn generateSymbol(
443 },443 },
444 }) byte.* |= mask else byte.* &= ~mask;444 }) byte.* |= mask else byte.* &= ~mask;
445 }445 }
446 } else switch (aggregate.storage) {446 } else {
447 .bytes => |bytes| try code.appendSlice(bytes),447 switch (aggregate.storage) {
448 .elems, .repeated_elem => {448 .bytes => |bytes| try code.appendSlice(bytes),
449 var index: u64 = 0;449 .elems, .repeated_elem => {
450 while (index < vector_type.len) : (index += 1) {450 var index: u64 = 0;
451 switch (try generateSymbol(bin_file, src_loc, .{451 while (index < vector_type.len) : (index += 1) {
452 .ty = Type.fromInterned(vector_type.child),452 switch (try generateSymbol(bin_file, src_loc, .{
453 .val = Value.fromInterned(switch (aggregate.storage) {453 .ty = Type.fromInterned(vector_type.child),
454 .bytes => unreachable,454 .val = Value.fromInterned(switch (aggregate.storage) {
455 .elems => |elems| elems[455 .bytes => unreachable,
456 math.cast(usize, index) orelse return error.Overflow456 .elems => |elems| elems[
457 ],457 math.cast(usize, index) orelse return error.Overflow
458 .repeated_elem => |elem| elem,458 ],
459 }),459 .repeated_elem => |elem| elem,
460 }, code, debug_output, reloc_info)) {460 }),
461 .ok => {},461 }, code, debug_output, reloc_info)) {
462 .fail => |em| return .{ .fail = em },462 .ok => {},
463 .fail => |em| return .{ .fail = em },
464 }
463 }465 }
464 }466 },
465 },467 }
466 }
467468
468 const padding = abi_size - (math.cast(usize, math.divCeil(469 const padding = abi_size -
469 u64,470 (math.cast(usize, Type.fromInterned(vector_type.child).abiSize(mod) * vector_type.len) orelse
470 Type.fromInterned(vector_type.child).bitSize(mod) * vector_type.len,471 return error.Overflow);
471 8,472 if (padding > 0) try code.appendNTimes(0, padding);
472 ) catch |err| switch (err) {473 }
473 error.DivisionByZero => unreachable,
474 else => |e| return e,
475 }) orelse return error.Overflow);
476 if (padding > 0) try code.appendNTimes(0, padding);
477 },474 },
478 .anon_struct_type => |tuple| {475 .anon_struct_type => |tuple| {
479 const struct_begin = code.items.len;476 const struct_begin = code.items.len;
src/type.zig+34-8
...@@ -905,11 +905,28 @@ pub const Type = struct {...@@ -905,11 +905,28 @@ pub const Type = struct {
905 return Type.fromInterned(array_type.child).abiAlignmentAdvanced(mod, strat);905 return Type.fromInterned(array_type.child).abiAlignmentAdvanced(mod, strat);
906 },906 },
907 .vector_type => |vector_type| {907 .vector_type => |vector_type| {
908 const bits_u64 = try bitSizeAdvanced(Type.fromInterned(vector_type.child), mod, opt_sema);908 if (vector_type.len == 0) return .{ .scalar = .@"1" };
909 const bits: u32 = @intCast(bits_u64);909 switch (mod.comp.getZigBackend()) {
910 const bytes = ((bits * vector_type.len) + 7) / 8;910 else => {
911 const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);911 const elem_bits: u32 = @intCast(try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema));
912 return .{ .scalar = Alignment.fromByteUnits(alignment) };912 if (elem_bits == 0) return .{ .scalar = .@"1" };
913 const bytes = ((elem_bits * vector_type.len) + 7) / 8;
914 const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
915 return .{ .scalar = Alignment.fromByteUnits(alignment) };
916 },
917 .stage2_x86_64 => {
918 if (vector_type.child == .bool_type) return .{ .scalar = intAbiAlignment(@intCast(vector_type.len), target) };
919 const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
920 if (elem_bytes == 0) return .{ .scalar = .@"1" };
921 const bytes = elem_bytes * vector_type.len;
922 if (bytes > 32 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };
923 if (bytes > 16 and std.Target.x86.featureSetHas(
924 target.cpu.features,
925 if (Type.fromInterned(vector_type.child).isRuntimeFloat()) .avx else .avx2,
926 )) return .{ .scalar = .@"32" };
927 return .{ .scalar = .@"16" };
928 },
929 }
913 },930 },
914931
915 .opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat),932 .opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat),
...@@ -1237,9 +1254,6 @@ pub const Type = struct {...@@ -1237,9 +1254,6 @@ pub const Type = struct {
1237 .storage = .{ .lazy_size = ty.toIntern() },1254 .storage = .{ .lazy_size = ty.toIntern() },
1238 } }))) },1255 } }))) },
1239 };1256 };
1240 const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
1241 const total_bits = elem_bits * vector_type.len;
1242 const total_bytes = (total_bits + 7) / 8;
1243 const alignment = switch (try ty.abiAlignmentAdvanced(mod, strat)) {1257 const alignment = switch (try ty.abiAlignmentAdvanced(mod, strat)) {
1244 .scalar => |x| x,1258 .scalar => |x| x,
1245 .val => return .{ .val = Value.fromInterned((try mod.intern(.{ .int = .{1259 .val => return .{ .val = Value.fromInterned((try mod.intern(.{ .int = .{
...@@ -1247,6 +1261,18 @@ pub const Type = struct {...@@ -1247,6 +1261,18 @@ pub const Type = struct {
1247 .storage = .{ .lazy_size = ty.toIntern() },1261 .storage = .{ .lazy_size = ty.toIntern() },
1248 } }))) },1262 } }))) },
1249 };1263 };
1264 const total_bytes = switch (mod.comp.getZigBackend()) {
1265 else => total_bytes: {
1266 const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
1267 const total_bits = elem_bits * vector_type.len;
1268 break :total_bytes (total_bits + 7) / 8;
1269 },
1270 .stage2_x86_64 => total_bytes: {
1271 if (vector_type.child == .bool_type) break :total_bytes std.math.divCeil(u32, vector_type.len, 8) catch unreachable;
1272 const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
1273 break :total_bytes elem_bytes * vector_type.len;
1274 },
1275 };
1250 return AbiSizeAdvanced{ .scalar = alignment.forward(total_bytes) };1276 return AbiSizeAdvanced{ .scalar = alignment.forward(total_bytes) };
1251 },1277 },
12521278
test/behavior/bitcast.zig+1-1
...@@ -336,7 +336,7 @@ test "comptime @bitCast packed struct to int and back" {...@@ -336,7 +336,7 @@ test "comptime @bitCast packed struct to int and back" {
336 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;336 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
337 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO337 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
338 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;338 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
339 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;339 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
340340
341 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {341 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
342 // https://github.com/ziglang/zig/issues/13782342 // https://github.com/ziglang/zig/issues/13782
test/behavior/cast.zig+1
...@@ -2441,6 +2441,7 @@ test "@intFromBool on vector" {...@@ -2441,6 +2441,7 @@ test "@intFromBool on vector" {
2441 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO2441 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
2442 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO2442 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
2443 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO2443 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
2444 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
24442445
2445 const S = struct {2446 const S = struct {
2446 fn doTheTest() !void {2447 fn doTheTest() !void {
test/behavior/shuffle.zig+2-1
...@@ -4,10 +4,11 @@ const mem = std.mem;...@@ -4,10 +4,11 @@ const mem = std.mem;
4const expect = std.testing.expect;4const expect = std.testing.expect;
55
6test "@shuffle int" {6test "@shuffle int" {
7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO7 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO8 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO9 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_x86_64 and
11 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
1112
12 const S = struct {13 const S = struct {
13 fn doTheTest() !void {14 fn doTheTest() !void {
test/behavior/vector.zig+29-13
...@@ -29,7 +29,7 @@ test "vector wrap operators" {...@@ -29,7 +29,7 @@ test "vector wrap operators" {
29 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO29 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
30 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO30 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
31 if (builtin.zig_backend == .stage2_x86_64 and31 if (builtin.zig_backend == .stage2_x86_64 and
32 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO32 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest;
3333
34 const S = struct {34 const S = struct {
35 fn doTheTest() !void {35 fn doTheTest() !void {
...@@ -906,22 +906,26 @@ test "vector @reduce comptime" {...@@ -906,22 +906,26 @@ test "vector @reduce comptime" {
906}906}
907907
908test "mask parameter of @shuffle is comptime scope" {908test "mask parameter of @shuffle is comptime scope" {
909 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
910 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO909 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
911 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO910 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
912 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO911 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
912 if (builtin.zig_backend == .stage2_x86_64 and
913 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
913914
914 const __v4hi = @Vector(4, i16);915 const __v4hi = @Vector(4, i16);
915 var v4_a = __v4hi{ 0, 0, 0, 0 };916 var v4_a = __v4hi{ 1, 2, 3, 4 };
916 var v4_b = __v4hi{ 0, 0, 0, 0 };917 var v4_b = __v4hi{ 5, 6, 7, 8 };
917 _ = .{ &v4_a, &v4_b };918 _ = .{ &v4_a, &v4_b };
918 const shuffled: __v4hi = @shuffle(i16, v4_a, v4_b, @Vector(4, i32){919 const shuffled: __v4hi = @shuffle(i16, v4_a, v4_b, @Vector(4, i32){
919 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),920 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
920 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),921 std.zig.c_translation.shuffleVectorIndex(2, @typeInfo(@TypeOf(v4_a)).Vector.len),
921 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),922 std.zig.c_translation.shuffleVectorIndex(4, @typeInfo(@TypeOf(v4_a)).Vector.len),
922 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),923 std.zig.c_translation.shuffleVectorIndex(6, @typeInfo(@TypeOf(v4_a)).Vector.len),
923 });924 });
924 _ = shuffled;925 try expect(shuffled[0] == 1);
926 try expect(shuffled[1] == 3);
927 try expect(shuffled[2] == 5);
928 try expect(shuffled[3] == 7);
925}929}
926930
927test "saturating add" {931test "saturating add" {
...@@ -1177,10 +1181,22 @@ test "@shlWithOverflow" {...@@ -1177,10 +1181,22 @@ test "@shlWithOverflow" {
1177}1181}
11781182
1179test "alignment of vectors" {1183test "alignment of vectors" {
1180 try expect(@alignOf(@Vector(2, u8)) == 2);1184 try expect(@alignOf(@Vector(2, u8)) == switch (builtin.zig_backend) {
1181 try expect(@alignOf(@Vector(2, u1)) == 1);1185 else => 2,
1182 try expect(@alignOf(@Vector(1, u1)) == 1);1186 .stage2_x86_64 => 16,
1183 try expect(@alignOf(@Vector(2, u16)) == 4);1187 });
1188 try expect(@alignOf(@Vector(2, u1)) == switch (builtin.zig_backend) {
1189 else => 1,
1190 .stage2_x86_64 => 16,
1191 });
1192 try expect(@alignOf(@Vector(1, u1)) == switch (builtin.zig_backend) {
1193 else => 1,
1194 .stage2_x86_64 => 16,
1195 });
1196 try expect(@alignOf(@Vector(2, u16)) == switch (builtin.zig_backend) {
1197 else => 4,
1198 .stage2_x86_64 => 16,
1199 });
1184}1200}
11851201
1186test "loading the second vector from a slice of vectors" {1202test "loading the second vector from a slice of vectors" {
...@@ -1316,10 +1332,10 @@ test "modRem with zero divisor" {...@@ -1316,10 +1332,10 @@ test "modRem with zero divisor" {
13161332
1317test "array operands to shuffle are coerced to vectors" {1333test "array operands to shuffle are coerced to vectors" {
1318 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1334 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1319 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1320 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1335 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1321 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1336 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1322 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1337 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1338 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
13231339
1324 const mask = [5]i32{ -1, 0, 1, 2, 3 };1340 const mask = [5]i32{ -1, 0, 1, 2, 3 };
13251341