authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-25 21:43:20-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-02-25 21:43:20-08:00
log91fb211faa3f37d08da55b0c8df92a6475624316
treead824c8fcdd386e378669c21a3e65923b52d0133
parentd656c2a7abe90d00ef6dbc3731b82bd26180038a
parent4fcc750ba58f51606c49310bdd7c81c156d48cfc
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #18906 from jacobly0/x86_64-tests

x86_64: pass more tests

24 files changed, 2239 insertions(+), 666 deletions(-)

lib/std/crypto/aes.zig+1-1
...@@ -6,7 +6,7 @@ const has_aesni = std.Target.x86.featureSetHas(builtin.cpu.features, .aes);...@@ -6,7 +6,7 @@ const has_aesni = std.Target.x86.featureSetHas(builtin.cpu.features, .aes);
6const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);6const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);
7const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);7const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);
8// C backend doesn't currently support passing vectors to inline asm.8// C backend doesn't currently support passing vectors to inline asm.
9const impl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and has_aesni and has_avx) impl: {9const impl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_c and has_aesni and has_avx) impl: {
10 break :impl @import("aes/aesni.zig");10 break :impl @import("aes/aesni.zig");
11} else if (builtin.cpu.arch == .aarch64 and builtin.zig_backend != .stage2_c and has_armaes)11} else if (builtin.cpu.arch == .aarch64 and builtin.zig_backend != .stage2_c and has_armaes)
12impl: {12impl: {
lib/std/crypto/blake3.zig+1-1
...@@ -200,7 +200,7 @@ const CompressGeneric = struct {...@@ -200,7 +200,7 @@ const CompressGeneric = struct {
200 }200 }
201};201};
202202
203const compress = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64)203const compress = if (builtin.cpu.arch == .x86_64)
204 CompressVectorized.compress204 CompressVectorized.compress
205else205else
206 CompressGeneric.compress;206 CompressGeneric.compress;
lib/std/crypto/salsa20.zig+4-1
...@@ -302,7 +302,10 @@ fn SalsaNonVecImpl(comptime rounds: comptime_int) type {...@@ -302,7 +302,10 @@ fn SalsaNonVecImpl(comptime rounds: comptime_int) type {
302 };302 };
303}303}
304304
305const SalsaImpl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64) SalsaVecImpl else SalsaNonVecImpl;305const SalsaImpl = if (builtin.cpu.arch == .x86_64)
306 SalsaVecImpl
307else
308 SalsaNonVecImpl;
306309
307fn keyToWords(key: [32]u8) [8]u32 {310fn keyToWords(key: [32]u8) [8]u32 {
308 var k: [8]u32 = undefined;311 var k: [8]u32 = undefined;
lib/std/crypto/sha2.zig+1-1
...@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {...@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238 return;238 return;
239 },239 },
240 // C backend doesn't currently support passing vectors to inline asm.240 // C backend doesn't currently support passing vectors to inline asm.
241 .x86_64 => if (builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {241 .x86_64 => if (builtin.zig_backend != .stage2_c and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
242 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };242 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
243 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };243 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
244 const s_v = @as(*[16]v4u32, @ptrCast(&s));244 const s_v = @as(*[16]v4u32, @ptrCast(&s));
lib/std/meta.zig+2-1
...@@ -1286,5 +1286,6 @@ test "hasUniqueRepresentation" {...@@ -1286,5 +1286,6 @@ test "hasUniqueRepresentation" {
1286 try testing.expect(!hasUniqueRepresentation([]u8));1286 try testing.expect(!hasUniqueRepresentation([]u8));
1287 try testing.expect(!hasUniqueRepresentation([]const u8));1287 try testing.expect(!hasUniqueRepresentation([]const u8));
12881288
1289 try testing.expect(hasUniqueRepresentation(@Vector(4, u16)));1289 try testing.expect(hasUniqueRepresentation(@Vector(std.simd.suggestVectorLength(u8) orelse 1, u8)));
1290 try testing.expect(@sizeOf(@Vector(3, u8)) == 3 or !hasUniqueRepresentation(@Vector(3, u8)));
1290}1291}
lib/std/unicode.zig+147-152
...@@ -239,18 +239,19 @@ pub fn utf8ValidateSlice(input: []const u8) bool {...@@ -239,18 +239,19 @@ pub fn utf8ValidateSlice(input: []const u8) bool {
239fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {239fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {
240 var remaining = input;240 var remaining = input;
241241
242 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;242 if (std.simd.suggestVectorLength(u8)) |chunk_len| {
243 const Chunk = @Vector(chunk_len, u8);243 const Chunk = @Vector(chunk_len, u8);
244244
245 // Fast path. Check for and skip ASCII characters at the start of the input.245 // Fast path. Check for and skip ASCII characters at the start of the input.
246 while (remaining.len >= chunk_len) {246 while (remaining.len >= chunk_len) {
247 const chunk: Chunk = remaining[0..chunk_len].*;247 const chunk: Chunk = remaining[0..chunk_len].*;
248 const mask: Chunk = @splat(0x80);248 const mask: Chunk = @splat(0x80);
249 if (@reduce(.Or, chunk & mask == mask)) {249 if (@reduce(.Or, chunk & mask == mask)) {
250 // found a non ASCII byte250 // found a non ASCII byte
251 break;251 break;
252 }
253 remaining = remaining[chunk_len..];
252 }254 }
253 remaining = remaining[chunk_len..];
254 }255 }
255256
256 // default lowest and highest continuation byte257 // default lowest and highest continuation byte
...@@ -601,9 +602,9 @@ fn testUtf8IteratorOnAscii() !void {...@@ -601,9 +602,9 @@ fn testUtf8IteratorOnAscii() !void {
601 const s = Utf8View.initComptime("abc");602 const s = Utf8View.initComptime("abc");
602603
603 var it1 = s.iterator();604 var it1 = s.iterator();
604 try testing.expect(std.mem.eql(u8, "a", it1.nextCodepointSlice().?));605 try testing.expect(mem.eql(u8, "a", it1.nextCodepointSlice().?));
605 try testing.expect(std.mem.eql(u8, "b", it1.nextCodepointSlice().?));606 try testing.expect(mem.eql(u8, "b", it1.nextCodepointSlice().?));
606 try testing.expect(std.mem.eql(u8, "c", it1.nextCodepointSlice().?));607 try testing.expect(mem.eql(u8, "c", it1.nextCodepointSlice().?));
607 try testing.expect(it1.nextCodepointSlice() == null);608 try testing.expect(it1.nextCodepointSlice() == null);
608609
609 var it2 = s.iterator();610 var it2 = s.iterator();
...@@ -631,9 +632,9 @@ fn testUtf8ViewOk() !void {...@@ -631,9 +632,9 @@ fn testUtf8ViewOk() !void {
631 const s = Utf8View.initComptime("東京市");632 const s = Utf8View.initComptime("東京市");
632633
633 var it1 = s.iterator();634 var it1 = s.iterator();
634 try testing.expect(std.mem.eql(u8, "東", it1.nextCodepointSlice().?));635 try testing.expect(mem.eql(u8, "東", it1.nextCodepointSlice().?));
635 try testing.expect(std.mem.eql(u8, "京", it1.nextCodepointSlice().?));636 try testing.expect(mem.eql(u8, "京", it1.nextCodepointSlice().?));
636 try testing.expect(std.mem.eql(u8, "市", it1.nextCodepointSlice().?));637 try testing.expect(mem.eql(u8, "市", it1.nextCodepointSlice().?));
637 try testing.expect(it1.nextCodepointSlice() == null);638 try testing.expect(it1.nextCodepointSlice() == null);
638639
639 var it2 = s.iterator();640 var it2 = s.iterator();
...@@ -771,20 +772,20 @@ fn testUtf8Peeking() !void {...@@ -771,20 +772,20 @@ fn testUtf8Peeking() !void {
771 const s = Utf8View.initComptime("noël");772 const s = Utf8View.initComptime("noël");
772 var it = s.iterator();773 var it = s.iterator();
773774
774 try testing.expect(std.mem.eql(u8, "n", it.nextCodepointSlice().?));775 try testing.expect(mem.eql(u8, "n", it.nextCodepointSlice().?));
775776
776 try testing.expect(std.mem.eql(u8, "o", it.peek(1)));777 try testing.expect(mem.eql(u8, "o", it.peek(1)));
777 try testing.expect(std.mem.eql(u8, "oë", it.peek(2)));778 try testing.expect(mem.eql(u8, "oë", it.peek(2)));
778 try testing.expect(std.mem.eql(u8, "oël", it.peek(3)));779 try testing.expect(mem.eql(u8, "oël", it.peek(3)));
779 try testing.expect(std.mem.eql(u8, "oël", it.peek(4)));780 try testing.expect(mem.eql(u8, "oël", it.peek(4)));
780 try testing.expect(std.mem.eql(u8, "oël", it.peek(10)));781 try testing.expect(mem.eql(u8, "oël", it.peek(10)));
781782
782 try testing.expect(std.mem.eql(u8, "o", it.nextCodepointSlice().?));783 try testing.expect(mem.eql(u8, "o", it.nextCodepointSlice().?));
783 try testing.expect(std.mem.eql(u8, "ë", it.nextCodepointSlice().?));784 try testing.expect(mem.eql(u8, "ë", it.nextCodepointSlice().?));
784 try testing.expect(std.mem.eql(u8, "l", it.nextCodepointSlice().?));785 try testing.expect(mem.eql(u8, "l", it.nextCodepointSlice().?));
785 try testing.expect(it.nextCodepointSlice() == null);786 try testing.expect(it.nextCodepointSlice() == null);
786787
787 try testing.expect(std.mem.eql(u8, &[_]u8{}, it.peek(1)));788 try testing.expect(mem.eql(u8, &[_]u8{}, it.peek(1)));
788}789}
789790
790fn testError(bytes: []const u8, expected_err: anyerror) !void {791fn testError(bytes: []const u8, expected_err: anyerror) !void {
...@@ -926,59 +927,50 @@ test "fmtUtf8" {...@@ -926,59 +927,50 @@ test "fmtUtf8" {
926}927}
927928
928fn utf16LeToUtf8ArrayListImpl(929fn utf16LeToUtf8ArrayListImpl(
929 array_list: *std.ArrayList(u8),930 result: *std.ArrayList(u8),
930 utf16le: []const u16,931 utf16le: []const u16,
931 comptime surrogates: Surrogates,932 comptime surrogates: Surrogates,
932) (switch (surrogates) {933) (switch (surrogates) {
933 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,934 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
934 .can_encode_surrogate_half => mem.Allocator.Error,935 .can_encode_surrogate_half => mem.Allocator.Error,
935})!void {936})!void {
936 // optimistically guess that it will all be ascii.937 assert(result.capacity >= utf16le.len);
937 try array_list.ensureTotalCapacityPrecise(utf16le.len);
938938
939 var remaining = utf16le;939 var remaining = utf16le;
940 if (builtin.zig_backend != .stage2_x86_64) {940 vectorized: {
941 const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;941 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
942 const Chunk = @Vector(chunk_len, u16);942 const Chunk = @Vector(chunk_len, u16);
943943
944 // Fast path. Check for and encode ASCII characters at the start of the input.944 // Fast path. Check for and encode ASCII characters at the start of the input.
945 while (remaining.len >= chunk_len) {945 while (remaining.len >= chunk_len) {
946 const chunk: Chunk = remaining[0..chunk_len].*;946 const chunk: Chunk = remaining[0..chunk_len].*;
947 const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));947 const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
948 if (@reduce(.Or, chunk | mask != mask)) {948 if (@reduce(.Or, chunk | mask != mask)) {
949 // found a non ASCII code unit949 // found a non ASCII code unit
950 break;950 break;
951 }951 }
952 const chunk_byte_len = chunk_len * 2;952 const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
953 const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
954 const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
955 const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
956 // We allocated enough space to encode every UTF-16 code unit953 // We allocated enough space to encode every UTF-16 code unit
957 // as ASCII, so if the entire string is ASCII then we are954 // as ASCII, so if the entire string is ASCII then we are
958 // guaranteed to have enough space allocated955 // guaranteed to have enough space allocated
959 array_list.appendSliceAssumeCapacity(&ascii_bytes);956 result.addManyAsArrayAssumeCapacity(chunk_len).* = ascii_chunk;
960 remaining = remaining[chunk_len..];957 remaining = remaining[chunk_len..];
961 }958 }
962 }959 }
963960
964 var out_index: usize = array_list.items.len;
965 switch (surrogates) {961 switch (surrogates) {
966 .cannot_encode_surrogate_half => {962 .cannot_encode_surrogate_half => {
967 var it = Utf16LeIterator.init(remaining);963 var it = Utf16LeIterator.init(remaining);
968 while (try it.nextCodepoint()) |codepoint| {964 while (try it.nextCodepoint()) |codepoint| {
969 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;965 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
970 try array_list.resize(array_list.items.len + utf8_len);966 assert((utf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
971 assert((utf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
972 out_index += utf8_len;
973 }967 }
974 },968 },
975 .can_encode_surrogate_half => {969 .can_encode_surrogate_half => {
976 var it = Wtf16LeIterator.init(remaining);970 var it = Wtf16LeIterator.init(remaining);
977 while (it.nextCodepoint()) |codepoint| {971 while (it.nextCodepoint()) |codepoint| {
978 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;972 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
979 try array_list.resize(array_list.items.len + utf8_len);973 assert((wtf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
980 assert((wtf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
981 out_index += utf8_len;
982 }974 }
983 },975 },
984 }976 }
...@@ -986,8 +978,9 @@ fn utf16LeToUtf8ArrayListImpl(...@@ -986,8 +978,9 @@ fn utf16LeToUtf8ArrayListImpl(
986978
987pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error;979pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error;
988980
989pub fn utf16LeToUtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {981pub fn utf16LeToUtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
990 return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .cannot_encode_surrogate_half);982 try result.ensureTotalCapacityPrecise(utf16le.len);
983 return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
991}984}
992985
993/// Deprecated; renamed to utf16LeToUtf8Alloc986/// Deprecated; renamed to utf16LeToUtf8Alloc
...@@ -999,8 +992,7 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L...@@ -999,8 +992,7 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L
999 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);992 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
1000 errdefer result.deinit();993 errdefer result.deinit();
1001994
1002 try utf16LeToUtf8ArrayList(&result, utf16le);995 try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
1003
1004 return result.toOwnedSlice();996 return result.toOwnedSlice();
1005}997}
1006998
...@@ -1013,8 +1005,7 @@ pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) Utf16...@@ -1013,8 +1005,7 @@ pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) Utf16
1013 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);1005 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);
1014 errdefer result.deinit();1006 errdefer result.deinit();
10151007
1016 try utf16LeToUtf8ArrayList(&result, utf16le);1008 try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
1017
1018 return result.toOwnedSliceSentinel(0);1009 return result.toOwnedSliceSentinel(0);
1019}1010}
10201011
...@@ -1026,27 +1017,24 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr...@@ -1026,27 +1017,24 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1026 .cannot_encode_surrogate_half => Utf16LeToUtf8Error,1017 .cannot_encode_surrogate_half => Utf16LeToUtf8Error,
1027 .can_encode_surrogate_half => error{},1018 .can_encode_surrogate_half => error{},
1028})!usize {1019})!usize {
1029 var end_index: usize = 0;1020 var dest_index: usize = 0;
10301021
1031 var remaining = utf16le;1022 var remaining = utf16le;
1032 if (builtin.zig_backend != .stage2_x86_64) {1023 vectorized: {
1033 const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;1024 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1034 const Chunk = @Vector(chunk_len, u16);1025 const Chunk = @Vector(chunk_len, u16);
10351026
1036 // Fast path. Check for and encode ASCII characters at the start of the input.1027 // Fast path. Check for and encode ASCII characters at the start of the input.
1037 while (remaining.len >= chunk_len) {1028 while (remaining.len >= chunk_len) {
1038 const chunk: Chunk = remaining[0..chunk_len].*;1029 const chunk: Chunk = remaining[0..chunk_len].*;
1039 const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));1030 const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
1040 if (@reduce(.Or, chunk | mask != mask)) {1031 if (@reduce(.Or, chunk | mask != mask)) {
1041 // found a non ASCII code unit1032 // found a non ASCII code unit
1042 break;1033 break;
1043 }1034 }
1044 const chunk_byte_len = chunk_len * 2;1035 const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
1045 const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;1036 utf8[dest_index..][0..chunk_len].* = ascii_chunk;
1046 const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);1037 dest_index += chunk_len;
1047 const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
1048 @memcpy(utf8[end_index .. end_index + chunk_len], &ascii_bytes);
1049 end_index += chunk_len;
1050 remaining = remaining[chunk_len..];1038 remaining = remaining[chunk_len..];
1051 }1039 }
1052 }1040 }
...@@ -1055,7 +1043,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr...@@ -1055,7 +1043,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1055 .cannot_encode_surrogate_half => {1043 .cannot_encode_surrogate_half => {
1056 var it = Utf16LeIterator.init(remaining);1044 var it = Utf16LeIterator.init(remaining);
1057 while (try it.nextCodepoint()) |codepoint| {1045 while (try it.nextCodepoint()) |codepoint| {
1058 end_index += utf8Encode(codepoint, utf8[end_index..]) catch |err| switch (err) {1046 dest_index += utf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
1059 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,1047 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
1060 // which is within the valid codepoint range.1048 // which is within the valid codepoint range.
1061 error.CodepointTooLarge => unreachable,1049 error.CodepointTooLarge => unreachable,
...@@ -1068,7 +1056,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr...@@ -1068,7 +1056,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1068 .can_encode_surrogate_half => {1056 .can_encode_surrogate_half => {
1069 var it = Wtf16LeIterator.init(remaining);1057 var it = Wtf16LeIterator.init(remaining);
1070 while (it.nextCodepoint()) |codepoint| {1058 while (it.nextCodepoint()) |codepoint| {
1071 end_index += wtf8Encode(codepoint, utf8[end_index..]) catch |err| switch (err) {1059 dest_index += wtf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
1072 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,1060 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
1073 // which is within the valid codepoint range.1061 // which is within the valid codepoint range.
1074 error.CodepointTooLarge => unreachable,1062 error.CodepointTooLarge => unreachable,
...@@ -1076,7 +1064,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr...@@ -1076,7 +1064,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1076 }1064 }
1077 },1065 },
1078 }1066 }
1079 return end_index;1067 return dest_index;
1080}1068}
10811069
1082/// Deprecated; renamed to utf16LeToUtf81070/// Deprecated; renamed to utf16LeToUtf8
...@@ -1149,14 +1137,12 @@ test utf16LeToUtf8 {...@@ -1149,14 +1137,12 @@ test utf16LeToUtf8 {
1149 }1137 }
1150}1138}
11511139
1152fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {1140fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
1153 // optimistically guess that it will not require surrogate pairs1141 assert(result.capacity >= utf8.len);
1154 try array_list.ensureTotalCapacityPrecise(utf8.len);
11551142
1156 var remaining = utf8;1143 var remaining = utf8;
1157 // Need support for std.simd.interlace1144 vectorized: {
1158 if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {1145 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1159 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
1160 const Chunk = @Vector(chunk_len, u8);1146 const Chunk = @Vector(chunk_len, u8);
11611147
1162 // Fast path. Check for and encode ASCII characters at the start of the input.1148 // Fast path. Check for and encode ASCII characters at the start of the input.
...@@ -1167,9 +1153,8 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,...@@ -1167,9 +1153,8 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,
1167 // found a non ASCII code unit1153 // found a non ASCII code unit
1168 break;1154 break;
1169 }1155 }
1170 const zeroes: Chunk = @splat(0);1156 const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
1171 const utf16_chunk: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });1157 result.addManyAsArrayAssumeCapacity(chunk_len).* = utf16_chunk;
1172 array_list.appendSliceAssumeCapacity(std.mem.bytesAsSlice(u16, &utf16_chunk));
1173 remaining = remaining[chunk_len..];1158 remaining = remaining[chunk_len..];
1174 }1159 }
1175 }1160 }
...@@ -1181,21 +1166,18 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,...@@ -1181,21 +1166,18 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,
1181 var it = view.iterator();1166 var it = view.iterator();
1182 while (it.nextCodepoint()) |codepoint| {1167 while (it.nextCodepoint()) |codepoint| {
1183 if (codepoint < 0x10000) {1168 if (codepoint < 0x10000) {
1184 const short = @as(u16, @intCast(codepoint));1169 try result.append(mem.nativeToLittle(u16, @intCast(codepoint)));
1185 try array_list.append(mem.nativeToLittle(u16, short));
1186 } else {1170 } else {
1187 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;1171 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
1188 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;1172 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
1189 var out: [2]u16 = undefined;1173 try result.appendSlice(&.{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) });
1190 out[0] = mem.nativeToLittle(u16, high);
1191 out[1] = mem.nativeToLittle(u16, low);
1192 try array_list.appendSlice(out[0..]);
1193 }1174 }
1194 }1175 }
1195}1176}
11961177
1197pub fn utf8ToUtf16LeArrayList(array_list: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {1178pub fn utf8ToUtf16LeArrayList(result: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
1198 return utf8ToUtf16LeArrayListImpl(array_list, utf8, .cannot_encode_surrogate_half);1179 try result.ensureTotalCapacityPrecise(utf8.len);
1180 return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
1199}1181}
12001182
1201pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {1183pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
...@@ -1204,7 +1186,6 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv...@@ -1204,7 +1186,6 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv
1204 errdefer result.deinit();1186 errdefer result.deinit();
12051187
1206 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);1188 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
1207
1208 return result.toOwnedSlice();1189 return result.toOwnedSlice();
1209}1190}
12101191
...@@ -1217,7 +1198,6 @@ pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) error{ In...@@ -1217,7 +1198,6 @@ pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) error{ In
1217 errdefer result.deinit();1198 errdefer result.deinit();
12181199
1219 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);1200 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
1220
1221 return result.toOwnedSliceSentinel(0);1201 return result.toOwnedSliceSentinel(0);
1222}1202}
12231203
...@@ -1228,12 +1208,11 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) error{InvalidUtf8}!usize...@@ -1228,12 +1208,11 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) error{InvalidUtf8}!usize
1228}1208}
12291209
1230pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize {1210pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize {
1231 var dest_i: usize = 0;1211 var dest_index: usize = 0;
12321212
1233 var remaining = utf8;1213 var remaining = utf8;
1234 // Need support for std.simd.interlace1214 vectorized: {
1235 if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {1215 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1236 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
1237 const Chunk = @Vector(chunk_len, u8);1216 const Chunk = @Vector(chunk_len, u8);
12381217
1239 // Fast path. Check for and encode ASCII characters at the start of the input.1218 // Fast path. Check for and encode ASCII characters at the start of the input.
...@@ -1244,57 +1223,60 @@ pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates:...@@ -1244,57 +1223,60 @@ pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates:
1244 // found a non ASCII code unit1223 // found a non ASCII code unit
1245 break;1224 break;
1246 }1225 }
1247 const zeroes: Chunk = @splat(0);1226 const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
1248 const utf16_bytes: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });1227 utf16le[dest_index..][0..chunk_len].* = utf16_chunk;
1249 @memcpy(utf16le[dest_i..][0..chunk_len], std.mem.bytesAsSlice(u16, &utf16_bytes));1228 dest_index += chunk_len;
1250 dest_i += chunk_len;
1251 remaining = remaining[chunk_len..];1229 remaining = remaining[chunk_len..];
1252 }1230 }
1253 }1231 }
12541232
1255 var src_i: usize = 0;1233 const view = switch (surrogates) {
1256 while (src_i < remaining.len) {1234 .cannot_encode_surrogate_half => try Utf8View.init(remaining),
1257 const n = utf8ByteSequenceLength(remaining[src_i]) catch return switch (surrogates) {1235 .can_encode_surrogate_half => try Wtf8View.init(remaining),
1258 .cannot_encode_surrogate_half => error.InvalidUtf8,1236 };
1259 .can_encode_surrogate_half => error.InvalidWtf8,1237 var it = view.iterator();
1260 };1238 while (it.nextCodepoint()) |codepoint| {
1261 const next_src_i = src_i + n;
1262 const codepoint = switch (surrogates) {
1263 .cannot_encode_surrogate_half => utf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8,
1264 .can_encode_surrogate_half => wtf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidWtf8,
1265 };
1266 if (codepoint < 0x10000) {1239 if (codepoint < 0x10000) {
1267 const short = @as(u16, @intCast(codepoint));1240 utf16le[dest_index] = mem.nativeToLittle(u16, @intCast(codepoint));
1268 utf16le[dest_i] = mem.nativeToLittle(u16, short);1241 dest_index += 1;
1269 dest_i += 1;
1270 } else {1242 } else {
1271 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;1243 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
1272 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;1244 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
1273 utf16le[dest_i] = mem.nativeToLittle(u16, high);1245 utf16le[dest_index..][0..2].* = .{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) };
1274 utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);1246 dest_index += 2;
1275 dest_i += 2;
1276 }1247 }
1277 src_i = next_src_i;
1278 }1248 }
1279 return dest_i;1249 return dest_index;
1280}1250}
12811251
1282test "utf8ToUtf16Le" {1252test "utf8ToUtf16Le" {
1283 var utf16le: [2]u16 = [_]u16{0} ** 2;1253 var utf16le: [128]u16 = undefined;
1284 {1254 {
1285 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");1255 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
1286 try testing.expectEqual(@as(usize, 2), length);1256 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..length]));
1287 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..]));
1288 }1257 }
1289 {1258 {
1290 const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");1259 const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
1291 try testing.expectEqual(@as(usize, 2), length);1260 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..length]));
1292 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..]));
1293 }1261 }
1294 {1262 {
1295 const result = utf8ToUtf16Le(utf16le[0..], "\xf4\x90\x80\x80");1263 const result = utf8ToUtf16Le(utf16le[0..], "\xf4\x90\x80\x80");
1296 try testing.expectError(error.InvalidUtf8, result);1264 try testing.expectError(error.InvalidUtf8, result);
1297 }1265 }
1266 {
1267 const length = try utf8ToUtf16Le(utf16le[0..], "This string has been designed to test the vectorized implementat" ++
1268 "ion by beginning with one hundred twenty-seven ASCII characters¡");
1269 try testing.expectEqualSlices(u8, &.{
1270 'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
1271 'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
1272 ' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
1273 'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
1274 'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
1275 ' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
1276 'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
1277 'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0, '¡', 0,
1278 }, mem.sliceAsBytes(utf16le[0..length]));
1279 }
1298}1280}
12991281
1300test utf8ToUtf16LeArrayList {1282test utf8ToUtf16LeArrayList {
...@@ -1339,25 +1321,40 @@ test utf8ToUtf16LeAllocZ {...@@ -1339,25 +1321,40 @@ test utf8ToUtf16LeAllocZ {
1339 {1321 {
1340 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");1322 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");
1341 defer testing.allocator.free(utf16);1323 defer testing.allocator.free(utf16);
1342 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));1324 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16));
1343 try testing.expect(utf16[2] == 0);1325 try testing.expect(utf16[2] == 0);
1344 }1326 }
1345 {1327 {
1346 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");1328 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");
1347 defer testing.allocator.free(utf16);1329 defer testing.allocator.free(utf16);
1348 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));1330 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16));
1349 try testing.expect(utf16[2] == 0);1331 try testing.expect(utf16[2] == 0);
1350 }1332 }
1351 {1333 {
1352 const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");1334 const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");
1353 try testing.expectError(error.InvalidUtf8, result);1335 try testing.expectError(error.InvalidUtf8, result);
1354 }1336 }
1337 {
1338 const utf16 = try utf8ToUtf16LeWithNull(testing.allocator, "This string has been designed to test the vectorized implementat" ++
1339 "ion by beginning with one hundred twenty-seven ASCII characters¡");
1340 defer testing.allocator.free(utf16);
1341 try testing.expectEqualSlices(u8, &.{
1342 'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
1343 'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
1344 ' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
1345 'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
1346 'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
1347 ' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
1348 'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
1349 'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0, '¡', 0,
1350 }, mem.sliceAsBytes(utf16));
1351 }
1355}1352}
13561353
1357/// Converts a UTF-8 string literal into a UTF-16LE string literal.1354/// Converts a UTF-8 string literal into a UTF-16LE string literal.
1358pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch unreachable:0]u16 {1355pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16 {
1359 return comptime blk: {1356 return comptime blk: {
1360 const len: usize = calcUtf16LeLen(utf8) catch |err| @compileError(err);1357 const len: usize = calcUtf16LeLen(utf8) catch unreachable;
1361 var utf16le: [len:0]u16 = [_:0]u16{0} ** len;1358 var utf16le: [len:0]u16 = [_:0]u16{0} ** len;
1362 const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err);1359 const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err);
1363 assert(len == utf16le_len);1360 assert(len == utf16le_len);
...@@ -1438,12 +1435,12 @@ test "fmtUtf16Le" {...@@ -1438,12 +1435,12 @@ test "fmtUtf16Le" {
1438 try expectFmt("", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});1435 try expectFmt("", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});
1439 try expectFmt("foo", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});1436 try expectFmt("foo", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});
1440 try expectFmt("𐐷", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("𐐷"))});1437 try expectFmt("𐐷", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("𐐷"))});
1441 try expectFmt("퟿", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xd7", native_endian)})});1438 try expectFmt("퟿", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xd7", native_endian)})});
1442 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xd8", native_endian)})});1439 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xd8", native_endian)})});
1443 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdb", native_endian)})});1440 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdb", native_endian)})});
1444 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xdc", native_endian)})});1441 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xdc", native_endian)})});
1445 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdf", native_endian)})});1442 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdf", native_endian)})});
1446 try expectFmt("", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xe0", native_endian)})});1443 try expectFmt("", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xe0", native_endian)})});
1447}1444}
14481445
1449test "utf8ToUtf16LeStringLiteral" {1446test "utf8ToUtf16LeStringLiteral" {
...@@ -1686,8 +1683,9 @@ pub const Wtf8Iterator = struct {...@@ -1686,8 +1683,9 @@ pub const Wtf8Iterator = struct {
1686 }1683 }
1687};1684};
16881685
1689pub fn wtf16LeToWtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {1686pub fn wtf16LeToWtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {
1690 return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .can_encode_surrogate_half);1687 try result.ensureTotalCapacityPrecise(utf16le.len);
1688 return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
1691}1689}
16921690
1693/// Caller must free returned memory.1691/// Caller must free returned memory.
...@@ -1696,8 +1694,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al...@@ -1696,8 +1694,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al
1696 var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len);1694 var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len);
1697 errdefer result.deinit();1695 errdefer result.deinit();
16981696
1699 try wtf16LeToWtf8ArrayList(&result, wtf16le);1697 try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
1700
1701 return result.toOwnedSlice();1698 return result.toOwnedSlice();
1702}1699}
17031700
...@@ -1707,8 +1704,7 @@ pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) mem.A...@@ -1707,8 +1704,7 @@ pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) mem.A
1707 var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len + 1);1704 var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len + 1);
1708 errdefer result.deinit();1705 errdefer result.deinit();
17091706
1710 try wtf16LeToWtf8ArrayList(&result, wtf16le);1707 try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
1711
1712 return result.toOwnedSliceSentinel(0);1708 return result.toOwnedSliceSentinel(0);
1713}1709}
17141710
...@@ -1716,8 +1712,9 @@ pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {...@@ -1716,8 +1712,9 @@ pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
1716 return utf16LeToUtf8Impl(wtf8, wtf16le, .can_encode_surrogate_half) catch |err| switch (err) {};1712 return utf16LeToUtf8Impl(wtf8, wtf16le, .can_encode_surrogate_half) catch |err| switch (err) {};
1717}1713}
17181714
1719pub fn wtf8ToWtf16LeArrayList(array_list: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {1715pub fn wtf8ToWtf16LeArrayList(result: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
1720 return utf8ToUtf16LeArrayListImpl(array_list, wtf8, .can_encode_surrogate_half);1716 try result.ensureTotalCapacityPrecise(wtf8.len);
1717 return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
1721}1718}
17221719
1723pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {1720pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
...@@ -1726,7 +1723,6 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv...@@ -1726,7 +1723,6 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv
1726 errdefer result.deinit();1723 errdefer result.deinit();
17271724
1728 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);1725 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1729
1730 return result.toOwnedSlice();1726 return result.toOwnedSlice();
1731}1727}
17321728
...@@ -1736,7 +1732,6 @@ pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ In...@@ -1736,7 +1732,6 @@ pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ In
1736 errdefer result.deinit();1732 errdefer result.deinit();
17371733
1738 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);1734 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1739
1740 return result.toOwnedSliceSentinel(0);1735 return result.toOwnedSliceSentinel(0);
1741}1736}
17421737
...@@ -1895,7 +1890,7 @@ pub const Wtf16LeIterator = struct {...@@ -1895,7 +1890,7 @@ pub const Wtf16LeIterator = struct {
18951890
1896 pub fn init(s: []const u16) Wtf16LeIterator {1891 pub fn init(s: []const u16) Wtf16LeIterator {
1897 return Wtf16LeIterator{1892 return Wtf16LeIterator{
1898 .bytes = std.mem.sliceAsBytes(s),1893 .bytes = mem.sliceAsBytes(s),
1899 .i = 0,1894 .i = 0,
1900 };1895 };
1901 }1896 }
...@@ -1908,12 +1903,12 @@ pub const Wtf16LeIterator = struct {...@@ -1908,12 +1903,12 @@ pub const Wtf16LeIterator = struct {
1908 assert(it.i <= it.bytes.len);1903 assert(it.i <= it.bytes.len);
1909 if (it.i == it.bytes.len) return null;1904 if (it.i == it.bytes.len) return null;
1910 var code_units: [2]u16 = undefined;1905 var code_units: [2]u16 = undefined;
1911 code_units[0] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);1906 code_units[0] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
1912 it.i += 2;1907 it.i += 2;
1913 surrogate_pair: {1908 surrogate_pair: {
1914 if (utf16IsHighSurrogate(code_units[0])) {1909 if (utf16IsHighSurrogate(code_units[0])) {
1915 if (it.i >= it.bytes.len) break :surrogate_pair;1910 if (it.i >= it.bytes.len) break :surrogate_pair;
1916 code_units[1] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);1911 code_units[1] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
1917 const codepoint = utf16DecodeSurrogatePair(&code_units) catch break :surrogate_pair;1912 const codepoint = utf16DecodeSurrogatePair(&code_units) catch break :surrogate_pair;
1918 it.i += 2;1913 it.i += 2;
1919 return codepoint;1914 return codepoint;
...@@ -2030,31 +2025,31 @@ fn testRoundtripWtf16(wtf16le: []const u16) !void {...@@ -2030,31 +2025,31 @@ fn testRoundtripWtf16(wtf16le: []const u16) !void {
20302025
2031test "well-formed WTF-16 roundtrips" {2026test "well-formed WTF-16 roundtrips" {
2032 try testRoundtripWtf16(&[_]u16{2027 try testRoundtripWtf16(&[_]u16{
2033 std.mem.nativeToLittle(u16, 0xD83D), // high surrogate2028 mem.nativeToLittle(u16, 0xD83D), // high surrogate
2034 std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate2029 mem.nativeToLittle(u16, 0xDCA9), // low surrogate
2035 });2030 });
2036 try testRoundtripWtf16(&[_]u16{2031 try testRoundtripWtf16(&[_]u16{
2037 std.mem.nativeToLittle(u16, 0xD83D), // high surrogate2032 mem.nativeToLittle(u16, 0xD83D), // high surrogate
2038 std.mem.nativeToLittle(u16, ' '), // not surrogate2033 mem.nativeToLittle(u16, ' '), // not surrogate
2039 std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate2034 mem.nativeToLittle(u16, 0xDCA9), // low surrogate
2040 });2035 });
2041 try testRoundtripWtf16(&[_]u16{2036 try testRoundtripWtf16(&[_]u16{
2042 std.mem.nativeToLittle(u16, 0xD800), // high surrogate2037 mem.nativeToLittle(u16, 0xD800), // high surrogate
2043 std.mem.nativeToLittle(u16, 0xDBFF), // high surrogate2038 mem.nativeToLittle(u16, 0xDBFF), // high surrogate
2044 });2039 });
2045 try testRoundtripWtf16(&[_]u16{2040 try testRoundtripWtf16(&[_]u16{
2046 std.mem.nativeToLittle(u16, 0xD800), // high surrogate2041 mem.nativeToLittle(u16, 0xD800), // high surrogate
2047 std.mem.nativeToLittle(u16, 0xE000), // not surrogate2042 mem.nativeToLittle(u16, 0xE000), // not surrogate
2048 });2043 });
2049 try testRoundtripWtf16(&[_]u16{2044 try testRoundtripWtf16(&[_]u16{
2050 std.mem.nativeToLittle(u16, 0xD7FF), // not surrogate2045 mem.nativeToLittle(u16, 0xD7FF), // not surrogate
2051 std.mem.nativeToLittle(u16, 0xDC00), // low surrogate2046 mem.nativeToLittle(u16, 0xDC00), // low surrogate
2052 });2047 });
2053 try testRoundtripWtf16(&[_]u16{2048 try testRoundtripWtf16(&[_]u16{
2054 std.mem.nativeToLittle(u16, 0x61), // not surrogate2049 mem.nativeToLittle(u16, 0x61), // not surrogate
2055 std.mem.nativeToLittle(u16, 0xDC00), // low surrogate2050 mem.nativeToLittle(u16, 0xDC00), // low surrogate
2056 });2051 });
2057 try testRoundtripWtf16(&[_]u16{2052 try testRoundtripWtf16(&[_]u16{
2058 std.mem.nativeToLittle(u16, 0xDC00), // low surrogate2053 mem.nativeToLittle(u16, 0xDC00), // low surrogate
2059 });2054 });
2060}2055}
lib/std/zig/c_translation.zig+3-5
...@@ -308,14 +308,12 @@ test "promoteIntLiteral" {...@@ -308,14 +308,12 @@ test "promoteIntLiteral" {
308308
309/// Convert from clang __builtin_shufflevector index to Zig @shuffle index309/// Convert from clang __builtin_shufflevector index to Zig @shuffle index
310/// clang requires __builtin_shufflevector index arguments to be integer constants.310/// clang requires __builtin_shufflevector index arguments to be integer constants.
311/// negative values for `this_index` indicate "don't care" so we arbitrarily choose 0311/// negative values for `this_index` indicate "don't care".
312/// clang enforces that `this_index` is less than the total number of vector elements312/// clang enforces that `this_index` is less than the total number of vector elements
313/// See https://ziglang.org/documentation/master/#shuffle313/// See https://ziglang.org/documentation/master/#shuffle
314/// See https://clang.llvm.org/docs/LanguageExtensions.html#langext-builtin-shufflevector314/// See https://clang.llvm.org/docs/LanguageExtensions.html#langext-builtin-shufflevector
315pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len: usize) i32 {315pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len: usize) i32 {
316 if (this_index <= 0) return 0;316 const positive_index = std.math.cast(usize, this_index) orelse return undefined;
317
318 const positive_index = @as(usize, @intCast(this_index));
319 if (positive_index < source_vector_len) return @as(i32, @intCast(this_index));317 if (positive_index < source_vector_len) return @as(i32, @intCast(this_index));
320 const b_index = positive_index - source_vector_len;318 const b_index = positive_index - source_vector_len;
321 return ~@as(i32, @intCast(b_index));319 return ~@as(i32, @intCast(b_index));
...@@ -324,7 +322,7 @@ pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len...@@ -324,7 +322,7 @@ pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len
324test "shuffleVectorIndex" {322test "shuffleVectorIndex" {
325 const vector_len: usize = 4;323 const vector_len: usize = 4;
326324
327 try testing.expect(shuffleVectorIndex(-1, vector_len) == 0);325 _ = shuffleVectorIndex(-1, vector_len);
328326
329 try testing.expect(shuffleVectorIndex(0, vector_len) == 0);327 try testing.expect(shuffleVectorIndex(0, vector_len) == 0);
330 try testing.expect(shuffleVectorIndex(1, vector_len) == 1);328 try testing.expect(shuffleVectorIndex(1, vector_len) == 1);
src/InternPool.zig+7-2
...@@ -3587,6 +3587,7 @@ pub const Alignment = enum(u6) {...@@ -3587,6 +3587,7 @@ pub const Alignment = enum(u6) {
3587 @"8" = 3,3587 @"8" = 3,
3588 @"16" = 4,3588 @"16" = 4,
3589 @"32" = 5,3589 @"32" = 5,
3590 @"64" = 6,
3590 none = std.math.maxInt(u6),3591 none = std.math.maxInt(u6),
3591 _,3592 _,
35923593
...@@ -7403,10 +7404,14 @@ pub fn isIntegerType(ip: *const InternPool, ty: Index) bool {...@@ -7403,10 +7404,14 @@ pub fn isIntegerType(ip: *const InternPool, ty: Index) bool {
7403 .c_ulong_type,7404 .c_ulong_type,
7404 .c_longlong_type,7405 .c_longlong_type,
7405 .c_ulonglong_type,7406 .c_ulonglong_type,
7406 .c_longdouble_type,
7407 .comptime_int_type,7407 .comptime_int_type,
7408 => true,7408 => true,
7409 else => ip.indexToKey(ty) == .int_type,7409 else => switch (ip.items.items(.tag)[@intFromEnum(ty)]) {
7410 .type_int_signed,
7411 .type_int_unsigned,
7412 => true,
7413 else => false,
7414 },
7410 };7415 };
7411}7416}
74127417
src/Sema.zig+31-2
...@@ -23315,7 +23315,8 @@ fn checkVectorElemType(...@@ -23315,7 +23315,8 @@ fn checkVectorElemType(
23315 const mod = sema.mod;23315 const mod = sema.mod;
23316 switch (ty.zigTypeTag(mod)) {23316 switch (ty.zigTypeTag(mod)) {
23317 .Int, .Float, .Bool => return,23317 .Int, .Float, .Bool => return,
23318 else => if (ty.isPtrAtRuntime(mod)) return,23318 .Optional, .Pointer => if (ty.isPtrAtRuntime(mod)) return,
23319 else => {},
23319 }23320 }
23320 return sema.fail(block, ty_src, "expected integer, float, bool, or pointer for the vector element type; found '{}'", .{ty.fmt(mod)});23321 return sema.fail(block, ty_src, "expected integer, float, bool, or pointer for the vector element type; found '{}'", .{ty.fmt(mod)});
23321}23322}
...@@ -28442,7 +28443,7 @@ const CoerceOpts = struct {...@@ -28442,7 +28443,7 @@ const CoerceOpts = struct {
28442 report_err: bool = true,28443 report_err: bool = true,
28443 /// Ignored if `report_err == false`.28444 /// Ignored if `report_err == false`.
28444 is_ret: bool = false,28445 is_ret: bool = false,
28445 /// Should coercion to comptime_int ermit an error message.28446 /// Should coercion to comptime_int emit an error message.
28446 no_cast_to_comptime_int: bool = false,28447 no_cast_to_comptime_int: bool = false,
2844728448
28448 param_src: struct {28449 param_src: struct {
...@@ -31845,6 +31846,34 @@ fn coerceArrayLike(...@@ -31845,6 +31846,34 @@ fn coerceArrayLike(
31845 }31846 }
3184631847
31847 const dest_elem_ty = dest_ty.childType(mod);31848 const dest_elem_ty = dest_ty.childType(mod);
31849 if (dest_ty.isVector(mod) and inst_ty.isVector(mod) and (try sema.resolveValue(inst)) == null) {
31850 const inst_elem_ty = inst_ty.childType(mod);
31851 switch (dest_elem_ty.zigTypeTag(mod)) {
31852 .Int => if (inst_elem_ty.isInt(mod)) {
31853 // integer widening
31854 const dst_info = dest_elem_ty.intInfo(mod);
31855 const src_info = inst_elem_ty.intInfo(mod);
31856 if ((src_info.signedness == dst_info.signedness and dst_info.bits >= src_info.bits) or
31857 // small enough unsigned ints can get casted to large enough signed ints
31858 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))
31859 {
31860 try sema.requireRuntimeBlock(block, inst_src, null);
31861 return block.addTyOp(.intcast, dest_ty, inst);
31862 }
31863 },
31864 .Float => if (inst_elem_ty.isRuntimeFloat()) {
31865 // float widening
31866 const src_bits = inst_elem_ty.floatBits(target);
31867 const dst_bits = dest_elem_ty.floatBits(target);
31868 if (dst_bits >= src_bits) {
31869 try sema.requireRuntimeBlock(block, inst_src, null);
31870 return block.addTyOp(.fpext, dest_ty, inst);
31871 }
31872 },
31873 else => {},
31874 }
31875 }
31876
31848 const element_vals = try sema.arena.alloc(InternPool.Index, dest_len);31877 const element_vals = try sema.arena.alloc(InternPool.Index, dest_len);
31849 const element_refs = try sema.arena.alloc(Air.Inst.Ref, dest_len);31878 const element_refs = try sema.arena.alloc(Air.Inst.Ref, dest_len);
31850 var runtime_src: ?LazySrcLoc = null;31879 var runtime_src: ?LazySrcLoc = null;
src/arch/x86_64/CodeGen.zig+1659-327
...@@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory(...@@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory(
1547 });1547 });
1548}1548}
15491549
1550fn asmRegisterRegisterMemoryRegister(
1551 self: *Self,
1552 tag: Mir.Inst.FixedTag,
1553 reg1: Register,
1554 reg2: Register,
1555 m: Memory,
1556 reg3: Register,
1557) !void {
1558 _ = try self.addInst(.{
1559 .tag = tag[1],
1560 .ops = .rrmr,
1561 .data = .{ .rrrx = .{
1562 .fixes = tag[0],
1563 .r1 = reg1,
1564 .r2 = reg2,
1565 .r3 = reg3,
1566 .payload = try self.addExtra(Mir.Memory.encode(m)),
1567 } },
1568 });
1569}
1570
1550fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {1571fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1551 _ = try self.addInst(.{1572 _ = try self.addInst(.{
1552 .tag = tag[1],1573 .tag = tag[1],
...@@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo...@@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
1570 });1591 });
1571}1592}
15721593
1594fn asmRegisterMemoryRegister(
1595 self: *Self,
1596 tag: Mir.Inst.FixedTag,
1597 reg1: Register,
1598 m: Memory,
1599 reg2: Register,
1600) !void {
1601 _ = try self.addInst(.{
1602 .tag = tag[1],
1603 .ops = .rmr,
1604 .data = .{ .rrx = .{
1605 .fixes = tag[0],
1606 .r1 = reg1,
1607 .r2 = reg2,
1608 .payload = try self.addExtra(Mir.Memory.encode(m)),
1609 } },
1610 });
1611}
1612
1573fn asmRegisterMemoryImmediate(1613fn asmRegisterMemoryImmediate(
1574 self: *Self,1614 self: *Self,
1575 tag: Mir.Inst.FixedTag,1615 tag: Mir.Inst.FixedTag,
...@@ -2570,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -2570,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
25702610
2571 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;2611 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
2572 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =2612 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2573 if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);2613 if (opts.update_tracking)
2614 {} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
25742615
2575 var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(2616 var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
2576 stack.get(),2617 stack.get(),
...@@ -2812,11 +2853,14 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2812,11 +2853,14 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2812}2853}
28132854
2814fn airFpext(self: *Self, inst: Air.Inst.Index) !void {2855fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2856 const mod = self.bin_file.comp.module.?;
2815 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2857 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2816 const dst_ty = self.typeOfIndex(inst);2858 const dst_ty = self.typeOfIndex(inst);
2817 const dst_bits = dst_ty.floatBits(self.target.*);2859 const dst_scalar_ty = dst_ty.scalarType(mod);
2860 const dst_bits = dst_scalar_ty.floatBits(self.target.*);
2818 const src_ty = self.typeOf(ty_op.operand);2861 const src_ty = self.typeOf(ty_op.operand);
2819 const src_bits = src_ty.floatBits(self.target.*);2862 const src_scalar_ty = src_ty.scalarType(mod);
2863 const src_bits = src_scalar_ty.floatBits(self.target.*);
28202864
2821 const result = result: {2865 const result = result: {
2822 if (switch (src_bits) {2866 if (switch (src_bits) {
...@@ -2840,94 +2884,290 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2840,94 +2884,290 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2840 },2884 },
2841 else => unreachable,2885 else => unreachable,
2842 }) {2886 }) {
2887 if (dst_ty.isVector(mod)) break :result null;
2843 var callee_buf: ["__extend?f?f2".len]u8 = undefined;2888 var callee_buf: ["__extend?f?f2".len]u8 = undefined;
2844 break :result try self.genCall(.{ .lib = .{2889 break :result try self.genCall(.{ .lib = .{
2845 .return_type = self.floatCompilerRtAbiType(dst_ty, src_ty).toIntern(),2890 .return_type = self.floatCompilerRtAbiType(dst_scalar_ty, src_scalar_ty).toIntern(),
2846 .param_types = &.{self.floatCompilerRtAbiType(src_ty, dst_ty).toIntern()},2891 .param_types = &.{self.floatCompilerRtAbiType(src_scalar_ty, dst_scalar_ty).toIntern()},
2847 .callee = std.fmt.bufPrint(&callee_buf, "__extend{c}f{c}f2", .{2892 .callee = std.fmt.bufPrint(&callee_buf, "__extend{c}f{c}f2", .{
2848 floatCompilerRtAbiName(src_bits),2893 floatCompilerRtAbiName(src_bits),
2849 floatCompilerRtAbiName(dst_bits),2894 floatCompilerRtAbiName(dst_bits),
2850 }) catch unreachable,2895 }) catch unreachable,
2851 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});2896 } }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }});
2852 }2897 }
28532898
2899 const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
2854 const src_mcv = try self.resolveInst(ty_op.operand);2900 const src_mcv = try self.resolveInst(ty_op.operand);
2855 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))2901 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2856 src_mcv2902 src_mcv
2857 else2903 else
2858 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);2904 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2859 const dst_reg = dst_mcv.getReg().?.to128();2905 const dst_reg = dst_mcv.getReg().?;
2906 const dst_alias = registerAlias(dst_reg, @intCast(@max(dst_ty.abiSize(mod), 16)));
2860 const dst_lock = self.register_manager.lockReg(dst_reg);2907 const dst_lock = self.register_manager.lockReg(dst_reg);
2861 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);2908 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
28622909
2910 const vec_len = if (dst_ty.isVector(mod)) dst_ty.vectorLen(mod) else 1;
2863 if (src_bits == 16) {2911 if (src_bits == 16) {
2864 assert(self.hasFeature(.f16c));2912 assert(self.hasFeature(.f16c));
2865 const mat_src_reg = if (src_mcv.isRegister())2913 const mat_src_reg = if (src_mcv.isRegister())
2866 src_mcv.getReg().?2914 src_mcv.getReg().?
2867 else2915 else
2868 try self.copyToTmpRegister(src_ty, src_mcv);2916 try self.copyToTmpRegister(src_ty, src_mcv);
2869 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128());2917 try self.asmRegisterRegister(
2918 .{ .v_ps, .cvtph2 },
2919 dst_alias,
2920 registerAlias(mat_src_reg, src_abi_size),
2921 );
2870 switch (dst_bits) {2922 switch (dst_bits) {
2871 32 => {},2923 32 => {},
2872 64 => try self.asmRegisterRegisterRegister(2924 64 => try self.asmRegisterRegisterRegister(
2873 .{ .v_sd, .cvtss2 },2925 .{ .v_sd, .cvtss2 },
2874 dst_reg,2926 dst_alias,
2875 dst_reg,2927 dst_alias,
2876 dst_reg,2928 dst_alias,
2877 ),2929 ),
2878 else => unreachable,2930 else => unreachable,
2879 }2931 }
2880 } else {2932 } else {
2881 assert(src_bits == 32 and dst_bits == 64);2933 assert(src_bits == 32 and dst_bits == 64);
2882 if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(2934 if (self.hasFeature(.avx)) switch (vec_len) {
2883 .{ .v_sd, .cvtss2 },2935 1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2884 dst_reg,2936 .{ .v_sd, .cvtss2 },
2885 dst_reg,2937 dst_alias,
2886 try src_mcv.mem(self, .dword),2938 dst_alias,
2887 ) else try self.asmRegisterRegisterRegister(2939 try src_mcv.mem(self, self.memSize(src_ty)),
2888 .{ .v_sd, .cvtss2 },2940 ) else try self.asmRegisterRegisterRegister(
2889 dst_reg,2941 .{ .v_sd, .cvtss2 },
2890 dst_reg,2942 dst_alias,
2891 (if (src_mcv.isRegister())2943 dst_alias,
2892 src_mcv.getReg().?2944 registerAlias(if (src_mcv.isRegister())
2893 else2945 src_mcv.getReg().?
2894 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),2946 else
2895 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(2947 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
2896 .{ ._sd, .cvtss2 },2948 ),
2897 dst_reg,2949 2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(
2898 try src_mcv.mem(self, .dword),2950 .{ .v_pd, .cvtps2 },
2951 dst_alias,
2952 try src_mcv.mem(self, self.memSize(src_ty)),
2953 ) else try self.asmRegisterRegister(
2954 .{ .v_pd, .cvtps2 },
2955 dst_alias,
2956 registerAlias(if (src_mcv.isRegister())
2957 src_mcv.getReg().?
2958 else
2959 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
2960 ),
2961 else => break :result null,
2962 } else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2963 switch (vec_len) {
2964 1 => .{ ._sd, .cvtss2 },
2965 2 => .{ ._pd, .cvtps2 },
2966 else => break :result null,
2967 },
2968 dst_alias,
2969 try src_mcv.mem(self, self.memSize(src_ty)),
2899 ) else try self.asmRegisterRegister(2970 ) else try self.asmRegisterRegister(
2900 .{ ._sd, .cvtss2 },2971 switch (vec_len) {
2901 dst_reg,2972 1 => .{ ._sd, .cvtss2 },
2902 (if (src_mcv.isRegister())2973 2 => .{ ._pd, .cvtps2 },
2974 else => break :result null,
2975 },
2976 dst_alias,
2977 registerAlias(if (src_mcv.isRegister())
2903 src_mcv.getReg().?2978 src_mcv.getReg().?
2904 else2979 else
2905 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),2980 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
2906 );2981 );
2907 }2982 }
2908 break :result dst_mcv;2983 break :result dst_mcv;
2909 };2984 } orelse return self.fail("TODO implement airFpext from {} to {}", .{
2985 src_ty.fmt(mod), dst_ty.fmt(mod),
2986 });
2910 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2987 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2911}2988}
29122989
2913fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {2990fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2914 const mod = self.bin_file.comp.module.?;2991 const mod = self.bin_file.comp.module.?;
2915 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2992 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2916 const result: MCValue = result: {2993 const src_ty = self.typeOf(ty_op.operand);
2917 const src_ty = self.typeOf(ty_op.operand);2994 const dst_ty = self.typeOfIndex(inst);
2918 const src_int_info = src_ty.intInfo(mod);
29192995
2920 const dst_ty = self.typeOfIndex(inst);2996 const result = @as(?MCValue, result: {
2921 const dst_int_info = dst_ty.intInfo(mod);2997 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
2922 const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
29232998
2924 const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;2999 const src_int_info = src_ty.intInfo(mod);
3000 const dst_int_info = dst_ty.intInfo(mod);
2925 const extend = switch (src_int_info.signedness) {3001 const extend = switch (src_int_info.signedness) {
2926 .signed => dst_int_info,3002 .signed => dst_int_info,
2927 .unsigned => src_int_info,3003 .unsigned => src_int_info,
2928 }.signedness;3004 }.signedness;
29293005
2930 const src_mcv = try self.resolveInst(ty_op.operand);3006 const src_mcv = try self.resolveInst(ty_op.operand);
3007 if (dst_ty.isVector(mod)) {
3008 const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
3009 const max_abi_size = @max(dst_abi_size, src_abi_size);
3010 if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null;
3011 const has_avx = self.hasFeature(.avx);
3012
3013 const dst_elem_abi_size = dst_ty.childType(mod).abiSize(mod);
3014 const src_elem_abi_size = src_ty.childType(mod).abiSize(mod);
3015 switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {
3016 .lt => {
3017 const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
3018 else => break :result null,
3019 1 => switch (src_elem_abi_size) {
3020 else => break :result null,
3021 2 => switch (dst_int_info.signedness) {
3022 .signed => if (has_avx) .{ .vp_b, .ackssw } else .{ .p_b, .ackssw },
3023 .unsigned => if (has_avx) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
3024 },
3025 },
3026 2 => switch (src_elem_abi_size) {
3027 else => break :result null,
3028 4 => switch (dst_int_info.signedness) {
3029 .signed => if (has_avx) .{ .vp_w, .ackssd } else .{ .p_w, .ackssd },
3030 .unsigned => if (has_avx)
3031 .{ .vp_w, .ackusd }
3032 else if (self.hasFeature(.sse4_1))
3033 .{ .p_w, .ackusd }
3034 else
3035 break :result null,
3036 },
3037 },
3038 };
3039
3040 const dst_mcv: MCValue = if (src_mcv.isRegister() and
3041 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3042 src_mcv
3043 else if (has_avx and src_mcv.isRegister())
3044 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
3045 else
3046 try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv);
3047 const dst_reg = dst_mcv.getReg().?;
3048 const dst_alias = registerAlias(dst_reg, dst_abi_size);
3049
3050 if (has_avx) try self.asmRegisterRegisterRegister(
3051 mir_tag,
3052 dst_alias,
3053 registerAlias(if (src_mcv.isRegister())
3054 src_mcv.getReg().?
3055 else
3056 dst_reg, src_abi_size),
3057 dst_alias,
3058 ) else try self.asmRegisterRegister(
3059 mir_tag,
3060 dst_alias,
3061 dst_alias,
3062 );
3063 break :result dst_mcv;
3064 },
3065 .eq => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3066 break :result src_mcv
3067 else {
3068 const dst_mcv = try self.allocRegOrMem(inst, true);
3069 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
3070 break :result dst_mcv;
3071 },
3072 .gt => if (self.hasFeature(.sse4_1)) {
3073 const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) {
3074 else => break :result null,
3075 2 => if (has_avx) .vp_w else .p_w,
3076 4 => if (has_avx) .vp_d else .p_d,
3077 8 => if (has_avx) .vp_q else .p_q,
3078 }, switch (src_elem_abi_size) {
3079 else => break :result null,
3080 1 => switch (extend) {
3081 .signed => .movsxb,
3082 .unsigned => .movzxb,
3083 },
3084 2 => switch (extend) {
3085 .signed => .movsxw,
3086 .unsigned => .movzxw,
3087 },
3088 4 => switch (extend) {
3089 .signed => .movsxd,
3090 .unsigned => .movzxd,
3091 },
3092 } };
3093
3094 const dst_mcv: MCValue = if (src_mcv.isRegister() and
3095 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3096 src_mcv
3097 else
3098 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) };
3099 const dst_reg = dst_mcv.getReg().?;
3100 const dst_alias = registerAlias(dst_reg, dst_abi_size);
3101
3102 if (src_mcv.isMemory()) try self.asmRegisterMemory(
3103 mir_tag,
3104 dst_alias,
3105 try src_mcv.mem(self, self.memSize(src_ty)),
3106 ) else try self.asmRegisterRegister(
3107 mir_tag,
3108 dst_alias,
3109 registerAlias(if (src_mcv.isRegister())
3110 src_mcv.getReg().?
3111 else
3112 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
3113 );
3114 break :result dst_mcv;
3115 } else {
3116 const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
3117 else => break :result null,
3118 2 => switch (src_elem_abi_size) {
3119 else => break :result null,
3120 1 => .{ .p_, .unpcklbw },
3121 },
3122 4 => switch (src_elem_abi_size) {
3123 else => break :result null,
3124 2 => .{ .p_, .unpcklwd },
3125 },
3126 8 => switch (src_elem_abi_size) {
3127 else => break :result null,
3128 2 => .{ .p_, .unpckldq },
3129 },
3130 };
3131
3132 const dst_mcv: MCValue = if (src_mcv.isRegister() and
3133 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3134 src_mcv
3135 else
3136 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
3137 const dst_reg = dst_mcv.getReg().?;
3138
3139 const ext_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
3140 const ext_alias = registerAlias(ext_reg, src_abi_size);
3141 const ext_lock = self.register_manager.lockRegAssumeUnused(ext_reg);
3142 defer self.register_manager.unlockReg(ext_lock);
3143
3144 try self.asmRegisterRegister(.{ .p_, .xor }, ext_alias, ext_alias);
3145 switch (extend) {
3146 .signed => try self.asmRegisterRegister(
3147 .{ switch (src_elem_abi_size) {
3148 else => unreachable,
3149 1 => .p_b,
3150 2 => .p_w,
3151 4 => .p_d,
3152 }, .cmpgt },
3153 ext_alias,
3154 registerAlias(dst_reg, src_abi_size),
3155 ),
3156 .unsigned => {},
3157 }
3158 try self.asmRegisterRegister(
3159 mir_tag,
3160 registerAlias(dst_reg, dst_abi_size),
3161 registerAlias(ext_reg, dst_abi_size),
3162 );
3163 break :result dst_mcv;
3164 },
3165 }
3166 @compileError("unreachable");
3167 }
3168
3169 const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
3170
2931 const src_storage_bits: u16 = switch (src_mcv) {3171 const src_storage_bits: u16 = switch (src_mcv) {
2932 .register, .register_offset => 64,3172 .register, .register_offset => 64,
2933 .register_pair => 128,3173 .register_pair => 128,
...@@ -2945,13 +3185,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2945,13 +3185,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2945 };3185 };
29463186
2947 if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())3187 if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())
2948 .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) }3188 .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }
2949 else3189 else
2950 dst_mcv;3190 dst_mcv;
29513191
2952 if (dst_mcv.isRegister()) {3192 if (dst_mcv.isRegister()) {
2953 try self.truncateRegister(src_ty, dst_mcv.getReg().?);3193 try self.truncateRegister(src_ty, dst_mcv.getReg().?);
2954 break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };3194 break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
2955 }3195 }
29563196
2957 const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;3197 const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
...@@ -2999,7 +3239,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2999,7 +3239,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2999 );3239 );
30003240
3001 break :result dst_mcv;3241 break :result dst_mcv;
3002 };3242 }) orelse return self.fail("TODO implement airIntCast from {} to {}", .{
3243 src_ty.fmt(mod), dst_ty.fmt(mod),
3244 });
3003 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3245 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3004}3246}
30053247
...@@ -3022,7 +3264,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3022,7 +3264,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3022 src_mcv3264 src_mcv
3023 else if (dst_abi_size <= 8)3265 else if (dst_abi_size <= 8)
3024 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)3266 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)
3025 else if (dst_abi_size <= 16) dst: {3267 else if (dst_abi_size <= 16 and !dst_ty.isVector(mod)) dst: {
3026 const dst_regs =3268 const dst_regs =
3027 try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);3269 try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
3028 const dst_mcv: MCValue = .{ .register_pair = dst_regs };3270 const dst_mcv: MCValue = .{ .register_pair = dst_regs };
...@@ -3032,26 +3274,29 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3032,26 +3274,29 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3032 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});3274 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
3033 break :dst dst_mcv;3275 break :dst dst_mcv;
3034 } else dst: {3276 } else dst: {
3035 const dst_mcv = try self.allocRegOrMem(inst, true);3277 const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true);
3036 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});3278 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
3037 break :dst dst_mcv;3279 break :dst dst_mcv;
3038 };3280 };
30393281
3040 if (dst_ty.zigTypeTag(mod) == .Vector) {3282 if (dst_ty.zigTypeTag(mod) == .Vector) {
3041 assert(src_ty.zigTypeTag(mod) == .Vector and dst_ty.vectorLen(mod) == src_ty.vectorLen(mod));3283 assert(src_ty.zigTypeTag(mod) == .Vector and dst_ty.vectorLen(mod) == src_ty.vectorLen(mod));
3042 const dst_info = dst_ty.childType(mod).intInfo(mod);3284 const dst_elem_ty = dst_ty.childType(mod);
3043 const src_info = src_ty.childType(mod).intInfo(mod);3285 const dst_elem_abi_size: u32 = @intCast(dst_elem_ty.abiSize(mod));
3044 const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_info.bits) {3286 const src_elem_ty = src_ty.childType(mod);
3045 8 => switch (src_info.bits) {3287 const src_elem_abi_size: u32 = @intCast(src_elem_ty.abiSize(mod));
3046 16 => switch (dst_ty.vectorLen(mod)) {3288
3289 const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_elem_abi_size) {
3290 1 => switch (src_elem_abi_size) {
3291 2 => switch (dst_ty.vectorLen(mod)) {
3047 1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },3292 1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
3048 9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,3293 9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,
3049 else => null,3294 else => null,
3050 },3295 },
3051 else => null,3296 else => null,
3052 },3297 },
3053 16 => switch (src_info.bits) {3298 2 => switch (src_elem_abi_size) {
3054 32 => switch (dst_ty.vectorLen(mod)) {3299 4 => switch (dst_ty.vectorLen(mod)) {
3055 1...4 => if (self.hasFeature(.avx))3300 1...4 => if (self.hasFeature(.avx))
3056 .{ .vp_w, .ackusd }3301 .{ .vp_w, .ackusd }
3057 else if (self.hasFeature(.sse4_1))3302 else if (self.hasFeature(.sse4_1))
...@@ -3066,12 +3311,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3066,12 +3311,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3066 else => null,3311 else => null,
3067 }) orelse return self.fail("TODO implement airTrunc for {}", .{dst_ty.fmt(mod)});3312 }) orelse return self.fail("TODO implement airTrunc for {}", .{dst_ty.fmt(mod)});
30683313
3069 const elem_ty = src_ty.childType(mod);3314 const dst_info = dst_elem_ty.intInfo(mod);
3070 const mask_val = try mod.intValue(elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));3315 const src_info = src_elem_ty.intInfo(mod);
3316
3317 const mask_val = try mod.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
30713318
3072 const splat_ty = try mod.vectorType(.{3319 const splat_ty = try mod.vectorType(.{
3073 .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),3320 .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
3074 .child = elem_ty.ip_index,3321 .child = src_elem_ty.ip_index,
3075 });3322 });
3076 const splat_abi_size: u32 = @intCast(splat_ty.abiSize(mod));3323 const splat_abi_size: u32 = @intCast(splat_ty.abiSize(mod));
30773324
...@@ -3086,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3086,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3086 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },3333 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
3087 };3334 };
30883335
3089 const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size);3336 const dst_reg = dst_mcv.getReg().?;
3337 const dst_alias = registerAlias(dst_reg, src_abi_size);
3090 if (self.hasFeature(.avx)) {3338 if (self.hasFeature(.avx)) {
3091 try self.asmRegisterRegisterMemory(3339 try self.asmRegisterRegisterMemory(
3092 .{ .vp_, .@"and" },3340 .{ .vp_, .@"and" },
3093 dst_reg,3341 dst_alias,
3094 dst_reg,3342 dst_alias,
3095 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),3343 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3096 );3344 );
3097 try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);3345 if (src_abi_size > 16) {
3346 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
3347 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
3348 defer self.register_manager.unlockReg(temp_lock);
3349
3350 try self.asmRegisterRegisterImmediate(
3351 .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
3352 registerAlias(temp_reg, dst_abi_size),
3353 dst_alias,
3354 Immediate.u(1),
3355 );
3356 try self.asmRegisterRegisterRegister(
3357 mir_tag,
3358 registerAlias(dst_reg, dst_abi_size),
3359 registerAlias(dst_reg, dst_abi_size),
3360 registerAlias(temp_reg, dst_abi_size),
3361 );
3362 } else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias);
3098 } else {3363 } else {
3099 try self.asmRegisterMemory(3364 try self.asmRegisterMemory(
3100 .{ .p_, .@"and" },3365 .{ .p_, .@"and" },
3101 dst_reg,3366 dst_alias,
3102 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),3367 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3103 );3368 );
3104 try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);3369 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
3105 }3370 }
3106 break :result dst_mcv;3371 break :result dst_mcv;
3107 }3372 }
...@@ -4045,7 +4310,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4045,7 +4310,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4045 if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {4310 if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
4046 const slow_inc = self.hasFeature(.slow_incdec);4311 const slow_inc = self.hasFeature(.slow_incdec);
4047 const abi_size: u32 = @intCast(dst_ty.abiSize(mod));4312 const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
4048 const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;4313 const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;
40494314
4050 try self.spillRegisters(&.{ .rax, .rcx, .rdx });4315 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
4051 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });4316 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
...@@ -4534,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4534,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4534 switch (lhs_ty.zigTypeTag(mod)) {4799 switch (lhs_ty.zigTypeTag(mod)) {
4535 .Int => {4800 .Int => {
4536 try self.spillRegisters(&.{.rcx});4801 try self.spillRegisters(&.{.rcx});
4537 try self.register_manager.getReg(.rcx, null);4802 try self.register_manager.getKnownReg(.rcx, null);
4538 const lhs_mcv = try self.resolveInst(bin_op.lhs);4803 const lhs_mcv = try self.resolveInst(bin_op.lhs);
4539 const rhs_mcv = try self.resolveInst(bin_op.rhs);4804 const rhs_mcv = try self.resolveInst(bin_op.rhs);
45404805
...@@ -6560,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)...@@ -6560,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
65606825
6561 const dst_mcv: MCValue = .{ .register = .st0 };6826 const dst_mcv: MCValue = .{ .register = .st0 };
6562 if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))6827 if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))
6563 try self.register_manager.getReg(.st0, inst);6828 try self.register_manager.getKnownReg(.st0, inst);
65646829
6565 try self.genCopy(ty, dst_mcv, src_mcv, .{});6830 try self.genCopy(ty, dst_mcv, src_mcv, .{});
6566 switch (tag) {6831 switch (tag) {
...@@ -6894,7 +7159,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -6894,7 +7159,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6894 },7159 },
6895 else => {7160 else => {
6896 const abi_size: u31 = @intCast(ty.abiSize(mod));7161 const abi_size: u31 = @intCast(ty.abiSize(mod));
6897 const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;7162 const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;
68987163
6899 const tmp_regs =7164 const tmp_regs =
6900 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);7165 try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
...@@ -8181,7 +8446,7 @@ fn genShiftBinOpMir(...@@ -8181,7 +8446,7 @@ fn genShiftBinOpMir(
8181 try self.asmRegisterImmediate(8446 try self.asmRegisterImmediate(
8182 .{ ._, .@"and" },8447 .{ ._, .@"and" },
8183 .cl,8448 .cl,
8184 Immediate.u(std.math.maxInt(u6)),8449 Immediate.u(math.maxInt(u6)),
8185 );8450 );
8186 try self.asmRegisterImmediate(8451 try self.asmRegisterImmediate(
8187 .{ ._r, .sh },8452 .{ ._r, .sh },
...@@ -8218,7 +8483,7 @@ fn genShiftBinOpMir(...@@ -8218,7 +8483,7 @@ fn genShiftBinOpMir(
8218 try self.asmRegisterImmediate(8483 try self.asmRegisterImmediate(
8219 .{ ._, .@"and" },8484 .{ ._, .@"and" },
8220 .cl,8485 .cl,
8221 Immediate.u(std.math.maxInt(u6)),8486 Immediate.u(math.maxInt(u6)),
8222 );8487 );
8223 try self.asmRegisterImmediate(8488 try self.asmRegisterImmediate(
8224 .{ ._r, .sh },8489 .{ ._r, .sh },
...@@ -8283,7 +8548,7 @@ fn genShiftBinOpMir(...@@ -8283,7 +8548,7 @@ fn genShiftBinOpMir(
8283 }, .sh },8548 }, .sh },
8284 temp_regs[2].to64(),8549 temp_regs[2].to64(),
8285 temp_regs[3].to64(),8550 temp_regs[3].to64(),
8286 Immediate.u(shift_imm & std.math.maxInt(u6)),8551 Immediate.u(shift_imm & math.maxInt(u6)),
8287 ),8552 ),
8288 else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {8553 else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
8289 ._l => ._ld,8554 ._l => ._ld,
...@@ -8338,7 +8603,7 @@ fn genShiftBinOpMir(...@@ -8338,7 +8603,7 @@ fn genShiftBinOpMir(
8338 .immediate => |shift_imm| try self.asmRegisterImmediate(8603 .immediate => |shift_imm| try self.asmRegisterImmediate(
8339 tag,8604 tag,
8340 temp_regs[2].to64(),8605 temp_regs[2].to64(),
8341 Immediate.u(shift_imm & std.math.maxInt(u6)),8606 Immediate.u(shift_imm & math.maxInt(u6)),
8342 ),8607 ),
8343 else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),8608 else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
8344 }8609 }
...@@ -8794,7 +9059,7 @@ fn genShiftBinOp(...@@ -8794,7 +9059,7 @@ fn genShiftBinOp(
8794 lhs_ty.fmt(mod),9059 lhs_ty.fmt(mod),
8795 });9060 });
87969061
8797 try self.register_manager.getReg(.rcx, null);9062 try self.register_manager.getKnownReg(.rcx, null);
8798 const rcx_lock = self.register_manager.lockReg(.rcx);9063 const rcx_lock = self.register_manager.lockReg(.rcx);
8799 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);9064 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
88009065
...@@ -8933,7 +9198,7 @@ fn genMulDivBinOp(...@@ -8933,7 +9198,7 @@ fn genMulDivBinOp(
8933 switch (tag) {9198 switch (tag) {
8934 .mul, .mul_wrap => {9199 .mul, .mul_wrap => {
8935 const slow_inc = self.hasFeature(.slow_incdec);9200 const slow_inc = self.hasFeature(.slow_incdec);
8936 const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;9201 const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;
89379202
8938 try self.spillRegisters(&.{ .rax, .rcx, .rdx });9203 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
8939 const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });9204 const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
...@@ -9117,8 +9382,8 @@ fn genMulDivBinOp(...@@ -9117,8 +9382,8 @@ fn genMulDivBinOp(
9117 .rem => maybe_inst,9382 .rem => maybe_inst,
9118 else => null,9383 else => null,
9119 };9384 };
9120 try self.register_manager.getReg(.rax, track_inst_rax);9385 try self.register_manager.getKnownReg(.rax, track_inst_rax);
9121 try self.register_manager.getReg(.rdx, track_inst_rdx);9386 try self.register_manager.getKnownReg(.rdx, track_inst_rdx);
91229387
9123 try self.genIntMulDivOpMir(switch (signedness) {9388 try self.genIntMulDivOpMir(switch (signedness) {
9124 .signed => switch (tag) {9389 .signed => switch (tag) {
...@@ -9158,8 +9423,11 @@ fn genMulDivBinOp(...@@ -9158,8 +9423,11 @@ fn genMulDivBinOp(
9158 },9423 },
91599424
9160 .mod => {9425 .mod => {
9161 try self.register_manager.getReg(.rax, null);9426 try self.register_manager.getKnownReg(.rax, null);
9162 try self.register_manager.getReg(.rdx, if (signedness == .unsigned) maybe_inst else null);9427 try self.register_manager.getKnownReg(
9428 .rdx,
9429 if (signedness == .unsigned) maybe_inst else null,
9430 );
91639431
9164 switch (signedness) {9432 switch (signedness) {
9165 .signed => {9433 .signed => {
...@@ -9200,8 +9468,11 @@ fn genMulDivBinOp(...@@ -9200,8 +9468,11 @@ fn genMulDivBinOp(
9200 },9468 },
92019469
9202 .div_floor => {9470 .div_floor => {
9203 try self.register_manager.getReg(.rax, if (signedness == .unsigned) maybe_inst else null);9471 try self.register_manager.getKnownReg(
9204 try self.register_manager.getReg(.rdx, null);9472 .rax,
9473 if (signedness == .unsigned) maybe_inst else null,
9474 );
9475 try self.register_manager.getKnownReg(.rdx, null);
92059476
9206 const lhs_lock: ?RegisterLock = switch (lhs_mcv) {9477 const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
9207 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),9478 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
...@@ -9445,7 +9716,7 @@ fn genBinOp(...@@ -9445,7 +9716,7 @@ fn genBinOp(
9445 .rem, .mod => unreachable,9716 .rem, .mod => unreachable,
9446 .max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias(9717 .max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias(
9447 if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {9718 if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {
9448 try self.register_manager.getReg(.xmm0, null);9719 try self.register_manager.getKnownReg(.xmm0, null);
9449 break :mask .xmm0;9720 break :mask .xmm0;
9450 } else try self.register_manager.allocReg(null, abi.RegisterClass.sse),9721 } else try self.register_manager.allocReg(null, abi.RegisterClass.sse),
9451 abi_size,9722 abi_size,
...@@ -10820,96 +11091,35 @@ fn genBinOp(...@@ -10820,96 +11091,35 @@ fn genBinOp(
10820 lhs_copy_reg.?,11091 lhs_copy_reg.?,
10821 mask_reg,11092 mask_reg,
10822 ) else {11093 ) else {
10823 try self.asmRegisterRegister(11094 const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(mod)) {
10824 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {11095 .Float => switch (lhs_ty.floatBits(self.target.*)) {
10825 .Float => switch (lhs_ty.floatBits(self.target.*)) {11096 32 => ._ps,
10826 32 => .{ ._ps, .@"and" },11097 64 => ._pd,
10827 64 => .{ ._pd, .@"and" },11098 16, 80, 128 => null,
10828 16, 80, 128 => null,
10829 else => unreachable,
10830 },
10831 .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10832 .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10833 32 => switch (lhs_ty.vectorLen(mod)) {
10834 1...4 => .{ ._ps, .@"and" },
10835 else => null,
10836 },
10837 64 => switch (lhs_ty.vectorLen(mod)) {
10838 1...2 => .{ ._pd, .@"and" },
10839 else => null,
10840 },
10841 16, 80, 128 => null,
10842 else => unreachable,
10843 },
10844 else => unreachable,
10845 },
10846 else => unreachable,11099 else => unreachable,
10847 }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{11100 },
10848 @tagName(air_tag), lhs_ty.fmt(mod),11101 .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10849 }),11102 .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10850 dst_reg,11103 32 => switch (lhs_ty.vectorLen(mod)) {
10851 mask_reg,11104 1...4 => ._ps,
10852 );11105 else => null,
10853 try self.asmRegisterRegister(
10854 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
10855 .Float => switch (lhs_ty.floatBits(self.target.*)) {
10856 32 => .{ ._ps, .andn },
10857 64 => .{ ._pd, .andn },
10858 16, 80, 128 => null,
10859 else => unreachable,
10860 },
10861 .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10862 .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10863 32 => switch (lhs_ty.vectorLen(mod)) {
10864 1...4 => .{ ._ps, .andn },
10865 else => null,
10866 },
10867 64 => switch (lhs_ty.vectorLen(mod)) {
10868 1...2 => .{ ._pd, .andn },
10869 else => null,
10870 },
10871 16, 80, 128 => null,
10872 else => unreachable,
10873 },11106 },
10874 else => unreachable,11107 64 => switch (lhs_ty.vectorLen(mod)) {
10875 },11108 1...2 => ._pd,
10876 else => unreachable,11109 else => null,
10877 }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
10878 @tagName(air_tag), lhs_ty.fmt(mod),
10879 }),
10880 mask_reg,
10881 lhs_copy_reg.?,
10882 );
10883 try self.asmRegisterRegister(
10884 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
10885 .Float => switch (lhs_ty.floatBits(self.target.*)) {
10886 32 => .{ ._ps, .@"or" },
10887 64 => .{ ._pd, .@"or" },
10888 16, 80, 128 => null,
10889 else => unreachable,
10890 },
10891 .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10892 .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10893 32 => switch (lhs_ty.vectorLen(mod)) {
10894 1...4 => .{ ._ps, .@"or" },
10895 else => null,
10896 },
10897 64 => switch (lhs_ty.vectorLen(mod)) {
10898 1...2 => .{ ._pd, .@"or" },
10899 else => null,
10900 },
10901 16, 80, 128 => null,
10902 else => unreachable,
10903 },11110 },
11111 16, 80, 128 => null,
10904 else => unreachable,11112 else => unreachable,
10905 },11113 },
10906 else => unreachable,11114 else => unreachable,
10907 }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{11115 },
10908 @tagName(air_tag), lhs_ty.fmt(mod),11116 else => unreachable,
10909 }),11117 }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
10910 dst_reg,11118 @tagName(air_tag), lhs_ty.fmt(mod),
10911 mask_reg,11119 });
10912 );11120 try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg);
11121 try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?);
11122 try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg);
10913 }11123 }
10914 },11124 },
10915 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {11125 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
...@@ -12192,48 +12402,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -12192,48 +12402,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
12192fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {12402fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12193 const mod = self.bin_file.comp.module.?;12403 const mod = self.bin_file.comp.module.?;
12194 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;12404 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
12195 const ty = self.typeOf(bin_op.lhs);12405 var ty = self.typeOf(bin_op.lhs);
12406 var null_compare: ?Mir.Inst.Index = null;
1219612407
12197 const result: Condition = result: {12408 const result: Condition = result: {
12198 switch (ty.zigTypeTag(mod)) {
12199 .Float => {
12200 const float_bits = ty.floatBits(self.target.*);
12201 if (switch (float_bits) {
12202 16 => !self.hasFeature(.f16c),
12203 32, 64 => false,
12204 80, 128 => true,
12205 else => unreachable,
12206 }) {
12207 var callee_buf: ["__???f2".len]u8 = undefined;
12208 const ret = try self.genCall(.{ .lib = .{
12209 .return_type = .i32_type,
12210 .param_types = &.{ ty.toIntern(), ty.toIntern() },
12211 .callee = std.fmt.bufPrint(&callee_buf, "__{s}{c}f2", .{
12212 switch (op) {
12213 .eq => "eq",
12214 .neq => "ne",
12215 .lt => "lt",
12216 .lte => "le",
12217 .gt => "gt",
12218 .gte => "ge",
12219 },
12220 floatCompilerRtAbiName(float_bits),
12221 }) catch unreachable,
12222 } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
12223 try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
12224 break :result switch (op) {
12225 .eq => .e,
12226 .neq => .ne,
12227 .lt => .l,
12228 .lte => .le,
12229 .gt => .g,
12230 .gte => .ge,
12231 };
12232 }
12233 },
12234 else => {},
12235 }
12236
12237 try self.spillEflagsIfOccupied();12409 try self.spillEflagsIfOccupied();
1223812410
12239 const lhs_mcv = try self.resolveInst(bin_op.lhs);12411 const lhs_mcv = try self.resolveInst(bin_op.lhs);
...@@ -12260,6 +12432,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -12260,6 +12432,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12260 };12432 };
12261 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);12433 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1226212434
12435 switch (ty.zigTypeTag(mod)) {
12436 .Float => {
12437 const float_bits = ty.floatBits(self.target.*);
12438 if (switch (float_bits) {
12439 16 => !self.hasFeature(.f16c),
12440 32, 64 => false,
12441 80, 128 => true,
12442 else => unreachable,
12443 }) {
12444 var callee_buf: ["__???f2".len]u8 = undefined;
12445 const ret = try self.genCall(.{ .lib = .{
12446 .return_type = .i32_type,
12447 .param_types = &.{ ty.toIntern(), ty.toIntern() },
12448 .callee = std.fmt.bufPrint(&callee_buf, "__{s}{c}f2", .{
12449 switch (op) {
12450 .eq => "eq",
12451 .neq => "ne",
12452 .lt => "lt",
12453 .lte => "le",
12454 .gt => "gt",
12455 .gte => "ge",
12456 },
12457 floatCompilerRtAbiName(float_bits),
12458 }) catch unreachable,
12459 } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
12460 try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
12461 break :result switch (op) {
12462 .eq => .e,
12463 .neq => .ne,
12464 .lt => .l,
12465 .lte => .le,
12466 .gt => .g,
12467 .gte => .ge,
12468 };
12469 }
12470 },
12471 .Optional => if (!ty.optionalReprIsPayload(mod)) {
12472 const opt_ty = ty;
12473 const opt_abi_size: u31 = @intCast(opt_ty.abiSize(mod));
12474 ty = opt_ty.optionalChild(mod);
12475 const payload_abi_size: u31 = @intCast(ty.abiSize(mod));
12476
12477 const temp_lhs_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
12478 const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);
12479 defer self.register_manager.unlockReg(temp_lhs_lock);
12480
12481 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
12482 .{ ._, .mov },
12483 temp_lhs_reg.to8(),
12484 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),
12485 ) else {
12486 try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
12487 try self.asmRegisterImmediate(
12488 .{ ._r, .sh },
12489 registerAlias(temp_lhs_reg, opt_abi_size),
12490 Immediate.u(payload_abi_size * 8),
12491 );
12492 }
12493
12494 const payload_compare = payload_compare: {
12495 if (rhs_mcv.isMemory()) {
12496 const rhs_mem =
12497 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);
12498 try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
12499 const payload_compare = try self.asmJccReloc(.nz, undefined);
12500 try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
12501 break :payload_compare payload_compare;
12502 }
12503
12504 const temp_rhs_reg = try self.copyToTmpRegister(opt_ty, rhs_mcv);
12505 const temp_rhs_lock = self.register_manager.lockRegAssumeUnused(temp_rhs_reg);
12506 defer self.register_manager.unlockReg(temp_rhs_lock);
12507
12508 try self.asmRegisterImmediate(
12509 .{ ._r, .sh },
12510 registerAlias(temp_rhs_reg, opt_abi_size),
12511 Immediate.u(payload_abi_size * 8),
12512 );
12513 try self.asmRegisterRegister(
12514 .{ ._, .@"test" },
12515 temp_lhs_reg.to8(),
12516 temp_rhs_reg.to8(),
12517 );
12518 const payload_compare = try self.asmJccReloc(.nz, undefined);
12519 try self.asmRegisterRegister(
12520 .{ ._, .cmp },
12521 temp_lhs_reg.to8(),
12522 temp_rhs_reg.to8(),
12523 );
12524 break :payload_compare payload_compare;
12525 };
12526 null_compare = try self.asmJmpReloc(undefined);
12527 self.performReloc(payload_compare);
12528 },
12529 else => {},
12530 }
12531
12263 switch (ty.zigTypeTag(mod)) {12532 switch (ty.zigTypeTag(mod)) {
12264 else => {12533 else => {
12265 const abi_size: u16 = @intCast(ty.abiSize(mod));12534 const abi_size: u16 = @intCast(ty.abiSize(mod));
...@@ -12571,6 +12840,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -12571,6 +12840,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12571 }12840 }
12572 };12841 };
1257312842
12843 if (null_compare) |reloc| self.performReloc(reloc);
12574 self.eflags_inst = inst;12844 self.eflags_inst = inst;
12575 return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });12845 return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
12576}12846}
...@@ -13521,6 +13791,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -13521,6 +13791,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
13521 } else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {13791 } else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {
13522 const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;13792 const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;
13523 if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint});13793 if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint});
13794 try self.genCopy(ty, args.items[index], input_mcv, .{});
13524 break :arg args.items[index];13795 break :arg args.items[index];
13525 } else return self.fail("invalid constraint: '{s}'", .{constraint});13796 } else return self.fail("invalid constraint: '{s}'", .{constraint});
13526 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {13797 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
...@@ -13619,25 +13890,26 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -13619,25 +13890,26 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
13619 label_gop.value_ptr.target = @intCast(self.mir_instructions.len);13890 label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
13620 } else continue;13891 } else continue;
1362113892
13622 var mnem_size: ?Memory.Size = null;13893 var mnem_size: ?Memory.Size = if (mem.endsWith(u8, mnem_str, "b"))
13623 const mnem_tag = mnem: {13894 .byte
13624 mnem_size = if (mem.endsWith(u8, mnem_str, "b"))13895 else if (mem.endsWith(u8, mnem_str, "w"))
13625 .byte13896 .word
13626 else if (mem.endsWith(u8, mnem_str, "w"))13897 else if (mem.endsWith(u8, mnem_str, "l"))
13627 .word13898 .dword
13628 else if (mem.endsWith(u8, mnem_str, "l"))13899 else if (mem.endsWith(u8, mnem_str, "q") and
13629 .dword13900 (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))
13630 else if (mem.endsWith(u8, mnem_str, "q"))13901 .qword
13631 .qword13902 else if (mem.endsWith(u8, mnem_str, "t"))
13632 else if (mem.endsWith(u8, mnem_str, "t"))13903 .tbyte
13633 .tbyte13904 else
13634 else13905 null;
13635 break :mnem null;13906 const mnem_tag = while (true) break std.meta.stringToEnum(
13636 break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str[0 .. mnem_str.len - 1]);13907 Instruction.Mnemonic,
13637 } orelse mnem: {13908 mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
13909 ) orelse if (mnem_size) |_| {
13638 mnem_size = null;13910 mnem_size = null;
13639 break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);13911 continue;
13640 } orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});13912 } else return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
13641 if (@as(?Memory.Size, switch (mnem_tag) {13913 if (@as(?Memory.Size, switch (mnem_tag) {
13642 .clflush => .byte,13914 .clflush => .byte,
13643 .fldenv, .fnstenv, .fstenv => .none,13915 .fldenv, .fnstenv, .fstenv => .none,
...@@ -14135,30 +14407,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14135,30 +14407,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14135 else => {},14407 else => {},
14136 },14408 },
14137 .Int => switch (ty.childType(mod).intInfo(mod).bits) {14409 .Int => switch (ty.childType(mod).intInfo(mod).bits) {
14138 8 => switch (ty.vectorLen(mod)) {14410 1...8 => switch (ty.vectorLen(mod)) {
14139 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{14411 1...16 => return .{ .move = if (self.hasFeature(.avx))
14140 .insert = .{ .vp_b, .insr },
14141 .extract = .{ .vp_b, .extr },
14142 } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
14143 .insert = .{ .p_b, .insr },
14144 .extract = .{ .p_b, .extr },
14145 } },
14146 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
14147 .insert = .{ .vp_w, .insr },
14148 .extract = .{ .vp_w, .extr },
14149 } } else .{ .insert_extract = .{
14150 .insert = .{ .p_w, .insr },
14151 .extract = .{ .p_w, .extr },
14152 } },
14153 3...4 => return .{ .move = if (self.hasFeature(.avx))
14154 .{ .v_d, .mov }
14155 else
14156 .{ ._d, .mov } },
14157 5...8 => return .{ .move = if (self.hasFeature(.avx))
14158 .{ .v_q, .mov }
14159 else
14160 .{ ._q, .mov } },
14161 9...16 => return .{ .move = if (self.hasFeature(.avx))
14162 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14412 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14163 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14413 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14164 17...32 => if (self.hasFeature(.avx))14414 17...32 => if (self.hasFeature(.avx))
...@@ -14168,23 +14418,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14168,23 +14418,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14168 .{ .v_, .movdqu } },14418 .{ .v_, .movdqu } },
14169 else => {},14419 else => {},
14170 },14420 },
14171 16 => switch (ty.vectorLen(mod)) {14421 9...16 => switch (ty.vectorLen(mod)) {
14172 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{14422 1...8 => return .{ .move = if (self.hasFeature(.avx))
14173 .insert = .{ .vp_w, .insr },
14174 .extract = .{ .vp_w, .extr },
14175 } } else .{ .insert_extract = .{
14176 .insert = .{ .p_w, .insr },
14177 .extract = .{ .p_w, .extr },
14178 } },
14179 2 => return .{ .move = if (self.hasFeature(.avx))
14180 .{ .v_d, .mov }
14181 else
14182 .{ ._d, .mov } },
14183 3...4 => return .{ .move = if (self.hasFeature(.avx))
14184 .{ .v_q, .mov }
14185 else
14186 .{ ._q, .mov } },
14187 5...8 => return .{ .move = if (self.hasFeature(.avx))
14188 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14423 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14189 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14424 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14190 9...16 => if (self.hasFeature(.avx))14425 9...16 => if (self.hasFeature(.avx))
...@@ -14194,16 +14429,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14194,16 +14429,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14194 .{ .v_, .movdqu } },14429 .{ .v_, .movdqu } },
14195 else => {},14430 else => {},
14196 },14431 },
14197 32 => switch (ty.vectorLen(mod)) {14432 17...32 => switch (ty.vectorLen(mod)) {
14198 1 => return .{ .move = if (self.hasFeature(.avx))14433 1...4 => return .{ .move = if (self.hasFeature(.avx))
14199 .{ .v_d, .mov }
14200 else
14201 .{ ._d, .mov } },
14202 2 => return .{ .move = if (self.hasFeature(.avx))
14203 .{ .v_q, .mov }
14204 else
14205 .{ ._q, .mov } },
14206 3...4 => return .{ .move = if (self.hasFeature(.avx))
14207 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14434 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14208 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14435 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14209 5...8 => if (self.hasFeature(.avx))14436 5...8 => if (self.hasFeature(.avx))
...@@ -14213,12 +14440,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14213,12 +14440,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14213 .{ .v_, .movdqu } },14440 .{ .v_, .movdqu } },
14214 else => {},14441 else => {},
14215 },14442 },
14216 64 => switch (ty.vectorLen(mod)) {14443 33...64 => switch (ty.vectorLen(mod)) {
14217 1 => return .{ .move = if (self.hasFeature(.avx))14444 1...2 => return .{ .move = if (self.hasFeature(.avx))
14218 .{ .v_q, .mov }
14219 else
14220 .{ ._q, .mov } },
14221 2 => return .{ .move = if (self.hasFeature(.avx))
14222 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14445 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14223 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14446 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14224 3...4 => if (self.hasFeature(.avx))14447 3...4 => if (self.hasFeature(.avx))
...@@ -14228,7 +14451,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14228,7 +14451,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14228 .{ .v_, .movdqu } },14451 .{ .v_, .movdqu } },
14229 else => {},14452 else => {},
14230 },14453 },
14231 128 => switch (ty.vectorLen(mod)) {14454 65...128 => switch (ty.vectorLen(mod)) {
14232 1 => return .{ .move = if (self.hasFeature(.avx))14455 1 => return .{ .move = if (self.hasFeature(.avx))
14233 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14456 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14234 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14457 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
...@@ -14239,7 +14462,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14239,7 +14462,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14239 .{ .v_, .movdqu } },14462 .{ .v_, .movdqu } },
14240 else => {},14463 else => {},
14241 },14464 },
14242 256 => switch (ty.vectorLen(mod)) {14465 129...256 => switch (ty.vectorLen(mod)) {
14243 1 => if (self.hasFeature(.avx))14466 1 => if (self.hasFeature(.avx))
14244 return .{ .move = if (aligned)14467 return .{ .move = if (aligned)
14245 .{ .v_, .movdqa }14468 .{ .v_, .movdqa }
...@@ -14251,11 +14474,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14251,11 +14474,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14251 },14474 },
14252 .Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))14475 .Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))
14253 switch (ty.vectorLen(mod)) {14476 switch (ty.vectorLen(mod)) {
14254 1 => return .{ .move = if (self.hasFeature(.avx))14477 1...2 => return .{ .move = if (self.hasFeature(.avx))
14255 .{ .v_q, .mov }
14256 else
14257 .{ ._q, .mov } },
14258 2 => return .{ .move = if (self.hasFeature(.avx))
14259 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14478 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14260 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14479 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14261 3...4 => if (self.hasFeature(.avx))14480 3...4 => if (self.hasFeature(.avx))
...@@ -14269,22 +14488,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14269,22 +14488,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14269 unreachable,14488 unreachable,
14270 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {14489 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {
14271 16 => switch (ty.vectorLen(mod)) {14490 16 => switch (ty.vectorLen(mod)) {
14272 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{14491 1...8 => return .{ .move = if (self.hasFeature(.avx))
14273 .insert = .{ .vp_w, .insr },
14274 .extract = .{ .vp_w, .extr },
14275 } } else .{ .insert_extract = .{
14276 .insert = .{ .p_w, .insr },
14277 .extract = .{ .p_w, .extr },
14278 } },
14279 2 => return .{ .move = if (self.hasFeature(.avx))
14280 .{ .v_d, .mov }
14281 else
14282 .{ ._d, .mov } },
14283 3...4 => return .{ .move = if (self.hasFeature(.avx))
14284 .{ .v_q, .mov }
14285 else
14286 .{ ._q, .mov } },
14287 5...8 => return .{ .move = if (self.hasFeature(.avx))
14288 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }14492 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14289 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },14493 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14290 9...16 => if (self.hasFeature(.avx))14494 9...16 => if (self.hasFeature(.avx))
...@@ -14295,15 +14499,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14295,15 +14499,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14295 else => {},14499 else => {},
14296 },14500 },
14297 32 => switch (ty.vectorLen(mod)) {14501 32 => switch (ty.vectorLen(mod)) {
14298 1 => return .{ .move = if (self.hasFeature(.avx))14502 1...4 => return .{ .move = if (self.hasFeature(.avx))
14299 .{ .v_ss, .mov }
14300 else
14301 .{ ._ss, .mov } },
14302 2 => return .{ .move = if (self.hasFeature(.avx))
14303 .{ .v_sd, .mov }
14304 else
14305 .{ ._sd, .mov } },
14306 3...4 => return .{ .move = if (self.hasFeature(.avx))
14307 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }14503 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
14308 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },14504 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
14309 5...8 => if (self.hasFeature(.avx))14505 5...8 => if (self.hasFeature(.avx))
...@@ -14314,11 +14510,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -14314,11 +14510,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14314 else => {},14510 else => {},
14315 },14511 },
14316 64 => switch (ty.vectorLen(mod)) {14512 64 => switch (ty.vectorLen(mod)) {
14317 1 => return .{ .move = if (self.hasFeature(.avx))14513 1...2 => return .{ .move = if (self.hasFeature(.avx))
14318 .{ .v_sd, .mov }
14319 else
14320 .{ ._sd, .mov } },
14321 2 => return .{ .move = if (self.hasFeature(.avx))
14322 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }14514 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
14323 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },14515 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
14324 3...4 => if (self.hasFeature(.avx))14516 3...4 => if (self.hasFeature(.avx))
...@@ -14633,7 +14825,7 @@ fn genSetReg(...@@ -14633,7 +14825,7 @@ fn genSetReg(
14633 ty,14825 ty,
14634 dst_reg.class(),14826 dst_reg.class(),
14635 self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(14827 self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(
14636 std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),14828 math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
14637 )),14829 )),
14638 ),14830 ),
14639 .lea_frame => .{ .move = .{ ._, .lea } },14831 .lea_frame => .{ .move = .{ ._, .lea } },
...@@ -16296,7 +16488,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -16296,7 +16488,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16296 },16488 },
16297 65...128 => switch (vector_len) {16489 65...128 => switch (vector_len) {
16298 else => null,16490 else => null,
16299 1...2 => .{ .vp_i128, .broadcast },16491 1...2 => .{ .v_i128, .broadcast },
16300 },16492 },
16301 }) orelse break :avx2;16493 }) orelse break :avx2;
1630216494
...@@ -16310,7 +16502,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -16310,7 +16502,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16310 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),16502 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
16311 try src_mcv.mem(self, self.memSize(scalar_ty)),16503 try src_mcv.mem(self, self.memSize(scalar_ty)),
16312 ) else {16504 ) else {
16313 if (mir_tag[0] == .vp_i128) break :avx2;16505 if (mir_tag[0] == .v_i128) break :avx2;
16314 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});16506 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
16315 try self.asmRegisterRegister(16507 try self.asmRegisterRegister(
16316 mir_tag,16508 mir_tag,
...@@ -16352,7 +16544,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -16352,7 +16544,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16352 .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },16544 .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
16353 dst_alias,16545 dst_alias,
16354 dst_alias,16546 dst_alias,
16355 Immediate.u(0),16547 Immediate.u(0b00_00_00_00),
16356 );16548 );
16357 if (switch (scalar_bits) {16549 if (switch (scalar_bits) {
16358 1...8 => vector_len > 4,16550 1...8 => vector_len > 4,
...@@ -16563,18 +16755,1158 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -16563,18 +16755,1158 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16563}16755}
1656416756
16565fn airSelect(self: *Self, inst: Air.Inst.Index) !void {16757fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
16758 const mod = self.bin_file.comp.module.?;
16566 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;16759 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
16567 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;16760 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
16568 _ = extra;16761 const ty = self.typeOfIndex(inst);
16569 return self.fail("TODO implement airSelect for x86_64", .{});16762 const vec_len = ty.vectorLen(mod);
16570 //return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });16763 const elem_ty = ty.childType(mod);
16764 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
16765 const abi_size: u32 = @intCast(ty.abiSize(mod));
16766 const pred_ty = self.typeOf(pl_op.operand);
16767
16768 const result = result: {
16769 const has_blend = self.hasFeature(.sse4_1);
16770 const has_avx = self.hasFeature(.avx);
16771 const need_xmm0 = has_blend and !has_avx;
16772 const pred_mcv = try self.resolveInst(pl_op.operand);
16773 const mask_reg = mask: {
16774 switch (pred_mcv) {
16775 .register => |pred_reg| switch (pred_reg.class()) {
16776 .general_purpose => {},
16777 .sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) {
16778 try self.register_manager.getKnownReg(.xmm0, null);
16779 try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{});
16780 break :mask .xmm0;
16781 } else break :mask if (has_blend)
16782 pred_reg
16783 else
16784 try self.copyToTmpRegister(pred_ty, pred_mcv),
16785 else => unreachable,
16786 },
16787 else => {},
16788 }
16789 const mask_reg: Register = if (need_xmm0) mask_reg: {
16790 try self.register_manager.getKnownReg(.xmm0, null);
16791 break :mask_reg .xmm0;
16792 } else try self.register_manager.allocReg(null, abi.RegisterClass.sse);
16793 const mask_alias = registerAlias(mask_reg, abi_size);
16794 const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
16795 defer self.register_manager.unlockReg(mask_lock);
16796
16797 const pred_fits_in_elem = vec_len <= elem_abi_size;
16798 if (self.hasFeature(.avx2) and abi_size <= 32) {
16799 if (pred_mcv.isRegister()) broadcast: {
16800 try self.asmRegisterRegister(
16801 .{ .v_d, .mov },
16802 mask_reg.to128(),
16803 pred_mcv.getReg().?.to32(),
16804 );
16805 if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister(
16806 .{ switch (elem_abi_size) {
16807 1 => .vp_b,
16808 2 => .vp_w,
16809 3...4 => .vp_d,
16810 5...8 => .vp_q,
16811 9...16 => {
16812 try self.asmRegisterRegisterRegisterImmediate(
16813 .{ .v_f128, .insert },
16814 mask_alias,
16815 mask_alias,
16816 mask_reg.to128(),
16817 Immediate.u(1),
16818 );
16819 break :broadcast;
16820 },
16821 17...32 => break :broadcast,
16822 else => unreachable,
16823 }, .broadcast },
16824 mask_alias,
16825 mask_reg.to128(),
16826 );
16827 } else try self.asmRegisterMemory(
16828 .{ switch (vec_len) {
16829 1...8 => .vp_b,
16830 9...16 => .vp_w,
16831 17...32 => .vp_d,
16832 else => unreachable,
16833 }, .broadcast },
16834 mask_alias,
16835 if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{
16836 .base = .{ .reg = (try self.copyToTmpRegister(
16837 Type.usize,
16838 pred_mcv.address(),
16839 )).to64() },
16840 .mod = .{ .rm = .{ .size = .byte } },
16841 },
16842 );
16843 } else if (abi_size <= 16) broadcast: {
16844 try self.asmRegisterRegister(
16845 .{ if (has_avx) .v_d else ._d, .mov },
16846 mask_alias,
16847 (if (pred_mcv.isRegister())
16848 pred_mcv.getReg().?
16849 else
16850 try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(),
16851 );
16852 if (!pred_fits_in_elem or vec_len == 1) break :broadcast;
16853 if (elem_abi_size <= 1) {
16854 if (has_avx) try self.asmRegisterRegisterRegister(
16855 .{ .vp_, .unpcklbw },
16856 mask_alias,
16857 mask_alias,
16858 mask_alias,
16859 ) else try self.asmRegisterRegister(
16860 .{ .p_, .unpcklbw },
16861 mask_alias,
16862 mask_alias,
16863 );
16864 if (abi_size <= 2) break :broadcast;
16865 }
16866 if (elem_abi_size <= 2) {
16867 try self.asmRegisterRegisterImmediate(
16868 .{ if (has_avx) .vp_w else .p_w, .shufl },
16869 mask_alias,
16870 mask_alias,
16871 Immediate.u(0b00_00_00_00),
16872 );
16873 if (abi_size <= 8) break :broadcast;
16874 }
16875 try self.asmRegisterRegisterImmediate(
16876 .{ if (has_avx) .vp_d else .p_d, .shuf },
16877 mask_alias,
16878 mask_alias,
16879 Immediate.u(switch (elem_abi_size) {
16880 1...2, 5...8 => 0b01_00_01_00,
16881 3...4 => 0b00_00_00_00,
16882 else => unreachable,
16883 }),
16884 );
16885 } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
16886 const elem_bits: u16 = @intCast(elem_abi_size * 8);
16887 const mask_elem_ty = try mod.intType(.unsigned, elem_bits);
16888 const mask_ty = try mod.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() });
16889 if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) {
16890 var mask_elems: [32]InternPool.Index = undefined;
16891 for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{
16892 .ty = mask_elem_ty.toIntern(),
16893 .storage = .{ .u64 = bit / elem_bits },
16894 } });
16895 const mask_mcv = try self.genTypedValue(.{
16896 .ty = mask_ty,
16897 .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
16898 .ty = mask_ty.toIntern(),
16899 .storage = .{ .elems = mask_elems[0..vec_len] },
16900 } })),
16901 });
16902 const mask_mem: Memory = .{
16903 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
16904 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
16905 };
16906 if (has_avx) try self.asmRegisterRegisterMemory(
16907 .{ .vp_b, .shuf },
16908 mask_alias,
16909 mask_alias,
16910 mask_mem,
16911 ) else try self.asmRegisterMemory(
16912 .{ .p_b, .shuf },
16913 mask_alias,
16914 mask_mem,
16915 );
16916 } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
16917 {
16918 var mask_elems: [32]InternPool.Index = undefined;
16919 for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{
16920 .ty = mask_elem_ty.toIntern(),
16921 .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
16922 } });
16923 const mask_mcv = try self.genTypedValue(.{
16924 .ty = mask_ty,
16925 .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
16926 .ty = mask_ty.toIntern(),
16927 .storage = .{ .elems = mask_elems[0..vec_len] },
16928 } })),
16929 });
16930 const mask_mem: Memory = .{
16931 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
16932 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
16933 };
16934 if (has_avx) {
16935 try self.asmRegisterRegisterMemory(
16936 .{ .vp_, .@"and" },
16937 mask_alias,
16938 mask_alias,
16939 mask_mem,
16940 );
16941 try self.asmRegisterRegisterMemory(
16942 .{ .vp_d, .cmpeq },
16943 mask_alias,
16944 mask_alias,
16945 mask_mem,
16946 );
16947 } else {
16948 try self.asmRegisterMemory(
16949 .{ .p_, .@"and" },
16950 mask_alias,
16951 mask_mem,
16952 );
16953 try self.asmRegisterMemory(
16954 .{ .p_d, .cmpeq },
16955 mask_alias,
16956 mask_mem,
16957 );
16958 }
16959 }
16960 break :mask mask_reg;
16961 };
16962 const mask_alias = registerAlias(mask_reg, abi_size);
16963 const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
16964 defer self.register_manager.unlockReg(mask_lock);
16965
16966 const lhs_mcv = try self.resolveInst(extra.lhs);
16967 const lhs_lock = switch (lhs_mcv) {
16968 .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg),
16969 else => null,
16970 };
16971 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
16972
16973 const rhs_mcv = try self.resolveInst(extra.rhs);
16974 const rhs_lock = switch (rhs_mcv) {
16975 .register => |rhs_reg| self.register_manager.lockReg(rhs_reg),
16976 else => null,
16977 };
16978 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
16979
16980 const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv;
16981 const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand(
16982 inst,
16983 if (has_blend) extra.rhs else extra.lhs,
16984 @intFromBool(has_blend),
16985 reuse_mcv,
16986 )) reuse_mcv else if (has_avx)
16987 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
16988 else
16989 try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv);
16990 const dst_reg = dst_mcv.getReg().?;
16991 const dst_alias = registerAlias(dst_reg, abi_size);
16992 const dst_lock = self.register_manager.lockReg(dst_reg);
16993 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
16994
16995 const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.childType(mod).zigTypeTag(mod)) {
16996 else => null,
16997 .Int => switch (abi_size) {
16998 0 => unreachable,
16999 1...16 => if (has_avx)
17000 .{ .vp_b, .blendv }
17001 else if (has_blend)
17002 .{ .p_b, .blendv }
17003 else
17004 .{ .p_, undefined },
17005 17...32 => if (self.hasFeature(.avx2))
17006 .{ .vp_b, .blendv }
17007 else
17008 null,
17009 else => null,
17010 },
17011 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {
17012 else => unreachable,
17013 16, 80, 128 => null,
17014 32 => switch (vec_len) {
17015 0 => unreachable,
17016 1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv },
17017 5...8 => if (has_avx) .{ .v_ps, .blendv } else null,
17018 else => null,
17019 },
17020 64 => switch (vec_len) {
17021 0 => unreachable,
17022 1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv },
17023 3...4 => if (has_avx) .{ .v_pd, .blendv } else null,
17024 else => null,
17025 },
17026 },
17027 }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
17028 if (has_avx) {
17029 const rhs_alias = if (rhs_mcv.isRegister())
17030 registerAlias(rhs_mcv.getReg().?, abi_size)
17031 else rhs: {
17032 try self.genSetReg(dst_reg, ty, rhs_mcv, .{});
17033 break :rhs dst_alias;
17034 };
17035 if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
17036 mir_tag,
17037 dst_alias,
17038 rhs_alias,
17039 try lhs_mcv.mem(self, self.memSize(ty)),
17040 mask_alias,
17041 ) else try self.asmRegisterRegisterRegisterRegister(
17042 mir_tag,
17043 dst_alias,
17044 rhs_alias,
17045 registerAlias(if (lhs_mcv.isRegister())
17046 lhs_mcv.getReg().?
17047 else
17048 try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
17049 mask_alias,
17050 );
17051 } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
17052 mir_tag,
17053 dst_alias,
17054 try lhs_mcv.mem(self, self.memSize(ty)),
17055 mask_alias,
17056 ) else try self.asmRegisterRegisterRegister(
17057 mir_tag,
17058 dst_alias,
17059 registerAlias(if (lhs_mcv.isRegister())
17060 lhs_mcv.getReg().?
17061 else
17062 try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
17063 mask_alias,
17064 ) else {
17065 const mir_fixes = @as(?Mir.Inst.Fixes, switch (elem_ty.zigTypeTag(mod)) {
17066 else => null,
17067 .Int => .p_,
17068 .Float => switch (elem_ty.floatBits(self.target.*)) {
17069 32 => ._ps,
17070 64 => ._pd,
17071 16, 80, 128 => null,
17072 else => unreachable,
17073 },
17074 }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
17075 try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
17076 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
17077 .{ mir_fixes, .andn },
17078 mask_alias,
17079 try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),
17080 ) else try self.asmRegisterRegister(
17081 .{ mir_fixes, .andn },
17082 mask_alias,
17083 if (rhs_mcv.isRegister())
17084 rhs_mcv.getReg().?
17085 else
17086 try self.copyToTmpRegister(ty, rhs_mcv),
17087 );
17088 try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
17089 }
17090 break :result dst_mcv;
17091 };
17092 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
16571}17093}
1657217094
16573fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {17095fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
17096 const mod = self.bin_file.comp.module.?;
16574 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;17097 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
16575 _ = ty_pl;17098 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
16576 return self.fail("TODO implement airShuffle for x86_64", .{});17099
16577 //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });17100 const dst_ty = self.typeOfIndex(inst);
17101 const elem_ty = dst_ty.childType(mod);
17102 const elem_abi_size: u16 = @intCast(elem_ty.abiSize(mod));
17103 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
17104 const lhs_ty = self.typeOf(extra.a);
17105 const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
17106 const rhs_ty = self.typeOf(extra.b);
17107 const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(mod));
17108 const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);
17109
17110 const ExpectedContents = [32]?i32;
17111 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
17112 std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
17113 const allocator = stack.get();
17114
17115 const mask_elems = try allocator.alloc(?i32, extra.mask_len);
17116 defer allocator.free(mask_elems);
17117 for (mask_elems, 0..) |*mask_elem, elem_index| {
17118 const mask_elem_val =
17119 Value.fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable;
17120 mask_elem.* = if (mask_elem_val.isUndef(mod))
17121 null
17122 else
17123 @intCast(mask_elem_val.toSignedInt(mod));
17124 }
17125
17126 const has_avx = self.hasFeature(.avx);
17127 const result = @as(?MCValue, result: {
17128 for (mask_elems) |mask_elem| {
17129 if (mask_elem) |_| break;
17130 } else break :result try self.allocRegOrMem(inst, true);
17131
17132 for (mask_elems, 0..) |mask_elem, elem_index| {
17133 if (mask_elem orelse continue != elem_index) break;
17134 } else {
17135 const lhs_mcv = try self.resolveInst(extra.a);
17136 if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;
17137 const dst_mcv = try self.allocRegOrMem(inst, true);
17138 try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});
17139 break :result dst_mcv;
17140 }
17141
17142 for (mask_elems, 0..) |mask_elem, elem_index| {
17143 if (~(mask_elem orelse continue) != elem_index) break;
17144 } else {
17145 const rhs_mcv = try self.resolveInst(extra.b);
17146 if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;
17147 const dst_mcv = try self.allocRegOrMem(inst, true);
17148 try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});
17149 break :result dst_mcv;
17150 }
17151
17152 for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: {
17153 if (elem_abi_size > 8) break :unpck;
17154 if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
17155 has_avx
17156 else
17157 self.hasFeature(.avx2)) 32 else 16)) break :unpck;
17158
17159 var sources = [1]?u1{null} ** 2;
17160 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
17161 const mask_elem = maybe_mask_elem orelse continue;
17162 const mask_elem_index =
17163 math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
17164 const elem_byte = (elem_index >> 1) * elem_abi_size;
17165 if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) {
17166 .unpckl => 0b0000,
17167 .unpckh => 0b1000,
17168 else => unreachable,
17169 }) | (elem_byte << 1 & 0b10000)) break :unpck;
17170
17171 const source = @intFromBool(mask_elem < 0);
17172 if (sources[elem_index & 0b00001]) |prev_source| {
17173 if (source != prev_source) break :unpck;
17174 } else sources[elem_index & 0b00001] = source;
17175 }
17176 if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck;
17177
17178 const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
17179 const operand_tys = [2]Type{ lhs_ty, rhs_ty };
17180 const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
17181 const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
17182
17183 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
17184 self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
17185 lhs_mcv
17186 else if (has_avx and lhs_mcv.isRegister())
17187 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
17188 else
17189 try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
17190 const dst_reg = dst_mcv.getReg().?;
17191 const dst_alias = registerAlias(dst_reg, max_abi_size);
17192
17193 const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
17194 (dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) {
17195 4 => if (has_avx) .v_ps else ._ps,
17196 8 => if (has_avx) .v_pd else ._pd,
17197 else => unreachable,
17198 }, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) {
17199 .unpckl => switch (elem_abi_size) {
17200 1 => .unpcklbw,
17201 2 => .unpcklwd,
17202 4 => .unpckldq,
17203 8 => .unpcklqdq,
17204 else => unreachable,
17205 },
17206 .unpckh => switch (elem_abi_size) {
17207 1 => .unpckhbw,
17208 2 => .unpckhwd,
17209 4 => .unpckhdq,
17210 8 => .unpckhqdq,
17211 else => unreachable,
17212 },
17213 else => unreachable,
17214 } };
17215 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory(
17216 mir_tag,
17217 dst_alias,
17218 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17219 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17220 ) else try self.asmRegisterRegisterRegister(
17221 mir_tag,
17222 dst_alias,
17223 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17224 registerAlias(if (rhs_mcv.isRegister())
17225 rhs_mcv.getReg().?
17226 else
17227 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17228 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
17229 mir_tag,
17230 dst_alias,
17231 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17232 ) else try self.asmRegisterRegister(
17233 mir_tag,
17234 dst_alias,
17235 registerAlias(if (rhs_mcv.isRegister())
17236 rhs_mcv.getReg().?
17237 else
17238 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17239 );
17240 break :result dst_mcv;
17241 }
17242
17243 pshufd: {
17244 if (elem_abi_size != 4) break :pshufd;
17245 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;
17246
17247 var control: u8 = 0b00_00_00_00;
17248 var sources = [1]?u1{null} ** 1;
17249 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
17250 const mask_elem = maybe_mask_elem orelse continue;
17251 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
17252 if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd;
17253
17254 const source = @intFromBool(mask_elem < 0);
17255 if (sources[0]) |prev_source| {
17256 if (source != prev_source) break :pshufd;
17257 } else sources[(elem_index & 0b010) >> 1] = source;
17258
17259 const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17260 const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17261 if (elem_index & 0b100 == 0)
17262 control |= select
17263 else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;
17264 }
17265
17266 const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
17267 const operand_tys = [2]Type{ lhs_ty, rhs_ty };
17268 const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]);
17269
17270 const dst_reg = if (src_mcv.isRegister() and
17271 self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv))
17272 src_mcv.getReg().?
17273 else
17274 try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
17275 const dst_alias = registerAlias(dst_reg, max_abi_size);
17276
17277 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17278 .{ if (has_avx) .vp_d else .p_d, .shuf },
17279 dst_alias,
17280 try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17281 Immediate.u(control),
17282 ) else try self.asmRegisterRegisterImmediate(
17283 .{ if (has_avx) .vp_d else .p_d, .shuf },
17284 dst_alias,
17285 registerAlias(if (src_mcv.isRegister())
17286 src_mcv.getReg().?
17287 else
17288 try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
17289 Immediate.u(control),
17290 );
17291 break :result .{ .register = dst_reg };
17292 }
17293
17294 shufps: {
17295 if (elem_abi_size != 4) break :shufps;
17296 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;
17297
17298 var control: u8 = 0b00_00_00_00;
17299 var sources = [1]?u1{null} ** 2;
17300 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
17301 const mask_elem = maybe_mask_elem orelse continue;
17302 const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
17303 if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps;
17304
17305 const source = @intFromBool(mask_elem < 0);
17306 if (sources[(elem_index & 0b010) >> 1]) |prev_source| {
17307 if (source != prev_source) break :shufps;
17308 } else sources[(elem_index & 0b010) >> 1] = source;
17309
17310 const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
17311 const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
17312 if (elem_index & 0b100 == 0)
17313 control |= select
17314 else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;
17315 }
17316 if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
17317
17318 const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
17319 const operand_tys = [2]Type{ lhs_ty, rhs_ty };
17320 const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
17321 const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
17322
17323 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
17324 self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
17325 lhs_mcv
17326 else if (has_avx and lhs_mcv.isRegister())
17327 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
17328 else
17329 try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
17330 const dst_reg = dst_mcv.getReg().?;
17331 const dst_alias = registerAlias(dst_reg, max_abi_size);
17332
17333 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
17334 .{ .v_ps, .shuf },
17335 dst_alias,
17336 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17337 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17338 Immediate.u(control),
17339 ) else try self.asmRegisterRegisterRegisterImmediate(
17340 .{ .v_ps, .shuf },
17341 dst_alias,
17342 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17343 registerAlias(if (rhs_mcv.isRegister())
17344 rhs_mcv.getReg().?
17345 else
17346 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17347 Immediate.u(control),
17348 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17349 .{ ._ps, .shuf },
17350 dst_alias,
17351 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17352 Immediate.u(control),
17353 ) else try self.asmRegisterRegisterImmediate(
17354 .{ ._ps, .shuf },
17355 dst_alias,
17356 registerAlias(if (rhs_mcv.isRegister())
17357 rhs_mcv.getReg().?
17358 else
17359 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17360 Immediate.u(control),
17361 );
17362 break :result dst_mcv;
17363 }
17364
17365 shufpd: {
17366 if (elem_abi_size != 8) break :shufpd;
17367 if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
17368
17369 var control: u4 = 0b0_0_0_0;
17370 var sources = [1]?u1{null} ** 2;
17371 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
17372 const mask_elem = maybe_mask_elem orelse continue;
17373 const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
17374 if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;
17375
17376 const source = @intFromBool(mask_elem < 0);
17377 if (sources[elem_index & 0b01]) |prev_source| {
17378 if (source != prev_source) break :shufpd;
17379 } else sources[elem_index & 0b01] = source;
17380
17381 control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);
17382 }
17383 if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;
17384
17385 const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b };
17386 const operand_tys: [2]Type = .{ lhs_ty, rhs_ty };
17387 const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
17388 const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
17389
17390 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
17391 self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
17392 lhs_mcv
17393 else if (has_avx and lhs_mcv.isRegister())
17394 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
17395 else
17396 try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
17397 const dst_reg = dst_mcv.getReg().?;
17398 const dst_alias = registerAlias(dst_reg, max_abi_size);
17399
17400 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
17401 .{ .v_pd, .shuf },
17402 dst_alias,
17403 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17404 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17405 Immediate.u(control),
17406 ) else try self.asmRegisterRegisterRegisterImmediate(
17407 .{ .v_pd, .shuf },
17408 dst_alias,
17409 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
17410 registerAlias(if (rhs_mcv.isRegister())
17411 rhs_mcv.getReg().?
17412 else
17413 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17414 Immediate.u(control),
17415 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17416 .{ ._pd, .shuf },
17417 dst_alias,
17418 try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
17419 Immediate.u(control),
17420 ) else try self.asmRegisterRegisterImmediate(
17421 .{ ._pd, .shuf },
17422 dst_alias,
17423 registerAlias(if (rhs_mcv.isRegister())
17424 rhs_mcv.getReg().?
17425 else
17426 try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
17427 Immediate.u(control),
17428 );
17429 break :result dst_mcv;
17430 }
17431
17432 blend: {
17433 if (elem_abi_size < 2) break :blend;
17434 if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend;
17435 if (!self.hasFeature(.sse4_1)) break :blend;
17436
17437 var control: u8 = 0b0_0_0_0_0_0_0_0;
17438 for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
17439 const mask_elem = maybe_mask_elem orelse continue;
17440 const mask_elem_index =
17441 math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
17442 if (mask_elem_index != elem_index) break :blend;
17443
17444 const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
17445 if (elem_index & 0b1000 == 0)
17446 control |= select
17447 else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;
17448 }
17449
17450 if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
17451 const expanded_control = switch (elem_abi_size) {
17452 4 => control,
17453 8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) |
17454 @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) |
17455 @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) |
17456 @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00),
17457 else => break :vpblendd,
17458 };
17459
17460 const lhs_mcv = try self.resolveInst(extra.a);
17461 const lhs_reg = if (lhs_mcv.isRegister())
17462 lhs_mcv.getReg().?
17463 else
17464 try self.copyToTmpRegister(dst_ty, lhs_mcv);
17465 const lhs_lock = self.register_manager.lockReg(lhs_reg);
17466 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
17467
17468 const rhs_mcv = try self.resolveInst(extra.b);
17469 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
17470 if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
17471 .{ .vp_d, .blend },
17472 registerAlias(dst_reg, dst_abi_size),
17473 registerAlias(lhs_reg, dst_abi_size),
17474 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17475 Immediate.u(expanded_control),
17476 ) else try self.asmRegisterRegisterRegisterImmediate(
17477 .{ .vp_d, .blend },
17478 registerAlias(dst_reg, dst_abi_size),
17479 registerAlias(lhs_reg, dst_abi_size),
17480 registerAlias(if (rhs_mcv.isRegister())
17481 rhs_mcv.getReg().?
17482 else
17483 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17484 Immediate.u(expanded_control),
17485 );
17486 break :result .{ .register = dst_reg };
17487 }
17488
17489 if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: {
17490 const expanded_control = switch (elem_abi_size) {
17491 2 => control,
17492 4 => if (dst_abi_size <= 16 or
17493 @as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0)))
17494 @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) |
17495 @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) |
17496 @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) |
17497 @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00)
17498 else
17499 break :pblendw,
17500 8 => if (dst_abi_size <= 16 or
17501 @as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0)))
17502 @as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) |
17503 @as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000)
17504 else
17505 break :pblendw,
17506 16 => break :pblendw,
17507 else => unreachable,
17508 };
17509
17510 const lhs_mcv = try self.resolveInst(extra.a);
17511 const rhs_mcv = try self.resolveInst(extra.b);
17512
17513 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
17514 self.reuseOperand(inst, extra.a, 0, lhs_mcv))
17515 lhs_mcv
17516 else if (has_avx and lhs_mcv.isRegister())
17517 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
17518 else
17519 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
17520 const dst_reg = dst_mcv.getReg().?;
17521
17522 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
17523 .{ .vp_w, .blend },
17524 registerAlias(dst_reg, dst_abi_size),
17525 registerAlias(if (lhs_mcv.isRegister())
17526 lhs_mcv.getReg().?
17527 else
17528 dst_reg, dst_abi_size),
17529 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17530 Immediate.u(expanded_control),
17531 ) else try self.asmRegisterRegisterRegisterImmediate(
17532 .{ .vp_w, .blend },
17533 registerAlias(dst_reg, dst_abi_size),
17534 registerAlias(if (lhs_mcv.isRegister())
17535 lhs_mcv.getReg().?
17536 else
17537 dst_reg, dst_abi_size),
17538 registerAlias(if (rhs_mcv.isRegister())
17539 rhs_mcv.getReg().?
17540 else
17541 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17542 Immediate.u(expanded_control),
17543 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17544 .{ .p_w, .blend },
17545 registerAlias(dst_reg, dst_abi_size),
17546 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17547 Immediate.u(expanded_control),
17548 ) else try self.asmRegisterRegisterImmediate(
17549 .{ .p_w, .blend },
17550 registerAlias(dst_reg, dst_abi_size),
17551 registerAlias(if (rhs_mcv.isRegister())
17552 rhs_mcv.getReg().?
17553 else
17554 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17555 Immediate.u(expanded_control),
17556 );
17557 break :result .{ .register = dst_reg };
17558 }
17559
17560 const expanded_control = switch (elem_abi_size) {
17561 4, 8 => control,
17562 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) |
17563 @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
17564 else => unreachable,
17565 };
17566
17567 const lhs_mcv = try self.resolveInst(extra.a);
17568 const rhs_mcv = try self.resolveInst(extra.b);
17569
17570 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
17571 self.reuseOperand(inst, extra.a, 0, lhs_mcv))
17572 lhs_mcv
17573 else if (has_avx and lhs_mcv.isRegister())
17574 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
17575 else
17576 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
17577 const dst_reg = dst_mcv.getReg().?;
17578
17579 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
17580 switch (elem_abi_size) {
17581 4 => .{ .v_ps, .blend },
17582 8, 16 => .{ .v_pd, .blend },
17583 else => unreachable,
17584 },
17585 registerAlias(dst_reg, dst_abi_size),
17586 registerAlias(if (lhs_mcv.isRegister())
17587 lhs_mcv.getReg().?
17588 else
17589 dst_reg, dst_abi_size),
17590 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17591 Immediate.u(expanded_control),
17592 ) else try self.asmRegisterRegisterRegisterImmediate(
17593 switch (elem_abi_size) {
17594 4 => .{ .v_ps, .blend },
17595 8, 16 => .{ .v_pd, .blend },
17596 else => unreachable,
17597 },
17598 registerAlias(dst_reg, dst_abi_size),
17599 registerAlias(if (lhs_mcv.isRegister())
17600 lhs_mcv.getReg().?
17601 else
17602 dst_reg, dst_abi_size),
17603 registerAlias(if (rhs_mcv.isRegister())
17604 rhs_mcv.getReg().?
17605 else
17606 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17607 Immediate.u(expanded_control),
17608 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
17609 switch (elem_abi_size) {
17610 4 => .{ ._ps, .blend },
17611 8, 16 => .{ ._pd, .blend },
17612 else => unreachable,
17613 },
17614 registerAlias(dst_reg, dst_abi_size),
17615 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17616 Immediate.u(expanded_control),
17617 ) else try self.asmRegisterRegisterImmediate(
17618 switch (elem_abi_size) {
17619 4 => .{ ._ps, .blend },
17620 8, 16 => .{ ._pd, .blend },
17621 else => unreachable,
17622 },
17623 registerAlias(dst_reg, dst_abi_size),
17624 registerAlias(if (rhs_mcv.isRegister())
17625 rhs_mcv.getReg().?
17626 else
17627 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17628 Immediate.u(expanded_control),
17629 );
17630 break :result .{ .register = dst_reg };
17631 }
17632
17633 blendv: {
17634 if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
17635 has_avx
17636 else
17637 self.hasFeature(.avx2)) 32 else 16)) break :blendv;
17638
17639 const select_mask_elem_ty = try mod.intType(.unsigned, elem_abi_size * 8);
17640 const select_mask_ty = try mod.vectorType(.{
17641 .len = @intCast(mask_elems.len),
17642 .child = select_mask_elem_ty.toIntern(),
17643 });
17644 var select_mask_elems: [32]InternPool.Index = undefined;
17645 for (
17646 select_mask_elems[0..mask_elems.len],
17647 mask_elems,
17648 0..,
17649 ) |*select_mask_elem, maybe_mask_elem, elem_index| {
17650 const mask_elem = maybe_mask_elem orelse continue;
17651 const mask_elem_index =
17652 math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
17653 if (mask_elem_index != elem_index) break :blendv;
17654
17655 select_mask_elem.* = (if (mask_elem < 0)
17656 try select_mask_elem_ty.maxIntScalar(mod, select_mask_elem_ty)
17657 else
17658 try select_mask_elem_ty.minIntScalar(mod, select_mask_elem_ty)).toIntern();
17659 }
17660 const select_mask_mcv = try self.genTypedValue(.{
17661 .ty = select_mask_ty,
17662 .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
17663 .ty = select_mask_ty.toIntern(),
17664 .storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
17665 } })),
17666 });
17667
17668 if (self.hasFeature(.sse4_1)) {
17669 const mir_tag: Mir.Inst.FixedTag = .{
17670 if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
17671 (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
17672 4 => if (has_avx) .v_ps else ._ps,
17673 8 => if (has_avx) .v_pd else ._pd,
17674 else => unreachable,
17675 } else if (has_avx) .vp_b else .p_b,
17676 .blendv,
17677 };
17678
17679 const select_mask_reg = if (!has_avx) reg: {
17680 try self.register_manager.getKnownReg(.xmm0, null);
17681 try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
17682 break :reg .xmm0;
17683 } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
17684 const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
17685 const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
17686 defer self.register_manager.unlockReg(select_mask_lock);
17687
17688 const lhs_mcv = try self.resolveInst(extra.a);
17689 const rhs_mcv = try self.resolveInst(extra.b);
17690
17691 const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
17692 self.reuseOperand(inst, extra.a, 0, lhs_mcv))
17693 lhs_mcv
17694 else if (has_avx and lhs_mcv.isRegister())
17695 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
17696 else
17697 try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
17698 const dst_reg = dst_mcv.getReg().?;
17699 const dst_alias = registerAlias(dst_reg, dst_abi_size);
17700
17701 if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
17702 mir_tag,
17703 dst_alias,
17704 if (lhs_mcv.isRegister())
17705 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
17706 else
17707 dst_alias,
17708 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17709 select_mask_alias,
17710 ) else try self.asmRegisterRegisterRegisterRegister(
17711 mir_tag,
17712 dst_alias,
17713 if (lhs_mcv.isRegister())
17714 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
17715 else
17716 dst_alias,
17717 registerAlias(if (rhs_mcv.isRegister())
17718 rhs_mcv.getReg().?
17719 else
17720 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17721 select_mask_alias,
17722 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
17723 mir_tag,
17724 dst_alias,
17725 try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17726 select_mask_alias,
17727 ) else try self.asmRegisterRegisterRegister(
17728 mir_tag,
17729 dst_alias,
17730 registerAlias(if (rhs_mcv.isRegister())
17731 rhs_mcv.getReg().?
17732 else
17733 try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
17734 select_mask_alias,
17735 );
17736 break :result dst_mcv;
17737 }
17738
17739 const lhs_mcv = try self.resolveInst(extra.a);
17740 const rhs_mcv = try self.resolveInst(extra.b);
17741
17742 const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
17743 self.reuseOperand(inst, extra.b, 1, rhs_mcv))
17744 rhs_mcv
17745 else
17746 try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
17747 const dst_reg = dst_mcv.getReg().?;
17748 const dst_alias = registerAlias(dst_reg, dst_abi_size);
17749
17750 const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
17751 const mask_alias = registerAlias(mask_reg, dst_abi_size);
17752 const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
17753 defer self.register_manager.unlockReg(mask_lock);
17754
17755 const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
17756 switch (elem_ty.floatBits(self.target.*)) {
17757 16, 80, 128 => .p_,
17758 32 => ._ps,
17759 64 => ._pd,
17760 else => unreachable,
17761 }
17762 else
17763 .p_;
17764 try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
17765 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
17766 .{ mir_fixes, .andn },
17767 mask_alias,
17768 try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
17769 ) else try self.asmRegisterRegister(
17770 .{ mir_fixes, .andn },
17771 mask_alias,
17772 if (lhs_mcv.isRegister())
17773 lhs_mcv.getReg().?
17774 else
17775 try self.copyToTmpRegister(dst_ty, lhs_mcv),
17776 );
17777 try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
17778 break :result dst_mcv;
17779 }
17780
17781 pshufb: {
17782 if (max_abi_size > 16) break :pshufb;
17783 if (!self.hasFeature(.ssse3)) break :pshufb;
17784
17785 const temp_regs =
17786 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
17787 const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
17788 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
17789
17790 const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
17791 try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
17792
17793 const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
17794 try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
17795
17796 var lhs_mask_elems: [16]InternPool.Index = undefined;
17797 for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| {
17798 const elem_index = byte_index / elem_abi_size;
17799 lhs_mask_elem.* = try mod.intern(.{ .int = .{
17800 .ty = .u8_type,
17801 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
17802 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
17803 if (mask_elem < 0) break :elem 0b1_00_00000;
17804 const mask_elem_index: u31 = @intCast(mask_elem);
17805 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
17806 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
17807 } },
17808 } });
17809 }
17810 const lhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
17811 const lhs_mask_mcv = try self.genTypedValue(.{
17812 .ty = lhs_mask_ty,
17813 .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
17814 .ty = lhs_mask_ty.toIntern(),
17815 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
17816 } })),
17817 });
17818 const lhs_mask_mem: Memory = .{
17819 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
17820 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
17821 };
17822 if (has_avx) try self.asmRegisterRegisterMemory(
17823 .{ .vp_b, .shuf },
17824 lhs_temp_alias,
17825 lhs_temp_alias,
17826 lhs_mask_mem,
17827 ) else try self.asmRegisterMemory(
17828 .{ .p_b, .shuf },
17829 lhs_temp_alias,
17830 lhs_mask_mem,
17831 );
17832
17833 var rhs_mask_elems: [16]InternPool.Index = undefined;
17834 for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| {
17835 const elem_index = byte_index / elem_abi_size;
17836 rhs_mask_elem.* = try mod.intern(.{ .int = .{
17837 .ty = .u8_type,
17838 .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
17839 const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
17840 if (mask_elem >= 0) break :elem 0b1_00_00000;
17841 const mask_elem_index: u31 = @intCast(~mask_elem);
17842 const byte_off: u32 = @intCast(byte_index % elem_abi_size);
17843 break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
17844 } },
17845 } });
17846 }
17847 const rhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
17848 const rhs_mask_mcv = try self.genTypedValue(.{
17849 .ty = rhs_mask_ty,
17850 .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
17851 .ty = rhs_mask_ty.toIntern(),
17852 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
17853 } })),
17854 });
17855 const rhs_mask_mem: Memory = .{
17856 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
17857 .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
17858 };
17859 if (has_avx) try self.asmRegisterRegisterMemory(
17860 .{ .vp_b, .shuf },
17861 rhs_temp_alias,
17862 rhs_temp_alias,
17863 rhs_mask_mem,
17864 ) else try self.asmRegisterMemory(
17865 .{ .p_b, .shuf },
17866 rhs_temp_alias,
17867 rhs_mask_mem,
17868 );
17869
17870 if (has_avx) try self.asmRegisterRegisterRegister(
17871 .{ switch (elem_ty.zigTypeTag(mod)) {
17872 else => break :result null,
17873 .Int => .vp_,
17874 .Float => switch (elem_ty.floatBits(self.target.*)) {
17875 32 => .v_ps,
17876 64 => .v_pd,
17877 16, 80, 128 => break :result null,
17878 else => unreachable,
17879 },
17880 }, .@"or" },
17881 lhs_temp_alias,
17882 lhs_temp_alias,
17883 rhs_temp_alias,
17884 ) else try self.asmRegisterRegister(
17885 .{ switch (elem_ty.zigTypeTag(mod)) {
17886 else => break :result null,
17887 .Int => .p_,
17888 .Float => switch (elem_ty.floatBits(self.target.*)) {
17889 32 => ._ps,
17890 64 => ._pd,
17891 16, 80, 128 => break :result null,
17892 else => unreachable,
17893 },
17894 }, .@"or" },
17895 lhs_temp_alias,
17896 rhs_temp_alias,
17897 );
17898 break :result .{ .register = temp_regs[0] };
17899 }
17900
17901 break :result null;
17902 }) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
17903 lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod),
17904 Value.fromInterned(extra.mask).fmtValue(
17905 Type.fromInterned(mod.intern_pool.typeOf(extra.mask)),
17906 mod,
17907 ),
17908 });
17909 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
16578}17910}
1657917911
16580fn airReduce(self: *Self, inst: Air.Inst.Index) !void {17912fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
...@@ -16751,7 +18083,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -16751,7 +18083,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
16751 },18083 },
16752 .Array, .Vector => {18084 .Array, .Vector => {
16753 const elem_ty = result_ty.childType(mod);18085 const elem_ty = result_ty.childType(mod);
16754 if (result_ty.isVector(mod) and elem_ty.bitSize(mod) == 1) {18086 if (result_ty.isVector(mod) and elem_ty.toIntern() == .bool_type) {
16755 const result_size: u32 = @intCast(result_ty.abiSize(mod));18087 const result_size: u32 = @intCast(result_ty.abiSize(mod));
16756 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);18088 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
16757 try self.asmRegisterRegister(18089 try self.asmRegisterRegister(
...@@ -17801,7 +19133,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {...@@ -17801,7 +19133,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {
17801 else => unreachable,19133 else => unreachable,
17802 },19134 },
17803 .float => Type.f32,19135 .float => Type.f32,
17804 .float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),19136 .float_combine => try mod.arrayType(.{ .len = 2, .child = .f32_type }),
17805 .sse => Type.f64,19137 .sse => Type.f64,
17806 else => break,19138 else => break,
17807 };19139 };
src/arch/x86_64/Encoding.zig+11-4
...@@ -324,16 +324,19 @@ pub const Mnemonic = enum {...@@ -324,16 +324,19 @@ pub const Mnemonic = enum {
324 // SSE3324 // SSE3
325 movddup, movshdup, movsldup,325 movddup, movshdup, movsldup,
326 // SSSE3326 // SSSE3
327 pabsb, pabsd, pabsw, palignr,327 pabsb, pabsd, pabsw, palignr, pshufb,
328 // SSE4.1328 // SSE4.1
329 blendpd, blendps, blendvpd, blendvps,329 blendpd, blendps, blendvpd, blendvps,
330 extractps,330 extractps,
331 insertps,331 insertps,
332 packusdw,332 packusdw,
333 pblendvb, pblendw,
333 pcmpeqq,334 pcmpeqq,
334 pextrb, pextrd, pextrq,335 pextrb, pextrd, pextrq,
335 pinsrb, pinsrd, pinsrq,336 pinsrb, pinsrd, pinsrq,
336 pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,337 pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,
338 pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,
339 pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,
337 pmulld,340 pmulld,
338 roundpd, roundps, roundsd, roundss,341 roundpd, roundps, roundsd, roundss,
339 // SSE4.2342 // SSE4.2
...@@ -377,7 +380,8 @@ pub const Mnemonic = enum {...@@ -377,7 +380,8 @@ pub const Mnemonic = enum {
377 vpabsb, vpabsd, vpabsw,380 vpabsb, vpabsd, vpabsw,
378 vpackssdw, vpacksswb, vpackusdw, vpackuswb,381 vpackssdw, vpacksswb, vpackusdw, vpackuswb,
379 vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,382 vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
380 vpalignr, vpand, vpandn, vpclmulqdq,383 vpalignr, vpand, vpandn,
384 vpblendvb, vpblendw, vpclmulqdq,
381 vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw,385 vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw,
382 vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw,386 vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw,
383 vpextrb, vpextrd, vpextrq, vpextrw,387 vpextrb, vpextrd, vpextrq, vpextrw,
...@@ -385,9 +389,11 @@ pub const Mnemonic = enum {...@@ -385,9 +389,11 @@ pub const Mnemonic = enum {
385 vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw,389 vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw,
386 vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw,390 vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw,
387 vpmovmskb,391 vpmovmskb,
392 vpmovsxbd, vpmovsxbq, vpmovsxbw, vpmovsxdq, vpmovsxwd, vpmovsxwq,
393 vpmovzxbd, vpmovzxbq, vpmovzxbw, vpmovzxdq, vpmovzxwd, vpmovzxwq,
388 vpmulhw, vpmulld, vpmullw,394 vpmulhw, vpmulld, vpmullw,
389 vpor,395 vpor,
390 vpshufd, vpshufhw, vpshuflw,396 vpshufb, vpshufd, vpshufhw, vpshuflw,
391 vpslld, vpslldq, vpsllq, vpsllw,397 vpslld, vpslldq, vpsllq, vpsllw,
392 vpsrad, vpsraq, vpsraw,398 vpsrad, vpsraq, vpsraw,
393 vpsrld, vpsrldq, vpsrlq, vpsrlw,399 vpsrld, vpsrldq, vpsrlq, vpsrlw,
...@@ -409,7 +415,8 @@ pub const Mnemonic = enum {...@@ -409,7 +415,8 @@ pub const Mnemonic = enum {
409 vfmadd132sd, vfmadd213sd, vfmadd231sd,415 vfmadd132sd, vfmadd213sd, vfmadd231sd,
410 vfmadd132ss, vfmadd213ss, vfmadd231ss,416 vfmadd132ss, vfmadd213ss, vfmadd231ss,
411 // AVX2417 // AVX2
412 vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,418 vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw,
419 vextracti128, vinserti128, vpblendd,
413 // zig fmt: on420 // zig fmt: on
414};421};
415422
src/arch/x86_64/Lower.zig+13-1
...@@ -477,8 +477,9 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -477,8 +477,9 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
477 .rri_s, .rri_u => inst.data.rri.fixes,477 .rri_s, .rri_u => inst.data.rri.fixes,
478 .ri_s, .ri_u => inst.data.ri.fixes,478 .ri_s, .ri_u => inst.data.ri.fixes,
479 .ri64, .rm, .rmi_s, .mr => inst.data.rx.fixes,479 .ri64, .rm, .rmi_s, .mr => inst.data.rx.fixes,
480 .mrr, .rrm => inst.data.rrx.fixes,480 .mrr, .rrm, .rmr => inst.data.rrx.fixes,
481 .rmi, .mri => inst.data.rix.fixes,481 .rmi, .mri => inst.data.rix.fixes,
482 .rrmr => inst.data.rrrx.fixes,
482 .rrmi => inst.data.rrix.fixes,483 .rrmi => inst.data.rrix.fixes,
483 .mi_u, .mi_s => inst.data.x.fixes,484 .mi_u, .mi_s => inst.data.x.fixes,
484 .m => inst.data.x.fixes,485 .m => inst.data.x.fixes,
...@@ -565,6 +566,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -565,6 +566,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
565 .{ .reg = inst.data.rx.r1 },566 .{ .reg = inst.data.rx.r1 },
566 .{ .mem = lower.mem(inst.data.rx.payload) },567 .{ .mem = lower.mem(inst.data.rx.payload) },
567 },568 },
569 .rmr => &.{
570 .{ .reg = inst.data.rrx.r1 },
571 .{ .mem = lower.mem(inst.data.rrx.payload) },
572 .{ .reg = inst.data.rrx.r2 },
573 },
568 .rmi => &.{574 .rmi => &.{
569 .{ .reg = inst.data.rix.r1 },575 .{ .reg = inst.data.rix.r1 },
570 .{ .mem = lower.mem(inst.data.rix.payload) },576 .{ .mem = lower.mem(inst.data.rix.payload) },
...@@ -597,6 +603,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -597,6 +603,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
597 .{ .reg = inst.data.rrx.r2 },603 .{ .reg = inst.data.rrx.r2 },
598 .{ .mem = lower.mem(inst.data.rrx.payload) },604 .{ .mem = lower.mem(inst.data.rrx.payload) },
599 },605 },
606 .rrmr => &.{
607 .{ .reg = inst.data.rrrx.r1 },
608 .{ .reg = inst.data.rrrx.r2 },
609 .{ .mem = lower.mem(inst.data.rrrx.payload) },
610 .{ .reg = inst.data.rrrx.r3 },
611 },
600 .rrmi => &.{612 .rrmi => &.{
601 .{ .reg = inst.data.rrix.r1 },613 .{ .reg = inst.data.rrix.r1 },
602 .{ .reg = inst.data.rrix.r2 },614 .{ .reg = inst.data.rrix.r2 },
src/arch/x86_64/Mir.zig+27-3
...@@ -230,6 +230,8 @@ pub const Inst = struct {...@@ -230,6 +230,8 @@ pub const Inst = struct {
230 v_d,230 v_d,
231 /// VEX-Encoded ___ QuadWord231 /// VEX-Encoded ___ QuadWord
232 v_q,232 v_q,
233 /// VEX-Encoded ___ Integer Data
234 v_i128,
233 /// VEX-Encoded Packed ___235 /// VEX-Encoded Packed ___
234 vp_,236 vp_,
235 /// VEX-Encoded Packed ___ Byte237 /// VEX-Encoded Packed ___ Byte
...@@ -242,8 +244,6 @@ pub const Inst = struct {...@@ -242,8 +244,6 @@ pub const Inst = struct {
242 vp_q,244 vp_q,
243 /// VEX-Encoded Packed ___ Double Quadword245 /// VEX-Encoded Packed ___ Double Quadword
244 vp_dq,246 vp_dq,
245 /// VEX-Encoded Packed ___ Integer Data
246 vp_i128,
247 /// VEX-Encoded ___ Scalar Single-Precision Values247 /// VEX-Encoded ___ Scalar Single-Precision Values
248 v_ss,248 v_ss,
249 /// VEX-Encoded ___ Packed Single-Precision Values249 /// VEX-Encoded ___ Packed Single-Precision Values
...@@ -654,10 +654,19 @@ pub const Inst = struct {...@@ -654,10 +654,19 @@ pub const Inst = struct {
654 /// Variable blend scalar double-precision floating-point values654 /// Variable blend scalar double-precision floating-point values
655 blendv,655 blendv,
656 /// Extract packed floating-point values656 /// Extract packed floating-point values
657 /// Extract packed integer values
657 extract,658 extract,
658 /// Insert scalar single-precision floating-point value659 /// Insert scalar single-precision floating-point value
659 /// Insert packed floating-point values660 /// Insert packed floating-point values
660 insert,661 insert,
662 /// Packed move with sign extend
663 movsxb,
664 movsxd,
665 movsxw,
666 /// Packed move with zero extend
667 movzxb,
668 movzxd,
669 movzxw,
661 /// Round packed single-precision floating-point values670 /// Round packed single-precision floating-point values
662 /// Round scalar single-precision floating-point value671 /// Round scalar single-precision floating-point value
663 /// Round packed double-precision floating-point values672 /// Round packed double-precision floating-point values
...@@ -688,6 +697,7 @@ pub const Inst = struct {...@@ -688,6 +697,7 @@ pub const Inst = struct {
688 sha256rnds2,697 sha256rnds2,
689698
690 /// Load with broadcast floating-point data699 /// Load with broadcast floating-point data
700 /// Load integer and broadcast
691 broadcast,701 broadcast,
692702
693 /// Convert 16-bit floating-point values to single-precision floating-point values703 /// Convert 16-bit floating-point values to single-precision floating-point values
...@@ -762,8 +772,11 @@ pub const Inst = struct {...@@ -762,8 +772,11 @@ pub const Inst = struct {
762 /// Uses `imm` payload.772 /// Uses `imm` payload.
763 rel,773 rel,
764 /// Register, memory operands.774 /// Register, memory operands.
765 /// Uses `rx` payload.775 /// Uses `rx` payload with extra data of type `Memory`.
766 rm,776 rm,
777 /// Register, memory, register operands.
778 /// Uses `rrx` payload with extra data of type `Memory`.
779 rmr,
767 /// Register, memory, immediate (word) operands.780 /// Register, memory, immediate (word) operands.
768 /// Uses `rix` payload with extra data of type `Memory`.781 /// Uses `rix` payload with extra data of type `Memory`.
769 rmi,782 rmi,
...@@ -776,6 +789,9 @@ pub const Inst = struct {...@@ -776,6 +789,9 @@ pub const Inst = struct {
776 /// Register, register, memory.789 /// Register, register, memory.
777 /// Uses `rrix` payload with extra data of type `Memory`.790 /// Uses `rrix` payload with extra data of type `Memory`.
778 rrm,791 rrm,
792 /// Register, register, memory, register.
793 /// Uses `rrrx` payload with extra data of type `Memory`.
794 rrmr,
779 /// Register, register, memory, immediate (byte) operands.795 /// Register, register, memory, immediate (byte) operands.
780 /// Uses `rrix` payload with extra data of type `Memory`.796 /// Uses `rrix` payload with extra data of type `Memory`.
781 rrmi,797 rrmi,
...@@ -953,6 +969,14 @@ pub const Inst = struct {...@@ -953,6 +969,14 @@ pub const Inst = struct {
953 r2: Register,969 r2: Register,
954 payload: u32,970 payload: u32,
955 },971 },
972 /// Register, register, register, followed by Custom payload found in extra.
973 rrrx: struct {
974 fixes: Fixes = ._,
975 r1: Register,
976 r2: Register,
977 r3: Register,
978 payload: u32,
979 },
956 /// Register, byte immediate, followed by Custom payload found in extra.980 /// Register, byte immediate, followed by Custom payload found in extra.
957 rix: struct {981 rix: struct {
958 fixes: Fixes = ._,982 fixes: Fixes = ._,
src/arch/x86_64/encodings.zig+68-1
...@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{...@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{
11851185
1186 .{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },1186 .{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },
11871187
1188 .{ .pshufb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 },
1189
1188 // SSE4.11190 // SSE4.1
1189 .{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },1191 .{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },
11901192
...@@ -1202,6 +1204,11 @@ pub const table = [_]Entry{...@@ -1202,6 +1204,11 @@ pub const table = [_]Entry{
12021204
1203 .{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 },1205 .{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 },
12041206
1207 .{ .pblendvb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
1208 .{ .pblendvb, .rm, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
1209
1210 .{ .pblendw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .none, .sse4_1 },
1211
1205 .{ .pcmpeqq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 },1212 .{ .pcmpeqq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 },
12061213
1207 .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },1214 .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
...@@ -1228,6 +1235,20 @@ pub const table = [_]Entry{...@@ -1228,6 +1235,20 @@ pub const table = [_]Entry{
12281235
1229 .{ .pminud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 },1236 .{ .pminud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 },
12301237
1238 .{ .pmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .none, .sse4_1 },
1239 .{ .pmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .none, .sse4_1 },
1240 .{ .pmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .none, .sse4_1 },
1241 .{ .pmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .none, .sse4_1 },
1242 .{ .pmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .none, .sse4_1 },
1243 .{ .pmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .none, .sse4_1 },
1244
1245 .{ .pmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .none, .sse4_1 },
1246 .{ .pmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .none, .sse4_1 },
1247 .{ .pmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .none, .sse4_1 },
1248 .{ .pmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .none, .sse4_1 },
1249 .{ .pmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .none, .sse4_1 },
1250 .{ .pmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .none, .sse4_1 },
1251
1231 .{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },1252 .{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
12321253
1233 .{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },1254 .{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
...@@ -1528,6 +1549,10 @@ pub const table = [_]Entry{...@@ -1528,6 +1549,10 @@ pub const table = [_]Entry{
15281549
1529 .{ .vpandn, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx },1550 .{ .vpandn, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx },
15301551
1552 .{ .vpblendvb, .rvmr, &.{ .xmm, .xmm, .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_128_w0, .avx },
1553
1554 .{ .vpblendw, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_128_wig, .avx },
1555
1531 .{ .vpclmulqdq, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" },1556 .{ .vpclmulqdq, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" },
15321557
1533 .{ .vpcmpeqb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx },1558 .{ .vpcmpeqb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx },
...@@ -1576,6 +1601,20 @@ pub const table = [_]Entry{...@@ -1576,6 +1601,20 @@ pub const table = [_]Entry{
1576 .{ .vpmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },1601 .{ .vpmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
1577 .{ .vpmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },1602 .{ .vpmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
15781603
1604 .{ .vpmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_128_wig, .avx },
1605 .{ .vpmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_128_wig, .avx },
1606 .{ .vpmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_128_wig, .avx },
1607 .{ .vpmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_128_wig, .avx },
1608 .{ .vpmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_128_wig, .avx },
1609 .{ .vpmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_128_wig, .avx },
1610
1611 .{ .vpmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_128_wig, .avx },
1612 .{ .vpmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_128_wig, .avx },
1613 .{ .vpmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_128_wig, .avx },
1614 .{ .vpmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_128_wig, .avx },
1615 .{ .vpmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_128_wig, .avx },
1616 .{ .vpmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_128_wig, .avx },
1617
1579 .{ .vpmulhw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },1618 .{ .vpmulhw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },
15801619
1581 .{ .vpmulld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },1620 .{ .vpmulld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },
...@@ -1584,6 +1623,8 @@ pub const table = [_]Entry{...@@ -1584,6 +1623,8 @@ pub const table = [_]Entry{
15841623
1585 .{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },1624 .{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },
15861625
1626 .{ .vpshufb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx },
1627
1587 .{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },1628 .{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
15881629
1589 .{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },1630 .{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
...@@ -1728,6 +1769,10 @@ pub const table = [_]Entry{...@@ -1728,6 +1769,10 @@ pub const table = [_]Entry{
1728 .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },1769 .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
1729 .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },1770 .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
17301771
1772 .{ .vextracti128, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 },
1773
1774 .{ .vinserti128, .rvmi, &.{ .ymm, .ymm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 },
1775
1731 .{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },1776 .{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },
1732 .{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },1777 .{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },
1733 .{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },1778 .{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },
...@@ -1756,6 +1801,13 @@ pub const table = [_]Entry{...@@ -1756,6 +1801,13 @@ pub const table = [_]Entry{
17561801
1757 .{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },1802 .{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
17581803
1804 .{ .vpblendd, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_128_w0, .avx2 },
1805 .{ .vpblendd, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_256_w0, .avx2 },
1806
1807 .{ .vpblendvb, .rvmr, &.{ .ymm, .ymm, .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_256_w0, .avx2 },
1808
1809 .{ .vpblendw, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_256_wig, .avx2 },
1810
1759 .{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },1811 .{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
1760 .{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },1812 .{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
1761 .{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },1813 .{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
...@@ -1764,7 +1816,7 @@ pub const table = [_]Entry{...@@ -1764,7 +1816,7 @@ pub const table = [_]Entry{
1764 .{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },1816 .{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
1765 .{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },1817 .{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
1766 .{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },1818 .{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
1767 .{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },1819 .{ .vbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
17681820
1769 .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },1821 .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
1770 .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },1822 .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
...@@ -1799,6 +1851,20 @@ pub const table = [_]Entry{...@@ -1799,6 +1851,20 @@ pub const table = [_]Entry{
1799 .{ .vpmovmskb, .rm, &.{ .r32, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },1851 .{ .vpmovmskb, .rm, &.{ .r32, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
1800 .{ .vpmovmskb, .rm, &.{ .r64, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },1852 .{ .vpmovmskb, .rm, &.{ .r64, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
18011853
1854 .{ .vpmovsxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_256_wig, .avx2 },
1855 .{ .vpmovsxbd, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_256_wig, .avx2 },
1856 .{ .vpmovsxbq, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_256_wig, .avx2 },
1857 .{ .vpmovsxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_256_wig, .avx2 },
1858 .{ .vpmovsxwq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_256_wig, .avx2 },
1859 .{ .vpmovsxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_256_wig, .avx2 },
1860
1861 .{ .vpmovzxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_256_wig, .avx2 },
1862 .{ .vpmovzxbd, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_256_wig, .avx2 },
1863 .{ .vpmovzxbq, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_256_wig, .avx2 },
1864 .{ .vpmovzxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_256_wig, .avx2 },
1865 .{ .vpmovzxwq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_256_wig, .avx2 },
1866 .{ .vpmovzxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_256_wig, .avx2 },
1867
1802 .{ .vpmulhw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 },1868 .{ .vpmulhw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 },
18031869
1804 .{ .vpmulld, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 },1870 .{ .vpmulld, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 },
...@@ -1807,6 +1873,7 @@ pub const table = [_]Entry{...@@ -1807,6 +1873,7 @@ pub const table = [_]Entry{
18071873
1808 .{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },1874 .{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },
18091875
1876 .{ .vpshufb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 },
1810 .{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },1877 .{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },
18111878
1812 .{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },1879 .{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },
src/codegen.zig+27-30
...@@ -405,7 +405,7 @@ pub fn generateSymbol(...@@ -405,7 +405,7 @@ pub fn generateSymbol(
405 .vector_type => |vector_type| {405 .vector_type => |vector_type| {
406 const abi_size = math.cast(usize, typed_value.ty.abiSize(mod)) orelse406 const abi_size = math.cast(usize, typed_value.ty.abiSize(mod)) orelse
407 return error.Overflow;407 return error.Overflow;
408 if (Type.fromInterned(vector_type.child).bitSize(mod) == 1) {408 if (vector_type.child == .bool_type) {
409 const bytes = try code.addManyAsSlice(abi_size);409 const bytes = try code.addManyAsSlice(abi_size);
410 @memset(bytes, 0xaa);410 @memset(bytes, 0xaa);
411 var index: usize = 0;411 var index: usize = 0;
...@@ -443,37 +443,34 @@ pub fn generateSymbol(...@@ -443,37 +443,34 @@ pub fn generateSymbol(
443 },443 },
444 }) byte.* |= mask else byte.* &= ~mask;444 }) byte.* |= mask else byte.* &= ~mask;
445 }445 }
446 } else switch (aggregate.storage) {446 } else {
447 .bytes => |bytes| try code.appendSlice(bytes),447 switch (aggregate.storage) {
448 .elems, .repeated_elem => {448 .bytes => |bytes| try code.appendSlice(bytes),
449 var index: u64 = 0;449 .elems, .repeated_elem => {
450 while (index < vector_type.len) : (index += 1) {450 var index: u64 = 0;
451 switch (try generateSymbol(bin_file, src_loc, .{451 while (index < vector_type.len) : (index += 1) {
452 .ty = Type.fromInterned(vector_type.child),452 switch (try generateSymbol(bin_file, src_loc, .{
453 .val = Value.fromInterned(switch (aggregate.storage) {453 .ty = Type.fromInterned(vector_type.child),
454 .bytes => unreachable,454 .val = Value.fromInterned(switch (aggregate.storage) {
455 .elems => |elems| elems[455 .bytes => unreachable,
456 math.cast(usize, index) orelse return error.Overflow456 .elems => |elems| elems[
457 ],457 math.cast(usize, index) orelse return error.Overflow
458 .repeated_elem => |elem| elem,458 ],
459 }),459 .repeated_elem => |elem| elem,
460 }, code, debug_output, reloc_info)) {460 }),
461 .ok => {},461 }, code, debug_output, reloc_info)) {
462 .fail => |em| return .{ .fail = em },462 .ok => {},
463 .fail => |em| return .{ .fail = em },
464 }
463 }465 }
464 }466 },
465 },467 }
466 }
467468
468 const padding = abi_size - (math.cast(usize, math.divCeil(469 const padding = abi_size -
469 u64,470 (math.cast(usize, Type.fromInterned(vector_type.child).abiSize(mod) * vector_type.len) orelse
470 Type.fromInterned(vector_type.child).bitSize(mod) * vector_type.len,471 return error.Overflow);
471 8,472 if (padding > 0) try code.appendNTimes(0, padding);
472 ) catch |err| switch (err) {473 }
473 error.DivisionByZero => unreachable,
474 else => |e| return e,
475 }) orelse return error.Overflow);
476 if (padding > 0) try code.appendNTimes(0, padding);
477 },474 },
478 .anon_struct_type => |tuple| {475 .anon_struct_type => |tuple| {
479 const struct_begin = code.items.len;476 const struct_begin = code.items.len;
src/codegen/c.zig+45-53
...@@ -4140,9 +4140,7 @@ fn airCmpOp(...@@ -4140,9 +4140,7 @@ fn airCmpOp(
4140 if (need_cast) try writer.writeAll("(void*)");4140 if (need_cast) try writer.writeAll("(void*)");
4141 try f.writeCValue(writer, lhs, .Other);4141 try f.writeCValue(writer, lhs, .Other);
4142 try v.elem(f, writer);4142 try v.elem(f, writer);
4143 try writer.writeByte(' ');
4144 try writer.writeAll(compareOperatorC(operator));4143 try writer.writeAll(compareOperatorC(operator));
4145 try writer.writeByte(' ');
4146 if (need_cast) try writer.writeAll("(void*)");4144 if (need_cast) try writer.writeAll("(void*)");
4147 try f.writeCValue(writer, rhs, .Other);4145 try f.writeCValue(writer, rhs, .Other);
4148 try v.elem(f, writer);4146 try v.elem(f, writer);
...@@ -4181,41 +4179,28 @@ fn airEquality(...@@ -4181,41 +4179,28 @@ fn airEquality(
4181 const writer = f.object.writer();4179 const writer = f.object.writer();
4182 const inst_ty = f.typeOfIndex(inst);4180 const inst_ty = f.typeOfIndex(inst);
4183 const local = try f.allocLocal(inst, inst_ty);4181 const local = try f.allocLocal(inst, inst_ty);
4182 const a = try Assignment.start(f, writer, inst_ty);
4184 try f.writeCValue(writer, local, .Other);4183 try f.writeCValue(writer, local, .Other);
4185 try writer.writeAll(" = ");4184 try a.assign(f, writer);
41864185
4187 if (operand_ty.zigTypeTag(mod) == .Optional and !operand_ty.optionalReprIsPayload(mod)) {4186 if (operand_ty.zigTypeTag(mod) == .Optional and !operand_ty.optionalReprIsPayload(mod)) {
4188 // (A && B) || (C && (A == B))4187 try f.writeCValueMember(writer, lhs, .{ .identifier = "is_null" });
4189 // A = lhs.is_null ; B = rhs.is_null ; C = rhs.payload == lhs.payload4188 try writer.writeAll(" || ");
41904189 try f.writeCValueMember(writer, rhs, .{ .identifier = "is_null" });
4191 switch (operator) {4190 try writer.writeAll(" ? ");
4192 .eq => {},4191 try f.writeCValueMember(writer, lhs, .{ .identifier = "is_null" });
4193 .neq => try writer.writeByte('!'),4192 try writer.writeAll(compareOperatorC(operator));
4194 else => unreachable,4193 try f.writeCValueMember(writer, rhs, .{ .identifier = "is_null" });
4195 }4194 try writer.writeAll(" : ");
4196 try writer.writeAll("((");4195 try f.writeCValueMember(writer, lhs, .{ .identifier = "payload" });
4197 try f.writeCValue(writer, lhs, .Other);4196 try writer.writeAll(compareOperatorC(operator));
4198 try writer.writeAll(".is_null && ");4197 try f.writeCValueMember(writer, rhs, .{ .identifier = "payload" });
4199 try f.writeCValue(writer, rhs, .Other);4198 } else {
4200 try writer.writeAll(".is_null) || (");
4201 try f.writeCValue(writer, lhs, .Other);
4202 try writer.writeAll(".payload == ");
4203 try f.writeCValue(writer, rhs, .Other);
4204 try writer.writeAll(".payload && ");
4205 try f.writeCValue(writer, lhs, .Other);4199 try f.writeCValue(writer, lhs, .Other);
4206 try writer.writeAll(".is_null == ");4200 try writer.writeAll(compareOperatorC(operator));
4207 try f.writeCValue(writer, rhs, .Other);4201 try f.writeCValue(writer, rhs, .Other);
4208 try writer.writeAll(".is_null));\n");
4209
4210 return local;
4211 }4202 }
42124203 try a.end(f, writer);
4213 try f.writeCValue(writer, lhs, .Other);
4214 try writer.writeByte(' ');
4215 try writer.writeAll(compareOperatorC(operator));
4216 try writer.writeByte(' ');
4217 try f.writeCValue(writer, rhs, .Other);
4218 try writer.writeAll(";\n");
42194204
4220 return local;4205 return local;
4221}4206}
...@@ -6109,41 +6094,48 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -6109,41 +6094,48 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue {
6109 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6094 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
61106095
6111 const inst_ty = f.typeOfIndex(inst);6096 const inst_ty = f.typeOfIndex(inst);
6097 const inst_scalar_ty = inst_ty.scalarType(mod);
6112 const operand = try f.resolveInst(ty_op.operand);6098 const operand = try f.resolveInst(ty_op.operand);
6113 try reap(f, inst, &.{ty_op.operand});6099 try reap(f, inst, &.{ty_op.operand});
6114 const operand_ty = f.typeOf(ty_op.operand);6100 const operand_ty = f.typeOf(ty_op.operand);
6101 const scalar_ty = operand_ty.scalarType(mod);
6115 const target = f.object.dg.module.getTarget();6102 const target = f.object.dg.module.getTarget();
6116 const operation = if (inst_ty.isRuntimeFloat() and operand_ty.isRuntimeFloat())6103 const operation = if (inst_scalar_ty.isRuntimeFloat() and scalar_ty.isRuntimeFloat())
6117 if (inst_ty.floatBits(target) < operand_ty.floatBits(target)) "trunc" else "extend"6104 if (inst_scalar_ty.floatBits(target) < scalar_ty.floatBits(target)) "trunc" else "extend"
6118 else if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat())6105 else if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat())
6119 if (inst_ty.isSignedInt(mod)) "fix" else "fixuns"6106 if (inst_scalar_ty.isSignedInt(mod)) "fix" else "fixuns"
6120 else if (inst_ty.isRuntimeFloat() and operand_ty.isInt(mod))6107 else if (inst_scalar_ty.isRuntimeFloat() and scalar_ty.isInt(mod))
6121 if (operand_ty.isSignedInt(mod)) "float" else "floatun"6108 if (scalar_ty.isSignedInt(mod)) "float" else "floatun"
6122 else6109 else
6123 unreachable;6110 unreachable;
61246111
6125 const writer = f.object.writer();6112 const writer = f.object.writer();
6126 const local = try f.allocLocal(inst, inst_ty);6113 const local = try f.allocLocal(inst, inst_ty);
6114 const v = try Vectorize.start(f, inst, writer, operand_ty);
6115 const a = try Assignment.start(f, writer, scalar_ty);
6127 try f.writeCValue(writer, local, .Other);6116 try f.writeCValue(writer, local, .Other);
61286117 try v.elem(f, writer);
6129 try writer.writeAll(" = ");6118 try a.assign(f, writer);
6130 if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat()) {6119 if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat()) {
6131 try writer.writeAll("zig_wrap_");6120 try writer.writeAll("zig_wrap_");
6132 try f.object.dg.renderTypeForBuiltinFnName(writer, inst_ty);6121 try f.object.dg.renderTypeForBuiltinFnName(writer, inst_scalar_ty);
6133 try writer.writeByte('(');6122 try writer.writeByte('(');
6134 }6123 }
6135 try writer.writeAll("zig_");6124 try writer.writeAll("zig_");
6136 try writer.writeAll(operation);6125 try writer.writeAll(operation);
6137 try writer.writeAll(compilerRtAbbrev(operand_ty, mod));6126 try writer.writeAll(compilerRtAbbrev(scalar_ty, mod));
6138 try writer.writeAll(compilerRtAbbrev(inst_ty, mod));6127 try writer.writeAll(compilerRtAbbrev(inst_scalar_ty, mod));
6139 try writer.writeByte('(');6128 try writer.writeByte('(');
6140 try f.writeCValue(writer, operand, .FunctionArgument);6129 try f.writeCValue(writer, operand, .FunctionArgument);
6130 try v.elem(f, writer);
6141 try writer.writeByte(')');6131 try writer.writeByte(')');
6142 if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat()) {6132 if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat()) {
6143 try f.object.dg.renderBuiltinInfo(writer, inst_ty, .bits);6133 try f.object.dg.renderBuiltinInfo(writer, inst_scalar_ty, .bits);
6144 try writer.writeByte(')');6134 try writer.writeByte(')');
6145 }6135 }
6146 try writer.writeAll(";\n");6136 try a.end(f, writer);
6137 try v.end(f, inst, writer);
6138
6147 return local;6139 return local;
6148}6140}
61496141
...@@ -6315,7 +6307,7 @@ fn airCmpBuiltinCall(...@@ -6315,7 +6307,7 @@ fn airCmpBuiltinCall(
6315 try v.elem(f, writer);6307 try v.elem(f, writer);
6316 try f.object.dg.renderBuiltinInfo(writer, scalar_ty, info);6308 try f.object.dg.renderBuiltinInfo(writer, scalar_ty, info);
6317 try writer.writeByte(')');6309 try writer.writeByte(')');
6318 if (!ref_ret) try writer.print(" {s} {}", .{6310 if (!ref_ret) try writer.print("{s}{}", .{
6319 compareOperatorC(operator),6311 compareOperatorC(operator),
6320 try f.fmtIntLiteral(Type.i32, try mod.intValue(Type.i32, 0)),6312 try f.fmtIntLiteral(Type.i32, try mod.intValue(Type.i32, 0)),
6321 });6313 });
...@@ -7661,12 +7653,12 @@ fn compareOperatorAbbrev(operator: std.math.CompareOperator) []const u8 {...@@ -7661,12 +7653,12 @@ fn compareOperatorAbbrev(operator: std.math.CompareOperator) []const u8 {
76617653
7662fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {7654fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {
7663 return switch (operator) {7655 return switch (operator) {
7664 .lt => "<",7656 .lt => " < ",
7665 .lte => "<=",7657 .lte => " <= ",
7666 .eq => "==",7658 .eq => " == ",
7667 .gte => ">=",7659 .gte => " >= ",
7668 .gt => ">",7660 .gt => " > ",
7669 .neq => "!=",7661 .neq => " != ",
7670 };7662 };
7671}7663}
76727664
src/codegen/llvm.zig+8-2
...@@ -8646,8 +8646,6 @@ pub const FuncGen = struct {...@@ -8646,8 +8646,6 @@ pub const FuncGen = struct {
8646 const operand_ty = self.typeOf(ty_op.operand);8646 const operand_ty = self.typeOf(ty_op.operand);
8647 const dest_ty = self.typeOfIndex(inst);8647 const dest_ty = self.typeOfIndex(inst);
8648 const target = mod.getTarget();8648 const target = mod.getTarget();
8649 const dest_bits = dest_ty.floatBits(target);
8650 const src_bits = operand_ty.floatBits(target);
86518649
8652 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {8650 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
8653 return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");8651 return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");
...@@ -8655,11 +8653,19 @@ pub const FuncGen = struct {...@@ -8655,11 +8653,19 @@ pub const FuncGen = struct {
8655 const operand_llvm_ty = try o.lowerType(operand_ty);8653 const operand_llvm_ty = try o.lowerType(operand_ty);
8656 const dest_llvm_ty = try o.lowerType(dest_ty);8654 const dest_llvm_ty = try o.lowerType(dest_ty);
86578655
8656 const dest_bits = dest_ty.scalarType(mod).floatBits(target);
8657 const src_bits = operand_ty.scalarType(mod).floatBits(target);
8658 const fn_name = try o.builder.fmt("__extend{s}f{s}f2", .{8658 const fn_name = try o.builder.fmt("__extend{s}f{s}f2", .{
8659 compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),8659 compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
8660 });8660 });
86618661
8662 const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);8662 const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
8663 if (dest_ty.isVector(mod)) return self.buildElementwiseCall(
8664 libc_fn,
8665 &.{operand},
8666 try o.builder.poisonValue(dest_llvm_ty),
8667 dest_ty.vectorLen(mod),
8668 );
8663 return self.wip.call(8669 return self.wip.call(
8664 .normal,8670 .normal,
8665 .ccc,8671 .ccc,
src/type.zig+40-9
...@@ -905,11 +905,32 @@ pub const Type = struct {...@@ -905,11 +905,32 @@ pub const Type = struct {
905 return Type.fromInterned(array_type.child).abiAlignmentAdvanced(mod, strat);905 return Type.fromInterned(array_type.child).abiAlignmentAdvanced(mod, strat);
906 },906 },
907 .vector_type => |vector_type| {907 .vector_type => |vector_type| {
908 const bits_u64 = try bitSizeAdvanced(Type.fromInterned(vector_type.child), mod, opt_sema);908 if (vector_type.len == 0) return .{ .scalar = .@"1" };
909 const bits: u32 = @intCast(bits_u64);909 switch (mod.comp.getZigBackend()) {
910 const bytes = ((bits * vector_type.len) + 7) / 8;910 else => {
911 const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);911 const elem_bits: u32 = @intCast(try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema));
912 return .{ .scalar = Alignment.fromByteUnits(alignment) };912 if (elem_bits == 0) return .{ .scalar = .@"1" };
913 const bytes = ((elem_bits * vector_type.len) + 7) / 8;
914 const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
915 return .{ .scalar = Alignment.fromByteUnits(alignment) };
916 },
917 .stage2_x86_64 => {
918 if (vector_type.child == .bool_type) {
919 if (vector_type.len > 256 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };
920 if (vector_type.len > 128 and std.Target.x86.featureSetHas(target.cpu.features, .avx2)) return .{ .scalar = .@"32" };
921 if (vector_type.len > 64) return .{ .scalar = .@"16" };
922 const bytes = std.math.divCeil(u32, vector_type.len, 8) catch unreachable;
923 const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
924 return .{ .scalar = Alignment.fromByteUnits(alignment) };
925 }
926 const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
927 if (elem_bytes == 0) return .{ .scalar = .@"1" };
928 const bytes = elem_bytes * vector_type.len;
929 if (bytes > 32 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };
930 if (bytes > 16 and std.Target.x86.featureSetHas(target.cpu.features, .avx)) return .{ .scalar = .@"32" };
931 return .{ .scalar = .@"16" };
932 },
933 }
913 },934 },
914935
915 .opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat),936 .opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat),
...@@ -1237,9 +1258,6 @@ pub const Type = struct {...@@ -1237,9 +1258,6 @@ pub const Type = struct {
1237 .storage = .{ .lazy_size = ty.toIntern() },1258 .storage = .{ .lazy_size = ty.toIntern() },
1238 } }))) },1259 } }))) },
1239 };1260 };
1240 const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
1241 const total_bits = elem_bits * vector_type.len;
1242 const total_bytes = (total_bits + 7) / 8;
1243 const alignment = switch (try ty.abiAlignmentAdvanced(mod, strat)) {1261 const alignment = switch (try ty.abiAlignmentAdvanced(mod, strat)) {
1244 .scalar => |x| x,1262 .scalar => |x| x,
1245 .val => return .{ .val = Value.fromInterned((try mod.intern(.{ .int = .{1263 .val => return .{ .val = Value.fromInterned((try mod.intern(.{ .int = .{
...@@ -1247,6 +1265,18 @@ pub const Type = struct {...@@ -1247,6 +1265,18 @@ pub const Type = struct {
1247 .storage = .{ .lazy_size = ty.toIntern() },1265 .storage = .{ .lazy_size = ty.toIntern() },
1248 } }))) },1266 } }))) },
1249 };1267 };
1268 const total_bytes = switch (mod.comp.getZigBackend()) {
1269 else => total_bytes: {
1270 const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
1271 const total_bits = elem_bits * vector_type.len;
1272 break :total_bytes (total_bits + 7) / 8;
1273 },
1274 .stage2_x86_64 => total_bytes: {
1275 if (vector_type.child == .bool_type) break :total_bytes std.math.divCeil(u32, vector_type.len, 8) catch unreachable;
1276 const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
1277 break :total_bytes elem_bytes * vector_type.len;
1278 },
1279 };
1250 return AbiSizeAdvanced{ .scalar = alignment.forward(total_bytes) };1280 return AbiSizeAdvanced{ .scalar = alignment.forward(total_bytes) };
1251 },1281 },
12521282
...@@ -2108,7 +2138,8 @@ pub const Type = struct {...@@ -2108,7 +2138,8 @@ pub const Type = struct {
21082138
2109 /// Returns true if and only if the type is a fixed-width integer.2139 /// Returns true if and only if the type is a fixed-width integer.
2110 pub fn isInt(self: Type, mod: *const Module) bool {2140 pub fn isInt(self: Type, mod: *const Module) bool {
2111 return self.isSignedInt(mod) or self.isUnsignedInt(mod);2141 return self.toIntern() != .comptime_int_type and
2142 mod.intern_pool.isIntegerType(self.toIntern());
2112 }2143 }
21132144
2114 /// Returns true if and only if the type is a fixed-width, signed integer.2145 /// Returns true if and only if the type is a fixed-width, signed integer.
test/behavior/bitcast.zig+1-1
...@@ -336,7 +336,7 @@ test "comptime @bitCast packed struct to int and back" {...@@ -336,7 +336,7 @@ test "comptime @bitCast packed struct to int and back" {
336 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;336 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
337 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO337 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
338 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;338 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
339 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;339 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
340340
341 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {341 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
342 // https://github.com/ziglang/zig/issues/13782342 // https://github.com/ziglang/zig/issues/13782
test/behavior/cast.zig+33-21
...@@ -601,25 +601,25 @@ test "cast *[1][*]const u8 to [*]const ?[*]const u8" {...@@ -601,25 +601,25 @@ test "cast *[1][*]const u8 to [*]const ?[*]const u8" {
601601
602test "@intCast on vector" {602test "@intCast on vector" {
603 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO603 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
604 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
605 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO604 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
606 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO605 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
607 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO606 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
607 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
608608
609 const S = struct {609 const S = struct {
610 fn doTheTest() !void {610 fn doTheTest() !void {
611 // Upcast (implicit, equivalent to @intCast)611 // Upcast (implicit, equivalent to @intCast)
612 var up0: @Vector(2, u8) = [_]u8{ 0x55, 0xaa };612 var up0: @Vector(2, u8) = [_]u8{ 0x55, 0xaa };
613 _ = &up0;613 _ = &up0;
614 const up1 = @as(@Vector(2, u16), up0);614 const up1: @Vector(2, u16) = up0;
615 const up2 = @as(@Vector(2, u32), up0);615 const up2: @Vector(2, u32) = up0;
616 const up3 = @as(@Vector(2, u64), up0);616 const up3: @Vector(2, u64) = up0;
617 // Downcast (safety-checked)617 // Downcast (safety-checked)
618 var down0 = up3;618 var down0 = up3;
619 _ = &down0;619 _ = &down0;
620 const down1 = @as(@Vector(2, u32), @intCast(down0));620 const down1: @Vector(2, u32) = @intCast(down0);
621 const down2 = @as(@Vector(2, u16), @intCast(down0));621 const down2: @Vector(2, u16) = @intCast(down0);
622 const down3 = @as(@Vector(2, u8), @intCast(down0));622 const down3: @Vector(2, u8) = @intCast(down0);
623623
624 try expect(mem.eql(u16, &@as([2]u16, up1), &[2]u16{ 0x55, 0xaa }));624 try expect(mem.eql(u16, &@as([2]u16, up1), &[2]u16{ 0x55, 0xaa }));
625 try expect(mem.eql(u32, &@as([2]u32, up2), &[2]u32{ 0x55, 0xaa }));625 try expect(mem.eql(u32, &@as([2]u32, up2), &[2]u32{ 0x55, 0xaa }));
...@@ -629,20 +629,10 @@ test "@intCast on vector" {...@@ -629,20 +629,10 @@ test "@intCast on vector" {
629 try expect(mem.eql(u16, &@as([2]u16, down2), &[2]u16{ 0x55, 0xaa }));629 try expect(mem.eql(u16, &@as([2]u16, down2), &[2]u16{ 0x55, 0xaa }));
630 try expect(mem.eql(u8, &@as([2]u8, down3), &[2]u8{ 0x55, 0xaa }));630 try expect(mem.eql(u8, &@as([2]u8, down3), &[2]u8{ 0x55, 0xaa }));
631 }631 }
632
633 fn doTheTestFloat() !void {
634 var vec: @Vector(2, f32) = @splat(1234.0);
635 _ = &vec;
636 const wider: @Vector(2, f64) = vec;
637 try expect(wider[0] == 1234.0);
638 try expect(wider[1] == 1234.0);
639 }
640 };632 };
641633
642 try S.doTheTest();634 try S.doTheTest();
643 try comptime S.doTheTest();635 try comptime S.doTheTest();
644 try S.doTheTestFloat();
645 try comptime S.doTheTestFloat();
646}636}
647637
648test "@floatCast cast down" {638test "@floatCast cast down" {
...@@ -2340,10 +2330,31 @@ test "@floatCast on vector" {...@@ -2340,10 +2330,31 @@ test "@floatCast on vector" {
23402330
2341 const S = struct {2331 const S = struct {
2342 fn doTheTest() !void {2332 fn doTheTest() !void {
2343 var a: @Vector(3, f64) = .{ 1.5, 2.5, 3.5 };2333 {
2344 _ = &a;2334 var a: @Vector(2, f64) = .{ 1.5, 2.5 };
2345 const b: @Vector(3, f32) = @floatCast(a);2335 _ = &a;
2346 try expectEqual(@Vector(3, f32){ 1.5, 2.5, 3.5 }, b);2336 const b: @Vector(2, f32) = @floatCast(a);
2337 try expectEqual(@Vector(2, f32){ 1.5, 2.5 }, b);
2338 }
2339 {
2340 var a: @Vector(2, f32) = .{ 3.25, 4.25 };
2341 _ = &a;
2342 const b: @Vector(2, f64) = @floatCast(a);
2343 try expectEqual(@Vector(2, f64){ 3.25, 4.25 }, b);
2344 }
2345 {
2346 var a: @Vector(2, f32) = .{ 5.75, 6.75 };
2347 _ = &a;
2348 const b: @Vector(2, f64) = a;
2349 try expectEqual(@Vector(2, f64){ 5.75, 6.75 }, b);
2350 }
2351 {
2352 var vec: @Vector(2, f32) = @splat(1234.0);
2353 _ = &vec;
2354 const wider: @Vector(2, f64) = vec;
2355 try expect(wider[0] == 1234.0);
2356 try expect(wider[1] == 1234.0);
2357 }
2347 }2358 }
2348 };2359 };
23492360
...@@ -2441,6 +2452,7 @@ test "@intFromBool on vector" {...@@ -2441,6 +2452,7 @@ test "@intFromBool on vector" {
2441 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO2452 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
2442 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO2453 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
2443 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO2454 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
2455 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
24442456
2445 const S = struct {2457 const S = struct {
2446 fn doTheTest() !void {2458 fn doTheTest() !void {
test/behavior/optional.zig+77-32
...@@ -110,44 +110,89 @@ test "nested optional field in struct" {...@@ -110,44 +110,89 @@ test "nested optional field in struct" {
110 try expect(s.x.?.y == 127);110 try expect(s.x.?.y == 127);
111}111}
112112
113test "equality compare optional with non-optional" {113test "equality compare optionals and non-optionals" {
114 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;114 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
115 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO115 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
116 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO116 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
117117
118 try test_cmp_optional_non_optional();118 const S = struct {
119 try comptime test_cmp_optional_non_optional();119 fn doTheTest() !void {
120 var five: isize = 5;
121 var ten: isize = 10;
122 var opt_null: ?isize = null;
123 var opt_ten: ?isize = 10;
124 _ = .{ &five, &ten, &opt_null, &opt_ten };
125 try expect(opt_null != five);
126 try expect(opt_null != ten);
127 try expect(opt_ten != five);
128 try expect(opt_ten == ten);
129
130 var opt_int: ?isize = null;
131 try expect(opt_int != five);
132 try expect(opt_int != ten);
133 try expect(opt_int == opt_null);
134 try expect(opt_int != opt_ten);
135
136 opt_int = 10;
137 try expect(opt_int != five);
138 try expect(opt_int == ten);
139 try expect(opt_int != opt_null);
140 try expect(opt_int == opt_ten);
141
142 opt_int = five;
143 try expect(opt_int == five);
144 try expect(opt_int != ten);
145 try expect(opt_int != opt_null);
146 try expect(opt_int != opt_ten);
147
148 // test evaluation is always lexical
149 // ensure that the optional isn't always computed before the non-optional
150 var mutable_state: i32 = 0;
151 _ = blk1: {
152 mutable_state += 1;
153 break :blk1 @as(?f64, 10.0);
154 } != blk2: {
155 try expect(mutable_state == 1);
156 break :blk2 @as(f64, 5.0);
157 };
158 _ = blk1: {
159 mutable_state += 1;
160 break :blk1 @as(f64, 10.0);
161 } != blk2: {
162 try expect(mutable_state == 2);
163 break :blk2 @as(?f64, 5.0);
164 };
165 }
166 };
167
168 try S.doTheTest();
169 try comptime S.doTheTest();
120}170}
121171
122fn test_cmp_optional_non_optional() !void {172test "compare optionals with modified payloads" {
123 var ten: i32 = 10;173 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
124 var opt_ten: ?i32 = 10;174
125 var five: i32 = 5;175 var lhs: ?bool = false;
126 var int_n: ?i32 = null;176 const lhs_payload = &lhs.?;
127177 var rhs: ?bool = true;
128 _ = .{ &ten, &opt_ten, &five, &int_n };178 const rhs_payload = &rhs.?;
129179 try expect(lhs != rhs and !(lhs == rhs));
130 try expect(int_n != ten);180
131 try expect(opt_ten == ten);181 lhs = null;
132 try expect(opt_ten != five);182 lhs_payload.* = false;
133183 rhs = false;
134 // test evaluation is always lexical184 try expect(lhs != rhs and !(lhs == rhs));
135 // ensure that the optional isn't always computed before the non-optional185
136 var mutable_state: i32 = 0;186 lhs = true;
137 _ = blk1: {187 rhs = null;
138 mutable_state += 1;188 rhs_payload.* = true;
139 break :blk1 @as(?f64, 10.0);189 try expect(lhs != rhs and !(lhs == rhs));
140 } != blk2: {190
141 try expect(mutable_state == 1);191 lhs = null;
142 break :blk2 @as(f64, 5.0);192 lhs_payload.* = false;
143 };193 rhs = null;
144 _ = blk1: {194 rhs_payload.* = true;
145 mutable_state += 1;195 try expect(lhs == rhs and !(lhs != rhs));
146 break :blk1 @as(f64, 10.0);
147 } != blk2: {
148 try expect(mutable_state == 2);
149 break :blk2 @as(?f64, 5.0);
150 };
151}196}
152197
153test "unwrap function call with optional pointer return value" {198test "unwrap function call with optional pointer return value" {
test/behavior/select.zig+2-2
...@@ -5,7 +5,6 @@ const expect = std.testing.expect;...@@ -5,7 +5,6 @@ const expect = std.testing.expect;
55
6test "@select vectors" {6test "@select vectors" {
7 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO7 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO8 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO9 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
11 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -36,11 +35,12 @@ fn selectVectors() !void {...@@ -36,11 +35,12 @@ fn selectVectors() !void {
3635
37test "@select arrays" {36test "@select arrays" {
38 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO37 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
39 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
40 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO38 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
41 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO39 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
42 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO40 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
43 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;41 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
42 if (builtin.zig_backend == .stage2_x86_64 and
43 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx2)) return error.SkipZigTest;
4444
45 try comptime selectArrays();45 try comptime selectArrays();
46 try selectArrays();46 try selectArrays();
test/behavior/shuffle.zig+2-1
...@@ -4,10 +4,11 @@ const mem = std.mem;...@@ -4,10 +4,11 @@ const mem = std.mem;
4const expect = std.testing.expect;4const expect = std.testing.expect;
55
6test "@shuffle int" {6test "@shuffle int" {
7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO7 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO8 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO9 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_x86_64 and
11 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
1112
12 const S = struct {13 const S = struct {
13 fn doTheTest() !void {14 fn doTheTest() !void {
test/behavior/vector.zig+29-13
...@@ -29,7 +29,7 @@ test "vector wrap operators" {...@@ -29,7 +29,7 @@ test "vector wrap operators" {
29 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO29 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
30 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO30 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
31 if (builtin.zig_backend == .stage2_x86_64 and31 if (builtin.zig_backend == .stage2_x86_64 and
32 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO32 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest;
3333
34 const S = struct {34 const S = struct {
35 fn doTheTest() !void {35 fn doTheTest() !void {
...@@ -906,22 +906,26 @@ test "vector @reduce comptime" {...@@ -906,22 +906,26 @@ test "vector @reduce comptime" {
906}906}
907907
908test "mask parameter of @shuffle is comptime scope" {908test "mask parameter of @shuffle is comptime scope" {
909 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
910 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO909 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
911 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO910 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
912 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO911 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
912 if (builtin.zig_backend == .stage2_x86_64 and
913 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
913914
914 const __v4hi = @Vector(4, i16);915 const __v4hi = @Vector(4, i16);
915 var v4_a = __v4hi{ 0, 0, 0, 0 };916 var v4_a = __v4hi{ 1, 2, 3, 4 };
916 var v4_b = __v4hi{ 0, 0, 0, 0 };917 var v4_b = __v4hi{ 5, 6, 7, 8 };
917 _ = .{ &v4_a, &v4_b };918 _ = .{ &v4_a, &v4_b };
918 const shuffled: __v4hi = @shuffle(i16, v4_a, v4_b, @Vector(4, i32){919 const shuffled: __v4hi = @shuffle(i16, v4_a, v4_b, @Vector(4, i32){
919 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),920 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
920 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),921 std.zig.c_translation.shuffleVectorIndex(2, @typeInfo(@TypeOf(v4_a)).Vector.len),
921 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),922 std.zig.c_translation.shuffleVectorIndex(4, @typeInfo(@TypeOf(v4_a)).Vector.len),
922 std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),923 std.zig.c_translation.shuffleVectorIndex(6, @typeInfo(@TypeOf(v4_a)).Vector.len),
923 });924 });
924 _ = shuffled;925 try expect(shuffled[0] == 1);
926 try expect(shuffled[1] == 3);
927 try expect(shuffled[2] == 5);
928 try expect(shuffled[3] == 7);
925}929}
926930
927test "saturating add" {931test "saturating add" {
...@@ -1177,10 +1181,22 @@ test "@shlWithOverflow" {...@@ -1177,10 +1181,22 @@ test "@shlWithOverflow" {
1177}1181}
11781182
1179test "alignment of vectors" {1183test "alignment of vectors" {
1180 try expect(@alignOf(@Vector(2, u8)) == 2);1184 try expect(@alignOf(@Vector(2, u8)) == switch (builtin.zig_backend) {
1181 try expect(@alignOf(@Vector(2, u1)) == 1);1185 else => 2,
1182 try expect(@alignOf(@Vector(1, u1)) == 1);1186 .stage2_x86_64 => 16,
1183 try expect(@alignOf(@Vector(2, u16)) == 4);1187 });
1188 try expect(@alignOf(@Vector(2, u1)) == switch (builtin.zig_backend) {
1189 else => 1,
1190 .stage2_x86_64 => 16,
1191 });
1192 try expect(@alignOf(@Vector(1, u1)) == switch (builtin.zig_backend) {
1193 else => 1,
1194 .stage2_x86_64 => 16,
1195 });
1196 try expect(@alignOf(@Vector(2, u16)) == switch (builtin.zig_backend) {
1197 else => 4,
1198 .stage2_x86_64 => 16,
1199 });
1184}1200}
11851201
1186test "loading the second vector from a slice of vectors" {1202test "loading the second vector from a slice of vectors" {
...@@ -1316,10 +1332,10 @@ test "modRem with zero divisor" {...@@ -1316,10 +1332,10 @@ test "modRem with zero divisor" {
13161332
1317test "array operands to shuffle are coerced to vectors" {1333test "array operands to shuffle are coerced to vectors" {
1318 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1334 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1319 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1320 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1335 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1321 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1336 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1322 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1337 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1338 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
13231339
1324 const mask = [5]i32{ -1, 0, 1, 2, 3 };1340 const mask = [5]i32{ -1, 0, 1, 2, 3 };
13251341