authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-11-04 14:24:59-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-11-04 14:24:59-04:00
loga7d8cd591c47536b0a1e359bf3b1806fc057ffe9
tree069ca321e822ea95d7265ece0b79173832a0745a
parentf6de3ec963e3a7d96cd4f6c72b0f076f0437c45d
parent095c4294aa8b275da0627adefad046923fcaae46
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17788 from jacobly0/x86_64

x86_64: pass more tests

49 files changed, 1650 insertions(+), 962 deletions(-)

lib/std/Thread/Condition.zig-10
...@@ -324,8 +324,6 @@ test "Condition - wait and signal" {...@@ -324,8 +324,6 @@ test "Condition - wait and signal" {
324 return error.SkipZigTest;324 return error.SkipZigTest;
325 }325 }
326326
327 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
328
329 const num_threads = 4;327 const num_threads = 4;
330328
331 const MultiWait = struct {329 const MultiWait = struct {
...@@ -371,8 +369,6 @@ test "Condition - signal" {...@@ -371,8 +369,6 @@ test "Condition - signal" {
371 return error.SkipZigTest;369 return error.SkipZigTest;
372 }370 }
373371
374 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
375
376 const num_threads = 4;372 const num_threads = 4;
377373
378 const SignalTest = struct {374 const SignalTest = struct {
...@@ -440,8 +436,6 @@ test "Condition - multi signal" {...@@ -440,8 +436,6 @@ test "Condition - multi signal" {
440 return error.SkipZigTest;436 return error.SkipZigTest;
441 }437 }
442438
443 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
444
445 const num_threads = 4;439 const num_threads = 4;
446 const num_iterations = 4;440 const num_iterations = 4;
447441
...@@ -504,8 +498,6 @@ test "Condition - broadcasting" {...@@ -504,8 +498,6 @@ test "Condition - broadcasting" {
504 return error.SkipZigTest;498 return error.SkipZigTest;
505 }499 }
506500
507 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
508
509 const num_threads = 10;501 const num_threads = 10;
510502
511 const BroadcastTest = struct {503 const BroadcastTest = struct {
...@@ -573,8 +565,6 @@ test "Condition - broadcasting - wake all threads" {...@@ -573,8 +565,6 @@ test "Condition - broadcasting - wake all threads" {
573 return error.SkipZigTest;565 return error.SkipZigTest;
574 }566 }
575567
576 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
577
578 var num_runs: usize = 1;568 var num_runs: usize = 1;
579 const num_threads = 10;569 const num_threads = 10;
580570
lib/std/Thread/Mutex.zig-2
...@@ -289,8 +289,6 @@ test "Mutex - many contended" {...@@ -289,8 +289,6 @@ test "Mutex - many contended" {
289 return error.SkipZigTest;289 return error.SkipZigTest;
290 }290 }
291291
292 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
293
294 const num_threads = 4;292 const num_threads = 4;
295 const num_increments = 1000;293 const num_increments = 1000;
296294
lib/std/Thread/RwLock.zig-2
...@@ -297,8 +297,6 @@ test "RwLock - concurrent access" {...@@ -297,8 +297,6 @@ test "RwLock - concurrent access" {
297 if (builtin.single_threaded)297 if (builtin.single_threaded)
298 return;298 return;
299299
300 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
301
302 const num_writers: usize = 2;300 const num_writers: usize = 2;
303 const num_readers: usize = 4;301 const num_readers: usize = 4;
304 const num_writes: usize = 10000;302 const num_writes: usize = 10000;
lib/std/Thread/Semaphore.zig-2
...@@ -39,8 +39,6 @@ test "Thread.Semaphore" {...@@ -39,8 +39,6 @@ test "Thread.Semaphore" {
39 return error.SkipZigTest;39 return error.SkipZigTest;
40 }40 }
4141
42 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
43
44 const TestContext = struct {42 const TestContext = struct {
45 sem: *Semaphore,43 sem: *Semaphore,
46 n: *i32,44 n: *i32,
lib/std/atomic/Atomic.zig-2
...@@ -467,8 +467,6 @@ test "Atomic.fetchSub" {...@@ -467,8 +467,6 @@ test "Atomic.fetchSub" {
467}467}
468468
469test "Atomic.fetchMin" {469test "Atomic.fetchMin" {
470 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
471
472 inline for (atomicIntTypes()) |Int| {470 inline for (atomicIntTypes()) |Int| {
473 inline for (atomic_rmw_orderings) |ordering| {471 inline for (atomic_rmw_orderings) |ordering| {
474 var x = Atomic(Int).init(5);472 var x = Atomic(Int).init(5);
lib/std/atomic/queue.zig-2
...@@ -175,8 +175,6 @@ const puts_per_thread = 500;...@@ -175,8 +175,6 @@ const puts_per_thread = 500;
175const put_thread_count = 3;175const put_thread_count = 3;
176176
177test "std.atomic.Queue" {177test "std.atomic.Queue" {
178 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
179
180 var plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);178 var plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);
181 defer std.heap.page_allocator.free(plenty_of_memory);179 defer std.heap.page_allocator.free(plenty_of_memory);
182180
lib/std/base64.zig-4
...@@ -355,8 +355,6 @@ pub const Base64DecoderWithIgnore = struct {...@@ -355,8 +355,6 @@ pub const Base64DecoderWithIgnore = struct {
355};355};
356356
357test "base64" {357test "base64" {
358 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
359
360 @setEvalBranchQuota(8000);358 @setEvalBranchQuota(8000);
361 try testBase64();359 try testBase64();
362 try comptime testAllApis(standard, "comptime", "Y29tcHRpbWU=");360 try comptime testAllApis(standard, "comptime", "Y29tcHRpbWU=");
...@@ -377,8 +375,6 @@ test "base64 padding dest overflow" {...@@ -377,8 +375,6 @@ test "base64 padding dest overflow" {
377}375}
378376
379test "base64 url_safe_no_pad" {377test "base64 url_safe_no_pad" {
380 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
381
382 @setEvalBranchQuota(8000);378 @setEvalBranchQuota(8000);
383 try testBase64UrlSafeNoPad();379 try testBase64UrlSafeNoPad();
384 try comptime testAllApis(url_safe_no_pad, "comptime", "Y29tcHRpbWU");380 try comptime testAllApis(url_safe_no_pad, "comptime", "Y29tcHRpbWU");
lib/std/bit_set.zig-3
...@@ -1638,7 +1638,6 @@ fn testStaticBitSet(comptime Set: type) !void {...@@ -1638,7 +1638,6 @@ fn testStaticBitSet(comptime Set: type) !void {
16381638
1639test "IntegerBitSet" {1639test "IntegerBitSet" {
1640 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;1640 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1641 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
16421641
1643 try testStaticBitSet(IntegerBitSet(0));1642 try testStaticBitSet(IntegerBitSet(0));
1644 try testStaticBitSet(IntegerBitSet(1));1643 try testStaticBitSet(IntegerBitSet(1));
...@@ -1651,8 +1650,6 @@ test "IntegerBitSet" {...@@ -1651,8 +1650,6 @@ test "IntegerBitSet" {
1651}1650}
16521651
1653test "ArrayBitSet" {1652test "ArrayBitSet" {
1654 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1655
1656 inline for (.{ 0, 1, 2, 31, 32, 33, 63, 64, 65, 254, 500, 3000 }) |size| {1653 inline for (.{ 0, 1, 2, 31, 32, 33, 63, 64, 65, 254, 500, 3000 }) |size| {
1657 try testStaticBitSet(ArrayBitSet(u8, size));1654 try testStaticBitSet(ArrayBitSet(u8, size));
1658 try testStaticBitSet(ArrayBitSet(u16, size));1655 try testStaticBitSet(ArrayBitSet(u16, size));
lib/std/compress/zstandard.zig-2
...@@ -264,8 +264,6 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {...@@ -264,8 +264,6 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {
264}264}
265265
266test "zstandard decompression" {266test "zstandard decompression" {
267 if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269 const uncompressed = @embedFile("testdata/rfc8478.txt");267 const uncompressed = @embedFile("testdata/rfc8478.txt");
270 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");268 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
271 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");269 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
lib/std/crypto/25519/ed25519.zig+2-13
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const crypto = std.crypto;2const crypto = std.crypto;
4const debug = std.debug;3const debug = std.debug;
5const fmt = std.fmt;4const fmt = std.fmt;
...@@ -276,8 +275,8 @@ pub const Ed25519 = struct {...@@ -276,8 +275,8 @@ pub const Ed25519 = struct {
276 pub fn fromSecretKey(secret_key: SecretKey) (NonCanonicalError || EncodingError || IdentityElementError)!KeyPair {275 pub fn fromSecretKey(secret_key: SecretKey) (NonCanonicalError || EncodingError || IdentityElementError)!KeyPair {
277 // It is critical for EdDSA to use the correct public key.276 // It is critical for EdDSA to use the correct public key.
278 // In order to enforce this, a SecretKey implicitly includes a copy of the public key.277 // In order to enforce this, a SecretKey implicitly includes a copy of the public key.
279 // In Debug mode, we can still afford checking that the public key is correct for extra safety.278 // With runtime safety, we can still afford checking that the public key is correct.
280 if (builtin.mode == .Debug) {279 if (std.debug.runtime_safety) {
281 const pk_p = try Curve.fromBytes(secret_key.publicKeyBytes());280 const pk_p = try Curve.fromBytes(secret_key.publicKeyBytes());
282 const recomputed_kp = try create(secret_key.seed());281 const recomputed_kp = try create(secret_key.seed());
283 debug.assert(mem.eql(u8, &recomputed_kp.public_key.toBytes(), &pk_p.toBytes()));282 debug.assert(mem.eql(u8, &recomputed_kp.public_key.toBytes(), &pk_p.toBytes()));
...@@ -493,8 +492,6 @@ test "ed25519 key pair creation" {...@@ -493,8 +492,6 @@ test "ed25519 key pair creation" {
493}492}
494493
495test "ed25519 signature" {494test "ed25519 signature" {
496 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
497
498 var seed: [32]u8 = undefined;495 var seed: [32]u8 = undefined;
499 _ = try fmt.hexToBytes(seed[0..], "8052030376d47112be7f73ed7a019293dd12ad910b654455798b4667d73de166");496 _ = try fmt.hexToBytes(seed[0..], "8052030376d47112be7f73ed7a019293dd12ad910b654455798b4667d73de166");
500 const key_pair = try Ed25519.KeyPair.create(seed);497 const key_pair = try Ed25519.KeyPair.create(seed);
...@@ -507,8 +504,6 @@ test "ed25519 signature" {...@@ -507,8 +504,6 @@ test "ed25519 signature" {
507}504}
508505
509test "ed25519 batch verification" {506test "ed25519 batch verification" {
510 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
511
512 var i: usize = 0;507 var i: usize = 0;
513 while (i < 100) : (i += 1) {508 while (i < 100) : (i += 1) {
514 const key_pair = try Ed25519.KeyPair.create(null);509 const key_pair = try Ed25519.KeyPair.create(null);
...@@ -538,8 +533,6 @@ test "ed25519 batch verification" {...@@ -538,8 +533,6 @@ test "ed25519 batch verification" {
538}533}
539534
540test "ed25519 test vectors" {535test "ed25519 test vectors" {
541 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
542
543 const Vec = struct {536 const Vec = struct {
544 msg_hex: *const [64:0]u8,537 msg_hex: *const [64:0]u8,
545 public_key_hex: *const [64:0]u8,538 public_key_hex: *const [64:0]u8,
...@@ -642,8 +635,6 @@ test "ed25519 test vectors" {...@@ -642,8 +635,6 @@ test "ed25519 test vectors" {
642}635}
643636
644test "ed25519 with blind keys" {637test "ed25519 with blind keys" {
645 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
646
647 const BlindKeyPair = Ed25519.key_blinding.BlindKeyPair;638 const BlindKeyPair = Ed25519.key_blinding.BlindKeyPair;
648639
649 // Create a standard Ed25519 key pair640 // Create a standard Ed25519 key pair
...@@ -667,8 +658,6 @@ test "ed25519 with blind keys" {...@@ -667,8 +658,6 @@ test "ed25519 with blind keys" {
667}658}
668659
669test "ed25519 signatures with streaming" {660test "ed25519 signatures with streaming" {
670 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
671
672 const kp = try Ed25519.KeyPair.create(null);661 const kp = try Ed25519.KeyPair.create(null);
673662
674 var signer = try kp.signer(null);663 var signer = try kp.signer(null);
lib/std/crypto/Certificate.zig+13-13
...@@ -614,18 +614,18 @@ const Date = struct {...@@ -614,18 +614,18 @@ const Date = struct {
614};614};
615615
616pub fn parseTimeDigits(text: *const [2]u8, min: u8, max: u8) !u8 {616pub fn parseTimeDigits(text: *const [2]u8, min: u8, max: u8) !u8 {
617 const nn: @Vector(2, u16) = .{ text[0], text[1] };617 const result = if (use_vectors) result: {
618 const zero: @Vector(2, u16) = .{ '0', '0' };618 const nn: @Vector(2, u16) = .{ text[0], text[1] };
619 const mm: @Vector(2, u16) = .{ 10, 1 };619 const zero: @Vector(2, u16) = .{ '0', '0' };
620 const result = @reduce(.Add, (nn -% zero) *% mm);620 const mm: @Vector(2, u16) = .{ 10, 1 };
621 break :result @reduce(.Add, (nn -% zero) *% mm);
622 } else std.fmt.parseInt(u8, text, 10) catch return error.CertificateTimeInvalid;
621 if (result < min) return error.CertificateTimeInvalid;623 if (result < min) return error.CertificateTimeInvalid;
622 if (result > max) return error.CertificateTimeInvalid;624 if (result > max) return error.CertificateTimeInvalid;
623 return @truncate(result);625 return @truncate(result);
624}626}
625627
626test parseTimeDigits {628test parseTimeDigits {
627 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
628
629 const expectEqual = std.testing.expectEqual;629 const expectEqual = std.testing.expectEqual;
630 try expectEqual(@as(u8, 0), try parseTimeDigits("00", 0, 99));630 try expectEqual(@as(u8, 0), try parseTimeDigits("00", 0, 99));
631 try expectEqual(@as(u8, 99), try parseTimeDigits("99", 0, 99));631 try expectEqual(@as(u8, 99), try parseTimeDigits("99", 0, 99));
...@@ -638,17 +638,17 @@ test parseTimeDigits {...@@ -638,17 +638,17 @@ test parseTimeDigits {
638}638}
639639
640pub fn parseYear4(text: *const [4]u8) !u16 {640pub fn parseYear4(text: *const [4]u8) !u16 {
641 const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };641 const result = if (use_vectors) result: {
642 const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };642 const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };
643 const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };643 const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };
644 const result = @reduce(.Add, (nnnn -% zero) *% mmmm);644 const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };
645 break :result @reduce(.Add, (nnnn -% zero) *% mmmm);
646 } else std.fmt.parseInt(u16, text, 10) catch return error.CertificateTimeInvalid;
645 if (result > 9999) return error.CertificateTimeInvalid;647 if (result > 9999) return error.CertificateTimeInvalid;
646 return @truncate(result);648 return @truncate(result);
647}649}
648650
649test parseYear4 {651test parseYear4 {
650 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
651
652 const expectEqual = std.testing.expectEqual;652 const expectEqual = std.testing.expectEqual;
653 try expectEqual(@as(u16, 0), try parseYear4("0000"));653 try expectEqual(@as(u16, 0), try parseYear4("0000"));
654 try expectEqual(@as(u16, 9999), try parseYear4("9999"));654 try expectEqual(@as(u16, 9999), try parseYear4("9999"));
...@@ -1124,4 +1124,4 @@ pub const rsa = struct {...@@ -1124,4 +1124,4 @@ pub const rsa = struct {
1124 }1124 }
1125};1125};
11261126
1127const builtin = @import("builtin");1127const use_vectors = @import("builtin").zig_backend != .stage2_x86_64;
lib/std/crypto/Certificate/Bundle.zig-2
...@@ -318,8 +318,6 @@ const MapContext = struct {...@@ -318,8 +318,6 @@ const MapContext = struct {
318test "scan for OS-provided certificates" {318test "scan for OS-provided certificates" {
319 if (builtin.os.tag == .wasi) return error.SkipZigTest;319 if (builtin.os.tag == .wasi) return error.SkipZigTest;
320320
321 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
322
323 var bundle: Bundle = .{};321 var bundle: Bundle = .{};
324 defer bundle.deinit(std.testing.allocator);322 defer bundle.deinit(std.testing.allocator);
325323
lib/std/crypto/aes.zig-2
...@@ -28,8 +28,6 @@ pub const Aes128 = impl.Aes128;...@@ -28,8 +28,6 @@ pub const Aes128 = impl.Aes128;
28pub const Aes256 = impl.Aes256;28pub const Aes256 = impl.Aes256;
2929
30test "ctr" {30test "ctr" {
31 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
32
33 // NIST SP 800-38A pp 55-5831 // NIST SP 800-38A pp 55-58
34 const ctr = @import("modes.zig").ctr;32 const ctr = @import("modes.zig").ctr;
3533
lib/std/crypto/aes_gcm.zig+2-11
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const assert = std.debug.assert;2const assert = std.debug.assert;
4const crypto = std.crypto;3const crypto = std.crypto;
5const debug = std.debug;4const debug = std.debug;
...@@ -42,7 +41,7 @@ fn AesGcm(comptime Aes: anytype) type {...@@ -42,7 +41,7 @@ fn AesGcm(comptime Aes: anytype) type {
42 mac.pad();41 mac.pad();
4342
44 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);43 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
45 modes.ctr(@TypeOf(aes), aes, c, m, j, std.builtin.Endian.big);44 modes.ctr(@TypeOf(aes), aes, c, m, j, .big);
46 mac.update(c[0..m.len][0..]);45 mac.update(c[0..m.len][0..]);
47 mac.pad();46 mac.pad();
4847
...@@ -104,7 +103,7 @@ fn AesGcm(comptime Aes: anytype) type {...@@ -104,7 +103,7 @@ fn AesGcm(comptime Aes: anytype) type {
104 }103 }
105104
106 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);105 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
107 modes.ctr(@TypeOf(aes), aes, m, c, j, std.builtin.Endian.big);106 modes.ctr(@TypeOf(aes), aes, m, c, j, .big);
108 }107 }
109 };108 };
110}109}
...@@ -113,8 +112,6 @@ const htest = @import("test.zig");...@@ -113,8 +112,6 @@ const htest = @import("test.zig");
113const testing = std.testing;112const testing = std.testing;
114113
115test "Aes256Gcm - Empty message and no associated data" {114test "Aes256Gcm - Empty message and no associated data" {
116 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
117
118 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;115 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
119 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;116 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
120 const ad = "";117 const ad = "";
...@@ -127,8 +124,6 @@ test "Aes256Gcm - Empty message and no associated data" {...@@ -127,8 +124,6 @@ test "Aes256Gcm - Empty message and no associated data" {
127}124}
128125
129test "Aes256Gcm - Associated data only" {126test "Aes256Gcm - Associated data only" {
130 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
131
132 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;127 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
133 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;128 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
134 const m = "";129 const m = "";
...@@ -141,8 +136,6 @@ test "Aes256Gcm - Associated data only" {...@@ -141,8 +136,6 @@ test "Aes256Gcm - Associated data only" {
141}136}
142137
143test "Aes256Gcm - Message only" {138test "Aes256Gcm - Message only" {
144 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
145
146 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;139 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
147 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;140 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
148 const m = "Test with message only";141 const m = "Test with message only";
...@@ -160,8 +153,6 @@ test "Aes256Gcm - Message only" {...@@ -160,8 +153,6 @@ test "Aes256Gcm - Message only" {
160}153}
161154
162test "Aes256Gcm - Message and associated data" {155test "Aes256Gcm - Message and associated data" {
163 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
164
165 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;156 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
166 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;157 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
167 const m = "Test with message";158 const m = "Test with message";
lib/std/crypto/argon2.zig-4
...@@ -896,8 +896,6 @@ test "kdf" {...@@ -896,8 +896,6 @@ test "kdf" {
896}896}
897897
898test "phc format hasher" {898test "phc format hasher" {
899 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
900
901 const allocator = std.testing.allocator;899 const allocator = std.testing.allocator;
902 const password = "testpass";900 const password = "testpass";
903901
...@@ -913,8 +911,6 @@ test "phc format hasher" {...@@ -913,8 +911,6 @@ test "phc format hasher" {
913}911}
914912
915test "password hash and password verify" {913test "password hash and password verify" {
916 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
917
918 const allocator = std.testing.allocator;914 const allocator = std.testing.allocator;
919 const password = "testpass";915 const password = "testpass";
920916
lib/std/crypto/bcrypt.zig-7
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const base64 = std.base64;2const base64 = std.base64;
4const crypto = std.crypto;3const crypto = std.crypto;
5const debug = std.debug;4const debug = std.debug;
...@@ -754,8 +753,6 @@ pub fn strVerify(...@@ -754,8 +753,6 @@ pub fn strVerify(
754}753}
755754
756test "bcrypt codec" {755test "bcrypt codec" {
757 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
758
759 var salt: [salt_length]u8 = undefined;756 var salt: [salt_length]u8 = undefined;
760 crypto.random.bytes(&salt);757 crypto.random.bytes(&salt);
761 var salt_str: [salt_str_length]u8 = undefined;758 var salt_str: [salt_str_length]u8 = undefined;
...@@ -766,8 +763,6 @@ test "bcrypt codec" {...@@ -766,8 +763,6 @@ test "bcrypt codec" {
766}763}
767764
768test "bcrypt crypt format" {765test "bcrypt crypt format" {
769 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
770
771 var hash_options = HashOptions{766 var hash_options = HashOptions{
772 .params = .{ .rounds_log = 5 },767 .params = .{ .rounds_log = 5 },
773 .encoding = .crypt,768 .encoding = .crypt,
...@@ -808,8 +803,6 @@ test "bcrypt crypt format" {...@@ -808,8 +803,6 @@ test "bcrypt crypt format" {
808}803}
809804
810test "bcrypt phc format" {805test "bcrypt phc format" {
811 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
812
813 var hash_options = HashOptions{806 var hash_options = HashOptions{
814 .params = .{ .rounds_log = 5 },807 .params = .{ .rounds_log = 5 },
815 .encoding = .phc,808 .encoding = .phc,
lib/std/crypto/cmac.zig-9
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const crypto = std.crypto;2const crypto = std.crypto;
4const mem = std.mem;3const mem = std.mem;
54
...@@ -94,8 +93,6 @@ pub fn Cmac(comptime BlockCipher: type) type {...@@ -94,8 +93,6 @@ pub fn Cmac(comptime BlockCipher: type) type {
94const testing = std.testing;93const testing = std.testing;
9594
96test "CmacAes128 - Example 1: len = 0" {95test "CmacAes128 - Example 1: len = 0" {
97 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
98
99 const key = [_]u8{96 const key = [_]u8{
100 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,97 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
101 };98 };
...@@ -109,8 +106,6 @@ test "CmacAes128 - Example 1: len = 0" {...@@ -109,8 +106,6 @@ test "CmacAes128 - Example 1: len = 0" {
109}106}
110107
111test "CmacAes128 - Example 2: len = 16" {108test "CmacAes128 - Example 2: len = 16" {
112 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
113
114 const key = [_]u8{109 const key = [_]u8{
115 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,110 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
116 };111 };
...@@ -126,8 +121,6 @@ test "CmacAes128 - Example 2: len = 16" {...@@ -126,8 +121,6 @@ test "CmacAes128 - Example 2: len = 16" {
126}121}
127122
128test "CmacAes128 - Example 3: len = 40" {123test "CmacAes128 - Example 3: len = 40" {
129 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
130
131 const key = [_]u8{124 const key = [_]u8{
132 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,125 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
133 };126 };
...@@ -145,8 +138,6 @@ test "CmacAes128 - Example 3: len = 40" {...@@ -145,8 +138,6 @@ test "CmacAes128 - Example 3: len = 40" {
145}138}
146139
147test "CmacAes128 - Example 4: len = 64" {140test "CmacAes128 - Example 4: len = 64" {
148 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
149
150 const key = [_]u8{141 const key = [_]u8{
151 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,142 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
152 };143 };
lib/std/crypto/ecdsa.zig-5
...@@ -373,7 +373,6 @@ pub fn Ecdsa(comptime Curve: type, comptime Hash: type) type {...@@ -373,7 +373,6 @@ pub fn Ecdsa(comptime Curve: type, comptime Hash: type) type {
373373
374test "ECDSA - Basic operations over EcdsaP384Sha384" {374test "ECDSA - Basic operations over EcdsaP384Sha384" {
375 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;375 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
376 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
377376
378 const Scheme = EcdsaP384Sha384;377 const Scheme = EcdsaP384Sha384;
379 const kp = try Scheme.KeyPair.create(null);378 const kp = try Scheme.KeyPair.create(null);
...@@ -407,7 +406,6 @@ test "ECDSA - Basic operations over Secp256k1" {...@@ -407,7 +406,6 @@ test "ECDSA - Basic operations over Secp256k1" {
407406
408test "ECDSA - Basic operations over EcdsaP384Sha256" {407test "ECDSA - Basic operations over EcdsaP384Sha256" {
409 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;408 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
410 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
411409
412 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);410 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
413 const kp = try Scheme.KeyPair.create(null);411 const kp = try Scheme.KeyPair.create(null);
...@@ -424,7 +422,6 @@ test "ECDSA - Basic operations over EcdsaP384Sha256" {...@@ -424,7 +422,6 @@ test "ECDSA - Basic operations over EcdsaP384Sha256" {
424422
425test "ECDSA - Verifying a existing signature with EcdsaP384Sha256" {423test "ECDSA - Verifying a existing signature with EcdsaP384Sha256" {
426 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;424 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
427 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
428425
429 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);426 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
430 // zig fmt: off427 // zig fmt: off
...@@ -469,7 +466,6 @@ const TestVector = struct {...@@ -469,7 +466,6 @@ const TestVector = struct {
469466
470test "ECDSA - Test vectors from Project Wycheproof" {467test "ECDSA - Test vectors from Project Wycheproof" {
471 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;468 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
472 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
473469
474 const vectors = [_]TestVector{470 const vectors = [_]TestVector{
475 .{ .key = "042927b10512bae3eddcfe467828128bad2903269919f7086069c8c4df6c732838c7787964eaac00e5921fb1498a60f4606766b3d9685001558d1a974e7341513e", .msg = "313233343030", .sig = "304402202ba3a8be6b94d5ec80a6d9d1190a436effe50d85a1eee859b8cc6af9bd5c2e1802204cd60b855d442f5b3c7b11eb6c4e0ae7525fe710fab9aa7c77a67f79e6fadd76", .result = .valid },471 .{ .key = "042927b10512bae3eddcfe467828128bad2903269919f7086069c8c4df6c732838c7787964eaac00e5921fb1498a60f4606766b3d9685001558d1a974e7341513e", .msg = "313233343030", .sig = "304402202ba3a8be6b94d5ec80a6d9d1190a436effe50d85a1eee859b8cc6af9bd5c2e1802204cd60b855d442f5b3c7b11eb6c4e0ae7525fe710fab9aa7c77a67f79e6fadd76", .result = .valid },
...@@ -884,7 +880,6 @@ fn tvTry(vector: TestVector) !void {...@@ -884,7 +880,6 @@ fn tvTry(vector: TestVector) !void {
884880
885test "ECDSA - Sec1 encoding/decoding" {881test "ECDSA - Sec1 encoding/decoding" {
886 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;882 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
887 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
888883
889 const Scheme = EcdsaP384Sha384;884 const Scheme = EcdsaP384Sha384;
890 const kp = try Scheme.KeyPair.create(null);885 const kp = try Scheme.KeyPair.create(null);
lib/std/crypto/ghash_polyval.zig-4
...@@ -422,8 +422,6 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {...@@ -422,8 +422,6 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {
422const htest = @import("test.zig");422const htest = @import("test.zig");
423423
424test "ghash" {424test "ghash" {
425 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
426
427 const key = [_]u8{0x42} ** 16;425 const key = [_]u8{0x42} ** 16;
428 const m = [_]u8{0x69} ** 256;426 const m = [_]u8{0x69} ** 256;
429427
...@@ -441,8 +439,6 @@ test "ghash" {...@@ -441,8 +439,6 @@ test "ghash" {
441}439}
442440
443test "ghash2" {441test "ghash2" {
444 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
445
446 var key: [16]u8 = undefined;442 var key: [16]u8 = undefined;
447 var i: usize = 0;443 var i: usize = 0;
448 while (i < key.len) : (i += 1) {444 while (i < key.len) : (i += 1) {
lib/std/crypto/pcurves/p256.zig-2
...@@ -478,7 +478,5 @@ pub const AffineCoordinates = struct {...@@ -478,7 +478,5 @@ pub const AffineCoordinates = struct {
478};478};
479479
480test {480test {
481 if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
482
483 _ = @import("tests/p256.zig");481 _ = @import("tests/p256.zig");
484}482}
lib/std/crypto/phc_encoding.zig-3
...@@ -1,7 +1,6 @@...@@ -1,7 +1,6 @@
1// https://github.com/P-H-C/phc-string-format1// https://github.com/P-H-C/phc-string-format
22
3const std = @import("std");3const std = @import("std");
4const builtin = @import("builtin");
5const fmt = std.fmt;4const fmt = std.fmt;
6const io = std.io;5const io = std.io;
7const mem = std.mem;6const mem = std.mem;
...@@ -264,8 +263,6 @@ fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {...@@ -264,8 +263,6 @@ fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {
264}263}
265264
266test "phc format - encoding/decoding" {265test "phc format - encoding/decoding" {
267 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269 const Input = struct {266 const Input = struct {
270 str: []const u8,267 str: []const u8,
271 HashResult: type,268 HashResult: type,
lib/std/crypto/sha2.zig+1-1
...@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {...@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238 return;238 return;
239 },239 },
240 // C backend doesn't currently support passing vectors to inline asm.240 // C backend doesn't currently support passing vectors to inline asm.
241 .x86_64 => if (builtin.zig_backend != .stage2_c and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {241 .x86_64 => if (builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
242 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };242 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
243 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };243 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
244 const s_v = @as(*[16]v4u32, @ptrCast(&s));244 const s_v = @as(*[16]v4u32, @ptrCast(&s));
lib/std/fmt/parse_float.zig-2
...@@ -83,8 +83,6 @@ test "fmt.parseFloat #11169" {...@@ -83,8 +83,6 @@ test "fmt.parseFloat #11169" {
83}83}
8484
85test "fmt.parseFloat hex.special" {85test "fmt.parseFloat hex.special" {
86 if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
87
88 try testing.expect(math.isNan(try parseFloat(f32, "nAn")));86 try testing.expect(math.isNan(try parseFloat(f32, "nAn")));
89 try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));87 try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));
90 try testing.expect(math.isPositiveInf(try parseFloat(f32, "+Inf")));88 try testing.expect(math.isPositiveInf(try parseFloat(f32, "+Inf")));
lib/std/hash/xxhash.zig+2-1
...@@ -2,6 +2,7 @@ const std = @import("std");...@@ -2,6 +2,7 @@ const std = @import("std");
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const mem = std.mem;3const mem = std.mem;
4const expectEqual = std.testing.expectEqual;4const expectEqual = std.testing.expectEqual;
5const native_endian = builtin.cpu.arch.endian();
56
6const rotl = std.math.rotl;7const rotl = std.math.rotl;
78
...@@ -472,7 +473,7 @@ pub const XxHash3 = struct {...@@ -472,7 +473,7 @@ pub const XxHash3 = struct {
472 }473 }
473474
474 inline fn swap(x: anytype) @TypeOf(x) {475 inline fn swap(x: anytype) @TypeOf(x) {
475 return if (builtin.cpu.arch.endian() == .big) @byteSwap(x) else x;476 return if (native_endian == .big) @byteSwap(x) else x;
476 }477 }
477478
478 inline fn disableAutoVectorization(x: anytype) void {479 inline fn disableAutoVectorization(x: anytype) void {
lib/std/http/Client.zig+16-10
...@@ -9,6 +9,7 @@ const net = std.net;...@@ -9,6 +9,7 @@ const net = std.net;
9const Uri = std.Uri;9const Uri = std.Uri;
10const Allocator = mem.Allocator;10const Allocator = mem.Allocator;
11const assert = std.debug.assert;11const assert = std.debug.assert;
12const use_vectors = builtin.zig_backend != .stage2_x86_64;
1213
13const Client = @This();14const Client = @This();
14const proto = @import("protocol.zig");15const proto = @import("protocol.zig");
...@@ -408,7 +409,7 @@ pub const Response = struct {...@@ -408,7 +409,7 @@ pub const Response = struct {
408 else => return error.HttpHeadersInvalid,409 else => return error.HttpHeadersInvalid,
409 };410 };
410 if (first_line[8] != ' ') return error.HttpHeadersInvalid;411 if (first_line[8] != ' ') return error.HttpHeadersInvalid;
411 const status = @as(http.Status, @enumFromInt(parseInt3(first_line[9..12].*)));412 const status: http.Status = @enumFromInt(parseInt3(first_line[9..12]));
412 const reason = mem.trimLeft(u8, first_line[12..], " ");413 const reason = mem.trimLeft(u8, first_line[12..], " ");
413414
414 res.version = version;415 res.version = version;
...@@ -481,20 +482,24 @@ pub const Response = struct {...@@ -481,20 +482,24 @@ pub const Response = struct {
481 }482 }
482483
483 inline fn int64(array: *const [8]u8) u64 {484 inline fn int64(array: *const [8]u8) u64 {
484 return @as(u64, @bitCast(array.*));485 return @bitCast(array.*);
485 }486 }
486487
487 fn parseInt3(nnn: @Vector(3, u8)) u10 {488 fn parseInt3(text: *const [3]u8) u10 {
488 const zero: @Vector(3, u8) = .{ '0', '0', '0' };489 if (use_vectors) {
489 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };490 const nnn: @Vector(3, u8) = text.*;
490 return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);491 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
492 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
493 return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
494 }
495 return std.fmt.parseInt(u10, text, 10) catch unreachable;
491 }496 }
492497
493 test parseInt3 {498 test parseInt3 {
494 const expectEqual = testing.expectEqual;499 const expectEqual = testing.expectEqual;
495 try expectEqual(@as(u10, 0), parseInt3("000".*));500 try expectEqual(@as(u10, 0), parseInt3("000"));
496 try expectEqual(@as(u10, 418), parseInt3("418".*));501 try expectEqual(@as(u10, 418), parseInt3("418"));
497 try expectEqual(@as(u10, 999), parseInt3("999".*));502 try expectEqual(@as(u10, 999), parseInt3("999"));
498 }503 }
499504
500 version: http.Version,505 version: http.Version,
...@@ -1588,7 +1593,8 @@ test {...@@ -1588,7 +1593,8 @@ test {
15881593
1589 if (builtin.os.tag == .wasi) return error.SkipZigTest;1594 if (builtin.os.tag == .wasi) return error.SkipZigTest;
15901595
1591 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;1596 if (builtin.zig_backend == .stage2_x86_64 and
1597 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest;
15921598
1593 std.testing.refAllDecls(@This());1599 std.testing.refAllDecls(@This());
1594}1600}
lib/std/http/Server.zig-2
...@@ -736,8 +736,6 @@ test "HTTP server handles a chunked transfer coding request" {...@@ -736,8 +736,6 @@ test "HTTP server handles a chunked transfer coding request" {
736 return error.SkipZigTest;736 return error.SkipZigTest;
737 }737 }
738738
739 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
740
741 const native_endian = comptime builtin.cpu.arch.endian();739 const native_endian = comptime builtin.cpu.arch.endian();
742 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {740 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
743 // https://github.com/ziglang/zig/issues/13782741 // https://github.com/ziglang/zig/issues/13782
lib/std/http/protocol.zig+23-12
...@@ -1,8 +1,10 @@...@@ -1,8 +1,10 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");
2const testing = std.testing;3const testing = std.testing;
3const mem = std.mem;4const mem = std.mem;
45
5const assert = std.debug.assert;6const assert = std.debug.assert;
7const use_vectors = builtin.zig_backend != .stage2_x86_64;
68
7pub const State = enum {9pub const State = enum {
8 /// Begin header parsing states.10 /// Begin header parsing states.
...@@ -83,7 +85,7 @@ pub const HeadersParser = struct {...@@ -83,7 +85,7 @@ pub const HeadersParser = struct {
83 /// first byte of content is located at `bytes[result]`.85 /// first byte of content is located at `bytes[result]`.
84 pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {86 pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
85 const vector_len: comptime_int = @max(std.simd.suggestVectorSize(u8) orelse 1, 8);87 const vector_len: comptime_int = @max(std.simd.suggestVectorSize(u8) orelse 1, 8);
86 const len = @as(u32, @intCast(bytes.len));88 const len: u32 = @intCast(bytes.len);
87 var index: u32 = 0;89 var index: u32 = 0;
8890
89 while (true) {91 while (true) {
...@@ -175,18 +177,27 @@ pub const HeadersParser = struct {...@@ -175,18 +177,27 @@ pub const HeadersParser = struct {
175 continue;177 continue;
176 },178 },
177 else => {179 else => {
178 const Vector = @Vector(vector_len, u8);
179 // const BoolVector = @Vector(vector_len, bool);
180 const BitVector = @Vector(vector_len, u1);
181 const SizeVector = @Vector(vector_len, u8);
182
183 const chunk = bytes[index..][0..vector_len];180 const chunk = bytes[index..][0..vector_len];
184 const v: Vector = chunk.*;181 const matches = if (use_vectors) matches: {
185 const matches_r = @as(BitVector, @bitCast(v == @as(Vector, @splat('\r'))));182 const Vector = @Vector(vector_len, u8);
186 const matches_n = @as(BitVector, @bitCast(v == @as(Vector, @splat('\n'))));183 // const BoolVector = @Vector(vector_len, bool);
187 const matches_or: SizeVector = matches_r | matches_n;184 const BitVector = @Vector(vector_len, u1);
188185 const SizeVector = @Vector(vector_len, u8);
189 const matches = @reduce(.Add, matches_or);186
187 const v: Vector = chunk.*;
188 const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
189 const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
190 const matches_or: SizeVector = matches_r | matches_n;
191
192 break :matches @reduce(.Add, matches_or);
193 } else matches: {
194 var matches: u8 = 0;
195 for (chunk) |byte| switch (byte) {
196 '\r', '\n' => matches += 1,
197 else => {},
198 };
199 break :matches matches;
200 };
190 switch (matches) {201 switch (matches) {
191 0 => {},202 0 => {},
192 1 => switch (chunk[vector_len - 1]) {203 1 => switch (chunk[vector_len - 1]) {
lib/std/math.zig+2-21
...@@ -492,8 +492,6 @@ pub fn shl(comptime T: type, a: T, shift_amt: anytype) T {...@@ -492,8 +492,6 @@ pub fn shl(comptime T: type, a: T, shift_amt: anytype) T {
492}492}
493493
494test "shl" {494test "shl" {
495 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
496
497 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {495 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
498 // https://github.com/ziglang/zig/issues/12012496 // https://github.com/ziglang/zig/issues/12012
499 return error.SkipZigTest;497 return error.SkipZigTest;
...@@ -539,8 +537,6 @@ pub fn shr(comptime T: type, a: T, shift_amt: anytype) T {...@@ -539,8 +537,6 @@ pub fn shr(comptime T: type, a: T, shift_amt: anytype) T {
539}537}
540538
541test "shr" {539test "shr" {
542 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
543
544 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {540 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
545 // https://github.com/ziglang/zig/issues/12012541 // https://github.com/ziglang/zig/issues/12012
546 return error.SkipZigTest;542 return error.SkipZigTest;
...@@ -587,8 +583,6 @@ pub fn rotr(comptime T: type, x: T, r: anytype) T {...@@ -587,8 +583,6 @@ pub fn rotr(comptime T: type, x: T, r: anytype) T {
587}583}
588584
589test "rotr" {585test "rotr" {
590 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
591
592 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {586 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
593 // https://github.com/ziglang/zig/issues/12012587 // https://github.com/ziglang/zig/issues/12012
594 return error.SkipZigTest;588 return error.SkipZigTest;
...@@ -634,8 +628,6 @@ pub fn rotl(comptime T: type, x: T, r: anytype) T {...@@ -634,8 +628,6 @@ pub fn rotl(comptime T: type, x: T, r: anytype) T {
634}628}
635629
636test "rotl" {630test "rotl" {
637 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
638
639 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {631 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
640 // https://github.com/ziglang/zig/issues/12012632 // https://github.com/ziglang/zig/issues/12012
641 return error.SkipZigTest;633 return error.SkipZigTest;
...@@ -764,8 +756,6 @@ pub fn divTrunc(comptime T: type, numerator: T, denominator: T) !T {...@@ -764,8 +756,6 @@ pub fn divTrunc(comptime T: type, numerator: T, denominator: T) !T {
764}756}
765757
766test "divTrunc" {758test "divTrunc" {
767 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
768
769 try testDivTrunc();759 try testDivTrunc();
770 try comptime testDivTrunc();760 try comptime testDivTrunc();
771}761}
...@@ -790,8 +780,6 @@ pub fn divFloor(comptime T: type, numerator: T, denominator: T) !T {...@@ -790,8 +780,6 @@ pub fn divFloor(comptime T: type, numerator: T, denominator: T) !T {
790}780}
791781
792test "divFloor" {782test "divFloor" {
793 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
794
795 try testDivFloor();783 try testDivFloor();
796 try comptime testDivFloor();784 try comptime testDivFloor();
797}785}
...@@ -829,8 +817,6 @@ pub fn divCeil(comptime T: type, numerator: T, denominator: T) !T {...@@ -829,8 +817,6 @@ pub fn divCeil(comptime T: type, numerator: T, denominator: T) !T {
829}817}
830818
831test "divCeil" {819test "divCeil" {
832 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
833
834 try testDivCeil();820 try testDivCeil();
835 try comptime testDivCeil();821 try comptime testDivCeil();
836}822}
...@@ -875,8 +861,6 @@ pub fn divExact(comptime T: type, numerator: T, denominator: T) !T {...@@ -875,8 +861,6 @@ pub fn divExact(comptime T: type, numerator: T, denominator: T) !T {
875}861}
876862
877test "divExact" {863test "divExact" {
878 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
879
880 try testDivExact();864 try testDivExact();
881 try comptime testDivExact();865 try comptime testDivExact();
882}866}
...@@ -903,8 +887,6 @@ pub fn mod(comptime T: type, numerator: T, denominator: T) !T {...@@ -903,8 +887,6 @@ pub fn mod(comptime T: type, numerator: T, denominator: T) !T {
903}887}
904888
905test "mod" {889test "mod" {
906 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
907
908 try testMod();890 try testMod();
909 try comptime testMod();891 try comptime testMod();
910}892}
...@@ -931,8 +913,6 @@ pub fn rem(comptime T: type, numerator: T, denominator: T) !T {...@@ -931,8 +913,6 @@ pub fn rem(comptime T: type, numerator: T, denominator: T) !T {
931}913}
932914
933test "rem" {915test "rem" {
934 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
935
936 try testRem();916 try testRem();
937 try comptime testRem();917 try comptime testRem();
938}918}
...@@ -1285,7 +1265,8 @@ pub fn lerp(a: anytype, b: anytype, t: anytype) @TypeOf(a, b, t) {...@@ -1285,7 +1265,8 @@ pub fn lerp(a: anytype, b: anytype, t: anytype) @TypeOf(a, b, t) {
1285}1265}
12861266
1287test "lerp" {1267test "lerp" {
1288 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;1268 if (builtin.zig_backend == .stage2_x86_64 and
1269 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .fma)) return error.SkipZigTest;
12891270
1290 try testing.expectEqual(@as(f64, 75), lerp(50, 100, 0.5));1271 try testing.expectEqual(@as(f64, 75), lerp(50, 100, 0.5));
1291 try testing.expectEqual(@as(f32, 43.75), lerp(50, 25, 0.25));1272 try testing.expectEqual(@as(f32, 43.75), lerp(50, 25, 0.25));
lib/std/math/big/int.zig+3-3
...@@ -1318,7 +1318,7 @@ pub const Mutable = struct {...@@ -1318,7 +1318,7 @@ pub const Mutable = struct {
1318 ///1318 ///
1319 /// `limbs_buffer` is used for temporary storage.1319 /// `limbs_buffer` is used for temporary storage.
1320 /// The amount required is given by `calcPowLimbsBufferLen`.1320 /// The amount required is given by `calcPowLimbsBufferLen`.
1321 pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) !void {1321 pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) void {
1322 assert(r.limbs.ptr != a.limbs.ptr); // illegal aliasing1322 assert(r.limbs.ptr != a.limbs.ptr); // illegal aliasing
13231323
1324 // Handle all the trivial cases first1324 // Handle all the trivial cases first
...@@ -3213,7 +3213,7 @@ pub const Managed = struct {...@@ -3213,7 +3213,7 @@ pub const Managed = struct {
3213 var m = try Managed.initCapacity(rma.allocator, needed_limbs);3213 var m = try Managed.initCapacity(rma.allocator, needed_limbs);
3214 errdefer m.deinit();3214 errdefer m.deinit();
3215 var m_mut = m.toMutable();3215 var m_mut = m.toMutable();
3216 try m_mut.pow(a.toConst(), b, limbs_buffer);3216 m_mut.pow(a.toConst(), b, limbs_buffer);
3217 m.setMetadata(m_mut.positive, m_mut.len);3217 m.setMetadata(m_mut.positive, m_mut.len);
32183218
3219 rma.deinit();3219 rma.deinit();
...@@ -3221,7 +3221,7 @@ pub const Managed = struct {...@@ -3221,7 +3221,7 @@ pub const Managed = struct {
3221 } else {3221 } else {
3222 try rma.ensureCapacity(needed_limbs);3222 try rma.ensureCapacity(needed_limbs);
3223 var rma_mut = rma.toMutable();3223 var rma_mut = rma.toMutable();
3224 try rma_mut.pow(a.toConst(), b, limbs_buffer);3224 rma_mut.pow(a.toConst(), b, limbs_buffer);
3225 rma.setMetadata(rma_mut.positive, rma_mut.len);3225 rma.setMetadata(rma_mut.positive, rma_mut.len);
3226 }3226 }
3227 }3227 }
lib/std/math/big/int_test.zig-6
...@@ -2568,8 +2568,6 @@ test "big.int const to managed" {...@@ -2568,8 +2568,6 @@ test "big.int const to managed" {
2568}2568}
25692569
2570test "big.int pow" {2570test "big.int pow" {
2571 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2572
2573 {2571 {
2574 var a = try Managed.initSet(testing.allocator, -3);2572 var a = try Managed.initSet(testing.allocator, -3);
2575 defer a.deinit();2573 defer a.deinit();
...@@ -2763,8 +2761,6 @@ fn popCountTest(val: *const Managed, bit_count: usize, expected: usize) !void {...@@ -2763,8 +2761,6 @@ fn popCountTest(val: *const Managed, bit_count: usize, expected: usize) !void {
2763}2761}
27642762
2765test "big int conversion read/write twos complement" {2763test "big int conversion read/write twos complement" {
2766 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2767
2768 var a = try Managed.initSet(testing.allocator, (1 << 493) - 1);2764 var a = try Managed.initSet(testing.allocator, (1 << 493) - 1);
2769 defer a.deinit();2765 defer a.deinit();
2770 var b = try Managed.initSet(testing.allocator, (1 << 493) - 1);2766 var b = try Managed.initSet(testing.allocator, (1 << 493) - 1);
...@@ -2863,8 +2859,6 @@ test "big int write twos complement +/- zero" {...@@ -2863,8 +2859,6 @@ test "big int write twos complement +/- zero" {
2863}2859}
28642860
2865test "big int conversion write twos complement with padding" {2861test "big int conversion write twos complement with padding" {
2866 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2867
2868 var a = try Managed.initSet(testing.allocator, 0x01_ffffffff_ffffffff_ffffffff);2862 var a = try Managed.initSet(testing.allocator, 0x01_ffffffff_ffffffff_ffffffff);
2869 defer a.deinit();2863 defer a.deinit();
28702864
lib/std/mem.zig-4
...@@ -315,8 +315,6 @@ pub fn zeroes(comptime T: type) T {...@@ -315,8 +315,6 @@ pub fn zeroes(comptime T: type) T {
315}315}
316316
317test "zeroes" {317test "zeroes" {
318 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
319
320 const C_struct = extern struct {318 const C_struct = extern struct {
321 x: u32,319 x: u32,
322 y: u32 align(128),320 y: u32 align(128),
...@@ -4342,8 +4340,6 @@ pub fn alignInSlice(slice: anytype, comptime new_alignment: usize) ?AlignedSlice...@@ -4342,8 +4340,6 @@ pub fn alignInSlice(slice: anytype, comptime new_alignment: usize) ?AlignedSlice
4342}4340}
43434341
4344test "read/write(Var)PackedInt" {4342test "read/write(Var)PackedInt" {
4345 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
4346
4347 switch (builtin.cpu.arch) {4343 switch (builtin.cpu.arch) {
4348 // This test generates too much code to execute on WASI.4344 // This test generates too much code to execute on WASI.
4349 // LLVM backend fails with "too many locals: locals exceed maximum"4345 // LLVM backend fails with "too many locals: locals exceed maximum"
lib/std/multi_array_list.zig+1-1
...@@ -106,7 +106,7 @@ pub fn MultiArrayList(comptime T: type) type {...@@ -106,7 +106,7 @@ pub fn MultiArrayList(comptime T: type) type {
106 }106 }
107107
108 pub fn toMultiArrayList(self: Slice) Self {108 pub fn toMultiArrayList(self: Slice) Self {
109 if (self.ptrs.len == 0) {109 if (self.ptrs.len == 0 or self.capacity == 0) {
110 return .{};110 return .{};
111 }111 }
112 const unaligned_ptr = self.ptrs[sizes.fields[0]];112 const unaligned_ptr = self.ptrs[sizes.fields[0]];
lib/std/net/test.zig+1-1
...@@ -60,7 +60,7 @@ test "parse and render IPv6 addresses" {...@@ -60,7 +60,7 @@ test "parse and render IPv6 addresses" {
60}60}
6161
62test "invalid but parseable IPv6 scope ids" {62test "invalid but parseable IPv6 scope ids" {
63 if (builtin.os.tag != .linux or comptime !builtin.os.tag.isDarwin()) {63 if (builtin.os.tag != .linux and comptime !builtin.os.tag.isDarwin()) {
64 // Currently, resolveIp6 with alphanumerical scope IDs only works on Linux.64 // Currently, resolveIp6 with alphanumerical scope IDs only works on Linux.
65 // TODO Make this test pass on other operating systems.65 // TODO Make this test pass on other operating systems.
66 return error.SkipZigTest;66 return error.SkipZigTest;
lib/std/once.zig-2
...@@ -46,8 +46,6 @@ fn incr() void {...@@ -46,8 +46,6 @@ fn incr() void {
46}46}
4747
48test "Once executes its function just once" {48test "Once executes its function just once" {
49 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
50
51 if (builtin.single_threaded) {49 if (builtin.single_threaded) {
52 global_once.call();50 global_once.call();
53 global_once.call();51 global_once.call();
lib/std/os/test.zig-4
...@@ -375,8 +375,6 @@ fn testThreadIdFn(thread_id: *Thread.Id) void {...@@ -375,8 +375,6 @@ fn testThreadIdFn(thread_id: *Thread.Id) void {
375test "std.Thread.getCurrentId" {375test "std.Thread.getCurrentId" {
376 if (builtin.single_threaded) return error.SkipZigTest;376 if (builtin.single_threaded) return error.SkipZigTest;
377377
378 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
379
380 var thread_current_id: Thread.Id = undefined;378 var thread_current_id: Thread.Id = undefined;
381 const thread = try Thread.spawn(.{}, testThreadIdFn, .{&thread_current_id});379 const thread = try Thread.spawn(.{}, testThreadIdFn, .{&thread_current_id});
382 thread.join();380 thread.join();
...@@ -420,8 +418,6 @@ test "cpu count" {...@@ -420,8 +418,6 @@ test "cpu count" {
420test "thread local storage" {418test "thread local storage" {
421 if (builtin.single_threaded) return error.SkipZigTest;419 if (builtin.single_threaded) return error.SkipZigTest;
422420
423 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
424
425 const thread1 = try Thread.spawn(.{}, testTls, .{});421 const thread1 = try Thread.spawn(.{}, testTls, .{});
426 const thread2 = try Thread.spawn(.{}, testTls, .{});422 const thread2 = try Thread.spawn(.{}, testTls, .{});
427 try testTls();423 try testTls();
lib/std/rand/test.zig-7
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const math = std.math;2const math = std.math;
4const DefaultPrng = std.rand.DefaultPrng;3const DefaultPrng = std.rand.DefaultPrng;
5const Random = std.rand.Random;4const Random = std.rand.Random;
...@@ -200,8 +199,6 @@ fn testRandomIntLessThan() !void {...@@ -200,8 +199,6 @@ fn testRandomIntLessThan() !void {
200}199}
201200
202test "Random intAtMost" {201test "Random intAtMost" {
203 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
204
205 @setEvalBranchQuota(10000);202 @setEvalBranchQuota(10000);
206 try testRandomIntAtMost();203 try testRandomIntAtMost();
207 try comptime testRandomIntAtMost();204 try comptime testRandomIntAtMost();
...@@ -242,8 +239,6 @@ fn testRandomIntAtMost() !void {...@@ -242,8 +239,6 @@ fn testRandomIntAtMost() !void {
242}239}
243240
244test "Random Biased" {241test "Random Biased" {
245 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
246
247 var prng = DefaultPrng.init(0);242 var prng = DefaultPrng.init(0);
248 const random = prng.random();243 const random = prng.random();
249 // Not thoroughly checking the logic here.244 // Not thoroughly checking the logic here.
...@@ -452,8 +447,6 @@ test "CSPRNG" {...@@ -452,8 +447,6 @@ test "CSPRNG" {
452}447}
453448
454test "Random weightedIndex" {449test "Random weightedIndex" {
455 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
456
457 // Make sure weightedIndex works for various integers and floats450 // Make sure weightedIndex works for various integers and floats
458 inline for (.{ u64, i4, f32, f64 }) |T| {451 inline for (.{ u64, i4, f32, f64 }) |T| {
459 var prng = DefaultPrng.init(0);452 var prng = DefaultPrng.init(0);
lib/std/zig/tokenizer.zig-9
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");
32
4pub const Token = struct {3pub const Token = struct {
5 tag: Tag,4 tag: Tag,
...@@ -1450,8 +1449,6 @@ test "chars" {...@@ -1450,8 +1449,6 @@ test "chars" {
1450}1449}
14511450
1452test "invalid token characters" {1451test "invalid token characters" {
1453 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1454
1455 try testTokenize("#", &.{.invalid});1452 try testTokenize("#", &.{.invalid});
1456 try testTokenize("`", &.{.invalid});1453 try testTokenize("`", &.{.invalid});
1457 try testTokenize("'c", &.{.invalid});1454 try testTokenize("'c", &.{.invalid});
...@@ -1571,8 +1568,6 @@ test "pipe and then invalid" {...@@ -1571,8 +1568,6 @@ test "pipe and then invalid" {
1571}1568}
15721569
1573test "line comment and doc comment" {1570test "line comment and doc comment" {
1574 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1575
1576 try testTokenize("//", &.{});1571 try testTokenize("//", &.{});
1577 try testTokenize("// a / b", &.{});1572 try testTokenize("// a / b", &.{});
1578 try testTokenize("// /", &.{});1573 try testTokenize("// /", &.{});
...@@ -1647,8 +1642,6 @@ test "range literals" {...@@ -1647,8 +1642,6 @@ test "range literals" {
1647}1642}
16481643
1649test "number literals decimal" {1644test "number literals decimal" {
1650 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1651
1652 try testTokenize("0", &.{.number_literal});1645 try testTokenize("0", &.{.number_literal});
1653 try testTokenize("1", &.{.number_literal});1646 try testTokenize("1", &.{.number_literal});
1654 try testTokenize("2", &.{.number_literal});1647 try testTokenize("2", &.{.number_literal});
...@@ -1897,8 +1890,6 @@ test "invalid token with unfinished escape right before eof" {...@@ -1897,8 +1890,6 @@ test "invalid token with unfinished escape right before eof" {
1897}1890}
18981891
1899test "saturating operators" {1892test "saturating operators" {
1900 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1901
1902 try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});1893 try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
1903 try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});1894 try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});
1904 try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});1895 try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});
src/Compilation.zig+3-1
...@@ -1121,7 +1121,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {...@@ -1121,7 +1121,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1121 const include_compiler_rt = options.want_compiler_rt orelse needs_c_symbols;1121 const include_compiler_rt = options.want_compiler_rt orelse needs_c_symbols;
11221122
1123 const must_single_thread = target_util.isSingleThreaded(options.target);1123 const must_single_thread = target_util.isSingleThreaded(options.target);
1124 const single_threaded = options.single_threaded orelse must_single_thread;1124 const single_threaded = options.single_threaded orelse must_single_thread or
1125 // x86_64 codegen doesn't support TLV for most object formats
1126 (!use_llvm and options.target.cpu.arch == .x86_64 and options.target.ofmt != .macho);
1125 if (must_single_thread and !single_threaded) {1127 if (must_single_thread and !single_threaded) {
1126 return error.TargetRequiresSingleThreaded;1128 return error.TargetRequiresSingleThreaded;
1127 }1129 }
src/arch/x86_64/CodeGen.zig+1440-681
...@@ -388,7 +388,7 @@ pub const MCValue = union(enum) {...@@ -388,7 +388,7 @@ pub const MCValue = union(enum) {
388 };388 };
389 }389 }
390390
391 fn mem(mcv: MCValue, size: Memory.Size) Memory {391 fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {
392 return switch (mcv) {392 return switch (mcv) {
393 .none,393 .none,
394 .unreach,394 .unreach,
...@@ -409,7 +409,6 @@ pub const MCValue = union(enum) {...@@ -409,7 +409,6 @@ pub const MCValue = union(enum) {
409 .lea_frame,409 .lea_frame,
410 .reserved_frame,410 .reserved_frame,
411 .air_ref,411 .air_ref,
412 .load_symbol,
413 .lea_symbol,412 .lea_symbol,
414 => unreachable,413 => unreachable,
415 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{414 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{
...@@ -433,6 +432,19 @@ pub const MCValue = union(enum) {...@@ -433,6 +432,19 @@ pub const MCValue = union(enum) {
433 .disp = frame_addr.off,432 .disp = frame_addr.off,
434 } },433 } },
435 },434 },
435 .load_symbol => |sym_off| {
436 assert(sym_off.off == 0);
437 return .{
438 .base = .{ .reloc = .{
439 .atom_index = try function.owner.getSymbolIndex(function),
440 .sym_index = sym_off.sym,
441 } },
442 .mod = .{ .rm = .{
443 .size = size,
444 .disp = sym_off.off,
445 } },
446 };
447 },
436 };448 };
437 }449 }
438450
...@@ -722,12 +734,14 @@ const InstTracking = struct {...@@ -722,12 +734,14 @@ const InstTracking = struct {
722734
723const FrameAlloc = struct {735const FrameAlloc = struct {
724 abi_size: u31,736 abi_size: u31,
737 spill_pad: u3,
725 abi_align: Alignment,738 abi_align: Alignment,
726 ref_count: u16,739 ref_count: u16,
727740
728 fn init(alloc_abi: struct { size: u64, alignment: Alignment }) FrameAlloc {741 fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
729 return .{742 return .{
730 .abi_size = @intCast(alloc_abi.size),743 .abi_size = @intCast(alloc_abi.size),
744 .spill_pad = alloc_abi.pad,
731 .abi_align = alloc_abi.alignment,745 .abi_align = alloc_abi.alignment,
732 .ref_count = 0,746 .ref_count = 0,
733 };747 };
...@@ -738,6 +752,20 @@ const FrameAlloc = struct {...@@ -738,6 +752,20 @@ const FrameAlloc = struct {
738 .alignment = ty.abiAlignment(mod),752 .alignment = ty.abiAlignment(mod),
739 });753 });
740 }754 }
755 fn initSpill(ty: Type, mod: *Module) FrameAlloc {
756 const abi_size = ty.abiSize(mod);
757 const spill_size = if (abi_size < 8)
758 math.ceilPowerOfTwoAssert(u64, abi_size)
759 else
760 std.mem.alignForward(u64, abi_size, 8);
761 return init(.{
762 .size = spill_size,
763 .pad = @intCast(spill_size - abi_size),
764 .alignment = ty.abiAlignment(mod).maxStrict(
765 Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
766 ),
767 });
768 }
741};769};
742770
743const StackAllocation = struct {771const StackAllocation = struct {
...@@ -1668,8 +1696,7 @@ fn gen(self: *Self) InnerError!void {...@@ -1668,8 +1696,7 @@ fn gen(self: *Self) InnerError!void {
1668 // The address where to store the return value for the caller is in a1696 // The address where to store the return value for the caller is in a
1669 // register which the callee is free to clobber. Therefore, we purposely1697 // register which the callee is free to clobber. Therefore, we purposely
1670 // spill it to stack immediately.1698 // spill it to stack immediately.
1671 const frame_index =1699 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, mod));
1672 try self.allocFrameIndex(FrameAlloc.initType(Type.usize, mod));
1673 try self.genSetMem(1700 try self.genSetMem(
1674 .{ .frame = frame_index },1701 .{ .frame = frame_index },
1675 0,1702 0,
...@@ -2434,7 +2461,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b...@@ -2434,7 +2461,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2434 }2461 }
2435 }2462 }
24362463
2437 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ty, mod));2464 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, mod));
2438 return .{ .load_frame = .{ .index = frame_index } };2465 return .{ .load_frame = .{ .index = frame_index } };
2439}2466}
24402467
...@@ -2445,7 +2472,10 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {...@@ -2445,7 +2472,10 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
2445 80 => abi.RegisterClass.x87,2472 80 => abi.RegisterClass.x87,
2446 else => abi.RegisterClass.sse,2473 else => abi.RegisterClass.sse,
2447 },2474 },
2448 .Vector => abi.RegisterClass.sse,2475 .Vector => switch (ty.childType(mod).toIntern()) {
2476 .bool_type => abi.RegisterClass.gp,
2477 else => abi.RegisterClass.sse,
2478 },
2449 else => abi.RegisterClass.gp,2479 else => abi.RegisterClass.gp,
2450 };2480 };
2451}2481}
...@@ -2503,6 +2533,19 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -2503,6 +2533,19 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2503 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);2533 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
2504 for (deaths) |death| try self.processDeath(death);2534 for (deaths) |death| try self.processDeath(death);
25052535
2536 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
2537 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2538 if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
2539
2540 var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
2541 stack.get(),
2542 @typeInfo(ExpectedContents).Array.len,
2543 );
2544 defer if (!opts.update_tracking) {
2545 for (reg_locks.items) |lock| self.register_manager.unlockReg(lock);
2546 reg_locks.deinit();
2547 };
2548
2506 for (0..state.registers.len) |index| {2549 for (0..state.registers.len) |index| {
2507 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))2550 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
2508 null2551 null
...@@ -2519,11 +2562,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -2519,11 +2562,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2519 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);2562 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
2520 }2563 }
2521 if (target_maybe_inst) |target_inst| {2564 if (target_maybe_inst) |target_inst| {
2522 try self.inst_tracking.getPtr(target_inst).?.materialize(2565 const target_tracking = self.inst_tracking.getPtr(target_inst).?;
2523 self,2566 try target_tracking.materialize(self, target_inst, state.reg_tracking[index]);
2524 target_inst,
2525 state.reg_tracking[index],
2526 );
2527 }2567 }
2528 }2568 }
2529 if (opts.update_tracking) {2569 if (opts.update_tracking) {
...@@ -2541,7 +2581,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -2541,7 +2581,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2541 state.reg_tracking[index],2581 state.reg_tracking[index],
2542 );2582 );
2543 }2583 }
2544 }2584 } else if (target_maybe_inst) |_|
2585 try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(@intCast(index)));
2545 }2586 }
2546 if (opts.emit_instructions) if (self.eflags_inst) |inst|2587 if (opts.emit_instructions) if (self.eflags_inst) |inst|
2547 try self.inst_tracking.getPtr(inst).?.spill(self, inst);2588 try self.inst_tracking.getPtr(inst).?.spill(self, inst);
...@@ -2578,8 +2619,17 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {...@@ -2578,8 +2619,17 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
2578 }2619 }
2579}2620}
25802621
2581pub fn spillRegisters(self: *Self, registers: []const Register) !void {2622pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention) !void {
2582 for (registers) |reg| try self.register_manager.getReg(reg, null);2623 switch (cc) {
2624 inline .SysV, .Win64 => |known_cc| try self.spillRegisters(
2625 comptime abi.getCallerPreservedRegs(known_cc),
2626 ),
2627 else => unreachable,
2628 }
2629}
2630
2631pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
2632 inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null);
2583}2633}
25842634
2585/// Copies a value to a register without tracking the register. The register is not considered2635/// Copies a value to a register without tracking the register. The register is not considered
...@@ -2699,7 +2749,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2699,7 +2749,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2699 .{ .v_ss, .cvtsd2 },2749 .{ .v_ss, .cvtsd2 },
2700 dst_reg,2750 dst_reg,
2701 dst_reg,2751 dst_reg,
2702 src_mcv.mem(.qword),2752 try src_mcv.mem(self, .qword),
2703 ) else try self.asmRegisterRegisterRegister(2753 ) else try self.asmRegisterRegisterRegister(
2704 .{ .v_ss, .cvtsd2 },2754 .{ .v_ss, .cvtsd2 },
2705 dst_reg,2755 dst_reg,
...@@ -2711,7 +2761,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2711,7 +2761,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2711 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(2761 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2712 .{ ._ss, .cvtsd2 },2762 .{ ._ss, .cvtsd2 },
2713 dst_reg,2763 dst_reg,
2714 src_mcv.mem(.qword),2764 try src_mcv.mem(self, .qword),
2715 ) else try self.asmRegisterRegister(2765 ) else try self.asmRegisterRegister(
2716 .{ ._ss, .cvtsd2 },2766 .{ ._ss, .cvtsd2 },
2717 dst_reg,2767 dst_reg,
...@@ -2798,7 +2848,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2798,7 +2848,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2798 .{ .v_sd, .cvtss2 },2848 .{ .v_sd, .cvtss2 },
2799 dst_reg,2849 dst_reg,
2800 dst_reg,2850 dst_reg,
2801 src_mcv.mem(.dword),2851 try src_mcv.mem(self, .dword),
2802 ) else try self.asmRegisterRegisterRegister(2852 ) else try self.asmRegisterRegisterRegister(
2803 .{ .v_sd, .cvtss2 },2853 .{ .v_sd, .cvtss2 },
2804 dst_reg,2854 dst_reg,
...@@ -2810,7 +2860,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2810,7 +2860,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2810 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(2860 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2811 .{ ._sd, .cvtss2 },2861 .{ ._sd, .cvtss2 },
2812 dst_reg,2862 dst_reg,
2813 src_mcv.mem(.dword),2863 try src_mcv.mem(self, .dword),
2814 ) else try self.asmRegisterRegister(2864 ) else try self.asmRegisterRegister(
2815 .{ ._sd, .cvtss2 },2865 .{ ._sd, .cvtss2 },
2816 dst_reg,2866 dst_reg,
...@@ -2851,8 +2901,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2851,8 +2901,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2851 };2901 };
28522902
2853 const dst_mcv = if (dst_int_info.bits <= src_storage_bits and2903 const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
2854 std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==2904 math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
2855 std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and2905 math.divCeil(u32, src_storage_bits, 64) catch unreachable and
2856 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {2906 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
2857 const dst_mcv = try self.allocRegOrMem(inst, true);2907 const dst_mcv = try self.allocRegOrMem(inst, true);
2858 try self.genCopy(min_ty, dst_mcv, src_mcv);2908 try self.genCopy(min_ty, dst_mcv, src_mcv);
...@@ -2869,22 +2919,28 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2869,22 +2919,28 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2869 break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };2919 break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };
2870 }2920 }
28712921
2872 const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable;2922 const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
2873 const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;2923 const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
28742924
2875 const high_mcv = dst_mcv.address().offset((src_limbs_len - 1) * 8).deref();2925 const high_mcv: MCValue = if (dst_mcv.isMemory())
2876 const high_reg = try self.copyToTmpRegister(switch (src_int_info.signedness) {2926 dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
2877 .signed => Type.isize,2927 else
2878 .unsigned => Type.usize,2928 .{ .register = dst_mcv.register_pair[1] };
2879 }, high_mcv);2929 const high_reg = if (high_mcv.isRegister())
2930 high_mcv.getReg().?
2931 else
2932 try self.copyToTmpRegister(switch (src_int_info.signedness) {
2933 .signed => Type.isize,
2934 .unsigned => Type.usize,
2935 }, high_mcv);
2880 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);2936 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
2881 defer self.register_manager.unlockReg(high_lock);2937 defer self.register_manager.unlockReg(high_lock);
28822938
2883 const high_bits = src_int_info.bits % 64;2939 const high_bits = src_int_info.bits % 64;
2884 if (high_bits > 0) {2940 if (high_bits > 0) {
2885 const high_ty = try mod.intType(extend, high_bits);2941 try self.truncateRegister(src_ty, high_reg);
2886 try self.truncateRegister(high_ty, high_reg);2942 const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;
2887 try self.genCopy(Type.usize, high_mcv, .{ .register = high_reg });2943 try self.genCopy(high_ty, high_mcv, .{ .register = high_reg });
2888 }2944 }
28892945
2890 if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(2946 if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(
...@@ -2995,14 +3051,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2995,14 +3051,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2995 .{ .vp_, .@"and" },3051 .{ .vp_, .@"and" },
2996 dst_reg,3052 dst_reg,
2997 dst_reg,3053 dst_reg,
2998 splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),3054 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
2999 );3055 );
3000 try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);3056 try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
3001 } else {3057 } else {
3002 try self.asmRegisterMemory(3058 try self.asmRegisterMemory(
3003 .{ .p_, .@"and" },3059 .{ .p_, .@"and" },
3004 dst_reg,3060 dst_reg,
3005 splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),3061 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3006 );3062 );
3007 try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);3063 try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
3008 }3064 }
...@@ -3048,7 +3104,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -3048,7 +3104,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
3048 const len = try self.resolveInst(bin_op.rhs);3104 const len = try self.resolveInst(bin_op.rhs);
3049 const len_ty = self.typeOf(bin_op.rhs);3105 const len_ty = self.typeOf(bin_op.rhs);
30503106
3051 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));3107 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
3052 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);3108 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
3053 try self.genSetMem(3109 try self.genSetMem(
3054 .{ .frame = frame_index },3110 .{ .frame = frame_index },
...@@ -3068,8 +3124,36 @@ fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {...@@ -3068,8 +3124,36 @@ fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3068}3124}
30693125
3070fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {3126fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3127 const mod = self.bin_file.options.module.?;
3071 const bin_op = self.air.instructions.items(.data)[inst].bin_op;3128 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3072 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);3129 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
3130
3131 const dst_ty = self.typeOfIndex(inst);
3132 if (dst_ty.isAbiInt(mod)) {
3133 const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
3134 const bit_size: u32 = @intCast(dst_ty.bitSize(mod));
3135 if (abi_size * 8 > bit_size) {
3136 const dst_lock = switch (dst_mcv) {
3137 .register => |dst_reg| self.register_manager.lockRegAssumeUnused(dst_reg),
3138 else => null,
3139 };
3140 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
3141
3142 if (dst_mcv.isRegister()) {
3143 try self.truncateRegister(dst_ty, dst_mcv.getReg().?);
3144 } else {
3145 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
3146 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3147 defer self.register_manager.unlockReg(tmp_lock);
3148
3149 const hi_ty = try mod.intType(.unsigned, @intCast((dst_ty.bitSize(mod) - 1) % 64 + 1));
3150 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
3151 try self.genSetReg(tmp_reg, hi_ty, hi_mcv);
3152 try self.truncateRegister(dst_ty, tmp_reg);
3153 try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg });
3154 }
3155 }
3156 }
3073 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3157 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
3074}3158}
30753159
...@@ -3176,7 +3260,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -3176,7 +3260,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3176 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(3260 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3177 .{ ._, .mov },3261 .{ ._, .mov },
3178 tmp_reg,3262 tmp_reg,
3179 mat_lhs_mcv.address().offset(8).deref().mem(.qword),3263 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3180 ) else try self.asmRegisterRegister(3264 ) else try self.asmRegisterRegister(
3181 .{ ._, .mov },3265 .{ ._, .mov },
3182 tmp_reg,3266 tmp_reg,
...@@ -3200,7 +3284,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -3200,7 +3284,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3200 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(3284 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3201 .{ ._, .xor },3285 .{ ._, .xor },
3202 tmp_reg,3286 tmp_reg,
3203 mat_rhs_mcv.address().offset(8).deref().mem(.qword),3287 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3204 ) else try self.asmRegisterRegister(3288 ) else try self.asmRegisterRegister(
3205 .{ ._, .xor },3289 .{ ._, .xor },
3206 tmp_reg,3290 tmp_reg,
...@@ -3300,12 +3384,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -3300,12 +3384,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3300 try self.asmRegisterMemory(3384 try self.asmRegisterMemory(
3301 .{ ._, .add },3385 .{ ._, .add },
3302 tmp_regs[0],3386 tmp_regs[0],
3303 mat_rhs_mcv.mem(.qword),3387 try mat_rhs_mcv.mem(self, .qword),
3304 );3388 );
3305 try self.asmRegisterMemory(3389 try self.asmRegisterMemory(
3306 .{ ._, .adc },3390 .{ ._, .adc },
3307 tmp_regs[1],3391 tmp_regs[1],
3308 mat_rhs_mcv.address().offset(8).deref().mem(.qword),3392 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3309 );3393 );
3310 } else for (3394 } else for (
3311 [_]Mir.Inst.Tag{ .add, .adc },3395 [_]Mir.Inst.Tag{ .add, .adc },
...@@ -3534,7 +3618,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3534,7 +3618,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3534 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(3618 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3535 .{ ._, .mov },3619 .{ ._, .mov },
3536 tmp_reg,3620 tmp_reg,
3537 mat_lhs_mcv.address().offset(8).deref().mem(.qword),3621 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3538 ) else try self.asmRegisterRegister(3622 ) else try self.asmRegisterRegister(
3539 .{ ._, .mov },3623 .{ ._, .mov },
3540 tmp_reg,3624 tmp_reg,
...@@ -3558,7 +3642,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3558,7 +3642,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3558 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(3642 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3559 .{ ._, .xor },3643 .{ ._, .xor },
3560 tmp_reg,3644 tmp_reg,
3561 mat_rhs_mcv.address().offset(8).deref().mem(.qword),3645 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3562 ) else try self.asmRegisterRegister(3646 ) else try self.asmRegisterRegister(
3563 .{ ._, .xor },3647 .{ ._, .xor },
3564 tmp_reg,3648 tmp_reg,
...@@ -3567,7 +3651,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3567,7 +3651,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
35673651
3568 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));3652 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
3569 try self.asmRegister(.{ ._, .not }, tmp_reg);3653 try self.asmRegister(.{ ._, .not }, tmp_reg);
3570 try self.asmMemoryImmediate(.{ ._, .cmp }, overflow.mem(.dword), Immediate.s(0));3654 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));
3571 try self.freeValue(overflow);3655 try self.freeValue(overflow);
3572 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);3656 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
3573 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));3657 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));
...@@ -3665,7 +3749,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3665,7 +3749,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3665 }3749 }
36663750
3667 const frame_index =3751 const frame_index =
3668 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3752 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3669 try self.genSetMem(3753 try self.genSetMem(
3670 .{ .frame = frame_index },3754 .{ .frame = frame_index },
3671 @intCast(tuple_ty.structFieldOffset(1, mod)),3755 @intCast(tuple_ty.structFieldOffset(1, mod)),
...@@ -3682,7 +3766,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3682,7 +3766,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3682 }3766 }
36833767
3684 const frame_index =3768 const frame_index =
3685 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3769 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3686 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);3770 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3687 break :result .{ .load_frame = .{ .index = frame_index } };3771 break :result .{ .load_frame = .{ .index = frame_index } };
3688 },3772 },
...@@ -3738,7 +3822,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3738,7 +3822,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3738 }3822 }
37393823
3740 const frame_index =3824 const frame_index =
3741 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3825 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3742 try self.genSetMem(3826 try self.genSetMem(
3743 .{ .frame = frame_index },3827 .{ .frame = frame_index },
3744 @intCast(tuple_ty.structFieldOffset(1, mod)),3828 @intCast(tuple_ty.structFieldOffset(1, mod)),
...@@ -3755,7 +3839,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3755,7 +3839,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3755 }3839 }
37563840
3757 const frame_index =3841 const frame_index =
3758 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3842 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3759 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);3843 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3760 break :result .{ .load_frame = .{ .index = frame_index } };3844 break :result .{ .load_frame = .{ .index = frame_index } };
3761 },3845 },
...@@ -3874,7 +3958,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3874,7 +3958,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3874 );3958 );
3875 try self.asmMemoryImmediate(3959 try self.asmMemoryImmediate(
3876 .{ ._, .cmp },3960 .{ ._, .cmp },
3877 overflow.mem(self.memSize(Type.c_int)),3961 try overflow.mem(self, self.memSize(Type.c_int)),
3878 Immediate.s(0),3962 Immediate.s(0),
3879 );3963 );
3880 try self.genSetMem(3964 try self.genSetMem(
...@@ -3926,14 +4010,19 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3926,14 +4010,19 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3926 };4010 };
3927 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);4011 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
39284012
3929 if (mat_lhs_mcv.isMemory())4013 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3930 try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))4014 .{ ._, .mov },
3931 else4015 .rax,
3932 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);4016 try mat_lhs_mcv.mem(self, .qword),
4017 ) else try self.asmRegisterRegister(
4018 .{ ._, .mov },
4019 .rax,
4020 mat_lhs_mcv.register_pair[0],
4021 );
3933 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(4022 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3934 .{ ._, .mov },4023 .{ ._, .mov },
3935 tmp_regs[0],4024 tmp_regs[0],
3936 mat_rhs_mcv.address().offset(8).deref().mem(.qword),4025 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3937 ) else try self.asmRegisterRegister(4026 ) else try self.asmRegisterRegister(
3938 .{ ._, .mov },4027 .{ ._, .mov },
3939 tmp_regs[0],4028 tmp_regs[0],
...@@ -3944,7 +4033,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3944,7 +4033,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3944 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);4033 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
3945 try self.asmSetccRegister(.o, tmp_regs[2].to8());4034 try self.asmSetccRegister(.o, tmp_regs[2].to8());
3946 if (mat_rhs_mcv.isMemory())4035 if (mat_rhs_mcv.isMemory())
3947 try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))4036 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
3948 else4037 else
3949 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);4038 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
3950 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);4039 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...@@ -3953,7 +4042,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3953,7 +4042,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3953 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(4042 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3954 .{ ._, .mov },4043 .{ ._, .mov },
3955 tmp_regs[0],4044 tmp_regs[0],
3956 mat_lhs_mcv.address().offset(8).deref().mem(.qword),4045 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3957 ) else try self.asmRegisterRegister(4046 ) else try self.asmRegisterRegister(
3958 .{ ._, .mov },4047 .{ ._, .mov },
3959 tmp_regs[0],4048 tmp_regs[0],
...@@ -3967,14 +4056,15 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3967,14 +4056,15 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3967 tmp_regs[3].to8(),4056 tmp_regs[3].to8(),
3968 );4057 );
3969 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());4058 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3970 if (mat_rhs_mcv.isMemory())4059 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3971 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_regs[0], mat_rhs_mcv.mem(.qword))4060 .{ .i_, .mul },
3972 else4061 tmp_regs[0],
3973 try self.asmRegisterRegister(4062 try mat_rhs_mcv.mem(self, .qword),
3974 .{ .i_, .mul },4063 ) else try self.asmRegisterRegister(
3975 tmp_regs[0],4064 .{ .i_, .mul },
3976 mat_rhs_mcv.register_pair[0],4065 tmp_regs[0],
3977 );4066 mat_rhs_mcv.register_pair[0],
4067 );
3978 try self.asmSetccRegister(.o, tmp_regs[2].to8());4068 try self.asmSetccRegister(.o, tmp_regs[2].to8());
3979 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());4069 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3980 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);4070 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...@@ -4020,8 +4110,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4020,8 +4110,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4020 self.eflags_inst = inst;4110 self.eflags_inst = inst;
4021 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };4111 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
4022 } else {4112 } else {
4023 const frame_index =4113 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4024 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
4025 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);4114 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4026 break :result .{ .load_frame = .{ .index = frame_index } };4115 break :result .{ .load_frame = .{ .index = frame_index } };
4027 },4116 },
...@@ -4032,8 +4121,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4032,8 +4121,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4032 src_ty.fmt(mod), dst_ty.fmt(mod),4121 src_ty.fmt(mod), dst_ty.fmt(mod),
4033 });4122 });
40344123
4035 const frame_index =4124 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4036 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
4037 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {4125 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
4038 try self.genSetMem(4126 try self.genSetMem(
4039 .{ .frame = frame_index },4127 .{ .frame = frame_index },
...@@ -4106,7 +4194,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue...@@ -4106,7 +4194,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
4106 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),4194 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
4107 .memory, .indirect, .load_frame => try self.asmMemory(4195 .memory, .indirect, .load_frame => try self.asmMemory(
4108 tag,4196 tag,
4109 mat_rhs.mem(Memory.Size.fromSize(abi_size)),4197 try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),
4110 ),4198 ),
4111 else => unreachable,4199 else => unreachable,
4112 }4200 }
...@@ -4160,8 +4248,8 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -4160,8 +4248,8 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
4160 );4248 );
4161 try self.asmCmovccRegisterRegister(4249 try self.asmCmovccRegisterRegister(
4162 .z,4250 .z,
4163 registerAlias(divisor, abi_size),4251 registerAlias(divisor, @max(abi_size, 2)),
4164 registerAlias(.rdx, abi_size),4252 registerAlias(.rdx, @max(abi_size, 2)),
4165 );4253 );
4166 try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });4254 try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
4167 return MCValue{ .register = divisor };4255 return MCValue{ .register = divisor };
...@@ -4171,47 +4259,268 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4171,47 +4259,268 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4171 const mod = self.bin_file.options.module.?;4259 const mod = self.bin_file.options.module.?;
4172 const bin_op = self.air.instructions.items(.data)[inst].bin_op;4260 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
41734261
4174 try self.spillRegisters(&.{.rcx});4262 const air_tags = self.air.instructions.items(.tag);
41754263 const tag = air_tags[inst];
4176 const tag = self.air.instructions.items(.tag)[inst];
4177 try self.register_manager.getReg(.rcx, null);
4178 const lhs = try self.resolveInst(bin_op.lhs);
4179 const rhs = try self.resolveInst(bin_op.rhs);
4180 const lhs_ty = self.typeOf(bin_op.lhs);4264 const lhs_ty = self.typeOf(bin_op.lhs);
4181 const rhs_ty = self.typeOf(bin_op.rhs);4265 const rhs_ty = self.typeOf(bin_op.rhs);
4266 const result: MCValue = result: {
4267 switch (lhs_ty.zigTypeTag(mod)) {
4268 .Int => {
4269 try self.spillRegisters(&.{.rcx});
4270 try self.register_manager.getReg(.rcx, null);
4271 const lhs_mcv = try self.resolveInst(bin_op.lhs);
4272 const rhs_mcv = try self.resolveInst(bin_op.rhs);
41824273
4183 const dst_mcv = try self.genShiftBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);4274 const dst_mcv = try self.genShiftBinOp(tag, inst, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty);
4184 switch (tag) {4275 switch (tag) {
4185 .shr, .shr_exact, .shl_exact => {},4276 .shr, .shr_exact, .shl_exact => {},
4186 .shl => switch (dst_mcv) {4277 .shl => switch (dst_mcv) {
4187 .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg),4278 .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg),
4188 .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]),4279 .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]),
4189 .load_frame => |frame_addr| {4280 .load_frame => |frame_addr| {
4190 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);4281 const tmp_reg =
4191 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);4282 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4192 defer self.register_manager.unlockReg(tmp_lock);4283 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4284 defer self.register_manager.unlockReg(tmp_lock);
41934285
4194 const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));4286 const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));
4195 const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;4287 const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
4196 const off = frame_addr.off + lhs_bits / 64 * 8;4288 const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
4197 try self.genSetReg(4289 try self.genSetReg(
4198 tmp_reg,4290 tmp_reg,
4199 tmp_ty,4291 tmp_ty,
4200 .{ .load_frame = .{ .index = frame_addr.index, .off = off } },4292 .{ .load_frame = .{ .index = frame_addr.index, .off = off } },
4201 );4293 );
4202 try self.truncateRegister(lhs_ty, tmp_reg);4294 try self.truncateRegister(lhs_ty, tmp_reg);
4203 try self.genSetMem(4295 try self.genSetMem(
4204 .{ .frame = frame_addr.index },4296 .{ .frame = frame_addr.index },
4205 off,4297 off,
4206 tmp_ty,4298 tmp_ty,
4207 .{ .register = tmp_reg },4299 .{ .register = tmp_reg },
4208 );4300 );
4301 },
4302 else => {},
4303 },
4304 else => unreachable,
4305 }
4306 break :result dst_mcv;
4307 },
4308 .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
4309 .Int => if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.childType(mod).intInfo(mod).bits) {
4310 else => null,
4311 16 => switch (lhs_ty.vectorLen(mod)) {
4312 else => null,
4313 1...8 => switch (tag) {
4314 else => unreachable,
4315 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4316 .signed => if (self.hasFeature(.avx))
4317 .{ .vp_w, .sra }
4318 else
4319 .{ .p_w, .sra },
4320 .unsigned => if (self.hasFeature(.avx))
4321 .{ .vp_w, .srl }
4322 else
4323 .{ .p_w, .srl },
4324 },
4325 .shl, .shl_exact => if (self.hasFeature(.avx))
4326 .{ .vp_w, .sll }
4327 else
4328 .{ .p_w, .sll },
4329 },
4330 9...16 => switch (tag) {
4331 else => unreachable,
4332 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4333 .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .sra } else null,
4334 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .srl } else null,
4335 },
4336 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_w, .sll } else null,
4337 },
4338 },
4339 32 => switch (lhs_ty.vectorLen(mod)) {
4340 else => null,
4341 1...4 => switch (tag) {
4342 else => unreachable,
4343 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4344 .signed => if (self.hasFeature(.avx))
4345 .{ .vp_d, .sra }
4346 else
4347 .{ .p_d, .sra },
4348 .unsigned => if (self.hasFeature(.avx))
4349 .{ .vp_d, .srl }
4350 else
4351 .{ .p_d, .srl },
4352 },
4353 .shl, .shl_exact => if (self.hasFeature(.avx))
4354 .{ .vp_d, .sll }
4355 else
4356 .{ .p_d, .sll },
4357 },
4358 5...8 => switch (tag) {
4359 else => unreachable,
4360 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4361 .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .sra } else null,
4362 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .srl } else null,
4363 },
4364 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_d, .sll } else null,
4365 },
4366 },
4367 64 => switch (lhs_ty.vectorLen(mod)) {
4368 else => null,
4369 1...2 => switch (tag) {
4370 else => unreachable,
4371 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4372 .signed => if (self.hasFeature(.avx))
4373 .{ .vp_q, .sra }
4374 else
4375 .{ .p_q, .sra },
4376 .unsigned => if (self.hasFeature(.avx))
4377 .{ .vp_q, .srl }
4378 else
4379 .{ .p_q, .srl },
4380 },
4381 .shl, .shl_exact => if (self.hasFeature(.avx))
4382 .{ .vp_q, .sll }
4383 else
4384 .{ .p_q, .sll },
4385 },
4386 3...4 => switch (tag) {
4387 else => unreachable,
4388 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4389 .signed => if (self.hasFeature(.avx2)) .{ .vp_q, .sra } else null,
4390 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_q, .srl } else null,
4391 },
4392 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_q, .sll } else null,
4393 },
4394 },
4395 })) |mir_tag| if (try self.air.value(bin_op.rhs, mod)) |rhs_val| {
4396 switch (mod.intern_pool.indexToKey(rhs_val.toIntern())) {
4397 .aggregate => |rhs_aggregate| switch (rhs_aggregate.storage) {
4398 .repeated_elem => |rhs_elem| {
4399 const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
4400
4401 const lhs_mcv = try self.resolveInst(bin_op.lhs);
4402 const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
4403 self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
4404 .{lhs_mcv.getReg().?} ** 2
4405 else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
4406 try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
4407 lhs_mcv.getReg().?,
4408 } else .{(try self.copyToRegisterWithInstTracking(
4409 inst,
4410 lhs_ty,
4411 lhs_mcv,
4412 )).register} ** 2;
4413 const reg_locks =
4414 self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
4415 defer for (reg_locks) |reg_lock| if (reg_lock) |lock|
4416 self.register_manager.unlockReg(lock);
4417
4418 const shift_imm =
4419 Immediate.u(@intCast(rhs_elem.toValue().toUnsignedInt(mod)));
4420 if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
4421 mir_tag,
4422 registerAlias(dst_reg, abi_size),
4423 registerAlias(lhs_reg, abi_size),
4424 shift_imm,
4425 ) else {
4426 assert(dst_reg.id() == lhs_reg.id());
4427 try self.asmRegisterImmediate(
4428 mir_tag,
4429 registerAlias(dst_reg, abi_size),
4430 shift_imm,
4431 );
4432 }
4433 break :result .{ .register = dst_reg };
4434 },
4435 else => {},
4436 },
4437 else => {},
4438 }
4439 } else if (Air.refToIndex(bin_op.rhs)) |rhs_inst| switch (air_tags[rhs_inst]) {
4440 .splat => {
4441 const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
4442
4443 const lhs_mcv = try self.resolveInst(bin_op.lhs);
4444 const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
4445 self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
4446 .{lhs_mcv.getReg().?} ** 2
4447 else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
4448 try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
4449 lhs_mcv.getReg().?,
4450 } else .{(try self.copyToRegisterWithInstTracking(
4451 inst,
4452 lhs_ty,
4453 lhs_mcv,
4454 )).register} ** 2;
4455 const reg_locks = self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
4456 defer for (reg_locks) |reg_lock| if (reg_lock) |lock|
4457 self.register_manager.unlockReg(lock);
4458
4459 const shift_reg =
4460 try self.copyToTmpRegister(rhs_ty, .{ .air_ref = bin_op.rhs });
4461 const shift_lock = self.register_manager.lockRegAssumeUnused(shift_reg);
4462 defer self.register_manager.unlockReg(shift_lock);
4463
4464 const mask_ty = try mod.vectorType(.{ .len = 16, .child = .u8_type });
4465 const mask_mcv = try self.genTypedValue(.{
4466 .ty = mask_ty,
4467 .val = (try mod.intern(.{ .aggregate = .{
4468 .ty = mask_ty.toIntern(),
4469 .storage = .{ .elems = &([1]InternPool.Index{
4470 (try rhs_ty.childType(mod).maxIntScalar(mod, Type.u8)).toIntern(),
4471 } ++ [1]InternPool.Index{
4472 (try mod.intValue(Type.u8, 0)).toIntern(),
4473 } ** 15) },
4474 } })).toValue(),
4475 });
4476 const mask_addr_reg =
4477 try self.copyToTmpRegister(Type.usize, mask_mcv.address());
4478 const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
4479 defer self.register_manager.unlockReg(mask_addr_lock);
4480
4481 if (self.hasFeature(.avx)) {
4482 try self.asmRegisterRegisterMemory(
4483 .{ .vp_, .@"and" },
4484 shift_reg.to128(),
4485 shift_reg.to128(),
4486 .{
4487 .base = .{ .reg = mask_addr_reg },
4488 .mod = .{ .rm = .{ .size = .xword } },
4489 },
4490 );
4491 try self.asmRegisterRegisterRegister(
4492 mir_tag,
4493 registerAlias(dst_reg, abi_size),
4494 registerAlias(lhs_reg, abi_size),
4495 shift_reg.to128(),
4496 );
4497 } else {
4498 try self.asmRegisterMemory(
4499 .{ .p_, .@"and" },
4500 shift_reg.to128(),
4501 .{
4502 .base = .{ .reg = mask_addr_reg },
4503 .mod = .{ .rm = .{ .size = .xword } },
4504 },
4505 );
4506 assert(dst_reg.id() == lhs_reg.id());
4507 try self.asmRegisterRegister(
4508 mir_tag,
4509 registerAlias(dst_reg, abi_size),
4510 shift_reg.to128(),
4511 );
4512 }
4513 break :result .{ .register = dst_reg };
4514 },
4515 else => {},
4516 },
4517 else => {},
4209 },4518 },
4210 else => {},4519 else => {},
4211 },4520 }
4212 else => unreachable,4521 return self.fail("TODO implement airShlShrBinOp for {}", .{lhs_ty.fmt(mod)});
4213 }4522 };
4214 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });4523 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4215}4524}
42164525
4217fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {4526fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
...@@ -4230,12 +4539,18 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -4230,12 +4539,18 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
42304539
4231 const opt_mcv = try self.resolveInst(ty_op.operand);4540 const opt_mcv = try self.resolveInst(ty_op.operand);
4232 if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {4541 if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
4233 switch (opt_mcv) {4542 const pl_mcv: MCValue = switch (opt_mcv) {
4234 .register => |reg| try self.truncateRegister(pl_ty, reg),4543 .register_overflow => |ro| pl: {
4235 .register_overflow => |ro| try self.truncateRegister(pl_ty, ro.reg),4544 self.eflags_inst = null; // actually stop tracking the overflow part
4545 break :pl .{ .register = ro.reg };
4546 },
4547 else => opt_mcv,
4548 };
4549 switch (pl_mcv) {
4550 .register => |pl_reg| try self.truncateRegister(pl_ty, pl_reg),
4236 else => {},4551 else => {},
4237 }4552 }
4238 break :result opt_mcv;4553 break :result pl_mcv;
4239 }4554 }
42404555
4241 const pl_mcv = try self.allocRegOrMem(inst, true);4556 const pl_mcv = try self.allocRegOrMem(inst, true);
...@@ -4472,8 +4787,9 @@ fn genUnwrapErrUnionPayloadMir(...@@ -4472,8 +4787,9 @@ fn genUnwrapErrUnionPayloadMir(
4472 const eu_lock = self.register_manager.lockReg(reg);4787 const eu_lock = self.register_manager.lockReg(reg);
4473 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);4788 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
44744789
4475 const result_mcv: MCValue = if (maybe_inst) |inst|4790 const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);
4476 try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)4791 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
4792 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
4477 else4793 else
4478 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };4794 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
4479 if (payload_off > 0) try self.genShiftBinOpMir(4795 if (payload_off > 0) try self.genShiftBinOpMir(
...@@ -4482,7 +4798,12 @@ fn genUnwrapErrUnionPayloadMir(...@@ -4482,7 +4798,12 @@ fn genUnwrapErrUnionPayloadMir(
4482 result_mcv,4798 result_mcv,
4483 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },4799 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },
4484 ) else try self.truncateRegister(payload_ty, result_mcv.register);4800 ) else try self.truncateRegister(payload_ty, result_mcv.register);
4485 break :result result_mcv;4801 break :result if (payload_in_gp)
4802 result_mcv
4803 else if (maybe_inst) |inst|
4804 try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv)
4805 else
4806 .{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) };
4486 },4807 },
4487 else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),4808 else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),
4488 }4809 }
...@@ -4593,7 +4914,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -4593,7 +4914,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
4593 const result: MCValue = result: {4914 const result: MCValue = result: {
4594 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .{ .immediate = 0 };4915 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .{ .immediate = 0 };
45954916
4596 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));4917 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4597 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));4918 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4598 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));4919 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4599 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);4920 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
...@@ -4615,7 +4936,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4615,7 +4936,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
4615 const result: MCValue = result: {4936 const result: MCValue = result: {
4616 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);4937 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);
46174938
4618 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));4939 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4619 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));4940 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4620 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));4941 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4621 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);4942 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
...@@ -4770,14 +5091,19 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -4770,14 +5091,19 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
4770fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {5091fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
4771 const mod = self.bin_file.options.module.?;5092 const mod = self.bin_file.options.module.?;
4772 const bin_op = self.air.instructions.items(.data)[inst].bin_op;5093 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4773 const slice_ty = self.typeOf(bin_op.lhs);
47745094
4775 const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);5095 const result: MCValue = result: {
4776 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);5096 const elem_ty = self.typeOfIndex(inst);
4777 const dst_mcv = try self.allocRegOrMem(inst, false);5097 if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;
4778 try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
47795098
4780 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });5099 const slice_ty = self.typeOf(bin_op.lhs);
5100 const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);
5101 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
5102 const dst_mcv = try self.allocRegOrMem(inst, false);
5103 try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
5104 break :result dst_mcv;
5105 };
5106 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4781}5107}
47825108
4783fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {5109fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
...@@ -4810,11 +5136,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4810,11 +5136,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4810 };5136 };
4811 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);5137 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
48125138
4813 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
4814 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4815 defer self.register_manager.unlockReg(offset_reg_lock);
4816
4817 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);5139 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5140 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
5141 defer self.register_manager.unlockReg(addr_lock);
5142
4818 switch (array) {5143 switch (array) {
4819 .register => {5144 .register => {
4820 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));5145 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));
...@@ -4843,6 +5168,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4843,6 +5168,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4843 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),5168 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
4844 }5169 }
48455170
5171 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
5172 const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
5173 defer self.register_manager.unlockReg(offset_lock);
5174
4846 // TODO we could allocate register here, but need to expect addr register and potentially5175 // TODO we could allocate register here, but need to expect addr register and potentially
4847 // offset register.5176 // offset register.
4848 try self.spillEflagsIfOccupied();5177 try self.spillEflagsIfOccupied();
...@@ -5093,7 +5422,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5093,7 +5422,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5093 .{ ._, .sub },5422 .{ ._, .sub },
5094 dst_ty,5423 dst_ty,
5095 dst_mcv,5424 dst_mcv,
5096 .{ .immediate = 8 + self.regExtraBits(src_ty) },5425 .{ .immediate = 32 - src_bits },
5097 );5426 );
5098 } else if (src_bits <= 64) {5427 } else if (src_bits <= 64) {
5099 try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);5428 try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
...@@ -5361,7 +5690,9 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {...@@ -5361,7 +5690,9 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
5361 mat_src_mcv5690 mat_src_mcv
5362 else5691 else
5363 .{ .register = mat_src_mcv.register_pair[0] }, false);5692 .{ .register = mat_src_mcv.register_pair[0] }, false);
5364 try self.genPopCount(tmp_regs[1], Type.usize, if (mat_src_mcv.isMemory())5693 const src_info = src_ty.intInfo(mod);
5694 const hi_ty = try mod.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);
5695 try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory())
5365 mat_src_mcv.address().offset(8).deref()5696 mat_src_mcv.address().offset(8).deref()
5366 else5697 else
5367 .{ .register = mat_src_mcv.register_pair[1] }, false);5698 .{ .register = mat_src_mcv.register_pair[1] }, false);
...@@ -5383,9 +5714,13 @@ fn genPopCount(...@@ -5383,9 +5714,13 @@ fn genPopCount(
5383 const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));5714 const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
5384 if (self.hasFeature(.popcnt)) return self.genBinOpMir(5715 if (self.hasFeature(.popcnt)) return self.genBinOpMir(
5385 .{ ._, .popcnt },5716 .{ ._, .popcnt },
5386 if (src_abi_size > 1) src_ty else Type.u16,5717 if (src_abi_size > 1) src_ty else Type.u32,
5387 .{ .register = dst_reg },5718 .{ .register = dst_reg },
5388 src_mcv,5719 if (src_abi_size > 1) src_mcv else src: {
5720 if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv);
5721 try self.truncateRegister(try src_ty.toUnsigned(mod), dst_reg);
5722 break :src .{ .register = dst_reg };
5723 },
5389 );5724 );
53905725
5391 const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);5726 const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
...@@ -5517,9 +5852,9 @@ fn genByteSwap(...@@ -5517,9 +5852,9 @@ fn genByteSwap(
5517 try self.asmRegisterMemory(5852 try self.asmRegisterMemory(
5518 .{ ._, .movbe },5853 .{ ._, .movbe },
5519 dst_regs[0],5854 dst_regs[0],
5520 src_mcv.address().offset(8).deref().mem(.qword),5855 try src_mcv.address().offset(8).deref().mem(self, .qword),
5521 );5856 );
5522 try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], src_mcv.mem(.qword));5857 try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], try src_mcv.mem(self, .qword));
5523 } else for (dst_regs, src_mcv.register_pair) |dst_reg, src_reg| {5858 } else for (dst_regs, src_mcv.register_pair) |dst_reg, src_reg| {
5524 try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to64(), src_reg.to64());5859 try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to64(), src_reg.to64());
5525 try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());5860 try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
...@@ -5762,7 +6097,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)...@@ -5762,7 +6097,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
5762 else => unreachable,6097 else => unreachable,
5763 } });6098 } });
5764 const sign_mem: Memory = if (sign_mcv.isMemory())6099 const sign_mem: Memory = if (sign_mcv.isMemory())
5765 sign_mcv.mem(Memory.Size.fromSize(abi_size))6100 try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))
5766 else6101 else
5767 .{6102 .{
5768 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },6103 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
...@@ -5945,7 +6280,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -5945,7 +6280,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5945 mir_tag,6280 mir_tag,
5946 dst_alias,6281 dst_alias,
5947 dst_alias,6282 dst_alias,
5948 src_mcv.mem(Memory.Size.fromSize(abi_size)),6283 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5949 Immediate.u(@as(u5, @bitCast(mode))),6284 Immediate.u(@as(u5, @bitCast(mode))),
5950 ) else try self.asmRegisterRegisterRegisterImmediate(6285 ) else try self.asmRegisterRegisterRegisterImmediate(
5951 mir_tag,6286 mir_tag,
...@@ -5960,7 +6295,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -5960,7 +6295,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5960 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(6295 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
5961 mir_tag,6296 mir_tag,
5962 dst_alias,6297 dst_alias,
5963 src_mcv.mem(Memory.Size.fromSize(abi_size)),6298 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5964 Immediate.u(@as(u5, @bitCast(mode))),6299 Immediate.u(@as(u5, @bitCast(mode))),
5965 ) else try self.asmRegisterRegisterImmediate(6300 ) else try self.asmRegisterRegisterImmediate(
5966 mir_tag,6301 mir_tag,
...@@ -6000,7 +6335,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -6000,7 +6335,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6000 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(6335 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
6001 .l,6336 .l,
6002 registerAlias(dst_mcv.register, cmov_abi_size),6337 registerAlias(dst_mcv.register, cmov_abi_size),
6003 src_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),6338 try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
6004 ),6339 ),
6005 else => {6340 else => {
6006 const val_reg = try self.copyToTmpRegister(ty, src_mcv);6341 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...@@ -6100,7 +6435,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -6100,7 +6435,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6100 if (src_mcv.isMemory()) try self.asmRegisterMemory(6435 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6101 mir_tag,6436 mir_tag,
6102 dst_alias,6437 dst_alias,
6103 src_mcv.mem(self.memSize(ty)),6438 try src_mcv.mem(self, self.memSize(ty)),
6104 ) else try self.asmRegisterRegister(6439 ) else try self.asmRegisterRegister(
6105 mir_tag,6440 mir_tag,
6106 dst_alias,6441 dst_alias,
...@@ -6206,7 +6541,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -6206,7 +6541,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6206 if (src_mcv.isMemory()) try self.asmRegisterMemory(6541 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6207 .{ .v_ps, .cvtph2 },6542 .{ .v_ps, .cvtph2 },
6208 wide_reg,6543 wide_reg,
6209 src_mcv.mem(Memory.Size.fromSize(6544 try src_mcv.mem(self, Memory.Size.fromSize(
6210 @intCast(@divExact(wide_reg.bitSize(), 16)),6545 @intCast(@divExact(wide_reg.bitSize(), 16)),
6211 )),6546 )),
6212 ) else try self.asmRegisterRegister(6547 ) else try self.asmRegisterRegister(
...@@ -6254,7 +6589,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -6254,7 +6589,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6254 mir_tag,6589 mir_tag,
6255 dst_reg,6590 dst_reg,
6256 dst_reg,6591 dst_reg,
6257 src_mcv.mem(Memory.Size.fromSize(abi_size)),6592 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6258 ) else try self.asmRegisterRegisterRegister(6593 ) else try self.asmRegisterRegisterRegister(
6259 mir_tag,6594 mir_tag,
6260 dst_reg,6595 dst_reg,
...@@ -6267,7 +6602,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -6267,7 +6602,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6267 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(6602 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
6268 mir_tag,6603 mir_tag,
6269 dst_reg,6604 dst_reg,
6270 src_mcv.mem(Memory.Size.fromSize(abi_size)),6605 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6271 ) else try self.asmRegisterRegister(6606 ) else try self.asmRegisterRegister(
6272 mir_tag,6607 mir_tag,
6273 dst_reg,6608 dst_reg,
...@@ -6332,7 +6667,7 @@ fn reuseOperandAdvanced(...@@ -6332,7 +6667,7 @@ fn reuseOperandAdvanced(
6332 return false;6667 return false;
63336668
6334 switch (mcv) {6669 switch (mcv) {
6335 .register, .register_pair => for (mcv.getRegs()) |reg| {6670 .register, .register_pair, .register_overflow => for (mcv.getRegs()) |reg| {
6336 // If it's in the registers table, need to associate the register(s) with the6671 // If it's in the registers table, need to associate the register(s) with the
6337 // new instruction.6672 // new instruction.
6338 if (maybe_tracked_inst) |tracked_inst| {6673 if (maybe_tracked_inst) |tracked_inst| {
...@@ -6346,6 +6681,10 @@ fn reuseOperandAdvanced(...@@ -6346,6 +6681,10 @@ fn reuseOperandAdvanced(
6346 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,6681 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
6347 else => return false,6682 else => return false,
6348 }6683 }
6684 switch (mcv) {
6685 .eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst,
6686 else => {},
6687 }
63496688
6350 // Prevent the operand deaths processing code from deallocating it.6689 // Prevent the operand deaths processing code from deallocating it.
6351 self.liveness.clearOperandDeath(inst, op_index);6690 self.liveness.clearOperandDeath(inst, op_index);
...@@ -6363,11 +6702,36 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -6363,11 +6702,36 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6363 if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return;6702 if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return;
6364 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));6703 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
63656704
6705 if (ptr_info.packed_offset.bit_offset % 8 == 0) {
6706 try self.load(
6707 dst_mcv,
6708 ptr_ty,
6709 ptr_mcv.offset(@intCast(@divExact(ptr_info.packed_offset.bit_offset, 8))),
6710 );
6711 const val_bit_size: u32 = @intCast(val_ty.bitSize(mod));
6712 if (val_abi_size * 8 > val_bit_size) {
6713 if (dst_mcv.isRegister()) {
6714 try self.truncateRegister(val_ty, dst_mcv.getReg().?);
6715 } else {
6716 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6717 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6718 defer self.register_manager.unlockReg(tmp_lock);
6719
6720 const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
6721 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
6722 try self.truncateRegister(val_ty, tmp_reg);
6723 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
6724 }
6725 }
6726 return;
6727 }
6728
6366 if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{val_ty.fmt(mod)});6729 if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{val_ty.fmt(mod)});
63676730
6368 const limb_abi_size: u32 = @min(val_abi_size, 8);6731 const limb_abi_size: u32 = @min(val_abi_size, 8);
6369 const limb_abi_bits = limb_abi_size * 8;6732 const limb_abi_bits = limb_abi_size * 8;
6370 const val_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);6733 const val_byte_off: i32 =
6734 @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
6371 const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;6735 const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;
6372 const val_extra_bits = self.regExtraBits(val_ty);6736 const val_extra_bits = self.regExtraBits(val_ty);
63736737
...@@ -6393,6 +6757,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -6393,6 +6757,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6393 .disp = val_byte_off,6757 .disp = val_byte_off,
6394 } },6758 } },
6395 });6759 });
6760 try self.spillEflagsIfOccupied();
6396 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));6761 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
6397 } else {6762 } else {
6398 const tmp_reg =6763 const tmp_reg =
...@@ -6415,6 +6780,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -6415,6 +6780,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6415 .disp = val_byte_off + 1,6780 .disp = val_byte_off + 1,
6416 } },6781 } },
6417 });6782 });
6783 try self.spillEflagsIfOccupied();
6418 try self.asmRegisterRegisterImmediate(6784 try self.asmRegisterRegisterImmediate(
6419 .{ ._rd, .sh },6785 .{ ._rd, .sh },
6420 dst_alias,6786 dst_alias,
...@@ -6498,6 +6864,27 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -6498,6 +6864,27 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
6498 } else {6864 } else {
6499 try self.load(dst_mcv, ptr_ty, ptr_mcv);6865 try self.load(dst_mcv, ptr_ty, ptr_mcv);
6500 }6866 }
6867
6868 if (elem_ty.isAbiInt(mod) and elem_size * 8 > elem_ty.bitSize(mod)) {
6869 const high_mcv: MCValue = switch (dst_mcv) {
6870 .register => |dst_reg| .{ .register = dst_reg },
6871 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
6872 else => dst_mcv.address().offset(@intCast((elem_size - 1) / 8 * 8)).deref(),
6873 };
6874 const high_reg = if (high_mcv.isRegister())
6875 high_mcv.getReg().?
6876 else
6877 try self.copyToTmpRegister(Type.usize, high_mcv);
6878 const high_lock = self.register_manager.lockReg(high_reg);
6879 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
6880
6881 try self.truncateRegister(elem_ty, high_reg);
6882 if (!high_mcv.isRegister()) try self.genCopy(
6883 if (elem_size <= 8) elem_ty else Type.usize,
6884 high_mcv,
6885 .{ .register = high_reg },
6886 );
6887 }
6501 break :result dst_mcv;6888 break :result dst_mcv;
6502 };6889 };
6503 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });6890 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...@@ -6530,7 +6917,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -6530,7 +6917,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6530 .base = .{ .reg = ptr_reg },6917 .base = .{ .reg = ptr_reg },
6531 .mod = .{ .rm = .{6918 .mod = .{ .rm = .{
6532 .size = Memory.Size.fromSize(limb_abi_size),6919 .size = Memory.Size.fromSize(limb_abi_size),
6533 .disp = src_byte_off + limb_i * limb_abi_bits,6920 .disp = src_byte_off + limb_i * limb_abi_size,
6534 } },6921 } },
6535 };6922 };
65366923
...@@ -6575,6 +6962,22 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -6575,6 +6962,22 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6575 limb_mem,6962 limb_mem,
6576 registerAlias(tmp_reg, limb_abi_size),6963 registerAlias(tmp_reg, limb_abi_size),
6577 );6964 );
6965 } else if (src_bit_size <= 128 and src_bit_off == 0) {
6966 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6967 const tmp_mcv = MCValue{ .register = tmp_reg };
6968 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6969 defer self.register_manager.unlockReg(tmp_lock);
6970
6971 try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {
6972 0 => src_mcv,
6973 else => src_mcv.address().offset(limb_i * limb_abi_size).deref(),
6974 });
6975 try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });
6976 try self.asmMemoryRegister(
6977 .{ ._, .@"or" },
6978 limb_mem,
6979 registerAlias(tmp_reg, limb_abi_size),
6980 );
6578 } else return self.fail("TODO: implement packed store of {}", .{src_ty.fmt(mod)});6981 } else return self.fail("TODO: implement packed store of {}", .{src_ty.fmt(mod)});
6579 }6982 }
6580}6983}
...@@ -6627,6 +7030,11 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -6627,6 +7030,11 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
6627 } else {7030 } else {
6628 // TODO if the value is undef, don't lower this instruction7031 // TODO if the value is undef, don't lower this instruction
6629 }7032 }
7033
7034 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
7035 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
7036 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
7037
6630 const bin_op = self.air.instructions.items(.data)[inst].bin_op;7038 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
6631 const ptr_mcv = try self.resolveInst(bin_op.lhs);7039 const ptr_mcv = try self.resolveInst(bin_op.lhs);
6632 const ptr_ty = self.typeOf(bin_op.lhs);7040 const ptr_ty = self.typeOf(bin_op.lhs);
...@@ -6719,12 +7127,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6719,12 +7127,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6719 const dst_lock = self.register_manager.lockReg(dst_reg);7127 const dst_lock = self.register_manager.lockReg(dst_reg);
6720 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);7128 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
67217129
6722 if (field_off > 0) try self.genShiftBinOpMir(7130 if (field_off > 0) {
6723 .{ ._r, .sh },7131 try self.spillEflagsIfOccupied();
6724 Type.usize,7132 try self.genShiftBinOpMir(
6725 dst_mcv,7133 .{ ._r, .sh },
6726 .{ .immediate = field_off },7134 Type.usize,
6727 );7135 dst_mcv,
7136 .{ .immediate = field_off },
7137 );
7138 }
6728 if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and7139 if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and
6729 container_ty.abiSize(mod) * 8 > field_ty.bitSize(mod))7140 container_ty.abiSize(mod) * 8 > field_ty.bitSize(mod))
6730 try self.truncateRegister(field_ty, dst_reg);7141 try self.truncateRegister(field_ty, dst_reg);
...@@ -6759,12 +7170,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6759,12 +7170,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6759 defer for (dst_locks) |dst_lock| if (dst_lock) |lock|7170 defer for (dst_locks) |dst_lock| if (dst_lock) |lock|
6760 self.register_manager.unlockReg(lock);7171 self.register_manager.unlockReg(lock);
67617172
6762 if (field_off > 0) try self.genShiftBinOpMir(7173 if (field_off > 0) {
6763 .{ ._r, .sh },7174 try self.spillEflagsIfOccupied();
6764 Type.u128,7175 try self.genShiftBinOpMir(
6765 dst_mcv,7176 .{ ._r, .sh },
6766 .{ .immediate = field_off },7177 Type.u128,
6767 );7178 dst_mcv,
7179 .{ .immediate = field_off },
7180 );
7181 }
67687182
6769 if (field_bit_size <= 64) {7183 if (field_bit_size <= 64) {
6770 if (self.regExtraBits(field_ty) > 0)7184 if (self.regExtraBits(field_ty) > 0)
...@@ -6792,15 +7206,18 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6792,15 +7206,18 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6792 const dst_lock = self.register_manager.lockReg(dst_reg);7206 const dst_lock = self.register_manager.lockReg(dst_reg);
6793 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);7207 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
67947208
6795 if (field_off % 64 > 0) try self.genShiftBinOpMir(7209 if (field_off % 64 > 0) {
6796 .{ ._r, .sh },7210 try self.spillEflagsIfOccupied();
6797 Type.usize,7211 try self.genShiftBinOpMir(
6798 dst_mcv,7212 .{ ._r, .sh },
6799 .{ .immediate = field_off % 64 },7213 Type.usize,
6800 );7214 dst_mcv,
6801 if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);7215 .{ .immediate = field_off % 64 },
68027216 );
6803 break :result if (field_rc.supersetOf(abi.RegisterClass.gp))7217 }
7218 if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
7219
7220 break :result if (field_rc.supersetOf(abi.RegisterClass.gp))
6804 dst_mcv7221 dst_mcv
6805 else7222 else
6806 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);7223 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
...@@ -6808,17 +7225,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6808,17 +7225,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6808 .register_overflow => |ro| {7225 .register_overflow => |ro| {
6809 switch (index) {7226 switch (index) {
6810 // Get wrapped value for overflow operation.7227 // Get wrapped value for overflow operation.
6811 0 => break :result if (self.liveness.operandDies(inst, 0))7228 0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {
6812 .{ .register = ro.reg }7229 self.eflags_inst = null; // actually stop tracking the overflow part
6813 else7230 break :result .{ .register = ro.reg };
6814 try self.copyToRegisterWithInstTracking(7231 } else break :result try self.copyToRegisterWithInstTracking(
6815 inst,7232 inst,
6816 Type.usize,7233 Type.usize,
6817 .{ .register = ro.reg },7234 .{ .register = ro.reg },
6818 ),7235 ),
6819 // Get overflow bit.7236 // Get overflow bit.
6820 1 => if (self.liveness.operandDies(inst, 0)) {7237 1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {
6821 self.eflags_inst = inst;7238 self.eflags_inst = inst; // actually keep tracking the overflow part
6822 break :result .{ .eflags = ro.eflags };7239 break :result .{ .eflags = ro.eflags };
6823 } else {7240 } else {
6824 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);7241 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
...@@ -6833,11 +7250,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6833,11 +7250,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6833 if (field_off % 8 == 0) {7250 if (field_off % 8 == 0) {
6834 const off_mcv =7251 const off_mcv =
6835 src_mcv.address().offset(@intCast(@divExact(field_off, 8))).deref();7252 src_mcv.address().offset(@intCast(@divExact(field_off, 8))).deref();
7253 const field_bit_size = field_ty.bitSize(mod);
68367254
6837 if (field_abi_size <= 8) {7255 if (field_abi_size <= 8) {
6838 const int_ty = try mod.intType(7256 const int_ty = try mod.intType(
6839 if (field_ty.isAbiInt(mod)) field_ty.intInfo(mod).signedness else .unsigned,7257 if (field_ty.isAbiInt(mod)) field_ty.intInfo(mod).signedness else .unsigned,
6840 @intCast(field_ty.bitSize(mod)),7258 @intCast(field_bit_size),
6841 );7259 );
68427260
6843 const dst_reg = try self.register_manager.allocReg(7261 const dst_reg = try self.register_manager.allocReg(
...@@ -6856,10 +7274,24 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6856,10 +7274,24 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6856 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);7274 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
6857 }7275 }
68587276
6859 if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;7277 const dst_mcv = if (self.reuseOperand(inst, operand, 0, src_mcv))
68607278 off_mcv
6861 const dst_mcv = try self.allocRegOrMem(inst, true);7279 else dst: {
6862 try self.genCopy(field_ty, dst_mcv, off_mcv);7280 const dst_mcv = try self.allocRegOrMem(inst, true);
7281 try self.genCopy(field_ty, dst_mcv, off_mcv);
7282 break :dst dst_mcv;
7283 };
7284 if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
7285 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
7286 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7287 defer self.register_manager.unlockReg(tmp_lock);
7288
7289 const hi_mcv =
7290 dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
7291 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
7292 try self.truncateRegister(field_ty, tmp_reg);
7293 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
7294 }
6863 break :result dst_mcv;7295 break :result dst_mcv;
6864 }7296 }
68657297
...@@ -6888,6 +7320,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6888,6 +7320,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6888 .disp = frame_addr.off + field_byte_off,7320 .disp = frame_addr.off + field_byte_off,
6889 } },7321 } },
6890 });7322 });
7323 try self.spillEflagsIfOccupied();
6891 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));7324 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
6892 } else {7325 } else {
6893 const tmp_reg = registerAlias(7326 const tmp_reg = registerAlias(
...@@ -6916,6 +7349,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6916,6 +7349,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6916 .disp = frame_addr.off + field_byte_off + limb_abi_size,7349 .disp = frame_addr.off + field_byte_off + limb_abi_size,
6917 } },7350 } },
6918 });7351 });
7352 try self.spillEflagsIfOccupied();
6919 try self.asmRegisterRegisterImmediate(7353 try self.asmRegisterRegisterImmediate(
6920 .{ ._rd, .sh },7354 .{ ._rd, .sh },
6921 dst_alias,7355 dst_alias,
...@@ -6960,13 +7394,18 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6960,13 +7394,18 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
6960fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {7394fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
6961 const mod = self.bin_file.options.module.?;7395 const mod = self.bin_file.options.module.?;
6962 const src_ty = self.typeOf(src_air);7396 const src_ty = self.typeOf(src_air);
6963 const src_mcv = try self.resolveInst(src_air);
6964 if (src_ty.zigTypeTag(mod) == .Vector)7397 if (src_ty.zigTypeTag(mod) == .Vector)
6965 return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(mod)});7398 return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(mod)});
69667399
7400 var src_mcv = try self.resolveInst(src_air);
6967 switch (src_mcv) {7401 switch (src_mcv) {
6968 .eflags => |cc| switch (tag) {7402 .eflags => |cc| switch (tag) {
6969 .not => return .{ .eflags = cc.negate() },7403 .not => {
7404 if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv))
7405 return .{ .eflags = cc.negate() };
7406 try self.spillEflagsIfOccupied();
7407 src_mcv = try self.resolveInst(src_air);
7408 },
6970 else => {},7409 else => {},
6971 },7410 },
6972 else => {},7411 else => {},
...@@ -7013,7 +7452,25 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:...@@ -7013,7 +7452,25 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
7013 } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);7452 } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
7014 }7453 }
7015 },7454 },
7016 .neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),7455 .neg => {
7456 try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv);
7457 const abi_size: u16 = @intCast(src_ty.abiSize(mod));
7458 const bit_size = src_ty.intInfo(mod).bits;
7459 if (abi_size * 8 > bit_size) {
7460 if (dst_mcv.isRegister()) {
7461 try self.truncateRegister(src_ty, dst_mcv.getReg().?);
7462 } else {
7463 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
7464 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7465 defer self.register_manager.unlockReg(tmp_lock);
7466
7467 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
7468 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
7469 try self.truncateRegister(src_ty, tmp_reg);
7470 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
7471 }
7472 }
7473 },
7017 else => unreachable,7474 else => unreachable,
7018 }7475 }
7019 return dst_mcv;7476 return dst_mcv;
...@@ -7054,7 +7511,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC...@@ -7054,7 +7511,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
7054 },7511 },
7055 .indirect, .load_frame => try self.asmMemory(7512 .indirect, .load_frame => try self.asmMemory(
7056 mir_tag,7513 mir_tag,
7057 dst_mcv.mem(Memory.Size.fromSize(abi_size)),7514 try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7058 ),7515 ),
7059 }7516 }
7060}7517}
...@@ -7552,27 +8009,27 @@ fn genMulDivBinOp(...@@ -7552,27 +8009,27 @@ fn genMulDivBinOp(
7552 defer self.register_manager.unlockReg(tmp_lock);8009 defer self.register_manager.unlockReg(tmp_lock);
75538010
7554 if (mat_lhs_mcv.isMemory())8011 if (mat_lhs_mcv.isMemory())
7555 try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))8012 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))
7556 else8013 else
7557 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);8014 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
7558 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(8015 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
7559 .{ ._, .mov },8016 .{ ._, .mov },
7560 tmp_reg,8017 tmp_reg,
7561 mat_rhs_mcv.address().offset(8).deref().mem(.qword),8018 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
7562 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);8019 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
7563 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);8020 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
7564 if (mat_rhs_mcv.isMemory())8021 if (mat_rhs_mcv.isMemory())
7565 try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))8022 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
7566 else8023 else
7567 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);8024 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
7568 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);8025 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
7569 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(8026 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
7570 .{ ._, .mov },8027 .{ ._, .mov },
7571 tmp_reg,8028 tmp_reg,
7572 mat_lhs_mcv.address().offset(8).deref().mem(.qword),8029 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
7573 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);8030 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
7574 if (mat_rhs_mcv.isMemory())8031 if (mat_rhs_mcv.isMemory())
7575 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.mem(.qword))8032 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))
7576 else8033 else
7577 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);8034 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
7578 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);8035 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
...@@ -7833,7 +8290,7 @@ fn genBinOp(...@@ -7833,7 +8290,7 @@ fn genBinOp(
7833 .{ .vp_w, .insr },8290 .{ .vp_w, .insr },
7834 dst_reg,8291 dst_reg,
7835 dst_reg,8292 dst_reg,
7836 rhs_mcv.mem(.word),8293 try rhs_mcv.mem(self, .word),
7837 Immediate.u(1),8294 Immediate.u(1),
7838 ) else try self.asmRegisterRegisterRegister(8295 ) else try self.asmRegisterRegisterRegister(
7839 .{ .vp_, .unpcklwd },8296 .{ .vp_, .unpcklwd },
...@@ -7858,7 +8315,7 @@ fn genBinOp(...@@ -7858,7 +8315,7 @@ fn genBinOp(
7858 mir_tag,8315 mir_tag,
7859 dst_reg,8316 dst_reg,
7860 dst_reg,8317 dst_reg,
7861 src_mcv.mem(Memory.Size.fromBitSize(float_bits)),8318 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7862 ) else try self.asmRegisterRegisterRegister(8319 ) else try self.asmRegisterRegisterRegister(
7863 mir_tag,8320 mir_tag,
7864 dst_reg,8321 dst_reg,
...@@ -7877,7 +8334,7 @@ fn genBinOp(...@@ -7877,7 +8334,7 @@ fn genBinOp(
7877 if (src_mcv.isMemory()) try self.asmRegisterMemory(8334 if (src_mcv.isMemory()) try self.asmRegisterMemory(
7878 mir_tag,8335 mir_tag,
7879 dst_reg,8336 dst_reg,
7880 src_mcv.mem(Memory.Size.fromBitSize(float_bits)),8337 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7881 ) else try self.asmRegisterRegister(8338 ) else try self.asmRegisterRegister(
7882 mir_tag,8339 mir_tag,
7883 dst_reg,8340 dst_reg,
...@@ -7919,12 +8376,18 @@ fn genBinOp(...@@ -7919,12 +8376,18 @@ fn genBinOp(
7919 };8376 };
7920 }8377 }
79218378
7922 if ((lhs_ty.scalarType(mod).isRuntimeFloat() and8379 const sse_op = switch (lhs_ty.zigTypeTag(mod)) {
8380 else => false,
8381 .Float => true,
8382 .Vector => switch (lhs_ty.childType(mod).toIntern()) {
8383 .bool_type => false,
8384 else => true,
8385 },
8386 };
8387 if (sse_op and ((lhs_ty.scalarType(mod).isRuntimeFloat() and
7923 lhs_ty.scalarType(mod).floatBits(self.target.*) == 80) or8388 lhs_ty.scalarType(mod).floatBits(self.target.*) == 80) or
7924 lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16))8389 lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16)))
7925 return self.fail("TODO implement genBinOp for {s} {}", .{8390 return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(mod) });
7926 @tagName(air_tag), lhs_ty.fmt(mod),
7927 });
79288391
7929 const maybe_mask_reg = switch (air_tag) {8392 const maybe_mask_reg = switch (air_tag) {
7930 else => null,8393 else => null,
...@@ -7941,10 +8404,16 @@ fn genBinOp(...@@ -7941,10 +8404,16 @@ fn genBinOp(
7941 if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null;8404 if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null;
7942 defer if (mask_lock) |lock| self.register_manager.unlockReg(lock);8405 defer if (mask_lock) |lock| self.register_manager.unlockReg(lock);
79438406
7944 const ordered_air = if (lhs_ty.isVector(mod) and lhs_ty.childType(mod).isAbiInt(mod) and8407 const ordered_air = if (lhs_ty.isVector(mod) and switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
7945 switch (air_tag) {8408 .Int => switch (air_tag) {
7946 .cmp_lt, .cmp_gte => true,8409 .cmp_lt, .cmp_gte => true,
7947 else => false,8410 else => false,
8411 },
8412 .Float => switch (air_tag) {
8413 .cmp_gte, .cmp_gt => true,
8414 else => false,
8415 },
8416 else => unreachable,
7948 }) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air };8417 }) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air };
79498418
7950 const lhs_mcv = try self.resolveInst(ordered_air.lhs);8419 const lhs_mcv = try self.resolveInst(ordered_air.lhs);
...@@ -7971,14 +8440,12 @@ fn genBinOp(...@@ -7971,14 +8440,12 @@ fn genBinOp(
7971 .xor,8440 .xor,
7972 .min,8441 .min,
7973 .max,8442 .max,
8443 .cmp_eq,
8444 .cmp_neq,
7974 => true,8445 => true,
79758446
7976 else => false,8447 else => false,
7977 };8448 };
7978 const vec_op = switch (lhs_ty.zigTypeTag(mod)) {
7979 else => false,
7980 .Float, .Vector => true,
7981 };
79828449
7983 const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {8450 const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
7984 .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },8451 .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
...@@ -8000,23 +8467,23 @@ fn genBinOp(...@@ -8000,23 +8467,23 @@ fn genBinOp(
8000 var flipped = false;8467 var flipped = false;
8001 var copied_to_dst = true;8468 var copied_to_dst = true;
8002 const dst_mcv: MCValue = dst: {8469 const dst_mcv: MCValue = dst: {
8470 const tracked_inst = switch (air_tag) {
8471 else => maybe_inst,
8472 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
8473 };
8003 if (maybe_inst) |inst| {8474 if (maybe_inst) |inst| {
8004 const tracked_inst = switch (air_tag) {8475 if ((!sse_op or lhs_mcv.isRegister()) and
8005 else => inst,
8006 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
8007 };
8008 if ((!vec_op or lhs_mcv.isRegister()) and
8009 self.reuseOperandAdvanced(inst, ordered_air.lhs, 0, lhs_mcv, tracked_inst))8476 self.reuseOperandAdvanced(inst, ordered_air.lhs, 0, lhs_mcv, tracked_inst))
8010 break :dst lhs_mcv;8477 break :dst lhs_mcv;
8011 if (is_commutative and (!vec_op or rhs_mcv.isRegister()) and8478 if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and
8012 self.reuseOperandAdvanced(inst, ordered_air.rhs, 1, rhs_mcv, tracked_inst))8479 self.reuseOperandAdvanced(inst, ordered_air.rhs, 1, rhs_mcv, tracked_inst))
8013 {8480 {
8014 flipped = true;8481 flipped = true;
8015 break :dst rhs_mcv;8482 break :dst rhs_mcv;
8016 }8483 }
8017 }8484 }
8018 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);8485 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, tracked_inst, true);
8019 if (vec_op and lhs_mcv.isRegister() and self.hasFeature(.avx))8486 if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
8020 copied_to_dst = false8487 copied_to_dst = false
8021 else8488 else
8022 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);8489 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
...@@ -8046,7 +8513,7 @@ fn genBinOp(...@@ -8046,7 +8513,7 @@ fn genBinOp(
8046 };8513 };
8047 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);8514 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
80488515
8049 if (!vec_op) {8516 if (!sse_op) {
8050 switch (air_tag) {8517 switch (air_tag) {
8051 .add,8518 .add,
8052 .add_wrap,8519 .add_wrap,
...@@ -8130,17 +8597,25 @@ fn genBinOp(...@@ -8130,17 +8597,25 @@ fn genBinOp(
81308597
8131 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);8598 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
8132 if (src_mcv.isMemory()) {8599 if (src_mcv.isMemory()) {
8133 try self.asmRegisterMemory(.{ ._, .cmp }, dst_regs[0], src_mcv.mem(.qword));8600 try self.asmRegisterMemory(
8601 .{ ._, .cmp },
8602 dst_regs[0],
8603 try src_mcv.mem(self, .qword),
8604 );
8134 try self.asmRegisterMemory(8605 try self.asmRegisterMemory(
8135 .{ ._, .sbb },8606 .{ ._, .sbb },
8136 tmp_reg,8607 tmp_reg,
8137 src_mcv.address().offset(8).deref().mem(.qword),8608 try src_mcv.address().offset(8).deref().mem(self, .qword),
8609 );
8610 try self.asmCmovccRegisterMemory(
8611 cc,
8612 dst_regs[0],
8613 try src_mcv.mem(self, .qword),
8138 );8614 );
8139 try self.asmCmovccRegisterMemory(cc, dst_regs[0], src_mcv.mem(.qword));
8140 try self.asmCmovccRegisterMemory(8615 try self.asmCmovccRegisterMemory(
8141 cc,8616 cc,
8142 dst_regs[1],8617 dst_regs[1],
8143 src_mcv.address().offset(8).deref().mem(.qword),8618 try src_mcv.address().offset(8).deref().mem(self, .qword),
8144 );8619 );
8145 } else {8620 } else {
8146 try self.asmRegisterRegister(8621 try self.asmRegisterRegister(
...@@ -8292,7 +8767,7 @@ fn genBinOp(...@@ -8292,7 +8767,7 @@ fn genBinOp(
8292 .{ .vp_w, .insr },8767 .{ .vp_w, .insr },
8293 dst_reg,8768 dst_reg,
8294 dst_reg,8769 dst_reg,
8295 src_mcv.mem(.word),8770 try src_mcv.mem(self, .word),
8296 Immediate.u(1),8771 Immediate.u(1),
8297 ) else try self.asmRegisterRegisterRegister(8772 ) else try self.asmRegisterRegisterRegister(
8298 .{ .vp_, .unpcklwd },8773 .{ .vp_, .unpcklwd },
...@@ -8738,7 +9213,7 @@ fn genBinOp(...@@ -8738,7 +9213,7 @@ fn genBinOp(
8738 .{ .vp_w, .insr },9213 .{ .vp_w, .insr },
8739 dst_reg,9214 dst_reg,
8740 dst_reg,9215 dst_reg,
8741 src_mcv.mem(.word),9216 try src_mcv.mem(self, .word),
8742 Immediate.u(1),9217 Immediate.u(1),
8743 ) else try self.asmRegisterRegisterRegister(9218 ) else try self.asmRegisterRegisterRegister(
8744 .{ .vp_, .unpcklwd },9219 .{ .vp_, .unpcklwd },
...@@ -8784,7 +9259,7 @@ fn genBinOp(...@@ -8784,7 +9259,7 @@ fn genBinOp(
8784 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(9259 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
8785 .{ .vp_d, .insr },9260 .{ .vp_d, .insr },
8786 dst_reg,9261 dst_reg,
8787 src_mcv.mem(.dword),9262 try src_mcv.mem(self, .dword),
8788 Immediate.u(1),9263 Immediate.u(1),
8789 ) else try self.asmRegisterRegisterRegister(9264 ) else try self.asmRegisterRegisterRegister(
8790 .{ .v_ps, .unpckl },9265 .{ .v_ps, .unpckl },
...@@ -8836,7 +9311,7 @@ fn genBinOp(...@@ -8836,7 +9311,7 @@ fn genBinOp(
8836 if (src_mcv.isMemory()) try self.asmRegisterMemory(9311 if (src_mcv.isMemory()) try self.asmRegisterMemory(
8837 .{ .v_ps, .cvtph2 },9312 .{ .v_ps, .cvtph2 },
8838 tmp_reg,9313 tmp_reg,
8839 src_mcv.mem(.qword),9314 try src_mcv.mem(self, .qword),
8840 ) else try self.asmRegisterRegister(9315 ) else try self.asmRegisterRegister(
8841 .{ .v_ps, .cvtph2 },9316 .{ .v_ps, .cvtph2 },
8842 tmp_reg,9317 tmp_reg,
...@@ -8879,7 +9354,7 @@ fn genBinOp(...@@ -8879,7 +9354,7 @@ fn genBinOp(
8879 if (src_mcv.isMemory()) try self.asmRegisterMemory(9354 if (src_mcv.isMemory()) try self.asmRegisterMemory(
8880 .{ .v_ps, .cvtph2 },9355 .{ .v_ps, .cvtph2 },
8881 tmp_reg,9356 tmp_reg,
8882 src_mcv.mem(.xword),9357 try src_mcv.mem(self, .xword),
8883 ) else try self.asmRegisterRegister(9358 ) else try self.asmRegisterRegister(
8884 .{ .v_ps, .cvtph2 },9359 .{ .v_ps, .cvtph2 },
8885 tmp_reg,9360 tmp_reg,
...@@ -8925,6 +9400,13 @@ fn genBinOp(...@@ -8925,6 +9400,13 @@ fn genBinOp(
8925 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },9400 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
8926 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },9401 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
8927 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },9402 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
9403 .cmp_lt,
9404 .cmp_lte,
9405 .cmp_eq,
9406 .cmp_gte,
9407 .cmp_gt,
9408 .cmp_neq,
9409 => if (self.hasFeature(.avx)) .{ .v_ss, .cmp } else .{ ._ss, .cmp },
8928 else => unreachable,9410 else => unreachable,
8929 },9411 },
8930 2...4 => switch (air_tag) {9412 2...4 => switch (air_tag) {
...@@ -8938,6 +9420,13 @@ fn genBinOp(...@@ -8938,6 +9420,13 @@ fn genBinOp(
8938 => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },9420 => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
8939 .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },9421 .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
8940 .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },9422 .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
9423 .cmp_lt,
9424 .cmp_lte,
9425 .cmp_eq,
9426 .cmp_gte,
9427 .cmp_gt,
9428 .cmp_neq,
9429 => if (self.hasFeature(.avx)) .{ .v_ps, .cmp } else .{ ._ps, .cmp },
8941 else => unreachable,9430 else => unreachable,
8942 },9431 },
8943 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {9432 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
...@@ -8947,6 +9436,7 @@ fn genBinOp(...@@ -8947,6 +9436,7 @@ fn genBinOp(
8947 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },9436 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
8948 .max => .{ .v_ps, .max },9437 .max => .{ .v_ps, .max },
8949 .min => .{ .v_ps, .min },9438 .min => .{ .v_ps, .min },
9439 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_ps, .cmp },
8950 else => unreachable,9440 else => unreachable,
8951 } else null,9441 } else null,
8952 else => null,9442 else => null,
...@@ -8963,6 +9453,13 @@ fn genBinOp(...@@ -8963,6 +9453,13 @@ fn genBinOp(
8963 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },9453 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
8964 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },9454 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
8965 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },9455 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
9456 .cmp_lt,
9457 .cmp_lte,
9458 .cmp_eq,
9459 .cmp_gte,
9460 .cmp_gt,
9461 .cmp_neq,
9462 => if (self.hasFeature(.avx)) .{ .v_sd, .cmp } else .{ ._sd, .cmp },
8966 else => unreachable,9463 else => unreachable,
8967 },9464 },
8968 2 => switch (air_tag) {9465 2 => switch (air_tag) {
...@@ -8976,6 +9473,13 @@ fn genBinOp(...@@ -8976,6 +9473,13 @@ fn genBinOp(
8976 => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },9473 => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
8977 .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },9474 .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
8978 .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },9475 .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
9476 .cmp_lt,
9477 .cmp_lte,
9478 .cmp_eq,
9479 .cmp_gte,
9480 .cmp_gt,
9481 .cmp_neq,
9482 => if (self.hasFeature(.avx)) .{ .v_pd, .cmp } else .{ ._pd, .cmp },
8979 else => unreachable,9483 else => unreachable,
8980 },9484 },
8981 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {9485 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
...@@ -8984,6 +9488,7 @@ fn genBinOp(...@@ -8984,6 +9488,7 @@ fn genBinOp(
8984 .mul => .{ .v_pd, .mul },9488 .mul => .{ .v_pd, .mul },
8985 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },9489 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
8986 .max => .{ .v_pd, .max },9490 .max => .{ .v_pd, .max },
9491 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_pd, .cmp },
8987 .min => .{ .v_pd, .min },9492 .min => .{ .v_pd, .min },
8988 else => unreachable,9493 else => unreachable,
8989 } else null,9494 } else null,
...@@ -9004,43 +9509,96 @@ fn genBinOp(...@@ -9004,43 +9509,96 @@ fn genBinOp(
9004 const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null;9509 const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null;
9005 defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock);9510 defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock);
90069511
9007 if (self.hasFeature(.avx)) {9512 switch (mir_tag[1]) {
9008 const lhs_reg =9513 else => if (self.hasFeature(.avx)) {
9009 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);9514 const lhs_reg =
9010 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(9515 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
9011 mir_tag,9516 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
9012 dst_reg,9517 mir_tag,
9013 lhs_reg,9518 dst_reg,
9014 src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {9519 lhs_reg,
9015 else => Memory.Size.fromSize(abi_size),9520 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9016 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),9521 else => Memory.Size.fromSize(abi_size),
9017 }),9522 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9018 ) else try self.asmRegisterRegisterRegister(9523 }),
9019 mir_tag,9524 ) else try self.asmRegisterRegisterRegister(
9020 dst_reg,9525 mir_tag,
9021 lhs_reg,9526 dst_reg,
9022 registerAlias(if (src_mcv.isRegister())9527 lhs_reg,
9023 src_mcv.getReg().?9528 registerAlias(if (src_mcv.isRegister())
9024 else9529 src_mcv.getReg().?
9025 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),9530 else
9026 );9531 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9027 } else {9532 );
9028 assert(copied_to_dst);9533 } else {
9029 if (src_mcv.isMemory()) try self.asmRegisterMemory(9534 assert(copied_to_dst);
9030 mir_tag,9535 if (src_mcv.isMemory()) try self.asmRegisterMemory(
9031 dst_reg,9536 mir_tag,
9032 src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {9537 dst_reg,
9033 else => Memory.Size.fromSize(abi_size),9538 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9034 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),9539 else => Memory.Size.fromSize(abi_size),
9035 }),9540 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9036 ) else try self.asmRegisterRegister(9541 }),
9037 mir_tag,9542 ) else try self.asmRegisterRegister(
9038 dst_reg,9543 mir_tag,
9039 registerAlias(if (src_mcv.isRegister())9544 dst_reg,
9040 src_mcv.getReg().?9545 registerAlias(if (src_mcv.isRegister())
9041 else9546 src_mcv.getReg().?
9042 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),9547 else
9043 );9548 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9549 );
9550 },
9551 .cmp => {
9552 const imm = Immediate.u(switch (air_tag) {
9553 .cmp_eq => 0,
9554 .cmp_lt, .cmp_gt => 1,
9555 .cmp_lte, .cmp_gte => 2,
9556 .cmp_neq => 4,
9557 else => unreachable,
9558 });
9559 if (self.hasFeature(.avx)) {
9560 const lhs_reg =
9561 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
9562 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
9563 mir_tag,
9564 dst_reg,
9565 lhs_reg,
9566 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9567 else => Memory.Size.fromSize(abi_size),
9568 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9569 }),
9570 imm,
9571 ) else try self.asmRegisterRegisterRegisterImmediate(
9572 mir_tag,
9573 dst_reg,
9574 lhs_reg,
9575 registerAlias(if (src_mcv.isRegister())
9576 src_mcv.getReg().?
9577 else
9578 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9579 imm,
9580 );
9581 } else {
9582 assert(copied_to_dst);
9583 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
9584 mir_tag,
9585 dst_reg,
9586 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9587 else => Memory.Size.fromSize(abi_size),
9588 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9589 }),
9590 imm,
9591 ) else try self.asmRegisterRegisterImmediate(
9592 mir_tag,
9593 dst_reg,
9594 registerAlias(if (src_mcv.isRegister())
9595 src_mcv.getReg().?
9596 else
9597 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9598 imm,
9599 );
9600 }
9601 },
9044 }9602 }
90459603
9046 switch (air_tag) {9604 switch (air_tag) {
...@@ -9281,48 +9839,46 @@ fn genBinOp(...@@ -9281,48 +9839,46 @@ fn genBinOp(
9281 );9839 );
9282 }9840 }
9283 },9841 },
9284 .cmp_lt,9842 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
9285 .cmp_lte,9843 switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
9286 .cmp_eq,9844 .Int => switch (air_tag) {
9287 .cmp_gte,9845 .cmp_lt,
9288 .cmp_gt,9846 .cmp_eq,
9289 .cmp_neq,9847 .cmp_gt,
9290 => {9848 => {},
9291 switch (air_tag) {9849 .cmp_lte,
9292 .cmp_lt,9850 .cmp_gte,
9293 .cmp_eq,9851 .cmp_neq,
9294 .cmp_gt,9852 => {
9295 => {},9853 const unsigned_ty = try lhs_ty.toUnsigned(mod);
9296 .cmp_lte,9854 const not_mcv = try self.genTypedValue(.{
9297 .cmp_gte,9855 .ty = lhs_ty,
9298 .cmp_neq,9856 .val = try unsigned_ty.maxInt(mod, unsigned_ty),
9299 => {9857 });
9300 const unsigned_ty = try lhs_ty.toUnsigned(mod);9858 const not_mem: Memory = if (not_mcv.isMemory())
9301 const not_mcv = try self.genTypedValue(.{9859 try not_mcv.mem(self, Memory.Size.fromSize(abi_size))
9302 .ty = lhs_ty,9860 else
9303 .val = try unsigned_ty.maxInt(mod, unsigned_ty),9861 .{ .base = .{
9304 });9862 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
9305 const not_mem: Memory = if (not_mcv.isMemory())9863 }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
9306 not_mcv.mem(Memory.Size.fromSize(abi_size))9864 switch (mir_tag[0]) {
9307 else9865 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
9308 .{ .base = .{9866 .{ .vp_, .xor },
9309 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),9867 dst_reg,
9310 }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };9868 dst_reg,
9311 switch (mir_tag[0]) {9869 not_mem,
9312 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(9870 ),
9313 .{ .vp_, .xor },9871 .p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(
9314 dst_reg,9872 .{ .p_, .xor },
9315 dst_reg,9873 dst_reg,
9316 not_mem,9874 not_mem,
9317 ),9875 ),
9318 .p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(9876 else => unreachable,
9319 .{ .p_, .xor },9877 }
9320 dst_reg,9878 },
9321 not_mem,9879 else => unreachable,
9322 ),
9323 else => unreachable,
9324 }
9325 },9880 },
9881 .Float => {},
9326 else => unreachable,9882 else => unreachable,
9327 }9883 }
93289884
...@@ -9331,8 +9887,12 @@ fn genBinOp(...@@ -9331,8 +9887,12 @@ fn genBinOp(
9331 defer self.register_manager.unlockReg(gp_lock);9887 defer self.register_manager.unlockReg(gp_lock);
93329888
9333 try self.asmRegisterRegister(switch (mir_tag[0]) {9889 try self.asmRegisterRegister(switch (mir_tag[0]) {
9334 .vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },9890 ._pd, ._sd => .{ ._pd, .movmsk },
9891 ._ps, ._ss => .{ ._ps, .movmsk },
9335 .p_b, .p_d, .p_q, .p_w => .{ .p_b, .movmsk },9892 .p_b, .p_d, .p_q, .p_w => .{ .p_b, .movmsk },
9893 .v_pd, .v_sd => .{ .v_pd, .movmsk },
9894 .v_ps, .v_ss => .{ .v_ps, .movmsk },
9895 .vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },
9336 else => unreachable,9896 else => unreachable,
9337 }, gp_reg.to32(), dst_reg);9897 }, gp_reg.to32(), dst_reg);
9338 return .{ .register = gp_reg };9898 return .{ .register = gp_reg };
...@@ -9459,13 +10019,13 @@ fn genBinOpMir(...@@ -9459,13 +10019,13 @@ fn genBinOpMir(
9459 .load_frame,10019 .load_frame,
9460 .lea_frame,10020 .lea_frame,
9461 => {10021 => {
9462 blk: {10022 direct: {
9463 return self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {10023 try self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
9464 .memory => |addr| .{10024 .memory => |addr| .{
9465 .base = .{ .reg = .ds },10025 .base = .{ .reg = .ds },
9466 .mod = .{ .rm = .{10026 .mod = .{ .rm = .{
9467 .size = Memory.Size.fromSize(limb_abi_size),10027 .size = Memory.Size.fromSize(limb_abi_size),
9468 .disp = math.cast(i32, addr + off) orelse break :blk,10028 .disp = math.cast(i32, addr + off) orelse break :direct,
9469 } },10029 } },
9470 },10030 },
9471 .indirect => |reg_off| .{10031 .indirect => |reg_off| .{
...@@ -9482,8 +10042,9 @@ fn genBinOpMir(...@@ -9482,8 +10042,9 @@ fn genBinOpMir(
9482 .disp = frame_addr.off + off,10042 .disp = frame_addr.off + off,
9483 } },10043 } },
9484 },10044 },
9485 else => break :blk,10045 else => break :direct,
9486 });10046 });
10047 continue;
9487 }10048 }
948810049
9489 switch (src_mcv) {10050 switch (src_mcv) {
...@@ -10142,30 +10703,30 @@ fn genCall(self: *Self, info: union(enum) {...@@ -10142,30 +10703,30 @@ fn genCall(self: *Self, info: union(enum) {
10142 }10703 }
1014310704
10144 try self.spillEflagsIfOccupied();10705 try self.spillEflagsIfOccupied();
10145 try self.spillRegisters(abi.getCallerPreservedRegs(resolved_cc));10706 try self.spillCallerPreservedRegs(resolved_cc);
1014610707
10147 // set stack arguments first because this can clobber registers10708 // set stack arguments first because this can clobber registers
10148 // also clobber spill arguments as we go10709 // also clobber spill arguments as we go
10149 switch (call_info.return_value.long) {10710 switch (call_info.return_value.long) {
10150 .none, .unreach => {},10711 .none, .unreach => {},
10151 .indirect => |reg_off| try self.spillRegisters(&.{reg_off.reg}),10712 .indirect => |reg_off| try self.register_manager.getReg(reg_off.reg, null),
10152 else => unreachable,10713 else => unreachable,
10153 }10714 }
10154 for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, *frame_index|10715 for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, *frame_index|
10155 switch (dst_arg) {10716 switch (dst_arg) {
10156 .none => {},10717 .none => {},
10157 .register => |reg| {10718 .register => |reg| {
10158 try self.spillRegisters(&.{reg});10719 try self.register_manager.getReg(reg, null);
10159 try reg_locks.append(self.register_manager.lockReg(reg));10720 try reg_locks.append(self.register_manager.lockReg(reg));
10160 },10721 },
10161 .register_pair => |regs| {10722 .register_pair => |regs| {
10162 try self.spillRegisters(&regs);10723 for (regs) |reg| try self.register_manager.getReg(reg, null);
10163 try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));10724 try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs));
10164 },10725 },
10165 .indirect => |reg_off| {10726 .indirect => |reg_off| {
10166 frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, mod));10727 frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, mod));
10167 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg);10728 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg);
10168 try self.spillRegisters(&.{reg_off.reg});10729 try self.register_manager.getReg(reg_off.reg, null);
10169 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));10730 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
10170 },10731 },
10171 .load_frame => {10732 .load_frame => {
...@@ -10180,7 +10741,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -10180,7 +10741,7 @@ fn genCall(self: *Self, info: union(enum) {
10180 .none, .unreach => {},10741 .none, .unreach => {},
10181 .indirect => |reg_off| {10742 .indirect => |reg_off| {
10182 const ret_ty = fn_info.return_type.toType();10743 const ret_ty = fn_info.return_type.toType();
10183 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, mod));10744 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, mod));
10184 try self.genSetReg(reg_off.reg, Type.usize, .{10745 try self.genSetReg(reg_off.reg, Type.usize, .{
10185 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },10746 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
10186 });10747 });
...@@ -10306,19 +10867,20 @@ fn genCall(self: *Self, info: union(enum) {...@@ -10306,19 +10867,20 @@ fn genCall(self: *Self, info: union(enum) {
10306fn airRet(self: *Self, inst: Air.Inst.Index) !void {10867fn airRet(self: *Self, inst: Air.Inst.Index) !void {
10307 const mod = self.bin_file.options.module.?;10868 const mod = self.bin_file.options.module.?;
10308 const un_op = self.air.instructions.items(.data)[inst].un_op;10869 const un_op = self.air.instructions.items(.data)[inst].un_op;
10309 const operand = try self.resolveInst(un_op);
1031010870
10311 const ret_ty = self.fn_type.fnReturnType(mod);10871 const ret_ty = self.fn_type.fnReturnType(mod);
10312 switch (self.ret_mcv.short) {10872 switch (self.ret_mcv.short) {
10313 .none => {},10873 .none => {},
10314 .register, .register_pair => try self.genCopy(ret_ty, self.ret_mcv.short, operand),10874 .register,
10875 .register_pair,
10876 => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }),
10315 .indirect => |reg_off| {10877 .indirect => |reg_off| {
10316 try self.register_manager.getReg(reg_off.reg, null);10878 try self.register_manager.getReg(reg_off.reg, null);
10317 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);10879 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
10318 defer self.register_manager.unlockReg(lock);10880 defer self.register_manager.unlockReg(lock);
1031910881
10320 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);10882 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
10321 try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);10883 try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, .{ .air_ref = un_op });
10322 },10884 },
10323 else => unreachable,10885 else => unreachable,
10324 }10886 }
...@@ -10593,7 +11155,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -10593,7 +11155,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10593 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);11155 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
10594 defer for (locks) |lock| self.register_manager.unlockReg(lock);11156 defer for (locks) |lock| self.register_manager.unlockReg(lock);
1059511157
10596 const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;11158 const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;
10597 var limb_i: u16 = 0;11159 var limb_i: u16 = 0;
10598 while (limb_i < limbs_len) : (limb_i += 1) {11160 while (limb_i < limbs_len) : (limb_i += 1) {
10599 const off = limb_i * 8;11161 const off = limb_i * 8;
...@@ -10688,7 +11250,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -10688,7 +11250,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10688 .{ .vp_w, .insr },11250 .{ .vp_w, .insr },
10689 tmp1_reg,11251 tmp1_reg,
10690 dst_reg.to128(),11252 dst_reg.to128(),
10691 src_mcv.mem(.word),11253 try src_mcv.mem(self, .word),
10692 Immediate.u(1),11254 Immediate.u(1),
10693 ) else try self.asmRegisterRegisterRegister(11255 ) else try self.asmRegisterRegisterRegister(
10694 .{ .vp_, .unpcklwd },11256 .{ .vp_, .unpcklwd },
...@@ -10892,8 +11454,8 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {...@@ -10892,8 +11454,8 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
10892 },11454 },
10893 .register => |reg| {11455 .register => |reg| {
10894 try self.spillEflagsIfOccupied();11456 try self.spillEflagsIfOccupied();
10895 try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));11457 try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));
10896 return self.asmJccReloc(.e, undefined);11458 return self.asmJccReloc(.z, undefined);
10897 },11459 },
10898 .immediate,11460 .immediate,
10899 .load_frame,11461 .load_frame,
...@@ -11366,8 +11928,18 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -11366,8 +11928,18 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
11366 try self.spillEflagsIfOccupied();11928 try self.spillEflagsIfOccupied();
11367 for (items, relocs, 0..) |item, *reloc, i| {11929 for (items, relocs, 0..) |item, *reloc, i| {
11368 const item_mcv = try self.resolveInst(item);11930 const item_mcv = try self.resolveInst(item);
11369 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);11931 const cc: Condition = switch (condition) {
11370 reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) .e else .ne, undefined);11932 .eflags => |cc| switch (item_mcv.immediate) {
11933 0 => cc.negate(),
11934 1 => cc,
11935 else => unreachable,
11936 },
11937 else => cc: {
11938 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
11939 break :cc .e;
11940 },
11941 };
11942 reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) cc else cc.negate(), undefined);
11371 }11943 }
1137211944
11373 for (liveness.deaths[case_i]) |operand| try self.processDeath(operand);11945 for (liveness.deaths[case_i]) |operand| try self.processDeath(operand);
...@@ -11433,12 +12005,12 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -11433,12 +12005,12 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
11433 if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {12005 if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
11434 if (first_br) break :result src_mcv;12006 if (first_br) break :result src_mcv;
1143512007
11436 if (block_tracking.getReg()) |block_reg|12008 for (block_tracking.getRegs()) |block_reg|
11437 try self.register_manager.getReg(block_reg, br.block_inst);12009 try self.register_manager.getReg(block_reg, br.block_inst);
11438 // .long = .none to avoid merging operand and block result stack frames.12010 // .long = .none to avoid merging operand and block result stack frames.
11439 var current_tracking = InstTracking{ .long = .none, .short = src_mcv };12011 var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
11440 try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);12012 try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
11441 if (src_mcv.getReg()) |src_reg| self.register_manager.freeReg(src_reg);12013 for (src_mcv.getRegs()) |src_reg| self.register_manager.freeReg(src_reg);
11442 break :result block_tracking.short;12014 break :result block_tracking.short;
11443 }12015 }
1144412016
...@@ -11492,8 +12064,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -11492,8 +12064,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11492 var args = std.ArrayList(MCValue).init(self.gpa);12064 var args = std.ArrayList(MCValue).init(self.gpa);
11493 try args.ensureTotalCapacity(outputs.len + inputs.len);12065 try args.ensureTotalCapacity(outputs.len + inputs.len);
11494 defer {12066 defer {
11495 for (args.items) |arg| if (arg.getReg()) |reg|12067 for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{
11496 self.register_manager.unlockReg(.{ .register = reg });12068 .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue,
12069 });
11497 args.deinit();12070 args.deinit();
11498 }12071 }
11499 var arg_map = std.StringHashMap(u8).init(self.gpa);12072 var arg_map = std.StringHashMap(u8).init(self.gpa);
...@@ -12177,16 +12750,87 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -12177,16 +12750,87 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12177 .general_purpose, .segment => return .{ .move = .{ ._, .mov } },12750 .general_purpose, .segment => return .{ .move = .{ ._, .mov } },
12178 .x87 => return .x87_load_store,12751 .x87 => return .x87_load_store,
12179 .mmx => {},12752 .mmx => {},
12180 .sse => {12753 .sse => switch (ty.zigTypeTag(mod)) {
12181 switch (ty.zigTypeTag(mod)) {12754 else => {
12182 else => {12755 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
12183 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);12756 assert(std.mem.indexOfNone(abi.Class, classes, &.{
12184 assert(std.mem.indexOfNone(abi.Class, classes, &.{12757 .integer, .sse, .float, .float_combine,
12185 .integer, .sse, .float, .float_combine,12758 }) == null);
12186 }) == null);12759 const abi_size = ty.abiSize(mod);
12187 const abi_size = ty.abiSize(mod);12760 if (abi_size < 4 or
12188 if (abi_size < 4 or12761 std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {
12189 std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {12762 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12763 .insert = .{ .vp_b, .insr },
12764 .extract = .{ .vp_b, .extr },
12765 } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
12766 .insert = .{ .p_b, .insr },
12767 .extract = .{ .p_b, .extr },
12768 } },
12769 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12770 .insert = .{ .vp_w, .insr },
12771 .extract = .{ .vp_w, .extr },
12772 } } else .{ .insert_extract = .{
12773 .insert = .{ .p_w, .insr },
12774 .extract = .{ .p_w, .extr },
12775 } },
12776 3...4 => return .{ .move = if (self.hasFeature(.avx))
12777 .{ .v_d, .mov }
12778 else
12779 .{ ._d, .mov } },
12780 5...8 => return .{ .move = if (self.hasFeature(.avx))
12781 .{ .v_q, .mov }
12782 else
12783 .{ ._q, .mov } },
12784 9...16 => return .{ .move = if (self.hasFeature(.avx))
12785 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12786 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12787 17...32 => if (self.hasFeature(.avx))
12788 return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
12789 else => {},
12790 } else switch (abi_size) {
12791 4 => return .{ .move = if (self.hasFeature(.avx))
12792 .{ .v_ss, .mov }
12793 else
12794 .{ ._ss, .mov } },
12795 5...8 => return .{ .move = if (self.hasFeature(.avx))
12796 .{ .v_sd, .mov }
12797 else
12798 .{ ._sd, .mov } },
12799 9...16 => return .{ .move = if (self.hasFeature(.avx))
12800 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12801 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12802 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
12803 .{ .v_pd, .mova }
12804 else
12805 .{ .v_pd, .movu } },
12806 else => {},
12807 }
12808 },
12809 .Float => switch (ty.floatBits(self.target.*)) {
12810 16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12811 .insert = .{ .vp_w, .insr },
12812 .extract = .{ .vp_w, .extr },
12813 } } else .{ .insert_extract = .{
12814 .insert = .{ .p_w, .insr },
12815 .extract = .{ .p_w, .extr },
12816 } },
12817 32 => return .{ .move = if (self.hasFeature(.avx))
12818 .{ .v_ss, .mov }
12819 else
12820 .{ ._ss, .mov } },
12821 64 => return .{ .move = if (self.hasFeature(.avx))
12822 .{ .v_sd, .mov }
12823 else
12824 .{ ._sd, .mov } },
12825 128 => return .{ .move = if (self.hasFeature(.avx))
12826 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12827 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12828 else => {},
12829 },
12830 .Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12831 .Bool => {},
12832 .Int => switch (ty.childType(mod).intInfo(mod).bits) {
12833 8 => switch (ty.vectorLen(mod)) {
12190 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{12834 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12191 .insert = .{ .vp_b, .insr },12835 .insert = .{ .vp_b, .insr },
12192 .extract = .{ .vp_b, .extr },12836 .extract = .{ .vp_b, .extr },
...@@ -12213,242 +12857,169 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -12213,242 +12857,169 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12213 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }12857 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12214 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },12858 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12215 17...32 => if (self.hasFeature(.avx))12859 17...32 => if (self.hasFeature(.avx))
12216 return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },12860 return .{ .move = if (aligned)
12861 .{ .v_, .movdqa }
12862 else
12863 .{ .v_, .movdqu } },
12217 else => {},12864 else => {},
12218 } else switch (abi_size) {12865 },
12219 4 => return .{ .move = if (self.hasFeature(.avx))12866 16 => switch (ty.vectorLen(mod)) {
12220 .{ .v_ss, .mov }12867 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12868 .insert = .{ .vp_w, .insr },
12869 .extract = .{ .vp_w, .extr },
12870 } } else .{ .insert_extract = .{
12871 .insert = .{ .p_w, .insr },
12872 .extract = .{ .p_w, .extr },
12873 } },
12874 2 => return .{ .move = if (self.hasFeature(.avx))
12875 .{ .v_d, .mov }
12221 else12876 else
12222 .{ ._ss, .mov } },12877 .{ ._d, .mov } },
12878 3...4 => return .{ .move = if (self.hasFeature(.avx))
12879 .{ .v_q, .mov }
12880 else
12881 .{ ._q, .mov } },
12223 5...8 => return .{ .move = if (self.hasFeature(.avx))12882 5...8 => return .{ .move = if (self.hasFeature(.avx))
12224 .{ .v_sd, .mov }12883 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12884 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12885 9...16 => if (self.hasFeature(.avx))
12886 return .{ .move = if (aligned)
12887 .{ .v_, .movdqa }
12888 else
12889 .{ .v_, .movdqu } },
12890 else => {},
12891 },
12892 32 => switch (ty.vectorLen(mod)) {
12893 1 => return .{ .move = if (self.hasFeature(.avx))
12894 .{ .v_d, .mov }
12225 else12895 else
12226 .{ ._sd, .mov } },12896 .{ ._d, .mov } },
12227 9...16 => return .{ .move = if (self.hasFeature(.avx))12897 2 => return .{ .move = if (self.hasFeature(.avx))
12228 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }12898 .{ .v_q, .mov }
12229 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12230 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
12231 .{ .v_pd, .mova }
12232 else12899 else
12233 .{ .v_pd, .movu } },12900 .{ ._q, .mov } },
12234 else => {},12901 3...4 => return .{ .move = if (self.hasFeature(.avx))
12235 }12902 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12236 },12903 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12237 .Float => switch (ty.floatBits(self.target.*)) {12904 5...8 => if (self.hasFeature(.avx))
12238 16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{12905 return .{ .move = if (aligned)
12239 .insert = .{ .vp_w, .insr },12906 .{ .v_, .movdqa }
12240 .extract = .{ .vp_w, .extr },
12241 } } else .{ .insert_extract = .{
12242 .insert = .{ .p_w, .insr },
12243 .extract = .{ .p_w, .extr },
12244 } },
12245 32 => return .{ .move = if (self.hasFeature(.avx))
12246 .{ .v_ss, .mov }
12247 else
12248 .{ ._ss, .mov } },
12249 64 => return .{ .move = if (self.hasFeature(.avx))
12250 .{ .v_sd, .mov }
12251 else
12252 .{ ._sd, .mov } },
12253 128 => return .{ .move = if (self.hasFeature(.avx))
12254 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12255 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12256 else => {},
12257 },
12258 .Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12259 .Bool => return .{ .move = .{ ._, .mov } },
12260 .Int => switch (ty.childType(mod).intInfo(mod).bits) {
12261 8 => switch (ty.vectorLen(mod)) {
12262 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12263 .insert = .{ .vp_b, .insr },
12264 .extract = .{ .vp_b, .extr },
12265 } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
12266 .insert = .{ .p_b, .insr },
12267 .extract = .{ .p_b, .extr },
12268 } },
12269 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12270 .insert = .{ .vp_w, .insr },
12271 .extract = .{ .vp_w, .extr },
12272 } } else .{ .insert_extract = .{
12273 .insert = .{ .p_w, .insr },
12274 .extract = .{ .p_w, .extr },
12275 } },
12276 3...4 => return .{ .move = if (self.hasFeature(.avx))
12277 .{ .v_d, .mov }
12278 else
12279 .{ ._d, .mov } },
12280 5...8 => return .{ .move = if (self.hasFeature(.avx))
12281 .{ .v_q, .mov }
12282 else
12283 .{ ._q, .mov } },
12284 9...16 => return .{ .move = if (self.hasFeature(.avx))
12285 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12286 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12287 17...32 => if (self.hasFeature(.avx))
12288 return .{ .move = if (aligned)
12289 .{ .v_, .movdqa }
12290 else
12291 .{ .v_, .movdqu } },
12292 else => {},
12293 },
12294 16 => switch (ty.vectorLen(mod)) {
12295 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12296 .insert = .{ .vp_w, .insr },
12297 .extract = .{ .vp_w, .extr },
12298 } } else .{ .insert_extract = .{
12299 .insert = .{ .p_w, .insr },
12300 .extract = .{ .p_w, .extr },
12301 } },
12302 2 => return .{ .move = if (self.hasFeature(.avx))
12303 .{ .v_d, .mov }
12304 else
12305 .{ ._d, .mov } },
12306 3...4 => return .{ .move = if (self.hasFeature(.avx))
12307 .{ .v_q, .mov }
12308 else
12309 .{ ._q, .mov } },
12310 5...8 => return .{ .move = if (self.hasFeature(.avx))
12311 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12312 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12313 9...16 => if (self.hasFeature(.avx))
12314 return .{ .move = if (aligned)
12315 .{ .v_, .movdqa }
12316 else
12317 .{ .v_, .movdqu } },
12318 else => {},
12319 },
12320 32 => switch (ty.vectorLen(mod)) {
12321 1 => return .{ .move = if (self.hasFeature(.avx))
12322 .{ .v_d, .mov }
12323 else12907 else
12324 .{ ._d, .mov } },12908 .{ .v_, .movdqu } },
12325 2 => return .{ .move = if (self.hasFeature(.avx))12909 else => {},
12326 .{ .v_q, .mov }12910 },
12911 64 => switch (ty.vectorLen(mod)) {
12912 1 => return .{ .move = if (self.hasFeature(.avx))
12913 .{ .v_q, .mov }
12914 else
12915 .{ ._q, .mov } },
12916 2 => return .{ .move = if (self.hasFeature(.avx))
12917 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12918 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12919 3...4 => if (self.hasFeature(.avx))
12920 return .{ .move = if (aligned)
12921 .{ .v_, .movdqa }
12327 else12922 else
12328 .{ ._q, .mov } },12923 .{ .v_, .movdqu } },
12329 3...4 => return .{ .move = if (self.hasFeature(.avx))12924 else => {},
12330 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }12925 },
12331 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },12926 128 => switch (ty.vectorLen(mod)) {
12332 5...8 => if (self.hasFeature(.avx))12927 1 => return .{ .move = if (self.hasFeature(.avx))
12333 return .{ .move = if (aligned)12928 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12334 .{ .v_, .movdqa }12929 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12335 else12930 2 => if (self.hasFeature(.avx))
12336 .{ .v_, .movdqu } },12931 return .{ .move = if (aligned)
12337 else => {},12932 .{ .v_, .movdqa }
12338 },
12339 64 => switch (ty.vectorLen(mod)) {
12340 1 => return .{ .move = if (self.hasFeature(.avx))
12341 .{ .v_q, .mov }
12342 else12933 else
12343 .{ ._q, .mov } },12934 .{ .v_, .movdqu } },
12344 2 => return .{ .move = if (self.hasFeature(.avx))
12345 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12346 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12347 3...4 => if (self.hasFeature(.avx))
12348 return .{ .move = if (aligned)
12349 .{ .v_, .movdqa }
12350 else
12351 .{ .v_, .movdqu } },
12352 else => {},
12353 },
12354 128 => switch (ty.vectorLen(mod)) {
12355 1 => return .{ .move = if (self.hasFeature(.avx))
12356 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12357 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12358 2 => if (self.hasFeature(.avx))
12359 return .{ .move = if (aligned)
12360 .{ .v_, .movdqa }
12361 else
12362 .{ .v_, .movdqu } },
12363 else => {},
12364 },
12365 256 => switch (ty.vectorLen(mod)) {
12366 1 => if (self.hasFeature(.avx))
12367 return .{ .move = if (aligned)
12368 .{ .v_, .movdqa }
12369 else
12370 .{ .v_, .movdqu } },
12371 else => {},
12372 },
12373 else => {},12935 else => {},
12374 },12936 },
12375 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {12937 256 => switch (ty.vectorLen(mod)) {
12376 16 => switch (ty.vectorLen(mod)) {12938 1 => if (self.hasFeature(.avx))
12377 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{12939 return .{ .move = if (aligned)
12378 .insert = .{ .vp_w, .insr },12940 .{ .v_, .movdqa }
12379 .extract = .{ .vp_w, .extr },
12380 } } else .{ .insert_extract = .{
12381 .insert = .{ .p_w, .insr },
12382 .extract = .{ .p_w, .extr },
12383 } },
12384 2 => return .{ .move = if (self.hasFeature(.avx))
12385 .{ .v_d, .mov }
12386 else12941 else
12387 .{ ._d, .mov } },12942 .{ .v_, .movdqu } },
12388 3...4 => return .{ .move = if (self.hasFeature(.avx))12943 else => {},
12389 .{ .v_q, .mov }12944 },
12945 else => {},
12946 },
12947 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {
12948 16 => switch (ty.vectorLen(mod)) {
12949 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12950 .insert = .{ .vp_w, .insr },
12951 .extract = .{ .vp_w, .extr },
12952 } } else .{ .insert_extract = .{
12953 .insert = .{ .p_w, .insr },
12954 .extract = .{ .p_w, .extr },
12955 } },
12956 2 => return .{ .move = if (self.hasFeature(.avx))
12957 .{ .v_d, .mov }
12958 else
12959 .{ ._d, .mov } },
12960 3...4 => return .{ .move = if (self.hasFeature(.avx))
12961 .{ .v_q, .mov }
12962 else
12963 .{ ._q, .mov } },
12964 5...8 => return .{ .move = if (self.hasFeature(.avx))
12965 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12966 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12967 9...16 => if (self.hasFeature(.avx))
12968 return .{ .move = if (aligned)
12969 .{ .v_, .movdqa }
12390 else12970 else
12391 .{ ._q, .mov } },12971 .{ .v_, .movdqu } },
12392 5...8 => return .{ .move = if (self.hasFeature(.avx))12972 else => {},
12393 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }12973 },
12394 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },12974 32 => switch (ty.vectorLen(mod)) {
12395 9...16 => if (self.hasFeature(.avx))12975 1 => return .{ .move = if (self.hasFeature(.avx))
12396 return .{ .move = if (aligned)12976 .{ .v_ss, .mov }
12397 .{ .v_, .movdqa }12977 else
12398 else12978 .{ ._ss, .mov } },
12399 .{ .v_, .movdqu } },12979 2 => return .{ .move = if (self.hasFeature(.avx))
12400 else => {},12980 .{ .v_sd, .mov }
12401 },12981 else
12402 32 => switch (ty.vectorLen(mod)) {12982 .{ ._sd, .mov } },
12403 1 => return .{ .move = if (self.hasFeature(.avx))12983 3...4 => return .{ .move = if (self.hasFeature(.avx))
12404 .{ .v_ss, .mov }12984 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
12985 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
12986 5...8 => if (self.hasFeature(.avx))
12987 return .{ .move = if (aligned)
12988 .{ .v_ps, .mova }
12405 else12989 else
12406 .{ ._ss, .mov } },12990 .{ .v_ps, .movu } },
12407 2 => return .{ .move = if (self.hasFeature(.avx))12991 else => {},
12408 .{ .v_sd, .mov }12992 },
12993 64 => switch (ty.vectorLen(mod)) {
12994 1 => return .{ .move = if (self.hasFeature(.avx))
12995 .{ .v_sd, .mov }
12996 else
12997 .{ ._sd, .mov } },
12998 2 => return .{ .move = if (self.hasFeature(.avx))
12999 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
13000 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
13001 3...4 => if (self.hasFeature(.avx))
13002 return .{ .move = if (aligned)
13003 .{ .v_pd, .mova }
12409 else13004 else
12410 .{ ._sd, .mov } },13005 .{ .v_pd, .movu } },
12411 3...4 => return .{ .move = if (self.hasFeature(.avx))13006 else => {},
12412 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }13007 },
12413 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },13008 128 => switch (ty.vectorLen(mod)) {
12414 5...8 => if (self.hasFeature(.avx))13009 1 => return .{ .move = if (self.hasFeature(.avx))
12415 return .{ .move = if (aligned)13010 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12416 .{ .v_ps, .mova }13011 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12417 else13012 2 => if (self.hasFeature(.avx))
12418 .{ .v_ps, .movu } },13013 return .{ .move = if (aligned)
12419 else => {},13014 .{ .v_, .movdqa }
12420 },
12421 64 => switch (ty.vectorLen(mod)) {
12422 1 => return .{ .move = if (self.hasFeature(.avx))
12423 .{ .v_sd, .mov }
12424 else13015 else
12425 .{ ._sd, .mov } },13016 .{ .v_, .movdqu } },
12426 2 => return .{ .move = if (self.hasFeature(.avx))
12427 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12428 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12429 3...4 => if (self.hasFeature(.avx))
12430 return .{ .move = if (aligned)
12431 .{ .v_pd, .mova }
12432 else
12433 .{ .v_pd, .movu } },
12434 else => {},
12435 },
12436 128 => switch (ty.vectorLen(mod)) {
12437 1 => return .{ .move = if (self.hasFeature(.avx))
12438 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12439 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12440 2 => if (self.hasFeature(.avx))
12441 return .{ .move = if (aligned)
12442 .{ .v_, .movdqa }
12443 else
12444 .{ .v_, .movdqu } },
12445 else => {},
12446 },
12447 else => {},13017 else => {},
12448 },13018 },
12449 else => {},13019 else => {},
12450 },13020 },
12451 }13021 else => {},
13022 },
12452 },13023 },
12453 }13024 }
12454 return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});13025 return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});
...@@ -12514,32 +13085,18 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError...@@ -12514,32 +13085,18 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
12514 };13085 };
12515 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);13086 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);
1251613087
12517 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);13088 var part_disp: i32 = 0;
12518 for (dst_regs, classes, 0..) |dst_reg, class, dst_reg_i| {13089 for (dst_regs, try self.splitType(ty), 0..) |dst_reg, dst_ty, part_i| {
12519 const class_ty = switch (class) {13090 try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
12520 .integer => Type.usize,13091 .register_pair => |src_regs| .{ .register = src_regs[part_i] },
12521 .sse, .float, .float_combine => Type.f64,13092 .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
12522 else => unreachable,13093 .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
12523 };13094 .reg = src_info.?.addr_reg,
12524 const off: i32 = @intCast(dst_reg_i * 8);13095 .off = part_disp,
12525 switch (src_mcv) {13096 } },
12526 .register_pair => |src_regs| try self.genSetReg(
12527 dst_reg,
12528 class_ty,
12529 .{ .register = src_regs[dst_reg_i] },
12530 ),
12531 .memory, .indirect, .load_frame => try self.genSetReg(
12532 dst_reg,
12533 class_ty,
12534 src_mcv.address().offset(off).deref(),
12535 ),
12536 .load_symbol, .load_direct, .load_got, .load_tlv => try self.genSetReg(
12537 dst_reg,
12538 class_ty,
12539 .{ .indirect = .{ .reg = src_info.?.addr_reg, .off = off } },
12540 ),
12541 else => unreachable,13097 else => unreachable,
12542 }13098 });
13099 part_disp += @intCast(dst_ty.abiSize(mod));
12543 }13100 }
12544 },13101 },
12545 .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),13102 .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
...@@ -12584,6 +13141,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -12584,6 +13141,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12584 if (imm == 0) {13141 if (imm == 0) {
12585 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit13142 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
12586 // register is the fastest way to zero a register.13143 // register is the fastest way to zero a register.
13144 try self.spillEflagsIfOccupied();
12587 try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());13145 try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12588 } else if (abi_size > 4 and math.cast(u32, imm) != null) {13146 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
12589 // 32-bit moves zero-extend to 64-bit.13147 // 32-bit moves zero-extend to 64-bit.
...@@ -12933,44 +13491,65 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal...@@ -12933,44 +13491,65 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12933 .eflags => |cc| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{13491 .eflags => |cc| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{
12934 .rm = .{ .size = .byte, .disp = disp },13492 .rm = .{ .size = .byte, .disp = disp },
12935 } }),13493 } }),
12936 .register => |src_reg| try (try self.moveStrategy(ty, src_reg.class(), switch (base) {13494 .register => |src_reg| {
12937 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),13495 const mem_size = switch (base) {
12938 .reg => |reg| switch (reg) {13496 .frame => |base_fi| mem_size: {
12939 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),13497 assert(disp >= 0);
12940 else => false,13498 const frame_abi_size = self.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)];
12941 },13499 const frame_spill_pad = self.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)];
12942 .frame => |frame_index| self.getFrameAddrAlignment(13500 assert(frame_abi_size - frame_spill_pad - disp >= abi_size);
12943 .{ .index = frame_index, .off = disp },13501 break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size)
12944 ).compare(.gte, ty.abiAlignment(mod)),13502 frame_abi_size
12945 .reloc => false,13503 else
12946 })).write(13504 abi_size;
12947 self,
12948 .{ .base = base, .mod = .{ .rm = .{
12949 .size = self.memSize(ty),
12950 .disp = disp,
12951 } } },
12952 registerAlias(src_reg, abi_size),
12953 ),
12954 .register_pair => |src_regs| for (src_regs, 0..) |src_reg, src_reg_i| {
12955 const part_size: u16 = @min(abi_size - src_reg_i * 8, 8);
12956 try (try self.moveStrategy(
12957 try mod.intType(.unsigned, part_size * 8),
12958 src_reg.class(),
12959 switch (base) {
12960 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12961 .reg => |reg| switch (reg) {
12962 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12963 else => false,
12964 },
12965 .frame => |frame_index| self.getFrameAddrAlignment(
12966 .{ .index = frame_index, .off = disp },
12967 ).compare(.gte, ty.abiAlignment(mod)),
12968 .reloc => false,
12969 },13505 },
12970 )).write(self, .{ .base = base, .mod = .{ .rm = .{13506 else => abi_size,
12971 .size = Memory.Size.fromSize(part_size),13507 };
12972 .disp = disp + @as(i32, @intCast(src_reg_i * 8)),13508 const src_alias = registerAlias(src_reg, abi_size);
12973 } } }, registerAlias(src_reg, part_size));13509 const src_size: u32 = @intCast(switch (src_alias.class()) {
13510 .general_purpose, .segment, .x87 => @divExact(src_alias.bitSize(), 8),
13511 .mmx, .sse => abi_size,
13512 });
13513 if (src_size > mem_size) {
13514 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
13515 .size = src_size,
13516 .alignment = Alignment.fromNonzeroByteUnits(src_size),
13517 }));
13518 const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } };
13519 try (try self.moveStrategy(ty, src_alias.class(), true)).write(
13520 self,
13521 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
13522 .size = Memory.Size.fromSize(src_size),
13523 } } },
13524 src_alias,
13525 );
13526 try self.genSetMem(base, disp, ty, frame_mcv);
13527 try self.freeValue(frame_mcv);
13528 } else try (try self.moveStrategy(ty, src_alias.class(), switch (base) {
13529 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
13530 .reg => |reg| switch (reg) {
13531 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
13532 else => false,
13533 },
13534 .frame => |frame_index| self.getFrameAddrAlignment(
13535 .{ .index = frame_index, .off = disp },
13536 ).compare(.gte, ty.abiAlignment(mod)),
13537 .reloc => false,
13538 })).write(
13539 self,
13540 .{ .base = base, .mod = .{ .rm = .{
13541 .size = self.memSize(ty),
13542 .disp = disp,
13543 } } },
13544 src_alias,
13545 );
13546 },
13547 .register_pair => |src_regs| {
13548 var part_disp: i32 = disp;
13549 for (try self.splitType(ty), src_regs) |src_ty, src_reg| {
13550 try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg });
13551 part_disp += @intCast(src_ty.abiSize(mod));
13552 }
12974 },13553 },
12975 .register_overflow => |ro| switch (ty.zigTypeTag(mod)) {13554 .register_overflow => |ro| switch (ty.zigTypeTag(mod)) {
12976 .Struct => {13555 .Struct => {
...@@ -13226,50 +13805,43 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -13226,50 +13805,43 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
13226 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;13805 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
13227 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);13806 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
1322813807
13229 const dst_mcv = if (dst_rc.supersetOf(src_rc) and13808 const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(mod) <= src_ty.abiSize(mod) and
13230 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))13809 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
13231 src_mcv
13232 else dst: {
13233 const dst_mcv = try self.allocRegOrMem(inst, true);13810 const dst_mcv = try self.allocRegOrMem(inst, true);
13234 try self.genCopy(13811 try self.genCopy(switch (math.order(dst_ty.abiSize(mod), src_ty.abiSize(mod))) {
13235 if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,13812 .lt => dst_ty,
13236 dst_mcv,13813 .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
13237 src_mcv,13814 .gt => src_ty,
13238 );13815 }, dst_mcv, src_mcv);
13239 break :dst dst_mcv;13816 break :dst dst_mcv;
13240 };13817 };
1324113818
13242 if (dst_ty.isRuntimeFloat()) break :result dst_mcv;13819 if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
1324313820
13244 const dst_signedness =13821 if (dst_ty.isAbiInt(mod) and src_ty.isAbiInt(mod) and
13245 if (dst_ty.isAbiInt(mod)) dst_ty.intInfo(mod).signedness else .unsigned;13822 dst_ty.intInfo(mod).signedness == src_ty.intInfo(mod).signedness) break :result dst_mcv;
13246 if (!src_ty.isRuntimeFloat() or src_ty.floatBits(self.target.*) != 80) {
13247 const src_signedness =
13248 if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
13249 if (dst_signedness == src_signedness) break :result dst_mcv;
13250 }
1325113823
13252 const abi_size: u16 = @intCast(dst_ty.abiSize(mod));13824 const abi_size = dst_ty.abiSize(mod);
13253 const bit_size: u16 = @intCast(dst_ty.bitSize(mod));13825 const bit_size = dst_ty.bitSize(mod);
13254 if (abi_size * 8 <= bit_size) break :result dst_mcv;13826 if (abi_size * 8 <= bit_size or dst_ty.isVector(mod)) break :result dst_mcv;
1325513827
13256 const dst_limbs_len = math.divCeil(i32, bit_size, 64) catch unreachable;13828 const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
13257 const high_reg = if (dst_mcv.isRegister())13829 const high_mcv: MCValue = switch (dst_mcv) {
13258 dst_mcv.getReg().?13830 .register => |dst_reg| .{ .register = dst_reg },
13831 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
13832 else => dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13833 };
13834 const high_reg = if (high_mcv.isRegister())
13835 high_mcv.getReg().?
13259 else13836 else
13260 try self.copyToTmpRegister(13837 try self.copyToTmpRegister(Type.usize, high_mcv);
13261 Type.usize,
13262 dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13263 );
13264 const high_lock = self.register_manager.lockReg(high_reg);13838 const high_lock = self.register_manager.lockReg(high_reg);
13265 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);13839 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
1326613840
13267 const high_ty = try mod.intType(dst_signedness, bit_size % 64);13841 try self.truncateRegister(dst_ty, high_reg);
1326813842 if (!high_mcv.isRegister()) try self.genCopy(
13269 try self.truncateRegister(high_ty, high_reg);13843 if (abi_size <= 8) dst_ty else Type.usize,
13270 if (!dst_mcv.isRegister()) try self.genCopy(13844 high_mcv,
13271 Type.usize,
13272 dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13273 .{ .register = high_reg },13845 .{ .register = high_reg },
13274 );13846 );
13275 break :result dst_mcv;13847 break :result dst_mcv;
...@@ -13287,7 +13859,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -13287,7 +13859,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
13287 const array_ty = ptr_ty.childType(mod);13859 const array_ty = ptr_ty.childType(mod);
13288 const array_len = array_ty.arrayLen(mod);13860 const array_len = array_ty.arrayLen(mod);
1328913861
13290 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));13862 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
13291 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);13863 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
13292 try self.genSetMem(13864 try self.genSetMem(
13293 .{ .frame = frame_index },13865 .{ .frame = frame_index },
...@@ -13497,7 +14069,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -13497,7 +14069,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
13497 const ptr_mcv = try self.resolveInst(extra.ptr);14069 const ptr_mcv = try self.resolveInst(extra.ptr);
13498 const mem_size = Memory.Size.fromSize(val_abi_size);14070 const mem_size = Memory.Size.fromSize(val_abi_size);
13499 const ptr_mem: Memory = switch (ptr_mcv) {14071 const ptr_mem: Memory = switch (ptr_mcv) {
13500 .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),14072 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13501 else => .{14073 else => .{
13502 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },14074 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13503 .mod = .{ .rm = .{ .size = mem_size } },14075 .mod = .{ .rm = .{ .size = mem_size } },
...@@ -13563,7 +14135,7 @@ fn atomicOp(...@@ -13563,7 +14135,7 @@ fn atomicOp(
13563 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));14135 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
13564 const mem_size = Memory.Size.fromSize(val_abi_size);14136 const mem_size = Memory.Size.fromSize(val_abi_size);
13565 const ptr_mem: Memory = switch (ptr_mcv) {14137 const ptr_mem: Memory = switch (ptr_mcv) {
13566 .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),14138 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13567 else => .{14139 else => .{
13568 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },14140 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13569 .mod = .{ .rm = .{ .size = mem_size } },14141 .mod = .{ .rm = .{ .size = mem_size } },
...@@ -13671,27 +14243,41 @@ fn atomicOp(...@@ -13671,27 +14243,41 @@ fn atomicOp(
13671 },14243 },
13672 };14244 };
1367314245
13674 try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13675 const cmov_abi_size = @max(val_abi_size, 2);14246 const cmov_abi_size = @max(val_abi_size, 2);
13676 switch (val_mcv) {14247 switch (val_mcv) {
13677 .register => |val_reg| try self.asmCmovccRegisterRegister(14248 .register => |val_reg| {
13678 cc,14249 try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13679 registerAlias(tmp_reg, cmov_abi_size),
13680 registerAlias(val_reg, cmov_abi_size),
13681 ),
13682 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
13683 cc,
13684 registerAlias(tmp_reg, cmov_abi_size),
13685 val_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
13686 ),
13687 else => {
13688 const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
13689 try self.asmCmovccRegisterRegister(14250 try self.asmCmovccRegisterRegister(
13690 cc,14251 cc,
13691 registerAlias(tmp_reg, cmov_abi_size),14252 registerAlias(tmp_reg, cmov_abi_size),
13692 registerAlias(val_reg, cmov_abi_size),14253 registerAlias(val_reg, cmov_abi_size),
13693 );14254 );
13694 },14255 },
14256 .memory, .indirect, .load_frame => {
14257 try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
14258 try self.asmCmovccRegisterMemory(
14259 cc,
14260 registerAlias(tmp_reg, cmov_abi_size),
14261 try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
14262 );
14263 },
14264 else => {
14265 const mat_reg = try self.copyToTmpRegister(val_ty, val_mcv);
14266 const mat_lock = self.register_manager.lockRegAssumeUnused(mat_reg);
14267 defer self.register_manager.unlockReg(mat_lock);
14268
14269 try self.genBinOpMir(
14270 .{ ._, .cmp },
14271 val_ty,
14272 tmp_mcv,
14273 .{ .register = mat_reg },
14274 );
14275 try self.asmCmovccRegisterRegister(
14276 cc,
14277 registerAlias(tmp_reg, cmov_abi_size),
14278 registerAlias(mat_reg, cmov_abi_size),
14279 );
14280 },
13695 }14281 }
13696 },14282 },
13697 };14283 };
...@@ -13728,8 +14314,8 @@ fn atomicOp(...@@ -13728,8 +14314,8 @@ fn atomicOp(
13728 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),14314 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
13729 } },14315 } },
13730 };14316 };
13731 const val_lo_mem = val_mem_mcv.mem(.qword);14317 const val_lo_mem = try val_mem_mcv.mem(self, .qword);
13732 const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);14318 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);
13733 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {14319 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
13734 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);14320 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
13735 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);14321 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
...@@ -13859,6 +14445,10 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -13859,6 +14445,10 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1385914445
13860 const bin_op = self.air.instructions.items(.data)[inst].bin_op;14446 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1386114447
14448 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
14449 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
14450 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
14451
13862 const dst_ptr = try self.resolveInst(bin_op.lhs);14452 const dst_ptr = try self.resolveInst(bin_op.lhs);
13863 const dst_ptr_ty = self.typeOf(bin_op.lhs);14453 const dst_ptr_ty = self.typeOf(bin_op.lhs);
13864 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {14454 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
...@@ -13976,6 +14566,10 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -13976,6 +14566,10 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
13976 const mod = self.bin_file.options.module.?;14566 const mod = self.bin_file.options.module.?;
13977 const bin_op = self.air.instructions.items(.data)[inst].bin_op;14567 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1397814568
14569 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
14570 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
14571 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
14572
13979 const dst_ptr = try self.resolveInst(bin_op.lhs);14573 const dst_ptr = try self.resolveInst(bin_op.lhs);
13980 const dst_ptr_ty = self.typeOf(bin_op.lhs);14574 const dst_ptr_ty = self.typeOf(bin_op.lhs);
13981 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {14575 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
...@@ -14000,7 +14594,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -14000,7 +14594,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
14000 try self.asmRegisterMemoryImmediate(14594 try self.asmRegisterMemoryImmediate(
14001 .{ .i_, .mul },14595 .{ .i_, .mul },
14002 len_reg,14596 len_reg,
14003 dst_ptr.address().offset(8).deref().mem(.qword),14597 try dst_ptr.address().offset(8).deref().mem(self, .qword),
14004 Immediate.s(@intCast(dst_ptr_ty.childType(mod).abiSize(mod))),14598 Immediate.s(@intCast(dst_ptr_ty.childType(mod).abiSize(mod))),
14005 );14599 );
14006 break :len .{ .register = len_reg };14600 break :len .{ .register = len_reg };
...@@ -14046,7 +14640,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {...@@ -14046,7 +14640,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
14046 }14640 }
1404714641
14048 try self.spillEflagsIfOccupied();14642 try self.spillEflagsIfOccupied();
14049 try self.spillRegisters(abi.getCallerPreservedRegs(resolved_cc));14643 try self.spillCallerPreservedRegs(resolved_cc);
1405014644
14051 const param_regs = abi.getCAbiIntParamRegs(resolved_cc);14645 const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
1405214646
...@@ -14171,28 +14765,162 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14171,28 +14765,162 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14171 const mod = self.bin_file.options.module.?;14765 const mod = self.bin_file.options.module.?;
14172 const ty_op = self.air.instructions.items(.data)[inst].ty_op;14766 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
14173 const vector_ty = self.typeOfIndex(inst);14767 const vector_ty = self.typeOfIndex(inst);
14768 const vector_len = vector_ty.vectorLen(mod);
14174 const dst_rc = self.regClassForType(vector_ty);14769 const dst_rc = self.regClassForType(vector_ty);
14175 const scalar_ty = vector_ty.scalarType(mod);14770 const scalar_ty = self.typeOf(ty_op.operand);
1417614771
14177 const src_mcv = try self.resolveInst(ty_op.operand);
14178 const result: MCValue = result: {14772 const result: MCValue = result: {
14179 switch (scalar_ty.zigTypeTag(mod)) {14773 switch (scalar_ty.zigTypeTag(mod)) {
14180 else => {},14774 else => {},
14775 .Bool => {
14776 const regs =
14777 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
14778 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
14779 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
14780
14781 try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 });
14782 try self.genSetReg(
14783 regs[1],
14784 vector_ty,
14785 .{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },
14786 );
14787 const src_mcv = try self.resolveInst(ty_op.operand);
14788 const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);
14789 try self.asmCmovccRegisterRegister(
14790 switch (src_mcv) {
14791 .eflags => |cc| cc,
14792 .register => |src_reg| cc: {
14793 try self.asmRegisterImmediate(
14794 .{ ._, .@"test" },
14795 src_reg.to8(),
14796 Immediate.u(1),
14797 );
14798 break :cc .nz;
14799 },
14800 else => cc: {
14801 try self.asmMemoryImmediate(
14802 .{ ._, .@"test" },
14803 try src_mcv.mem(self, .byte),
14804 Immediate.u(1),
14805 );
14806 break :cc .nz;
14807 },
14808 },
14809 registerAlias(regs[0], abi_size),
14810 registerAlias(regs[1], abi_size),
14811 );
14812 break :result .{ .register = regs[0] };
14813 },
14814 .Int => if (self.hasFeature(.avx2)) avx2: {
14815 const mir_tag = @as(?Mir.Inst.FixedTag, switch (scalar_ty.intInfo(mod).bits) {
14816 else => null,
14817 1...8 => switch (vector_len) {
14818 else => null,
14819 1...32 => .{ .vp_b, .broadcast },
14820 },
14821 9...16 => switch (vector_len) {
14822 else => null,
14823 1...16 => .{ .vp_w, .broadcast },
14824 },
14825 17...32 => switch (vector_len) {
14826 else => null,
14827 1...8 => .{ .vp_d, .broadcast },
14828 },
14829 33...64 => switch (vector_len) {
14830 else => null,
14831 1...4 => .{ .vp_q, .broadcast },
14832 },
14833 65...128 => switch (vector_len) {
14834 else => null,
14835 1...2 => .{ .vp_i128, .broadcast },
14836 },
14837 }) orelse break :avx2;
14838
14839 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
14840 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
14841 defer self.register_manager.unlockReg(dst_lock);
14842
14843 const src_mcv = try self.resolveInst(ty_op.operand);
14844 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14845 mir_tag,
14846 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
14847 try src_mcv.mem(self, self.memSize(scalar_ty)),
14848 ) else {
14849 if (mir_tag[0] == .vp_i128) break :avx2;
14850 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14851 try self.asmRegisterRegister(
14852 mir_tag,
14853 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
14854 registerAlias(dst_reg, @intCast(scalar_ty.abiSize(mod))),
14855 );
14856 }
14857 break :result .{ .register = dst_reg };
14858 } else {
14859 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
14860 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
14861 defer self.register_manager.unlockReg(dst_lock);
14862
14863 try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand });
14864 if (vector_len == 1) break :result .{ .register = dst_reg };
14865
14866 const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod)));
14867 const scalar_bits = scalar_ty.intInfo(mod).bits;
14868 if (switch (scalar_bits) {
14869 1...8 => true,
14870 9...128 => false,
14871 else => unreachable,
14872 }) if (self.hasFeature(.avx)) try self.asmRegisterRegisterRegister(
14873 .{ .vp_, .unpcklbw },
14874 dst_alias,
14875 dst_alias,
14876 dst_alias,
14877 ) else try self.asmRegisterRegister(
14878 .{ .p_, .unpcklbw },
14879 dst_alias,
14880 dst_alias,
14881 );
14882 if (switch (scalar_bits) {
14883 1...8 => vector_len > 2,
14884 9...16 => true,
14885 17...128 => false,
14886 else => unreachable,
14887 }) try self.asmRegisterRegisterImmediate(
14888 .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
14889 dst_alias,
14890 dst_alias,
14891 Immediate.u(0),
14892 );
14893 if (switch (scalar_bits) {
14894 1...8 => vector_len > 4,
14895 9...16 => vector_len > 2,
14896 17...64 => true,
14897 65...128 => false,
14898 else => unreachable,
14899 }) try self.asmRegisterRegisterImmediate(
14900 .{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
14901 dst_alias,
14902 dst_alias,
14903 Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
14904 );
14905 break :result .{ .register = dst_reg };
14906 },
14181 .Float => switch (scalar_ty.floatBits(self.target.*)) {14907 .Float => switch (scalar_ty.floatBits(self.target.*)) {
14182 32 => switch (vector_ty.vectorLen(mod)) {14908 32 => switch (vector_len) {
14183 1 => {14909 1 => {
14910 const src_mcv = try self.resolveInst(ty_op.operand);
14184 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;14911 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14185 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14912 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14186 try self.genSetReg(dst_reg, scalar_ty, src_mcv);14913 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14187 break :result .{ .register = dst_reg };14914 break :result .{ .register = dst_reg };
14188 },14915 },
14189 2...4 => {14916 2...4 => {
14917 const src_mcv = try self.resolveInst(ty_op.operand);
14190 if (self.hasFeature(.avx)) {14918 if (self.hasFeature(.avx)) {
14191 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14919 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14192 if (src_mcv.isMemory()) try self.asmRegisterMemory(14920 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14193 .{ .v_ss, .broadcast },14921 .{ .v_ss, .broadcast },
14194 dst_reg.to128(),14922 dst_reg.to128(),
14195 src_mcv.mem(.dword),14923 try src_mcv.mem(self, .dword),
14196 ) else {14924 ) else {
14197 const src_reg = if (src_mcv.isRegister())14925 const src_reg = if (src_mcv.isRegister())
14198 src_mcv.getReg().?14926 src_mcv.getReg().?
...@@ -14224,11 +14952,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14224,11 +14952,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14224 }14952 }
14225 },14953 },
14226 5...8 => if (self.hasFeature(.avx)) {14954 5...8 => if (self.hasFeature(.avx)) {
14955 const src_mcv = try self.resolveInst(ty_op.operand);
14227 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14956 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14228 if (src_mcv.isMemory()) try self.asmRegisterMemory(14957 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14229 .{ .v_ss, .broadcast },14958 .{ .v_ss, .broadcast },
14230 dst_reg.to256(),14959 dst_reg.to256(),
14231 src_mcv.mem(.dword),14960 try src_mcv.mem(self, .dword),
14232 ) else {14961 ) else {
14233 const src_reg = if (src_mcv.isRegister())14962 const src_reg = if (src_mcv.isRegister())
14234 src_mcv.getReg().?14963 src_mcv.getReg().?
...@@ -14259,20 +14988,22 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14259,20 +14988,22 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14259 },14988 },
14260 else => {},14989 else => {},
14261 },14990 },
14262 64 => switch (vector_ty.vectorLen(mod)) {14991 64 => switch (vector_len) {
14263 1 => {14992 1 => {
14993 const src_mcv = try self.resolveInst(ty_op.operand);
14264 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;14994 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14265 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14995 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14266 try self.genSetReg(dst_reg, scalar_ty, src_mcv);14996 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14267 break :result .{ .register = dst_reg };14997 break :result .{ .register = dst_reg };
14268 },14998 },
14269 2 => {14999 2 => {
15000 const src_mcv = try self.resolveInst(ty_op.operand);
14270 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);15001 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14271 if (self.hasFeature(.sse3)) {15002 if (self.hasFeature(.sse3)) {
14272 if (src_mcv.isMemory()) try self.asmRegisterMemory(15003 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14273 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },15004 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14274 dst_reg.to128(),15005 dst_reg.to128(),
14275 src_mcv.mem(.qword),15006 try src_mcv.mem(self, .qword),
14276 ) else try self.asmRegisterRegister(15007 ) else try self.asmRegisterRegister(
14277 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },15008 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14278 dst_reg.to128(),15009 dst_reg.to128(),
...@@ -14292,11 +15023,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14292,11 +15023,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14292 );15023 );
14293 },15024 },
14294 3...4 => if (self.hasFeature(.avx)) {15025 3...4 => if (self.hasFeature(.avx)) {
15026 const src_mcv = try self.resolveInst(ty_op.operand);
14295 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);15027 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14296 if (src_mcv.isMemory()) try self.asmRegisterMemory(15028 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14297 .{ .v_sd, .broadcast },15029 .{ .v_sd, .broadcast },
14298 dst_reg.to256(),15030 dst_reg.to256(),
14299 src_mcv.mem(.qword),15031 try src_mcv.mem(self, .qword),
14300 ) else {15032 ) else {
14301 const src_reg = if (src_mcv.isRegister())15033 const src_reg = if (src_mcv.isRegister())
14302 src_mcv.getReg().?15034 src_mcv.getReg().?
...@@ -14325,19 +15057,21 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14325,19 +15057,21 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14325 },15057 },
14326 else => {},15058 else => {},
14327 },15059 },
14328 128 => switch (vector_ty.vectorLen(mod)) {15060 128 => switch (vector_len) {
14329 1 => {15061 1 => {
15062 const src_mcv = try self.resolveInst(ty_op.operand);
14330 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;15063 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14331 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);15064 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14332 try self.genSetReg(dst_reg, scalar_ty, src_mcv);15065 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14333 break :result .{ .register = dst_reg };15066 break :result .{ .register = dst_reg };
14334 },15067 },
14335 2 => if (self.hasFeature(.avx)) {15068 2 => if (self.hasFeature(.avx)) {
15069 const src_mcv = try self.resolveInst(ty_op.operand);
14336 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);15070 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14337 if (src_mcv.isMemory()) try self.asmRegisterMemory(15071 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14338 .{ .v_f128, .broadcast },15072 .{ .v_f128, .broadcast },
14339 dst_reg.to256(),15073 dst_reg.to256(),
14340 src_mcv.mem(.xword),15074 try src_mcv.mem(self, .xword),
14341 ) else {15075 ) else {
14342 const src_reg = if (src_mcv.isRegister())15076 const src_reg = if (src_mcv.isRegister())
14343 src_mcv.getReg().?15077 src_mcv.getReg().?
...@@ -14389,7 +15123,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {...@@ -14389,7 +15123,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14389 try self.spillEflagsIfOccupied();15123 try self.spillEflagsIfOccupied();
1439015124
14391 const operand_mcv = try self.resolveInst(reduce.operand);15125 const operand_mcv = try self.resolveInst(reduce.operand);
14392 const mask_len = (std.math.cast(u6, operand_ty.vectorLen(mod)) orelse15126 const mask_len = (math.cast(u6, operand_ty.vectorLen(mod)) orelse
14393 return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(mod)}));15127 return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(mod)}));
14394 const mask = (@as(u64, 1) << mask_len) - 1;15128 const mask = (@as(u64, 1) << mask_len) - 1;
14395 const abi_size: u32 = @intCast(operand_ty.abiSize(mod));15129 const abi_size: u32 = @intCast(operand_ty.abiSize(mod));
...@@ -14397,7 +15131,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {...@@ -14397,7 +15131,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14397 .Or => {15131 .Or => {
14398 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(15132 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
14399 .{ ._, .@"test" },15133 .{ ._, .@"test" },
14400 operand_mcv.mem(Memory.Size.fromSize(abi_size)),15134 try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
14401 Immediate.u(mask),15135 Immediate.u(mask),
14402 ) else {15136 ) else {
14403 const operand_reg = registerAlias(if (operand_mcv.isRegister())15137 const operand_reg = registerAlias(if (operand_mcv.isRegister())
...@@ -14445,8 +15179,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -14445,8 +15179,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14445 const result: MCValue = result: {15179 const result: MCValue = result: {
14446 switch (result_ty.zigTypeTag(mod)) {15180 switch (result_ty.zigTypeTag(mod)) {
14447 .Struct => {15181 .Struct => {
14448 const frame_index =15182 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14449 try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
14450 if (result_ty.containerLayout(mod) == .Packed) {15183 if (result_ty.containerLayout(mod) == .Packed) {
14451 const struct_type = mod.typeToStruct(result_ty).?;15184 const struct_type = mod.typeToStruct(result_ty).?;
14452 try self.genInlineMemset(15185 try self.genInlineMemset(
...@@ -14542,8 +15275,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -14542,8 +15275,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14542 break :result .{ .load_frame = .{ .index = frame_index } };15275 break :result .{ .load_frame = .{ .index = frame_index } };
14543 },15276 },
14544 .Array => {15277 .Array => {
14545 const frame_index =15278 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14546 try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
14547 const elem_ty = result_ty.childType(mod);15279 const elem_ty = result_ty.childType(mod);
14548 const elem_size: u32 = @intCast(elem_ty.abiSize(mod));15280 const elem_size: u32 = @intCast(elem_ty.abiSize(mod));
1454915281
...@@ -14789,7 +15521,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -14789,7 +15521,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
14789 mir_tag,15521 mir_tag,
14790 mop1_reg,15522 mop1_reg,
14791 mop2_reg,15523 mop2_reg,
14792 mops[2].mem(Memory.Size.fromSize(abi_size)),15524 try mops[2].mem(self, Memory.Size.fromSize(abi_size)),
14793 );15525 );
14794 break :result mops[0];15526 break :result mops[0];
14795 };15527 };
...@@ -14807,7 +15539,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {...@@ -14807,7 +15539,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
14807 )) {15539 )) {
14808 .SysV => result: {15540 .SysV => result: {
14809 const info = self.va_info.sysv;15541 const info = self.va_info.sysv;
14810 const dst_fi = try self.allocFrameIndex(FrameAlloc.initType(va_list_ty, mod));15542 const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, mod));
14811 var field_off: u31 = 0;15543 var field_off: u31 = 0;
14812 // gp_offset: c_uint,15544 // gp_offset: c_uint,
14813 try self.genSetMem(15545 try self.genSetMem(
...@@ -15015,7 +15747,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -15015,7 +15747,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15015 .{ .v_ss, .cvtsd2 },15747 .{ .v_ss, .cvtsd2 },
15016 dst_reg,15748 dst_reg,
15017 dst_reg,15749 dst_reg,
15018 promote_mcv.mem(.qword),15750 try promote_mcv.mem(self, .qword),
15019 ) else try self.asmRegisterRegisterRegister(15751 ) else try self.asmRegisterRegisterRegister(
15020 .{ .v_ss, .cvtsd2 },15752 .{ .v_ss, .cvtsd2 },
15021 dst_reg,15753 dst_reg,
...@@ -15027,7 +15759,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -15027,7 +15759,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15027 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(15759 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
15028 .{ ._ss, .cvtsd2 },15760 .{ ._ss, .cvtsd2 },
15029 dst_reg,15761 dst_reg,
15030 promote_mcv.mem(.qword),15762 try promote_mcv.mem(self, .qword),
15031 ) else try self.asmRegisterRegister(15763 ) else try self.asmRegisterRegister(
15032 .{ ._ss, .cvtsd2 },15764 .{ ._ss, .cvtsd2 },
15033 dst_reg,15765 dst_reg,
...@@ -15473,6 +16205,33 @@ fn memSize(self: *Self, ty: Type) Memory.Size {...@@ -15473,6 +16205,33 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
15473 };16205 };
15474}16206}
1547516207
16208fn splitType(self: *Self, ty: Type) ![2]Type {
16209 const mod = self.bin_file.options.module.?;
16210 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
16211 var parts: [2]Type = undefined;
16212 if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| {
16213 part.* = switch (class) {
16214 .integer => switch (part_i) {
16215 0 => Type.u64,
16216 1 => part: {
16217 const elem_size = ty.abiAlignment(mod).minStrict(.@"8").toByteUnitsOptional().?;
16218 const elem_ty = try mod.intType(.unsigned, @intCast(elem_size * 8));
16219 break :part switch (@divExact(ty.abiSize(mod) - 8, elem_size)) {
16220 1 => elem_ty,
16221 else => |len| try mod.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
16222 };
16223 },
16224 else => unreachable,
16225 },
16226 .float => Type.f32,
16227 .float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),
16228 .sse => Type.f64,
16229 else => break,
16230 };
16231 } else if (parts[0].abiSize(mod) + parts[1].abiSize(mod) == ty.abiSize(mod)) return parts;
16232 return self.fail("TODO implement splitType for {}", .{ty.fmt(mod)});
16233}
16234
15476/// Truncates the value in the register in place.16235/// Truncates the value in the register in place.
15477/// Clobbers any remaining bits.16236/// Clobbers any remaining bits.
15478fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {16237fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
src/arch/x86_64/Encoding.zig+13-17
...@@ -62,8 +62,7 @@ pub fn findByMnemonic(...@@ -62,8 +62,7 @@ pub fn findByMnemonic(
62 .rex, .rex_short => if (!rex_required) continue,62 .rex, .rex_short => if (!rex_required) continue,
63 else => {},63 else => {},
64 }64 }
65 for (input_ops, data.ops) |input_op, data_op|65 for (input_ops, data.ops) |input_op, data_op| if (!input_op.isSubset(data_op)) continue :next;
66 if (!input_op.isSubset(data_op)) continue :next;
6766
68 const enc = Encoding{ .mnemonic = mnemonic, .data = data };67 const enc = Encoding{ .mnemonic = mnemonic, .data = data };
69 if (shortest_enc) |previous_shortest_enc| {68 if (shortest_enc) |previous_shortest_enc| {
...@@ -410,6 +409,8 @@ pub const Mnemonic = enum {...@@ -410,6 +409,8 @@ pub const Mnemonic = enum {
410 vfmadd132ps, vfmadd213ps, vfmadd231ps,409 vfmadd132ps, vfmadd213ps, vfmadd231ps,
411 vfmadd132sd, vfmadd213sd, vfmadd231sd,410 vfmadd132sd, vfmadd213sd, vfmadd231sd,
412 vfmadd132ss, vfmadd213ss, vfmadd231ss,411 vfmadd132ss, vfmadd213ss, vfmadd231ss,
412 // AVX2
413 vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
413 // zig fmt: on414 // zig fmt: on
414};415};
415416
...@@ -444,7 +445,7 @@ pub const Op = enum {...@@ -444,7 +445,7 @@ pub const Op = enum {
444 moffs,445 moffs,
445 sreg,446 sreg,
446 st, mm, mm_m64,447 st, mm, mm_m64,
447 xmm0, xmm, xmm_m32, xmm_m64, xmm_m128,448 xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128,
448 ymm, ymm_m256,449 ymm, ymm_m256,
449 // zig fmt: on450 // zig fmt: on
450451
...@@ -534,7 +535,7 @@ pub const Op = enum {...@@ -534,7 +535,7 @@ pub const Op = enum {
534 .eax, .r32, .rm32, .r32_m16 => unreachable,535 .eax, .r32, .rm32, .r32_m16 => unreachable,
535 .rax, .r64, .rm64, .r64_m16 => unreachable,536 .rax, .r64, .rm64, .r64_m16 => unreachable,
536 .st, .mm, .mm_m64 => unreachable,537 .st, .mm, .mm_m64 => unreachable,
537 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,538 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
538 .ymm, .ymm_m256 => unreachable,539 .ymm, .ymm_m256 => unreachable,
539 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,540 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
540 .unity => 1,541 .unity => 1,
...@@ -556,7 +557,7 @@ pub const Op = enum {...@@ -556,7 +557,7 @@ pub const Op = enum {
556 .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,557 .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
557 .rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,558 .rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,
558 .st => 80,559 .st => 80,
559 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,560 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
560 .ymm, .ymm_m256 => 256,561 .ymm, .ymm_m256 => 256,
561 };562 };
562 }563 }
...@@ -568,8 +569,8 @@ pub const Op = enum {...@@ -568,8 +569,8 @@ pub const Op = enum {
568 .rel8, .rel16, .rel32 => unreachable,569 .rel8, .rel16, .rel32 => unreachable,
569 .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64 => unreachable,570 .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64 => unreachable,
570 .st, .mm, .xmm0, .xmm, .ymm => unreachable,571 .st, .mm, .xmm0, .xmm, .ymm => unreachable,
571 .m8, .rm8, .r32_m8 => 8,572 .m8, .rm8, .r32_m8, .xmm_m8 => 8,
572 .m16, .rm16, .r32_m16, .r64_m16 => 16,573 .m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16,
573 .m32, .rm32, .xmm_m32 => 32,574 .m32, .rm32, .xmm_m32 => 32,
574 .m64, .rm64, .mm_m64, .xmm_m64 => 64,575 .m64, .rm64, .mm_m64, .xmm_m64 => 64,
575 .m80 => 80,576 .m80 => 80,
...@@ -600,7 +601,7 @@ pub const Op = enum {...@@ -600,7 +601,7 @@ pub const Op = enum {
600 .rm8, .rm16, .rm32, .rm64,601 .rm8, .rm16, .rm32, .rm64,
601 .r32_m8, .r32_m16, .r64_m16,602 .r32_m8, .r32_m16, .r64_m16,
602 .st, .mm, .mm_m64,603 .st, .mm, .mm_m64,
603 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128,604 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
604 .ymm, .ymm_m256,605 .ymm, .ymm_m256,
605 => true,606 => true,
606 else => false,607 else => false,
...@@ -629,7 +630,7 @@ pub const Op = enum {...@@ -629,7 +630,7 @@ pub const Op = enum {
629 .m8, .m16, .m32, .m64, .m80, .m128, .m256,630 .m8, .m16, .m32, .m64, .m80, .m128, .m256,
630 .m,631 .m,
631 .mm_m64,632 .mm_m64,
632 .xmm_m32, .xmm_m64, .xmm_m128,633 .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
633 .ymm_m256,634 .ymm_m256,
634 => true,635 => true,
635 else => false,636 else => false,
...@@ -654,7 +655,7 @@ pub const Op = enum {...@@ -654,7 +655,7 @@ pub const Op = enum {
654 .sreg => .segment,655 .sreg => .segment,
655 .st => .x87,656 .st => .x87,
656 .mm, .mm_m64 => .mmx,657 .mm, .mm_m64 => .mmx,
657 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,658 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
658 .ymm, .ymm_m256 => .sse,659 .ymm, .ymm_m256 => .sse,
659 };660 };
660 }661 }
...@@ -826,18 +827,13 @@ const mnemonic_to_encodings_map = init: {...@@ -826,18 +827,13 @@ const mnemonic_to_encodings_map = init: {
826 for (&data_storage, entries, 0..) |*data, entry, data_index| {827 for (&data_storage, entries, 0..) |*data, entry, data_index| {
827 data.* = .{828 data.* = .{
828 .op_en = entry[1],829 .op_en = entry[1],
829 .ops = undefined,830 .ops = (entry[2] ++ .{.none} ** (data.ops.len - entry[2].len)).*,
830 .opc_len = entry[3].len,831 .opc_len = entry[3].len,
831 .opc = undefined,832 .opc = (entry[3] ++ .{undefined} ** (data.opc.len - entry[3].len)).*,
832 .modrm_ext = entry[4],833 .modrm_ext = entry[4],
833 .mode = entry[5],834 .mode = entry[5],
834 .feature = entry[6],835 .feature = entry[6],
835 };836 };
836 // TODO: use `@memcpy` for these. When I did that, I got a false positive
837 // compile error for this copy happening at compile time.
838 std.mem.copyForwards(Op, &data.ops, entry[2]);
839 std.mem.copyForwards(u8, &data.opc, entry[3]);
840
841 while (mnemonic_int < @intFromEnum(entry[0])) : (mnemonic_int += 1) {837 while (mnemonic_int < @intFromEnum(entry[0])) : (mnemonic_int += 1) {
842 mnemonic_map[mnemonic_int] = data_storage[mnemonic_start..data_index];838 mnemonic_map[mnemonic_int] = data_storage[mnemonic_start..data_index];
843 mnemonic_start = data_index;839 mnemonic_start = data_index;
src/arch/x86_64/Mir.zig+2
...@@ -255,6 +255,8 @@ pub const Inst = struct {...@@ -255,6 +255,8 @@ pub const Inst = struct {
255 vp_q,255 vp_q,
256 /// VEX-Encoded Packed ___ Double Quadword256 /// VEX-Encoded Packed ___ Double Quadword
257 vp_dq,257 vp_dq,
258 /// VEX-Encoded Packed ___ Integer Data
259 vp_i128,
258 /// VEX-Encoded ___ Scalar Single-Precision Values260 /// VEX-Encoded ___ Scalar Single-Precision Values
259 v_ss,261 v_ss,
260 /// VEX-Encoded ___ Packed Single-Precision Values262 /// VEX-Encoded ___ Packed Single-Precision Values
src/arch/x86_64/bits.zig+2-2
...@@ -222,7 +222,7 @@ pub const Register = enum(u7) {...@@ -222,7 +222,7 @@ pub const Register = enum(u7) {
222 @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax),222 @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax),
223 @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax),223 @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax),
224 @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al),224 @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al),
225 @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah) - 4,225 @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah),
226226
227 @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0) - 16,227 @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0) - 16,
228 @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0) - 16,228 @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0) - 16,
...@@ -237,7 +237,7 @@ pub const Register = enum(u7) {...@@ -237,7 +237,7 @@ pub const Register = enum(u7) {
237 return @intCast(@intFromEnum(reg) - base);237 return @intCast(@intFromEnum(reg) - base);
238 }238 }
239239
240 pub fn bitSize(reg: Register) u64 {240 pub fn bitSize(reg: Register) u10 {
241 return switch (@intFromEnum(reg)) {241 return switch (@intFromEnum(reg)) {
242 // zig fmt: off242 // zig fmt: off
243 @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => 64,243 @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => 64,
src/arch/x86_64/encodings.zig+10
...@@ -1742,6 +1742,16 @@ pub const table = [_]Entry{...@@ -1742,6 +1742,16 @@ pub const table = [_]Entry{
17421742
1743 .{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },1743 .{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
17441744
1745 .{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
1746 .{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
1747 .{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
1748 .{ .vpbroadcastw, .rm, &.{ .ymm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_256_w0, .avx2 },
1749 .{ .vpbroadcastd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_128_w0, .avx2 },
1750 .{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
1751 .{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
1752 .{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
1753 .{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
1754
1745 .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },1755 .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
1746 .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },1756 .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
1747 .{ .vpcmpeqd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 },1757 .{ .vpcmpeqd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 },
src/codegen.zig+4-1
...@@ -376,7 +376,10 @@ pub fn generateSymbol(...@@ -376,7 +376,10 @@ pub fn generateSymbol(
376 .val = switch (aggregate.storage) {376 .val = switch (aggregate.storage) {
377 .bytes => unreachable,377 .bytes => unreachable,
378 .elems => |elems| elems[@as(usize, @intCast(index))],378 .elems => |elems| elems[@as(usize, @intCast(index))],
379 .repeated_elem => |elem| elem,379 .repeated_elem => |elem| if (index < array_type.len)
380 elem
381 else
382 array_type.sentinel,
380 }.toValue(),383 }.toValue(),
381 }, code, debug_output, reloc_info)) {384 }, code, debug_output, reloc_info)) {
382 .ok => {},385 .ok => {},
src/link/Coff.zig+1
...@@ -388,6 +388,7 @@ fn populateMissingMetadata(self: *Coff) !void {...@@ -388,6 +388,7 @@ fn populateMissingMetadata(self: *Coff) !void {
388 self.rdata_section_index = try self.allocateSection(".rdata", file_size, .{388 self.rdata_section_index = try self.allocateSection(".rdata", file_size, .{
389 .CNT_INITIALIZED_DATA = 1,389 .CNT_INITIALIZED_DATA = 1,
390 .MEM_READ = 1,390 .MEM_READ = 1,
391 .MEM_WRITE = 1,
391 });392 });
392 }393 }
393394
src/main.zig+1
...@@ -6492,6 +6492,7 @@ pub fn cmdDumpZir(...@@ -6492,6 +6492,7 @@ pub fn cmdDumpZir(
6492 .mod = undefined,6492 .mod = undefined,
6493 .root_decl = .none,6493 .root_decl = .none,
6494 };6494 };
6495 defer file.zir.deinit(gpa);
64956496
6496 {6497 {
6497 const instruction_bytes = file.zir.instructions.len *6498 const instruction_bytes = file.zir.instructions.len *
src/print_zir.zig+6-9
...@@ -2456,20 +2456,15 @@ const Writer = struct {...@@ -2456,20 +2456,15 @@ const Writer = struct {
2456 const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].save_err_ret_index;2456 const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].save_err_ret_index;
24572457
2458 try self.writeInstRef(stream, inst_data.operand);2458 try self.writeInstRef(stream, inst_data.operand);
2459
2459 try stream.writeAll(")");2460 try stream.writeAll(")");
2460 }2461 }
24612462
2462 fn writeRestoreErrRetIndex(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {2463 fn writeRestoreErrRetIndex(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {
2463 const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].restore_err_ret_index;2464 const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].restore_err_ret_index;
24642465
2465 if (inst_data.block != .none) {2466 try self.writeInstRef(stream, inst_data.block);
2466 try self.writeInstRef(stream, inst_data.block);2467 try self.writeInstRef(stream, inst_data.operand);
2467 }
2468
2469 if (inst_data.operand != .none) {
2470 if (inst_data.block != .none) try stream.writeAll(", ");
2471 try self.writeInstRef(stream, inst_data.operand);
2472 }
24732468
2474 try stream.writeAll(")");2469 try stream.writeAll(")");
2475 }2470 }
...@@ -2615,7 +2610,9 @@ const Writer = struct {...@@ -2615,7 +2610,9 @@ const Writer = struct {
2615 }2610 }
26162611
2617 fn writeInstRef(self: *Writer, stream: anytype, ref: Zir.Inst.Ref) !void {2612 fn writeInstRef(self: *Writer, stream: anytype, ref: Zir.Inst.Ref) !void {
2618 if (ref.toIndex()) |i| {2613 if (ref == .none) {
2614 return stream.writeAll(".none");
2615 } else if (ref.toIndex()) |i| {
2619 return self.writeInstIndex(stream, i);2616 return self.writeInstIndex(stream, i);
2620 } else {2617 } else {
2621 const val: InternPool.Index = @enumFromInt(@intFromEnum(ref));2618 const val: InternPool.Index = @enumFromInt(@intFromEnum(ref));
src/register_manager.zig+102-60
...@@ -55,6 +55,7 @@ pub fn RegisterManager(...@@ -55,6 +55,7 @@ pub fn RegisterManager(
55 const Self = @This();55 const Self = @This();
5656
57 pub const TrackedRegisters = [tracked_registers.len]Air.Inst.Index;57 pub const TrackedRegisters = [tracked_registers.len]Air.Inst.Index;
58 pub const TrackedIndex = std.math.IntFittingRange(0, tracked_registers.len - 1);
58 pub const RegisterBitSet = StaticBitSet(tracked_registers.len);59 pub const RegisterBitSet = StaticBitSet(tracked_registers.len);
5960
60 fn getFunction(self: *Self) *Function {61 fn getFunction(self: *Self) *Function {
...@@ -66,45 +67,64 @@ pub fn RegisterManager(...@@ -66,45 +67,64 @@ pub fn RegisterManager(
66 return !register_class.isSet(index);67 return !register_class.isSet(index);
67 }68 }
6869
70 fn markRegIndexAllocated(self: *Self, tracked_index: TrackedIndex) void {
71 self.allocated_registers.set(tracked_index);
72 }
69 fn markRegAllocated(self: *Self, reg: Register) void {73 fn markRegAllocated(self: *Self, reg: Register) void {
70 const index = indexOfRegIntoTracked(reg) orelse return;74 self.markRegIndexAllocated(indexOfRegIntoTracked(reg) orelse return);
71 self.allocated_registers.set(index);
72 }75 }
7376
77 fn markRegIndexUsed(self: *Self, tracked_index: TrackedIndex) void {
78 self.free_registers.unset(tracked_index);
79 }
74 fn markRegUsed(self: *Self, reg: Register) void {80 fn markRegUsed(self: *Self, reg: Register) void {
75 const index = indexOfRegIntoTracked(reg) orelse return;81 self.markRegIndexUsed(indexOfRegIntoTracked(reg) orelse return);
76 self.free_registers.unset(index);
77 }82 }
7883
84 fn markRegIndexFree(self: *Self, tracked_index: TrackedIndex) void {
85 self.free_registers.set(tracked_index);
86 }
79 fn markRegFree(self: *Self, reg: Register) void {87 fn markRegFree(self: *Self, reg: Register) void {
80 const index = indexOfRegIntoTracked(reg) orelse return;88 self.markRegIndexFree(indexOfRegIntoTracked(reg) orelse return);
81 self.free_registers.set(index);
82 }89 }
8390
84 pub fn indexOfReg(91 pub fn indexOfReg(
85 comptime registers: []const Register,92 comptime set: []const Register,
86 reg: Register,93 reg: Register,
87 ) ?std.math.IntFittingRange(0, registers.len - 1) {94 ) ?std.math.IntFittingRange(0, set.len - 1) {
88 inline for (tracked_registers, 0..) |cpreg, i| {95 const Id = @TypeOf(reg.id());
89 if (reg.id() == cpreg.id()) return i;96 comptime var min_id: Id = std.math.maxInt(Id);
97 comptime var max_id: Id = std.math.minInt(Id);
98 inline for (set) |elem| {
99 const elem_id = comptime elem.id();
100 min_id = @min(elem_id, min_id);
101 max_id = @max(elem_id, max_id);
90 }102 }
91 return null;103
104 const OptionalIndex = std.math.IntFittingRange(0, set.len);
105 comptime var map = [1]OptionalIndex{set.len} ** (max_id + 1 - min_id);
106 inline for (set, 0..) |elem, elem_index| map[comptime elem.id() - min_id] = elem_index;
107
108 const id_index = reg.id() -% min_id;
109 if (id_index >= map.len) return null;
110 const set_index = map[id_index];
111 return if (set_index < set.len) @intCast(set_index) else null;
92 }112 }
93113
94 pub fn indexOfRegIntoTracked(114 pub fn indexOfRegIntoTracked(reg: Register) ?TrackedIndex {
95 reg: Register,
96 ) ?std.math.IntFittingRange(0, tracked_registers.len) {
97 return indexOfReg(tracked_registers, reg);115 return indexOfReg(tracked_registers, reg);
98 }116 }
99117
100 pub fn regAtTrackedIndex(index: std.math.IntFittingRange(0, tracked_registers.len)) Register {118 pub fn regAtTrackedIndex(tracked_index: TrackedIndex) Register {
101 return tracked_registers[index];119 return tracked_registers[tracked_index];
102 }120 }
103121
104 /// Returns true when this register is not tracked122 /// Returns true when this register is not tracked
123 pub fn isRegIndexFree(self: Self, tracked_index: TrackedIndex) bool {
124 return self.free_registers.isSet(tracked_index);
125 }
105 pub fn isRegFree(self: Self, reg: Register) bool {126 pub fn isRegFree(self: Self, reg: Register) bool {
106 const index = indexOfRegIntoTracked(reg) orelse return true;127 return self.isRegIndexFree(indexOfRegIntoTracked(reg) orelse return true);
107 return self.free_registers.isSet(index);
108 }128 }
109129
110 /// Returns whether this register was allocated in the course130 /// Returns whether this register was allocated in the course
...@@ -119,14 +139,14 @@ pub fn RegisterManager(...@@ -119,14 +139,14 @@ pub fn RegisterManager(
119 /// Returns whether this register is locked139 /// Returns whether this register is locked
120 ///140 ///
121 /// Returns false when this register is not tracked141 /// Returns false when this register is not tracked
142 fn isRegIndexLocked(self: Self, tracked_index: TrackedIndex) bool {
143 return self.locked_registers.isSet(tracked_index);
144 }
122 pub fn isRegLocked(self: Self, reg: Register) bool {145 pub fn isRegLocked(self: Self, reg: Register) bool {
123 const index = indexOfRegIntoTracked(reg) orelse return false;146 return self.isRegIndexLocked(indexOfRegIntoTracked(reg) orelse return false);
124 return self.locked_registers.isSet(index);
125 }147 }
126148
127 pub const RegisterLock = struct {149 pub const RegisterLock = struct { tracked_index: TrackedIndex };
128 register: Register,
129 };
130150
131 /// Prevents the register from being allocated until they are151 /// Prevents the register from being allocated until they are
132 /// unlocked again.152 /// unlocked again.
...@@ -134,25 +154,29 @@ pub fn RegisterManager(...@@ -134,25 +154,29 @@ pub fn RegisterManager(
134 /// locked, or `null` otherwise.154 /// locked, or `null` otherwise.
135 /// Only the owner of the `RegisterLock` can unlock the155 /// Only the owner of the `RegisterLock` can unlock the
136 /// register later.156 /// register later.
137 pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {157 pub fn lockRegIndex(self: *Self, tracked_index: TrackedIndex) ?RegisterLock {
138 log.debug("locking {}", .{reg});158 log.debug("locking {}", .{regAtTrackedIndex(tracked_index)});
139 if (self.isRegLocked(reg)) {159 if (self.isRegIndexLocked(tracked_index)) {
140 log.debug(" register already locked", .{});160 log.debug(" register already locked", .{});
141 return null;161 return null;
142 }162 }
143 const index = indexOfRegIntoTracked(reg) orelse return null;163 self.locked_registers.set(tracked_index);
144 self.locked_registers.set(index);164 return RegisterLock{ .tracked_index = tracked_index };
145 return RegisterLock{ .register = reg };165 }
166 pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
167 return self.lockRegIndex(indexOfRegIntoTracked(reg) orelse return null);
146 }168 }
147169
148 /// Like `lockReg` but asserts the register was unused always170 /// Like `lockReg` but asserts the register was unused always
149 /// returning a valid lock.171 /// returning a valid lock.
172 pub fn lockRegIndexAssumeUnused(self: *Self, tracked_index: TrackedIndex) RegisterLock {
173 log.debug("locking asserting free {}", .{regAtTrackedIndex(tracked_index)});
174 assert(!self.isRegIndexLocked(tracked_index));
175 self.locked_registers.set(tracked_index);
176 return RegisterLock{ .tracked_index = tracked_index };
177 }
150 pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {178 pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
151 log.debug("locking asserting free {}", .{reg});179 return self.lockRegIndexAssumeUnused(indexOfRegIntoTracked(reg) orelse unreachable);
152 assert(!self.isRegLocked(reg));
153 const index = indexOfRegIntoTracked(reg) orelse unreachable;
154 self.locked_registers.set(index);
155 return RegisterLock{ .register = reg };
156 }180 }
157181
158 /// Like `lockReg` but locks multiple registers.182 /// Like `lockReg` but locks multiple registers.
...@@ -181,9 +205,8 @@ pub fn RegisterManager(...@@ -181,9 +205,8 @@ pub fn RegisterManager(
181 /// Requires `RegisterLock` to unlock a register.205 /// Requires `RegisterLock` to unlock a register.
182 /// Call `lockReg` to obtain the lock first.206 /// Call `lockReg` to obtain the lock first.
183 pub fn unlockReg(self: *Self, lock: RegisterLock) void {207 pub fn unlockReg(self: *Self, lock: RegisterLock) void {
184 log.debug("unlocking {}", .{lock.register});208 log.debug("unlocking {}", .{regAtTrackedIndex(lock.tracked_index)});
185 const index = indexOfRegIntoTracked(lock.register) orelse return;209 self.locked_registers.unset(lock.tracked_index);
186 self.locked_registers.unset(index);
187 }210 }
188211
189 /// Returns true when at least one register is locked212 /// Returns true when at least one register is locked
...@@ -319,44 +342,63 @@ pub fn RegisterManager(...@@ -319,44 +342,63 @@ pub fn RegisterManager(
319 /// Spills the register if it is currently allocated. If a342 /// Spills the register if it is currently allocated. If a
320 /// corresponding instruction is passed, will also track this343 /// corresponding instruction is passed, will also track this
321 /// register.344 /// register.
322 pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {345 fn getRegIndex(
323 const index = indexOfRegIntoTracked(reg) orelse return;346 self: *Self,
324 log.debug("getReg {} for inst {?}", .{ reg, inst });347 tracked_index: TrackedIndex,
325348 inst: ?Air.Inst.Index,
326 if (!self.isRegFree(reg)) {349 ) AllocateRegistersError!void {
327 self.markRegAllocated(reg);350 log.debug("getReg {} for inst {?}", .{ regAtTrackedIndex(tracked_index), inst });
351 if (!self.isRegIndexFree(tracked_index)) {
352 self.markRegIndexAllocated(tracked_index);
328353
329 // Move the instruction that was previously there to a354 // Move the instruction that was previously there to a
330 // stack allocation.355 // stack allocation.
331 const spilled_inst = self.registers[index];356 const spilled_inst = self.registers[tracked_index];
332 if (inst) |tracked_inst| self.registers[index] = tracked_inst;357 if (inst) |tracked_inst| self.registers[tracked_index] = tracked_inst;
333 try self.getFunction().spillInstruction(reg, spilled_inst);358 try self.getFunction().spillInstruction(regAtTrackedIndex(tracked_index), spilled_inst);
334 if (inst == null) self.freeReg(reg);359 if (inst == null) self.freeRegIndex(tracked_index);
335 } else self.getRegAssumeFree(reg, inst);360 } else self.getRegIndexAssumeFree(tracked_index, inst);
361 }
362 pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
363 return self.getRegIndex(indexOfRegIntoTracked(reg) orelse return, inst);
364 }
365 pub fn getKnownReg(
366 self: *Self,
367 comptime reg: Register,
368 inst: ?Air.Inst.Index,
369 ) AllocateRegistersError!void {
370 return self.getRegIndex((comptime indexOfRegIntoTracked(reg)) orelse return, inst);
336 }371 }
337372
338 /// Allocates the specified register with the specified373 /// Allocates the specified register with the specified
339 /// instruction. Asserts that the register is free and no374 /// instruction. Asserts that the register is free and no
340 /// spilling is necessary.375 /// spilling is necessary.
341 pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {376 fn getRegIndexAssumeFree(
342 const index = indexOfRegIntoTracked(reg) orelse return;377 self: *Self,
343 log.debug("getRegAssumeFree {} for inst {?}", .{ reg, inst });378 tracked_index: TrackedIndex,
344 self.markRegAllocated(reg);379 inst: ?Air.Inst.Index,
380 ) void {
381 log.debug("getRegAssumeFree {} for inst {?}", .{ regAtTrackedIndex(tracked_index), inst });
382 self.markRegIndexAllocated(tracked_index);
345383
346 assert(self.isRegFree(reg));384 assert(self.isRegIndexFree(tracked_index));
347 if (inst) |tracked_inst| {385 if (inst) |tracked_inst| {
348 self.registers[index] = tracked_inst;386 self.registers[tracked_index] = tracked_inst;
349 self.markRegUsed(reg);387 self.markRegIndexUsed(tracked_index);
350 }388 }
351 }389 }
390 pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
391 self.getRegIndexAssumeFree(indexOfRegIntoTracked(reg) orelse return, inst);
392 }
352393
353 /// Marks the specified register as free394 /// Marks the specified register as free
395 fn freeRegIndex(self: *Self, tracked_index: TrackedIndex) void {
396 log.debug("freeing register {}", .{regAtTrackedIndex(tracked_index)});
397 self.registers[tracked_index] = undefined;
398 self.markRegIndexFree(tracked_index);
399 }
354 pub fn freeReg(self: *Self, reg: Register) void {400 pub fn freeReg(self: *Self, reg: Register) void {
355 const index = indexOfRegIntoTracked(reg) orelse return;401 self.freeRegIndex(indexOfRegIntoTracked(reg) orelse return);
356 log.debug("freeing register {}", .{reg});
357
358 self.registers[index] = undefined;
359 self.markRegFree(reg);
360 }402 }
361 };403 };
362}404}
test/behavior/vector.zig-1
...@@ -1260,7 +1260,6 @@ test "zero multiplicand" {...@@ -1260,7 +1260,6 @@ test "zero multiplicand" {
12601260
1261test "@intCast to u0" {1261test "@intCast to u0" {
1262 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1262 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1263 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1264 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1263 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1265 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1264 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1266 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1265 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO